From dfcac3750c28a1be42b59e794e630157858220c4 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Wed, 9 Aug 2023 06:03:09 -0400 Subject: [PATCH] perf: read timelines from scylla --- .../api/endpoints/notes/global-timeline.ts | 82 ++++++++++-- .../api/endpoints/notes/hybrid-timeline.ts | 102 ++++++++++++++- .../endpoints/notes/recommended-timeline.ts | 118 ++++++++++++++++-- 3 files changed, 277 insertions(+), 25 deletions(-) diff --git a/packages/backend/src/server/api/endpoints/notes/global-timeline.ts b/packages/backend/src/server/api/endpoints/notes/global-timeline.ts index 03a5535a8c..f660dd8f2b 100644 --- a/packages/backend/src/server/api/endpoints/notes/global-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/global-timeline.ts @@ -1,5 +1,5 @@ import { fetchMeta } from "@/misc/fetch-meta.js"; -import { Notes } from "@/models/index.js"; +import { Notes, UserProfiles } from "@/models/index.js"; import { activeUsersChart } from "@/services/chart/index.js"; import define from "../../define.js"; import { ApiError } from "../../error.js"; @@ -9,6 +9,23 @@ import { generateRepliesQuery } from "../../common/generate-replies-query.js"; import { generateMutedNoteQuery } from "../../common/generate-muted-note-query.js"; import { generateBlockedUserQuery } from "../../common/generate-block-query.js"; import { generateMutedUserRenotesQueryForNotes } from "../../common/generated-muted-renote-query.js"; +import { + ScyllaNote, + execTimelineQuery, + filterBlockedUser, + filterMutedNote, + filterMutedRenotes, + filterMutedUser, + filterReply, + scyllaClient, +} from "@/db/scylla.js"; +import { + InstanceMutingsCache, + RenoteMutingsCache, + UserBlockedCache, + UserMutingsCache, + userWordMuteCache, +} from "@/misc/cache.js"; export const meta = { tags: ["notes"], @@ -70,6 +87,63 @@ export default define(meta, paramDef, async (ps, user) => { } } + process.nextTick(() => { + if (user) { + activeUsersChart.read(user); + } + }); + + if (scyllaClient) { + let [mutedUserIds, mutedInstances, blockerIds, renoteMutedIds]: string[][] = + []; + let mutedWords: string[][]; + if (user) { + [mutedUserIds, mutedInstances, mutedWords, blockerIds, renoteMutedIds] = + await Promise.all([ + UserMutingsCache.init(user.id).then((cache) => cache.getAll()), + InstanceMutingsCache.init(user.id).then((cache) => cache.getAll()), + userWordMuteCache + .fetchMaybe(user.id, () => + UserProfiles.findOne({ + select: ["mutedWords"], + where: { userId: user.id }, + }).then((profile) => profile?.mutedWords), + ) + .then((words) => words ?? []), + UserBlockedCache.init(user.id).then((cache) => cache.getAll()), + RenoteMutingsCache.init(user.id).then((cache) => cache.getAll()), + ]); + } + + const filter = async (notes: ScyllaNote[]) => { + let filtered = notes.filter( + (n) => n.visibility === "public" && !n.channelId, + ); + filtered = await filterReply(filtered, ps.withReplies, user); + if (user) { + filtered = await filterMutedUser( + filtered, + user, + mutedUserIds, + mutedInstances, + ); + filtered = await filterMutedNote(filtered, user, mutedWords); + filtered = await filterBlockedUser(filtered, user, blockerIds); + filtered = await filterMutedRenotes(filtered, user, renoteMutedIds); + } + if (ps.withFiles) { + filtered = filtered.filter((n) => n.files.length > 0); + } + filtered = filtered.filter((n) => n.visibility !== "hidden"); + return filtered; + }; + + const foundNotes = await execTimelineQuery(ps, filter); + return await Notes.packMany(foundNotes.slice(0, ps.limit), user, { + scyllaNote: true, + }); + } + //#region Construct query const query = makePaginationQuery( Notes.createQueryBuilder("note"), @@ -95,12 +169,6 @@ export default define(meta, paramDef, async (ps, user) => { query.andWhere("note.visibility != 'hidden'"); //#endregion - process.nextTick(() => { - if (user) { - activeUsersChart.read(user); - } - }); - // We fetch more than requested because some may be filtered out, and if there's less than // requested, the pagination stops. const found = []; diff --git a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts index ca2c516f3a..4443946509 100644 --- a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts @@ -1,6 +1,6 @@ import { Brackets } from "typeorm"; import { fetchMeta } from "@/misc/fetch-meta.js"; -import { Followings, Notes } from "@/models/index.js"; +import { Followings, Notes, UserProfiles } from "@/models/index.js"; import { activeUsersChart } from "@/services/chart/index.js"; import define from "../../define.js"; import { ApiError } from "../../error.js"; @@ -12,6 +12,27 @@ import { generateMutedNoteQuery } from "../../common/generate-muted-note-query.j import { generateChannelQuery } from "../../common/generate-channel-query.js"; import { generateBlockedUserQuery } from "../../common/generate-block-query.js"; import { generateMutedUserRenotesQueryForNotes } from "../../common/generated-muted-renote-query.js"; +import { + ScyllaNote, + execTimelineQuery, + filterBlockedUser, + filterChannel, + filterMutedNote, + filterMutedRenotes, + filterMutedUser, + filterReply, + filterVisibility, + scyllaClient, +} from "@/db/scylla.js"; +import { + ChannelFollowingsCache, + InstanceMutingsCache, + LocalFollowingsCache, + RenoteMutingsCache, + UserBlockedCache, + UserMutingsCache, + userWordMuteCache, +} from "@/misc/cache.js"; export const meta = { tags: ["notes"], @@ -75,6 +96,81 @@ export default define(meta, paramDef, async (ps, user) => { throw new ApiError(meta.errors.stlDisabled); } + process.nextTick(() => { + activeUsersChart.read(user); + }); + + if (scyllaClient) { + const [ + followingChannelIds, + followingUserIds, + mutedUserIds, + mutedInstances, + mutedWords, + blockerIds, + renoteMutedIds, + ] = await Promise.all([ + ChannelFollowingsCache.init(user.id).then((cache) => cache.getAll()), + LocalFollowingsCache.init(user.id).then((cache) => cache.getAll()), + UserMutingsCache.init(user.id).then((cache) => cache.getAll()), + InstanceMutingsCache.init(user.id).then((cache) => cache.getAll()), + userWordMuteCache + .fetchMaybe(user.id, () => + UserProfiles.findOne({ + select: ["mutedWords"], + where: { userId: user.id }, + }).then((profile) => profile?.mutedWords), + ) + .then((words) => words ?? []), + UserBlockedCache.init(user.id).then((cache) => cache.getAll()), + RenoteMutingsCache.init(user.id).then((cache) => cache.getAll()), + ]); + const validUserIds = [user.id].concat(followingUserIds); + const optFilter = (n: ScyllaNote) => + !n.renoteId || !!n.text || n.files.length > 0 || n.hasPoll; + + const filter = async (notes: ScyllaNote[]) => { + let filtered = notes.filter( + (n) => + validUserIds.includes(n.userId) || + (n.visibility === "public" && !n.userHost), + ); + filtered = await filterChannel(filtered, user, followingChannelIds); + filtered = await filterReply(filtered, ps.withReplies, user); + filtered = await filterVisibility(filtered, user, followingUserIds); + filtered = await filterMutedUser( + filtered, + user, + mutedUserIds, + mutedInstances, + ); + filtered = await filterMutedNote(filtered, user, mutedWords); + filtered = await filterBlockedUser(filtered, user, blockerIds); + filtered = await filterMutedRenotes(filtered, user, renoteMutedIds); + if (!ps.includeMyRenotes) { + filtered = filtered.filter((n) => n.userId !== user.id || optFilter(n)); + } + if (!ps.includeRenotedMyNotes) { + filtered = filtered.filter( + (n) => n.renoteUserId !== user.id || optFilter(n), + ); + } + if (!ps.includeLocalRenotes) { + filtered = filtered.filter((n) => n.renoteUserHost || optFilter(n)); + } + if (ps.withFiles) { + filtered = filtered.filter((n) => n.files.length > 0); + } + filtered = filtered.filter((n) => n.visibility !== "hidden"); + return filtered; + }; + + const foundNotes = await execTimelineQuery(ps, filter); + return await Notes.packMany(foundNotes.slice(0, ps.limit), user, { + scyllaNote: true, + }); + } + //#region Construct query const followingQuery = Followings.createQueryBuilder("following") .select("following.followeeId") @@ -154,10 +250,6 @@ export default define(meta, paramDef, async (ps, user) => { query.andWhere("note.visibility != 'hidden'"); //#endregion - process.nextTick(() => { - activeUsersChart.read(user); - }); - // We fetch more than requested because some may be filtered out, and if there's less than // requested, the pagination stops. const found = []; diff --git a/packages/backend/src/server/api/endpoints/notes/recommended-timeline.ts b/packages/backend/src/server/api/endpoints/notes/recommended-timeline.ts index 009151ed84..6bdc017482 100644 --- a/packages/backend/src/server/api/endpoints/notes/recommended-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/recommended-timeline.ts @@ -1,6 +1,6 @@ import { Brackets } from "typeorm"; import { fetchMeta } from "@/misc/fetch-meta.js"; -import { Notes } from "@/models/index.js"; +import { Notes, UserProfiles } from "@/models/index.js"; import { activeUsersChart } from "@/services/chart/index.js"; import define from "../../define.js"; import { ApiError } from "../../error.js"; @@ -12,6 +12,25 @@ import { generateMutedNoteQuery } from "../../common/generate-muted-note-query.j import { generateChannelQuery } from "../../common/generate-channel-query.js"; import { generateBlockedUserQuery } from "../../common/generate-block-query.js"; import { generateMutedUserRenotesQueryForNotes } from "../../common/generated-muted-renote-query.js"; +import { + ScyllaNote, + execTimelineQuery, + filterBlockedUser, + filterMutedNote, + filterMutedRenotes, + filterMutedUser, + filterReply, + filterVisibility, + scyllaClient, +} from "@/db/scylla.js"; +import { + InstanceMutingsCache, + LocalFollowingsCache, + RenoteMutingsCache, + UserBlockedCache, + UserMutingsCache, + userWordMuteCache, +} from "@/misc/cache.js"; export const meta = { tags: ["notes"], @@ -80,6 +99,90 @@ export default define(meta, paramDef, async (ps, user) => { } } + process.nextTick(() => { + if (user) { + activeUsersChart.read(user); + } + }); + + if (scyllaClient) { + let [ + followingUserIds, + mutedUserIds, + mutedInstances, + blockerIds, + renoteMutedIds, + ]: string[][] = []; + let mutedWords: string[][]; + if (user) { + [ + followingUserIds, + mutedUserIds, + mutedInstances, + mutedWords, + blockerIds, + renoteMutedIds, + ] = await Promise.all([ + LocalFollowingsCache.init(user.id).then((cache) => cache.getAll()), + UserMutingsCache.init(user.id).then((cache) => cache.getAll()), + InstanceMutingsCache.init(user.id).then((cache) => cache.getAll()), + userWordMuteCache + .fetchMaybe(user.id, () => + UserProfiles.findOne({ + select: ["mutedWords"], + where: { userId: user.id }, + }).then((profile) => profile?.mutedWords), + ) + .then((words) => words ?? []), + UserBlockedCache.init(user.id).then((cache) => cache.getAll()), + RenoteMutingsCache.init(user.id).then((cache) => cache.getAll()), + ]); + } + + const filter = async (notes: ScyllaNote[]) => { + let filtered = notes.filter( + (n) => + n.visibility === "public" && + n.userHost && + m.recommendedInstances.includes(n.userHost) && + !n.channelId, + ); + filtered = await filterReply(filtered, ps.withReplies, user); + filtered = await filterVisibility(filtered, user, followingUserIds); + if (user) { + filtered = await filterMutedUser( + filtered, + user, + mutedUserIds, + mutedInstances, + ); + filtered = await filterMutedNote(filtered, user, mutedWords); + filtered = await filterBlockedUser(filtered, user, blockerIds); + filtered = await filterMutedRenotes(filtered, user, renoteMutedIds); + } + if (ps.withFiles) { + filtered = filtered.filter((n) => n.files.length > 0); + } + if (ps.fileType) { + filtered = filtered.filter((n) => + n.files.some((f) => ps.fileType?.includes(f.type)), + ); + } + if (ps.excludeNsfw) { + filtered = filtered.filter( + (n) => !n.cw && n.files.every((f) => !f.isSensitive), + ); + } + filtered = filtered.filter((n) => n.visibility !== "hidden"); + return filtered; + }; + + const foundNotes = await execTimelineQuery(ps, filter); + return await Notes.packMany(foundNotes.slice(0, ps.limit), user, { + scyllaNote: true, + }); + } + //#region Construct query const query = makePaginationQuery( Notes.createQueryBuilder("note"), @@ -91,12 +194,7 @@ export default define(meta, paramDef, async (ps, user) => { .andWhere( `(note.userHost = ANY ('{"${m.recommendedInstances.join('","')}"}'))`, ) - .andWhere("(note.visibility = 'public')") - .innerJoinAndSelect("note.user", "user") - .leftJoinAndSelect("note.reply", "reply") - .leftJoinAndSelect("note.renote", "renote") - .leftJoinAndSelect("reply.user", "replyUser") - .leftJoinAndSelect("renote.user", "renoteUser"); + .andWhere("(note.visibility = 'public')"); generateChannelQuery(query, user); generateRepliesQuery(query, ps.withReplies, user); @@ -133,12 +231,6 @@ export default define(meta, paramDef, async (ps, user) => { query.andWhere("note.visibility != 'hidden'"); //#endregion - process.nextTick(() => { - if (user) { - activeUsersChart.read(user); - } - }); - // We fetch more than requested because some may be filtered out, and if there's less than // requested, the pagination stops. const found = [];