From 618dc8349c6ee9fbc0d18c1f87d908b3ff7ef4d5 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Thu, 10 Aug 2023 02:40:02 -0400 Subject: [PATCH] fix: take more than requested --- packages/backend/src/db/scylla.ts | 2 ++ .../api/endpoints/notes/global-timeline.ts | 18 ++++++++++++++---- .../api/endpoints/notes/hybrid-timeline.ts | 18 ++++++++++++++---- .../api/endpoints/notes/local-timeline.ts | 18 ++++++++++++++---- .../src/server/api/endpoints/notes/renotes.ts | 19 +++++++++++++++---- .../server/api/endpoints/notes/timeline.ts | 18 ++++++++++++++---- 6 files changed, 73 insertions(+), 20 deletions(-) diff --git a/packages/backend/src/db/scylla.ts b/packages/backend/src/db/scylla.ts index 41f667f1f2..8808d0692f 100644 --- a/packages/backend/src/db/scylla.ts +++ b/packages/backend/src/db/scylla.ts @@ -280,6 +280,8 @@ export function prepareNoteQuery(ps: { queryParts.push(`AND "createdAt" > ?`); } + queryParts.push("LIMIT 100"); + const query = queryParts.join(" "); return { diff --git a/packages/backend/src/server/api/endpoints/notes/global-timeline.ts b/packages/backend/src/server/api/endpoints/notes/global-timeline.ts index 5f504aeed8..2220ea7e18 100644 --- a/packages/backend/src/server/api/endpoints/notes/global-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/global-timeline.ts @@ -138,10 +138,20 @@ export default define(meta, paramDef, async (ps, user) => { return filtered; }; - const foundNotes = await execNotePaginationQuery(ps, filter); - return await Notes.packMany(foundNotes.slice(0, ps.limit), user, { - scyllaNote: true, - }); + const foundPacked = []; + while (foundPacked.length < ps.limit) { + const foundNotes = (await execNotePaginationQuery(ps, filter)).slice( + 0, + ps.limit * 1.5, + ); // Some may filtered out by Notes.packMany, thus we take more than ps.limit. + foundPacked.push( + ...(await Notes.packMany(foundNotes, user, { scyllaNote: true })), + ); + if (foundNotes.length < ps.limit) break; + ps.untilDate = foundNotes[foundNotes.length - 1].createdAt.getTime(); + } + + return foundPacked.slice(0, ps.limit); } //#region Construct query diff --git a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts index 9b97b20b28..ab30adc5b2 100644 --- a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts @@ -165,10 +165,20 @@ export default define(meta, paramDef, async (ps, user) => { return filtered; }; - const foundNotes = await execNotePaginationQuery(ps, filter); - return await Notes.packMany(foundNotes.slice(0, ps.limit), user, { - scyllaNote: true, - }); + const foundPacked = []; + while (foundPacked.length < ps.limit) { + const foundNotes = (await execNotePaginationQuery(ps, filter)).slice( + 0, + ps.limit * 1.5, + ); // Some may filtered out by Notes.packMany, thus we take more than ps.limit. + foundPacked.push( + ...(await Notes.packMany(foundNotes, user, { scyllaNote: true })), + ); + if (foundNotes.length < ps.limit) break; + ps.untilDate = foundNotes[foundNotes.length - 1].createdAt.getTime(); + } + + return foundPacked.slice(0, ps.limit); } //#region Construct query diff --git a/packages/backend/src/server/api/endpoints/notes/local-timeline.ts b/packages/backend/src/server/api/endpoints/notes/local-timeline.ts index 246548c983..89a913edae 100644 --- a/packages/backend/src/server/api/endpoints/notes/local-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/local-timeline.ts @@ -180,10 +180,20 @@ export default define(meta, paramDef, async (ps, user) => { return filtered; }; - const foundNotes = await execNotePaginationQuery(ps, filter); - return await Notes.packMany(foundNotes.slice(0, ps.limit), user, { - scyllaNote: true, - }); + const foundPacked = []; + while (foundPacked.length < ps.limit) { + const foundNotes = (await execNotePaginationQuery(ps, filter)).slice( + 0, + ps.limit * 1.5, + ); // Some may filtered out by Notes.packMany, thus we take more than ps.limit. + foundPacked.push( + ...(await Notes.packMany(foundNotes, user, { scyllaNote: true })), + ); + if (foundNotes.length < ps.limit) break; + ps.untilDate = foundNotes[foundNotes.length - 1].createdAt.getTime(); + } + + return foundPacked.slice(0, ps.limit); } //#region Construct query diff --git a/packages/backend/src/server/api/endpoints/notes/renotes.ts b/packages/backend/src/server/api/endpoints/notes/renotes.ts index 5eb320e247..e5da64c7fe 100644 --- a/packages/backend/src/server/api/endpoints/notes/renotes.ts +++ b/packages/backend/src/server/api/endpoints/notes/renotes.ts @@ -102,10 +102,21 @@ export default define(meta, paramDef, async (ps, user) => { return filtered; }; - const foundNotes = await execNotePaginationQuery(ps, filter, 1); - return await Notes.packMany(foundNotes.slice(0, ps.limit), user, { - scyllaNote: true, - }); + const foundPacked = []; + let untilDate: number | undefined; + while (foundPacked.length < ps.limit) { + const foundNotes = (await execNotePaginationQuery({...ps, untilDate}, filter)).slice( + 0, + ps.limit * 1.5, + ); // Some may filtered out by Notes.packMany, thus we take more than ps.limit. + foundPacked.push( + ...(await Notes.packMany(foundNotes, user, { scyllaNote: true })), + ); + if (foundNotes.length < ps.limit) break; + untilDate = foundNotes[foundNotes.length - 1].createdAt.getTime(); + } + + return foundPacked.slice(0, ps.limit); } const query = makePaginationQuery( diff --git a/packages/backend/src/server/api/endpoints/notes/timeline.ts b/packages/backend/src/server/api/endpoints/notes/timeline.ts index 50e4adbe89..131bfa333c 100644 --- a/packages/backend/src/server/api/endpoints/notes/timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/timeline.ts @@ -152,10 +152,20 @@ export default define(meta, paramDef, async (ps, user) => { return filtered; }; - const foundNotes = await execNotePaginationQuery(ps, filter); - return await Notes.packMany(foundNotes.slice(0, ps.limit), user, { - scyllaNote: true, - }); + const foundPacked = []; + while (foundPacked.length < ps.limit) { + const foundNotes = (await execNotePaginationQuery(ps, filter)).slice( + 0, + ps.limit * 1.5, + ); // Some may filtered out by Notes.packMany, thus we take more than ps.limit. + foundPacked.push( + ...(await Notes.packMany(foundNotes, user, { scyllaNote: true })), + ); + if (foundNotes.length < ps.limit) break; + ps.untilDate = foundNotes[foundNotes.length - 1].createdAt.getTime(); + } + + return foundPacked.slice(0, ps.limit); } const hasFollowing = await followingsCache.exists();