From 2937c7fa432daa50264f5bc43c4e22a7840c5324 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Tue, 1 Aug 2023 02:26:19 -0400 Subject: [PATCH] wip: read row timeline from scylla --- .../backend/src/models/repositories/note.ts | 61 ++++++++++-------- .../backend/src/server/api/common/getters.ts | 2 +- .../server/api/endpoints/notes/timeline.ts | 64 ++++++++++--------- 3 files changed, 69 insertions(+), 58 deletions(-) diff --git a/packages/backend/src/models/repositories/note.ts b/packages/backend/src/models/repositories/note.ts index 65d8c58f84..f014f897d7 100644 --- a/packages/backend/src/models/repositories/note.ts +++ b/packages/backend/src/models/repositories/note.ts @@ -183,24 +183,32 @@ export const NoteRepository = db.getRepository(Note).extend({ const meId = me ? me.id : null; let note: Note | null = null; - const noteId = typeof src === "object" ? src.id : src; - if (scyllaClient) { - const result = await scyllaClient.execute( - prepared.note.select.byId, - [[noteId]], - { prepare: true }, - ); - if (result.rowLength > 0) { - note = parseScyllaNote(result.first()); + let foundScyllaNote = false; + const isSrcNote = typeof src === "object"; + + // Always lookup from ScyllaDB if enabled + if (isSrcNote && !scyllaClient) { + note = src; + } else { + const noteId = isSrcNote ? src.id : src; + if (scyllaClient) { + const result = await scyllaClient.execute( + prepared.note.select.byId, + [[noteId]], + { prepare: true }, + ); + if (result.rowLength > 0) { + note = parseScyllaNote(result.first()); + foundScyllaNote = true; + } + } + if (!foundScyllaNote) { + // Fallback to Postgres + note = await this.findOneBy({ id: noteId }); } } if (!note) { - // Fallback to Postgres - note = await this.findOneBy({ id: noteId }); - } - - if (note === null) { throw new IdentifiableError( "9725d0ce-ba28-4dde-95a7-2cbb2c15de24", "No such note.", @@ -260,18 +268,19 @@ export const NoteRepository = db.getRepository(Note).extend({ emojis: noteEmoji, tags: note.tags.length > 0 ? note.tags : undefined, fileIds: note.fileIds, - files: scyllaClient - ? (note as ScyllaNote).files.map((file) => ({ - ...file, - createdAt: file.createdAt.toISOString(), - properties: { - width: file.width ?? undefined, - height: file.height ?? undefined, - }, - userId: null, - folderId: null, - })) - : DriveFiles.packMany(note.fileIds), + files: + scyllaClient && foundScyllaNote + ? (note as ScyllaNote).files.map((file) => ({ + ...file, + createdAt: file.createdAt.toISOString(), + properties: { + width: file.width ?? undefined, + height: file.height ?? undefined, + }, + userId: null, + folderId: null, + })) + : DriveFiles.packMany(note.fileIds), replyId: note.replyId, renoteId: note.renoteId, channelId: note.channelId || undefined, diff --git a/packages/backend/src/server/api/common/getters.ts b/packages/backend/src/server/api/common/getters.ts index b873914294..6279936d41 100644 --- a/packages/backend/src/server/api/common/getters.ts +++ b/packages/backend/src/server/api/common/getters.ts @@ -32,7 +32,7 @@ export async function getNote( } } - // For legacy notes + // Fallback to Postgres if (!note) { const query = Notes.createQueryBuilder("note").where("note.id = :id", { id: noteId, diff --git a/packages/backend/src/server/api/endpoints/notes/timeline.ts b/packages/backend/src/server/api/endpoints/notes/timeline.ts index a5243fd1fa..fb542f1d68 100644 --- a/packages/backend/src/server/api/endpoints/notes/timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/timeline.ts @@ -11,7 +11,12 @@ import { generateChannelQuery } from "../../common/generate-channel-query.js"; import { generateBlockedUserQuery } from "../../common/generate-block-query.js"; import { generateMutedUserRenotesQueryForNotes } from "../../common/generated-muted-renote-query.js"; import { ApiError } from "../../error.js"; -import { parseScyllaNote, prepared, scyllaClient } from "@/db/scylla.js"; +import { + type ScyllaNote, + parseScyllaNote, + prepared, + scyllaClient, +} from "@/db/scylla.js"; import { LocalFollowingsCache } from "@/misc/cache.js"; export const meta = { @@ -69,27 +74,35 @@ export default define(meta, paramDef, async (ps, user) => { const followingsCache = await LocalFollowingsCache.init(user.id); if (scyllaClient) { - const untilDate = ps.untilDate ? new Date(ps.untilDate) : new Date(); - const query = [`${prepared.note.select.byDate} AND "createdAt" <= ?`]; - const params: (Date | string | string[])[] = [untilDate, untilDate]; - if (ps.sinceDate) { - query.push(`AND "createdAt" >= ?`); - params.push(new Date(ps.sinceDate)); - } - if (ps.untilId) { - query.push(`AND "id" <= ?`); - params.push(ps.untilId); - } - if (ps.sinceId) { - query.push(`AND "id" >= ?`); - params.push(ps.sinceId); + let untilDate = new Date(); + const foundNotes: ScyllaNote[] = []; + const validIds = [user.id].concat(await followingsCache.getAll()); + + while (foundNotes.length < ps.limit) { + const query = [`${prepared.note.select.byDate} AND "createdAt" < ?`]; + const params: (Date | string | string[])[] = [untilDate, untilDate]; + if (ps.untilId) { + query.push(`AND "id" < ?`); + params.push(ps.untilId); + } + query.push("LIMIT 50"); // Hardcoded to enable prepared query for performance + + const result = await scyllaClient.execute(query.join(" "), params, { + prepare: true, + }); + + if (result.rowLength === 0) { + break; + } + + const notes = result.rows.map(parseScyllaNote); + const filtered = notes.filter((note) => validIds.includes(note.userId)); + foundNotes.push(...filtered); + + untilDate = notes[notes.length - 1].createdAt; } - const result = await scyllaClient.execute(query.join(" "), params, { - prepare: true, - }); - const notes = result.rows.map(parseScyllaNote); - return Notes.packMany(notes, user); + return Notes.packMany(foundNotes, user); } const hasFollowing = await followingsCache.hasFollowing(); @@ -113,17 +126,6 @@ export default define(meta, paramDef, async (ps, user) => { qb.orWhere(`note.userId IN (${followingQuery.getQuery()})`); }), ) - .innerJoinAndSelect("note.user", "user") - .leftJoinAndSelect("user.avatar", "avatar") - .leftJoinAndSelect("user.banner", "banner") - .leftJoinAndSelect("note.reply", "reply") - .leftJoinAndSelect("note.renote", "renote") - .leftJoinAndSelect("reply.user", "replyUser") - .leftJoinAndSelect("replyUser.avatar", "replyUserAvatar") - .leftJoinAndSelect("replyUser.banner", "replyUserBanner") - .leftJoinAndSelect("renote.user", "renoteUser") - .leftJoinAndSelect("renoteUser.avatar", "renoteUserAvatar") - .leftJoinAndSelect("renoteUser.banner", "renoteUserBanner") .setParameters(followingQuery.getParameters()); generateChannelQuery(query, user);