From 37215ae0fa5e6f8f5048546cf3dda2001c6a4e41 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Sun, 13 Aug 2023 20:38:26 -0400 Subject: [PATCH] fix: include quotes --- packages/backend/src/db/cql.ts | 2 +- packages/backend/src/db/scylla.ts | 40 +++--- .../backend/src/server/api/endpoints/notes.ts | 13 +- .../server/api/endpoints/notes/children.ts | 61 +++++---- .../src/server/api/endpoints/users/notes.ts | 117 +++++++++++++++--- 5 files changed, 167 insertions(+), 66 deletions(-) diff --git a/packages/backend/src/db/cql.ts b/packages/backend/src/db/cql.ts index 102367ee8a..c2c4f52213 100644 --- a/packages/backend/src/db/cql.ts +++ b/packages/backend/src/db/cql.ts @@ -46,7 +46,7 @@ export const scyllaQueries = { byUri: `SELECT * FROM note WHERE "uri" = ?`, byUrl: `SELECT * FROM note WHERE "url" = ?`, byId: `SELECT * FROM note WHERE "id" = ?`, - byUserId: `SELECT * FROM note_by_user_id WHERE "userId" IN ?`, + byUserId: `SELECT * FROM note_by_user_id WHERE "userId" = ?`, byRenoteId: `SELECT * FROM note_by_renote_id WHERE "renoteId" = ?`, byReplyId: `SELECT * FROM note WHERE "replyId" = ?` }, diff --git a/packages/backend/src/db/scylla.ts b/packages/backend/src/db/scylla.ts index 0cfd098cd3..3527bca42c 100644 --- a/packages/backend/src/db/scylla.ts +++ b/packages/backend/src/db/scylla.ts @@ -82,7 +82,10 @@ export interface ScyllaDriveFile { height: number | null; } -export function getScyllaDrivePublicUrl(file: ScyllaDriveFile, thumbnail = false): string | null { +export function getScyllaDrivePublicUrl( + file: ScyllaDriveFile, + thumbnail = false, +): string | null { const isImage = file.type && [ @@ -190,12 +193,13 @@ export interface ScyllaNoteReaction extends NoteReaction { const QUERY_LIMIT = 1000; // TODO: should this be configurable? -export type TimelineKind = +export type FeedType = | "home" | "local" | "recommended" | "global" - | "renotes"; + | "renotes" + | "user"; export function parseScyllaReaction(row: types.Row): ScyllaNoteReaction { return { @@ -209,7 +213,7 @@ export function parseScyllaReaction(row: types.Row): ScyllaNoteReaction { } export function prepareNoteQuery( - kind: TimelineKind, + kind: FeedType, ps: { untilId?: string; untilDate?: number; @@ -233,6 +237,9 @@ export function prepareNoteQuery( case "renotes": queryParts.push(prepared.note.select.byRenoteId); break; + case "user": + queryParts.push(prepared.note.select.byUserId); + break; default: queryParts.push(prepared.note.select.byDate); } @@ -269,7 +276,7 @@ export function prepareNoteQuery( } export async function execNotePaginationQuery( - kind: TimelineKind, + kind: FeedType, ps: { limit: number; untilId?: string; @@ -284,11 +291,15 @@ export async function execNotePaginationQuery( ): Promise { if (!scyllaClient) return []; - if (kind === "home" && !userId) { - throw new Error("Query of home timeline needs userId"); - } - if (kind === "renotes" && !ps.noteId) { - throw new Error("Query of renotes needs noteId"); + switch (kind) { + case "home": + case "user": + if (!userId) + throw new Error("Query of home and user timelines needs userId"); + break; + case "renotes": + if (!ps.noteId) throw new Error("Query of renotes needs noteId"); + break; } let { query, untilDate, sinceDate } = prepareNoteQuery(kind, ps); @@ -300,14 +311,15 @@ export async function execNotePaginationQuery( while (foundNotes.length < ps.limit && scannedPartitions < maxPartitions) { const params: (Date | string | string[] | number)[] = []; if (kind === "home" && userId) { - params.push(userId); - } - - if (kind === "renotes" && ps.noteId) { + params.push(userId, untilDate, untilDate); + } else if (kind === "user" && userId) { + params.push(userId, untilDate); + } else if (kind === "renotes" && ps.noteId) { params.push(ps.noteId, untilDate); } else { params.push(untilDate, untilDate); } + if (sinceDate) { params.push(sinceDate); } diff --git a/packages/backend/src/server/api/endpoints/notes.ts b/packages/backend/src/server/api/endpoints/notes.ts index 9787740ab0..61ac8f4048 100644 --- a/packages/backend/src/server/api/endpoints/notes.ts +++ b/packages/backend/src/server/api/endpoints/notes.ts @@ -41,18 +41,7 @@ export default define(meta, paramDef, async (ps) => { ps.untilId, ) .andWhere("note.visibility = 'public'") - .andWhere("note.localOnly = FALSE") - .innerJoinAndSelect("note.user", "user") - .leftJoinAndSelect("user.avatar", "avatar") - .leftJoinAndSelect("user.banner", "banner") - .leftJoinAndSelect("note.reply", "reply") - .leftJoinAndSelect("note.renote", "renote") - .leftJoinAndSelect("reply.user", "replyUser") - .leftJoinAndSelect("replyUser.avatar", "replyUserAvatar") - .leftJoinAndSelect("replyUser.banner", "replyUserBanner") - .leftJoinAndSelect("renote.user", "renoteUser") - .leftJoinAndSelect("renoteUser.avatar", "renoteUserAvatar") - .leftJoinAndSelect("renoteUser.banner", "renoteUserBanner"); + .andWhere("note.localOnly = FALSE"); if (ps.local) { query.andWhere("note.userHost IS NULL"); diff --git a/packages/backend/src/server/api/endpoints/notes/children.ts b/packages/backend/src/server/api/endpoints/notes/children.ts index aacfa0b331..9989154e24 100644 --- a/packages/backend/src/server/api/endpoints/notes/children.ts +++ b/packages/backend/src/server/api/endpoints/notes/children.ts @@ -5,6 +5,7 @@ import { generateVisibilityQuery } from "../../common/generate-visibility-query. import { generateMutedUserQuery } from "../../common/generate-muted-user-query.js"; import { generateBlockedUserQuery } from "../../common/generate-block-query.js"; import { + ScyllaNote, filterBlockedUser, filterMutedNote, filterMutedUser, @@ -89,44 +90,54 @@ export default define(meta, paramDef, async (ps, user) => { return await Notes.packMany([]); } - // Find replies in BFS manner - const queue = [root]; - const foundReplies: Note[] = []; - let depth = 0; + const filter = async (notes: ScyllaNote[]) => { + let filtered = await filterVisibility(notes, user, followingUserIds); + if (user) { + filtered = await filterMutedUser( + filtered, + user, + mutedUserIds, + mutedInstances, + ); + filtered = await filterMutedNote(filtered, user, mutedWords); + filtered = await filterBlockedUser(filtered, user, blockerIds); + } + return filtered; + }; - while ( - queue.length > 0 && - foundReplies.length < ps.limit && - depth < ps.depth - ) { + // Find quotes first + const renoteResult = await scyllaClient.execute( + prepared.note.select.byRenoteId, + [root.id], + { prepare: true }, + ); + const foundNotes = await filter( + renoteResult.rows.map(parseScyllaNote).filter((note) => !!note.text), + ); + + // Then find replies in BFS manner + const queue = [root]; + let depth = 0; + const limit = ps.limit + foundNotes.length; + + while (queue.length > 0 && foundNotes.length < limit && depth < ps.depth) { const note = queue.shift(); if (note) { - const result = await scyllaClient.execute( + const replyResult = await scyllaClient.execute( prepared.note.select.byReplyId, [note.id], { prepare: true }, ); - if (result.rowLength > 0) { - let replies = result.rows.map(parseScyllaNote); - replies = await filterVisibility(replies, user, followingUserIds); - if (user) { - replies = await filterMutedUser( - replies, - user, - mutedUserIds, - mutedInstances, - ); - replies = await filterMutedNote(replies, user, mutedWords); - replies = await filterBlockedUser(replies, user, blockerIds); - } - foundReplies.push(...replies); + const replies = await filter(replyResult.rows.map(parseScyllaNote)); + if (replies.length > 0) { + foundNotes.push(...replies); queue.push(...replies); depth++; } } } - return await Notes.packMany(foundReplies, user, { + return await Notes.packMany(foundNotes, user, { detail: false, scyllaNote: true, }); diff --git a/packages/backend/src/server/api/endpoints/users/notes.ts b/packages/backend/src/server/api/endpoints/users/notes.ts index 724cfc9af1..4420eea788 100644 --- a/packages/backend/src/server/api/endpoints/users/notes.ts +++ b/packages/backend/src/server/api/endpoints/users/notes.ts @@ -1,5 +1,5 @@ import { Brackets } from "typeorm"; -import { Notes } from "@/models/index.js"; +import { Notes, UserProfiles } from "@/models/index.js"; import define from "../../define.js"; import { ApiError } from "../../error.js"; import { getUser } from "../../common/getters.js"; @@ -7,6 +7,22 @@ import { makePaginationQuery } from "../../common/make-pagination-query.js"; import { generateVisibilityQuery } from "../../common/generate-visibility-query.js"; import { generateMutedUserQuery } from "../../common/generate-muted-user-query.js"; import { generateBlockedUserQuery } from "../../common/generate-block-query.js"; +import { + ScyllaNote, + execNotePaginationQuery, + filterBlockedUser, + filterMutedNote, + filterMutedUser, + filterVisibility, + scyllaClient, +} from "@/db/scylla.js"; +import { + InstanceMutingsCache, + LocalFollowingsCache, + UserBlockedCache, + UserMutingsCache, + userWordMuteCache, +} from "@/misc/cache.js"; export const meta = { tags: ["users", "notes"], @@ -66,6 +82,91 @@ export default define(meta, paramDef, async (ps, me) => { throw e; }); + if (scyllaClient) { + const [ + followingUserIds, + mutedUserIds, + mutedInstances, + mutedWords, + blockerIds, + ] = await Promise.all([ + LocalFollowingsCache.init(user.id).then((cache) => cache.getAll()), + UserMutingsCache.init(user.id).then((cache) => cache.getAll()), + InstanceMutingsCache.init(user.id).then((cache) => cache.getAll()), + userWordMuteCache + .fetchMaybe(user.id, () => + UserProfiles.findOne({ + select: ["mutedWords"], + where: { userId: user.id }, + }).then((profile) => profile?.mutedWords), + ) + .then((words) => words ?? []), + UserBlockedCache.init(user.id).then((cache) => cache.getAll()), + ]); + + if ( + mutedUserIds.includes(user.id) || + blockerIds.includes(user.id) || + (user.host && mutedInstances.includes(user.host)) + ) { + return Notes.packMany([]); + } + + const filter = async (notes: ScyllaNote[]) => { + let filtered = notes.filter((n) => n.userId === ps.userId); + filtered = await filterVisibility(filtered, user, followingUserIds); + filtered = await filterMutedUser( + filtered, + user, + mutedUserIds, + mutedInstances, + ); + filtered = await filterMutedNote(filtered, user, mutedWords); + filtered = await filterBlockedUser(filtered, user, blockerIds); + if (ps.withFiles) { + filtered = filtered.filter((n) => n.files.length > 0); + } + if (ps.fileType) { + filtered = filtered.filter((n) => + n.files.some((f) => ps.fileType?.includes(f.type)), + ); + } + if (ps.excludeNsfw) { + filtered = filtered.filter( + (n) => !n.cw && n.files.every((f) => !f.isSensitive), + ); + } + if (!ps.includeMyRenotes) { + filtered = filtered.filter( + (n) => + n.userId !== user.id || + !n.renoteId || + !!n.text || + n.files.length > 0 || + n.hasPoll, + ); + } + if (!ps.includeReplies) { + filtered = filtered.filter((n) => !n.replyId); + } + return filtered; + }; + + const foundPacked = []; + while (foundPacked.length < ps.limit) { + const foundNotes = ( + await execNotePaginationQuery("user", ps, filter, user.id) + ).slice(0, ps.limit * 1.5); // Some may filtered out by Notes.packMany, thus we take more than ps.limit. + foundPacked.push( + ...(await Notes.packMany(foundNotes, user, { scyllaNote: true })), + ); + if (foundNotes.length < ps.limit) break; + ps.untilDate = foundNotes[foundNotes.length - 1].createdAt.getTime(); + } + + return foundPacked.slice(0, ps.limit); + } + //#region Construct query const query = makePaginationQuery( Notes.createQueryBuilder("note"), @@ -73,19 +174,7 @@ export default define(meta, paramDef, async (ps, me) => { ps.untilId, ps.sinceDate, ps.untilDate, - ) - .andWhere("note.userId = :userId", { userId: user.id }) - .innerJoinAndSelect("note.user", "user") - .leftJoinAndSelect("user.avatar", "avatar") - .leftJoinAndSelect("user.banner", "banner") - .leftJoinAndSelect("note.reply", "reply") - .leftJoinAndSelect("note.renote", "renote") - .leftJoinAndSelect("reply.user", "replyUser") - .leftJoinAndSelect("replyUser.avatar", "replyUserAvatar") - .leftJoinAndSelect("replyUser.banner", "replyUserBanner") - .leftJoinAndSelect("renote.user", "renoteUser") - .leftJoinAndSelect("renoteUser.avatar", "renoteUserAvatar") - .leftJoinAndSelect("renoteUser.banner", "renoteUserBanner"); + ).andWhere("note.userId = :userId", { userId: user.id }); generateVisibilityQuery(query, me); if (me) {