From a472d2027f70f6d089586d51a0771f877afff9bf Mon Sep 17 00:00:00 2001 From: Namekuji Date: Thu, 17 Aug 2023 03:56:42 -0400 Subject: [PATCH] add channel timeline --- .../cql/1689400417034_timeline/up.cql | 11 +++ packages/backend/src/db/cql.ts | 1 + packages/backend/src/db/scylla.ts | 13 +++- .../server/api/endpoints/channels/timeline.ts | 78 +++++++++++++++---- 4 files changed, 88 insertions(+), 15 deletions(-) diff --git a/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/up.cql b/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/up.cql index de3693dd02..3afd48c95a 100644 --- a/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/up.cql +++ b/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/up.cql @@ -119,6 +119,17 @@ CREATE MATERIALIZED VIEW IF NOT EXISTS note_by_renote_id_and_user_id AS PRIMARY KEY (("renoteId", "userId"), "createdAt", "createdAtDate", "userHost", "visibility") WITH CLUSTERING ORDER BY ("createdAt" DESC); +CREATE MATERIALIZED VIEW IF NOT EXISTS note_by_channel_id AS + SELECT * FROM note + WHERE "channelId" IS NOT NULL + AND "createdAt" IS NOT NULL + AND "createdAtDate" IS NOT NULL + AND "userId" IS NOT NULL + AND "userHost" IS NOT NULL + AND "visibility" IS NOT NULL + PRIMARY KEY ("channelId", "createdAt", "createdAtDate", "userId", "userHost", "visibility") + WITH CLUSTERING ORDER BY ("createdAt" DESC); + CREATE MATERIALIZED VIEW IF NOT EXISTS global_timeline AS SELECT * FROM note WHERE "createdAtDate" IS NOT NULL diff --git a/packages/backend/src/db/cql.ts b/packages/backend/src/db/cql.ts index 62772c6ea5..906b6cf270 100644 --- a/packages/backend/src/db/cql.ts +++ b/packages/backend/src/db/cql.ts @@ -50,6 +50,7 @@ export const scyllaQueries = { byUserId: `SELECT * FROM note_by_user_id WHERE "userId" = ?`, byRenoteId: `SELECT * FROM note_by_renote_id WHERE "renoteId" = ?`, byReplyId: `SELECT * FROM note WHERE "replyId" = ?`, + byChannelId: `SELECT * FROM note_by_channel_id WHERE "channelId" = ?`, }, delete: `DELETE FROM note WHERE "createdAtDate" = ? AND "createdAt" = ? AND "userId" = ? AND "userHost" = ? AND "visibility" = ?`, update: { diff --git a/packages/backend/src/db/scylla.ts b/packages/backend/src/db/scylla.ts index 75b68d63a6..bd4837b1d1 100644 --- a/packages/backend/src/db/scylla.ts +++ b/packages/backend/src/db/scylla.ts @@ -200,7 +200,8 @@ export type FeedType = | "recommended" | "global" | "renotes" - | "user"; + | "user" + | "channel"; export function parseScyllaReaction(row: types.Row): ScyllaNoteReaction { return { @@ -241,6 +242,9 @@ export function prepareNoteQuery( case "user": queryParts.push(prepared.note.select.byUserId); break; + case "channel": + queryParts.push(prepared.note.select.byChannelId); + break; default: queryParts.push(prepared.note.select.byDate); } @@ -285,6 +289,7 @@ export async function execNotePaginationQuery( sinceId?: string; sinceDate?: number; noteId?: string; + channelId?: string; }, filter?: (_: ScyllaNote[]) => Promise, userId?: User["id"], @@ -301,6 +306,10 @@ export async function execNotePaginationQuery( case "renotes": if (!ps.noteId) throw new Error("Query of renotes needs noteId"); break; + case "channel": + if (!ps.channelId) + throw new Error("Query of channel timeline needs channelId"); + break; } let { query, untilDate, sinceDate } = prepareNoteQuery(kind, ps); @@ -317,6 +326,8 @@ export async function execNotePaginationQuery( params.push(userId, untilDate); } else if (kind === "renotes" && ps.noteId) { params.push(ps.noteId, untilDate); + } else if (kind === "channel" && ps.channelId) { + params.push(ps.channelId, untilDate); } else { params.push(untilDate, untilDate); } diff --git a/packages/backend/src/server/api/endpoints/channels/timeline.ts b/packages/backend/src/server/api/endpoints/channels/timeline.ts index b5d5325234..f1dca17f0a 100644 --- a/packages/backend/src/server/api/endpoints/channels/timeline.ts +++ b/packages/backend/src/server/api/endpoints/channels/timeline.ts @@ -1,8 +1,23 @@ import define from "../../define.js"; import { ApiError } from "../../error.js"; -import { Notes, Channels } from "@/models/index.js"; +import { Notes, Channels, UserProfiles } from "@/models/index.js"; import { makePaginationQuery } from "../../common/make-pagination-query.js"; import { activeUsersChart } from "@/services/chart/index.js"; +import { + ScyllaNote, + execNotePaginationQuery, + filterBlockUser, + filterMutedNote, + filterMutedUser, + filterVisibility, + scyllaClient, +} from "@/db/scylla.js"; +import { + UserBlockedCache, + UserBlockingCache, + UserMutingsCache, + userWordMuteCache, +} from "@/misc/cache.js"; export const meta = { tags: ["notes", "channels"], @@ -49,10 +64,57 @@ export default define(meta, paramDef, async (ps, user) => { id: ps.channelId, }); - if (channel == null) { + if (!channel) { throw new ApiError(meta.errors.noSuchChannel); } + if (user) activeUsersChart.read(user); + + if (scyllaClient) { + let [mutedUserIds, blockerIds, blockingIds]: string[][] = []; + let mutedWords: string[][]; + if (user) { + [mutedUserIds, mutedWords, blockerIds, blockingIds] = await Promise.all([ + UserMutingsCache.init(user.id).then((cache) => cache.getAll()), + userWordMuteCache + .fetchMaybe(user.id, () => + UserProfiles.findOne({ + select: ["mutedWords"], + where: { userId: user.id }, + }).then((profile) => profile?.mutedWords), + ) + .then((words) => words ?? []), + UserBlockedCache.init(user.id).then((cache) => cache.getAll()), + UserBlockingCache.init(user.id).then((cache) => cache.getAll()), + ]); + } + + const filter = async (notes: ScyllaNote[]) => { + if (!user) return notes; + let filtered = await filterMutedUser(notes, user, mutedUserIds, []); + filtered = await filterMutedNote(filtered, user, mutedWords); + filtered = await filterBlockUser(filtered, user, [ + ...blockerIds, + ...blockingIds, + ]); + return filtered; + }; + + const foundPacked = []; + while (foundPacked.length < ps.limit) { + const foundNotes = ( + await execNotePaginationQuery("channel", ps, filter) + ).slice(0, ps.limit * 1.5); // Some may filtered out by Notes.packMany, thus we take more than ps.limit. + foundPacked.push( + ...(await Notes.packMany(foundNotes, user, { scyllaNote: true })), + ); + if (foundNotes.length < ps.limit) break; + ps.untilDate = foundNotes[foundNotes.length - 1].createdAt.getTime(); + } + + return foundPacked.slice(0, ps.limit); + } + //#region Construct query const query = makePaginationQuery( Notes.createQueryBuilder("note"), @@ -63,22 +125,10 @@ export default define(meta, paramDef, async (ps, user) => { ) .andWhere("note.channelId = :channelId", { channelId: channel.id }) .innerJoinAndSelect("note.user", "user") - .leftJoinAndSelect("user.avatar", "avatar") - .leftJoinAndSelect("user.banner", "banner") - .leftJoinAndSelect("note.reply", "reply") - .leftJoinAndSelect("note.renote", "renote") - .leftJoinAndSelect("reply.user", "replyUser") - .leftJoinAndSelect("replyUser.avatar", "replyUserAvatar") - .leftJoinAndSelect("replyUser.banner", "replyUserBanner") - .leftJoinAndSelect("renote.user", "renoteUser") - .leftJoinAndSelect("renoteUser.avatar", "renoteUserAvatar") - .leftJoinAndSelect("renoteUser.banner", "renoteUserBanner") .leftJoinAndSelect("note.channel", "channel"); //#endregion const timeline = await query.take(ps.limit).getMany(); - if (user) activeUsersChart.read(user); - return await Notes.packMany(timeline, user); });