perf: no postgres word filter

packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/up.cql

@@ -57,9 +57,15 @@ CREATE TABLE IF NOT EXISTS note ( -- Models timeline
 	"replyId" ascii, -- Reply
 	"replyUserId" ascii,
 	"replyUserHost" text,
+	"replyContent" text,
+	"replyCw" text,
+	"replyFiles" set<frozen<drive_file>>,
 	"renoteId" ascii, -- Boost
 	"renoteUserId" ascii,
 	"renoteUserHost" text,
+	"renoteContent" text,
+	"renoteCw" text,
+	"renoteFiles" set<frozen<drive_file>>,
 	"reactions" map<text, int>, -- Reactions
 	"noteEdit" set<frozen<note_edit_history>>, -- Edit History
 	"updatedAt" timestamp,

packages/backend/src/db/scylla.ts

@@ -10,9 +10,12 @@ import {
 	InstanceMutingsCache,
 	LocalFollowingsCache,
 	UserMutingsCache,
+	userWordMuteCache,
 } from "@/misc/cache.js";
 import { getTimestamp } from "@/misc/gen-id.js";
 import Logger from "@/services/logger.js";
+import { UserProfiles } from "@/models/index.js";
+import { getWordHardMute } from "@/misc/check-word-mute";
 
 function newClient(): Client | null {
 	if (!config.scylla) {
@@ -86,15 +89,21 @@ export const prepared = {
 				"replyId",
 				"replyUserId",
 				"replyUserHost",
+				"replyContent",
+				"replyCw",
+				"replyFiles",
 				"renoteId",
 				"renoteUserId",
 				"renoteUserHost",
+				"renoteContent",
+				"renoteCw",
+				"renoteFiles",
 				"reactions",
 				"noteEdit",
 				"updatedAt"
 			)
 			VALUES
-			(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+			(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
 		select: {
 			byDate: `SELECT * FROM note WHERE "createdAtDate" = ?`,
 			byUri: `SELECT * FROM note WHERE "uri" IN ?`,
@@ -108,6 +117,9 @@ export const prepared = {
 				"renoteCount" = ?,
 				"score" = ?
 				WHERE "createdAtDate" = ? AND "createdAt" = ? AND "id" = ? IF EXISTS`,
+			repliesCount: `UPDATE note SET
+				"repliesCount" = ?,
+				WHERE "createdAtDate" = ? AND "createdAt" = ? AND "id" = ? IF EXISTS`,
 			reactions: `UPDATE note SET
 				"emojis" = ?,
 				"reactions" = ?,
@@ -157,6 +169,12 @@ export type ScyllaNote = Note & {
 	createdAtDate: Date;
 	files: ScyllaDriveFile[];
 	noteEdit: ScyllaNoteEditHistory[];
+	replyText: string | null;
+	replyCw: string | null;
+	replyFiles: ScyllaDriveFile[];
+	renoteText: string | null;
+	renoteCw: string | null;
+	renoteFiles: ScyllaDriveFile[];
 };
 
 export function parseScyllaNote(row: types.Row): ScyllaNote {
@@ -191,9 +209,15 @@ export function parseScyllaNote(row: types.Row): ScyllaNote {
 		replyId: row.get("replyId") ?? null,
 		replyUserId: row.get("replyUserId") ?? null,
 		replyUserHost: row.get("replyUserHost") ?? null,
+		replyText: row.get("replyContent") ?? null,
+		replyCw: row.get("replyCw") ?? null,
+		replyFiles: row.get("replyFiles") ?? [],
 		renoteId: row.get("renoteId") ?? null,
 		renoteUserId: row.get("renoteUserId") ?? null,
 		renoteUserHost: row.get("renoteUserHost") ?? null,
+		renoteText: row.get("renoteContent") ?? null,
+		renoteCw: row.get("renoteCw") ?? null,
+		renoteFiles: row.get("renoteFiles") ?? [],
 		reactions: row.get("reactions") ?? {},
 		noteEdit: row.get("noteEdit") ?? [],
 		updatedAt: row.get("updatedAt") ?? null,
@@ -424,3 +448,21 @@ export async function filterMutedUser(
 			!(note.renoteUserHost && mutedInstances.includes(note.renoteUserHost)),
 	);
 }
+
+export async function filterMutedNote(
+	notes: ScyllaNote[],
+	user: { id: User["id"] },
+): Promise<ScyllaNote[]> {
+	const mutedWords = await userWordMuteCache.fetchMaybe(user.id, () =>
+		UserProfiles.findOne({
+			select: ["mutedWords"],
+			where: { userId: user.id },
+		}).then((profile) => profile?.mutedWords),
+	);
+
+	if (!mutedWords) {
+		return notes;
+	}
+
+	return notes.filter((note) => !getWordHardMute(note, user, mutedWords));
+}

packages/backend/src/misc/cache.ts

@@ -4,6 +4,7 @@ import { ChainableCommander } from "ioredis";
 import {
 	ChannelFollowings,
 	Followings,
+	MutedNotes,
 	Mutings,
 	UserProfiles,
 } from "@/models/index.js";
@@ -257,7 +258,7 @@ export class LocalFollowingsCache extends SetCache {
 	private constructor(userId: string) {
 		const fetcher = () =>
 			Followings.find({
-				select: { followeeId: true },
+				select: ["followeeId"],
 				where: { followerId: userId, followerHost: IsNull() },
 			}).then((follows) => follows.map(({ followeeId }) => followeeId));
 
@@ -276,7 +277,7 @@ export class ChannelFollowingsCache extends SetCache {
 	private constructor(userId: string) {
 		const fetcher = () =>
 			ChannelFollowings.find({
-				select: { followeeId: true },
+				select: ["followeeId"],
 				where: {
 					followerId: userId,
 				},
@@ -297,7 +298,7 @@ export class UserMutingsCache extends HashCache {
 	private constructor(userId: string) {
 		const fetcher = () =>
 			Mutings.find({
-				select: { muteeId: true, expiresAt: true },
+				select: ["muteeId", "expiresAt"],
 				where: { muterId: userId },
 			}).then(
 				(mutes) =>
@@ -364,7 +365,7 @@ export class InstanceMutingsCache extends SetCache {
 	private constructor(userId: string) {
 		const fetcher = () =>
 			UserProfiles.findOne({
-				select: { mutedInstances: true },
+				select: ["mutedInstances"],
 				where: { userId },
 			}).then((profile) => (profile ? profile.mutedInstances : []));
 
@@ -378,3 +379,5 @@ export class InstanceMutingsCache extends SetCache {
 		return cache;
 	}
 }
+
+export const userWordMuteCache = new Cache<string[][]>("mutedWord", 60 * 30);

packages/backend/src/misc/check-word-mute.ts

@@ -1,11 +1,13 @@
 import RE2 from "re2";
 import type { Note } from "@/models/entities/note.js";
 import type { User } from "@/models/entities/user.js";
+import { DriveFile } from "@/models/entities/drive-file";
+import { scyllaClient, type ScyllaNote } from "@/db/scylla.js";
 
 type NoteLike = {
 	userId: Note["userId"];
 	text: Note["text"];
-	files?: Note["files"];
+	files?: DriveFile[];
 	cw?: Note["cw"];
 };
 
@@ -14,14 +16,30 @@ type UserLike = {
 };
 
 function checkWordMute(
-	note: NoteLike,
+	note: NoteLike | ScyllaNote,
 	mutedWords: Array<string | string[]>,
 ): boolean {
 	if (note == null) return false;
 
 	let text = `${note.cw ?? ""} ${note.text ?? ""}`;
-	if (note.files != null)
+	if (note.files && note.files.length > 0)
 		text += ` ${note.files.map((f) => f.comment ?? "").join(" ")}`;
+
+	if (scyllaClient) {
+		const scyllaNote = note as ScyllaNote;
+		text += `${scyllaNote.replyCw ?? ""} ${scyllaNote.replyText ?? ""} ${
+			scyllaNote.renoteCw ?? ""
+		} ${scyllaNote.renoteText ?? ""}`;
+
+		if (scyllaNote.replyFiles.length > 0) {
+			text += ` ${scyllaNote.replyFiles.map((f) => f.comment ?? "").join(" ")}`;
+		}
+		if (scyllaNote.renoteFiles.length > 0) {
+			text += ` ${scyllaNote.renoteFiles
+				.map((f) => f.comment ?? "")
+				.join(" ")}`;
+		}
+	}
 	text = text.trim();
 
 	if (text === "") return false;
@@ -57,23 +75,28 @@ function checkWordMute(
 	return false;
 }
 
-export async function getWordHardMute(
+export function getWordHardMute(
-	note: NoteLike,
+	note: NoteLike | ScyllaNote,
 	me: UserLike | null | undefined,
 	mutedWords: Array<string | string[]>,
-): Promise<boolean> {
+): boolean {
 	// 自分自身
 	if (me && note.userId === me.id) {
 		return false;
 	}
 
+	let ng = false;
+
 	if (mutedWords.length > 0) {
-		return (
+		ng = checkWordMute(note, mutedWords);
-			checkWordMute(note, mutedWords) ||
+
-			checkWordMute(note.reply, mutedWords) ||
+		if (!scyllaClient) {
-			checkWordMute(note.renote, mutedWords)
+			ng =
-		);
+				ng ||
+				checkWordMute(note.reply, mutedWords) ||
+				checkWordMute(note.renote, mutedWords);
+		}
 	}
 
-	return false;
+	return ng;
 }

packages/backend/src/server/api/endpoints/i/update.ts

@@ -16,7 +16,7 @@ import { verifyLink } from "@/services/fetch-rel-me.js";
 import { ApiError } from "../../error.js";
 import define from "../../define.js";
 import { userByIdCache, userDenormalizedCache } from "@/services/user-cache.js";
-import { InstanceMutingsCache } from "@/misc/cache.js";
+import { InstanceMutingsCache, userWordMuteCache } from "@/misc/cache.js";
 
 export const meta = {
 	tags: ["account"],
@@ -332,6 +332,11 @@ export default define(meta, paramDef, async (ps, _user, token) => {
 			await cache.clear();
 			await cache.add(...profileUpdates.mutedInstances);
 		}
+		if (profileUpdates.enableWordMute && profileUpdates.mutedWords) {
+			await userWordMuteCache.set(user.id, profileUpdates.mutedWords)
+		} else {
+			await userWordMuteCache.delete(user.id);
+		}
 	}
 
 	const iObj = await Users.pack<true, true>(user.id, user, {

packages/backend/src/server/api/endpoints/notes/timeline.ts

@@ -19,6 +19,7 @@ import {
 	filterVisibility,
 	execTimelineQuery,
 	filterMutedUser,
+	filterMutedNote,
 } from "@/db/scylla.js";
 import { ChannelFollowingsCache, LocalFollowingsCache } from "@/misc/cache.js";
 
@@ -88,6 +89,7 @@ export default define(meta, paramDef, async (ps, user) => {
 			filtered = await filterReply(filtered, ps.withReplies, user);
 			filtered = await filterVisibility(filtered, user, followingUserIds);
 			filtered = await filterMutedUser(filtered, user);
+			filtered = await filterMutedNote(filtered, user);
 			return filtered;
 		};
 

packages/backend/src/services/note/create.ts

@@ -33,6 +33,7 @@ import {
 	Channels,
 	ChannelFollowings,
 	NoteThreadMutings,
+	DriveFiles,
 } from "@/models/index.js";
 import type { DriveFile } from "@/models/entities/drive-file.js";
 import type { App } from "@/models/entities/app.js";
@@ -68,9 +69,8 @@ import meilisearch from "../../db/meilisearch.js";
 import { redisClient } from "@/db/redis.js";
 import { Mutex } from "redis-semaphore";
 import { parseScyllaNote, prepared, scyllaClient } from "@/db/scylla.js";
-import { populateEmojis } from "@/misc/populate-emojis.js";
 
-const mutedWordsCache = new Cache<
+export const mutedWordsCache = new Cache<
 	{ userId: UserProfile["userId"]; mutedWords: UserProfile["mutedWords"] }[]
 >("mutedWords", 60 * 5);
 
@@ -358,31 +358,34 @@ export default async (
 		incNotesCountOfUser(user);
 
 		// Word mute
-		mutedWordsCache
+		if (!scyllaClient) {
-			.fetch(null, () =>
+			mutedWordsCache
-				UserProfiles.find({
+				.fetch(null, () =>
-					where: {
+					UserProfiles.find({
-						enableWordMute: true,
+						where: {
-					},
+							enableWordMute: true,
-					select: ["userId", "mutedWords"],
-				}),
-			)
-			.then((us) => {
-				for (const u of us) {
-					getWordHardMute(data, { id: u.userId }, u.mutedWords).then(
-						(shouldMute) => {
-							if (shouldMute) {
-								MutedNotes.insert({
-									id: genId(),
-									userId: u.userId,
-									noteId: note.id,
-									reason: "word",
-								});
-							}
 						},
-					);
+						select: ["userId", "mutedWords"],
-				}
+					}),
-			});
+				)
+				.then((us) => {
+					for (const u of us) {
+						const shouldMute = getWordHardMute(
+							data,
+							{ id: u.userId },
+							u.mutedWords,
+						);
+						if (shouldMute) {
+							MutedNotes.insert({
+								id: genId(),
+								userId: u.userId,
+								noteId: note.id,
+								reason: "word",
+							});
+						}
+					}
+				});
+		}
 
 		// Antenna
 		for (const antenna of await getAntennas()) {
@@ -408,7 +411,7 @@ export default async (
 		}
 
 		if (data.reply) {
-			saveReply(data.reply, note);
+			saveReply(data.reply);
 		}
 
 		if (
@@ -775,6 +778,12 @@ async function insertNote(
 	// 投稿を作成
 	try {
 		if (scyllaClient) {
+			const fileMapper = (file: DriveFile) => ({
+				...file,
+				width: file.properties.width ?? null,
+				height: file.properties.height ?? null,
+			});
+
 			await scyllaClient.execute(
 				prepared.note.insert,
 				[
@@ -791,11 +800,7 @@ async function insertNote(
 					insert.uri,
 					insert.url,
 					insert.score ?? 0,
-					data.files?.map((file) => ({
+					data.files?.map(fileMapper),
-						...file,
-						width: file.properties.width ?? null,
-						height: file.properties.height ?? null,
-					})),
 					insert.visibleUserIds,
 					insert.mentions,
 					insert.mentionedRemoteUsers,
@@ -809,9 +814,23 @@ async function insertNote(
 					insert.replyId,
 					insert.replyUserId,
 					insert.replyUserHost,
+					data.reply?.text ?? null,
+					data.reply?.cw ?? null,
+					data.reply?.fileIds
+						? await DriveFiles.findBy({ id: In(data.reply.fileIds) }).then(
+								(files) => files.map(fileMapper),
+						  )
+						: null,
 					insert.renoteId,
 					insert.renoteUserId,
 					insert.renoteUserHost,
+					data.renote?.text ?? null,
+					data.renote?.cw ?? null,
+					data.renote?.fileIds
+						? await DriveFiles.findBy({ id: In(data.renote.fileIds) }).then(
+								(files) => files.map(fileMapper),
+						  )
+						: null,
 					null,
 					null,
 					null,
@@ -982,8 +1001,16 @@ async function createMentionedEvents(
 	}
 }
 
-function saveReply(reply: Note, note: Note) {
+async function saveReply(reply: Note) {
-	Notes.increment({ id: reply.id }, "repliesCount", 1);
+	if (scyllaClient) {
+		await scyllaClient.execute(
+			prepared.note.update.repliesCount,
+			[reply.repliesCount + 1, reply.createdAt, reply.createdAt, reply.id],
+			{ prepare: true },
+		);
+	} else {
+		await Notes.increment({ id: reply.id }, "repliesCount", 1);
+	}
 }
 
 function incNotesCountOfUser(user: { id: User["id"] }) {