From ffa08748d044de130e21f7b75585ec3237af3081 Mon Sep 17 00:00:00 2001 From: naskya Date: Fri, 17 May 2024 17:59:45 +0900 Subject: [PATCH] refactor (backend): don't check word mute twice --- packages/backend-rs/index.d.ts | 3 +- packages/backend-rs/index.js | 5 +-- packages/backend-rs/src/database/cache.rs | 2 -- .../backend-rs/src/misc/check_hit_antenna.rs | 34 +----------------- .../backend-rs/src/misc/check_word_mute.rs | 4 +-- packages/backend-rs/src/misc/is_quote.rs | 9 +++++ packages/backend-rs/src/misc/mod.rs | 1 + packages/backend-rs/src/service/antenna.rs | 34 +++++++++++++++--- packages/backend/src/misc/antenna-cache.ts | 36 ------------------- packages/backend/src/services/note/create.ts | 26 ++++++++------ 10 files changed, 62 insertions(+), 92 deletions(-) create mode 100644 packages/backend-rs/src/misc/is_quote.rs delete mode 100644 packages/backend/src/misc/antenna-cache.ts diff --git a/packages/backend-rs/index.d.ts b/packages/backend-rs/index.d.ts index 25ffd0b509..854a1f894b 100644 --- a/packages/backend-rs/index.d.ts +++ b/packages/backend-rs/index.d.ts @@ -267,6 +267,7 @@ export interface NoteLikeForGetNoteSummary { hasPoll: boolean } export function getNoteSummary(note: NoteLikeForGetNoteSummary): string +export function isQuote(note: Note): boolean export function isSafeUrl(url: string): boolean export function latestVersion(): Promise export function toMastodonId(firefishId: string): string | null @@ -1175,7 +1176,7 @@ export interface Webhook { latestSentAt: Date | null latestStatus: number | null } -export function updateAntennaOnCreateNote(antenna: Antenna, note: Note, noteAuthor: Acct): Promise +export function updateAntennasOnNewNote(note: Note, noteAuthor: Acct, noteMutedUsers: Array): Promise export function fetchNodeinfo(host: string): Promise export function nodeinfo_2_1(): Promise export function nodeinfo_2_0(): Promise diff --git a/packages/backend-rs/index.js b/packages/backend-rs/index.js index dd64d767ff..64b4e0d9d6 100644 --- a/packages/backend-rs/index.js +++ b/packages/backend-rs/index.js @@ -310,7 +310,7 @@ if (!nativeBinding) { throw new Error(`Failed to load native binding`) } -const { SECOND, MINUTE, HOUR, DAY, USER_ONLINE_THRESHOLD, USER_ACTIVE_THRESHOLD, FILE_TYPE_BROWSERSAFE, loadEnv, loadConfig, stringToAcct, acctToString, initializeRustLogger, showServerInfo, isBlockedServer, isSilencedServer, isAllowedServer, checkWordMute, getFullApAccount, isSelfHost, isSameOrigin, extractHost, toPuny, isUnicodeEmoji, sqlLikeEscape, safeForSql, formatMilliseconds, getImageSizeFromUrl, getNoteSummary, isSafeUrl, latestVersion, toMastodonId, fromMastodonId, fetchMeta, metaToPugArgs, nyaify, hashPassword, verifyPassword, isOldPasswordAlgorithm, decodeReaction, countReactions, toDbReaction, removeOldAttestationChallenges, cpuInfo, cpuUsage, memoryUsage, storageUsage, AntennaSrcEnum, DriveFileUsageHintEnum, MutedNoteReasonEnum, NoteVisibilityEnum, NotificationTypeEnum, PageVisibilityEnum, PollNotevisibilityEnum, RelayStatusEnum, UserEmojimodpermEnum, UserProfileFfvisibilityEnum, UserProfileMutingnotificationtypesEnum, updateAntennaOnCreateNote, fetchNodeinfo, nodeinfo_2_1, nodeinfo_2_0, Protocol, Inbound, Outbound, watchNote, unwatchNote, PushNotificationKind, sendPushNotification, publishToChannelStream, ChatEvent, publishToChatStream, ChatIndexEvent, publishToChatIndexStream, publishToBroadcastStream, publishToGroupChatStream, publishToModerationStream, getTimestamp, genId, genIdAt, generateSecureRandomString, generateUserToken } = nativeBinding +const { SECOND, MINUTE, HOUR, DAY, USER_ONLINE_THRESHOLD, USER_ACTIVE_THRESHOLD, FILE_TYPE_BROWSERSAFE, loadEnv, loadConfig, stringToAcct, acctToString, initializeRustLogger, showServerInfo, isBlockedServer, isSilencedServer, isAllowedServer, checkWordMute, getFullApAccount, isSelfHost, isSameOrigin, extractHost, toPuny, isUnicodeEmoji, sqlLikeEscape, safeForSql, formatMilliseconds, getImageSizeFromUrl, getNoteSummary, isQuote, isSafeUrl, latestVersion, toMastodonId, fromMastodonId, fetchMeta, metaToPugArgs, nyaify, hashPassword, verifyPassword, isOldPasswordAlgorithm, decodeReaction, countReactions, toDbReaction, removeOldAttestationChallenges, cpuInfo, cpuUsage, memoryUsage, storageUsage, AntennaSrcEnum, DriveFileUsageHintEnum, MutedNoteReasonEnum, NoteVisibilityEnum, NotificationTypeEnum, PageVisibilityEnum, PollNotevisibilityEnum, RelayStatusEnum, UserEmojimodpermEnum, UserProfileFfvisibilityEnum, UserProfileMutingnotificationtypesEnum, updateAntennasOnNewNote, fetchNodeinfo, nodeinfo_2_1, nodeinfo_2_0, Protocol, Inbound, Outbound, watchNote, unwatchNote, PushNotificationKind, sendPushNotification, publishToChannelStream, ChatEvent, publishToChatStream, ChatIndexEvent, publishToChatIndexStream, publishToBroadcastStream, publishToGroupChatStream, publishToModerationStream, getTimestamp, genId, genIdAt, generateSecureRandomString, generateUserToken } = nativeBinding module.exports.SECOND = SECOND module.exports.MINUTE = MINUTE @@ -340,6 +340,7 @@ module.exports.safeForSql = safeForSql module.exports.formatMilliseconds = formatMilliseconds module.exports.getImageSizeFromUrl = getImageSizeFromUrl module.exports.getNoteSummary = getNoteSummary +module.exports.isQuote = isQuote module.exports.isSafeUrl = isSafeUrl module.exports.latestVersion = latestVersion module.exports.toMastodonId = toMastodonId @@ -369,7 +370,7 @@ module.exports.RelayStatusEnum = RelayStatusEnum module.exports.UserEmojimodpermEnum = UserEmojimodpermEnum module.exports.UserProfileFfvisibilityEnum = UserProfileFfvisibilityEnum module.exports.UserProfileMutingnotificationtypesEnum = UserProfileMutingnotificationtypesEnum -module.exports.updateAntennaOnCreateNote = updateAntennaOnCreateNote +module.exports.updateAntennasOnNewNote = updateAntennasOnNewNote module.exports.fetchNodeinfo = fetchNodeinfo module.exports.nodeinfo_2_1 = nodeinfo_2_1 module.exports.nodeinfo_2_0 = nodeinfo_2_0 diff --git a/packages/backend-rs/src/database/cache.rs b/packages/backend-rs/src/database/cache.rs index f6f465ab00..6760004c2d 100644 --- a/packages/backend-rs/src/database/cache.rs +++ b/packages/backend-rs/src/database/cache.rs @@ -10,8 +10,6 @@ pub enum Category { Block, #[strum(serialize = "following")] Follow, - #[strum(serialize = "wordMute")] - WordMute, #[cfg(test)] #[strum(serialize = "usedOnlyForTesting")] Test, diff --git a/packages/backend-rs/src/misc/check_hit_antenna.rs b/packages/backend-rs/src/misc/check_hit_antenna.rs index d8fa7b925f..0a6871837f 100644 --- a/packages/backend-rs/src/misc/check_hit_antenna.rs +++ b/packages/backend-rs/src/misc/check_hit_antenna.rs @@ -1,11 +1,8 @@ use crate::config::CONFIG; use crate::database::{cache, db_conn}; use crate::federation::acct::Acct; -use crate::misc::check_word_mute::check_word_mute_bare; use crate::misc::get_note_all_texts::{all_texts, NoteLike}; -use crate::model::entity::{ - antenna, blocking, following, note, sea_orm_active_enums::*, user_profile, -}; +use crate::model::entity::{antenna, blocking, following, note, sea_orm_active_enums::*}; use sea_orm::{ColumnTrait, DbErr, EntityTrait, QueryFilter, QuerySelect}; #[derive(thiserror::Error, Debug)] @@ -152,35 +149,6 @@ pub async fn check_hit_antenna( } } - type WordMute = ( - Vec, // muted words - Vec, // muted patterns - ); - - let word_mute: WordMute = cache::get_one(cache::Category::WordMute, &antenna.user_id)? - .unwrap_or({ - // cache miss - let mute = user_profile::Entity::find() - .select_only() - .columns([ - user_profile::Column::MutedWords, - user_profile::Column::MutedPatterns, - ]) - .into_tuple::() - .one(db) - .await? - .ok_or({ - tracing::warn!("there is no user_profile for user {}", &antenna.user_id); - AntennaCheckError::UserProfileNotFoundErr(antenna.user_id.clone()) - })?; - cache::set_one(cache::Category::WordMute, &antenna.user_id, &mute, 10 * 60)?; - mute - }); - - if check_word_mute_bare(¬e_texts, &word_mute.0, &word_mute.1) { - return Ok(false); - } - Ok(true) } diff --git a/packages/backend-rs/src/misc/check_word_mute.rs b/packages/backend-rs/src/misc/check_word_mute.rs index fd51765ac4..1abeeaf33c 100644 --- a/packages/backend-rs/src/misc/check_word_mute.rs +++ b/packages/backend-rs/src/misc/check_word_mute.rs @@ -8,7 +8,7 @@ fn convert_regex(js_regex: &str) -> String { RE.replace(js_regex, "(?$2)$1").to_string() } -pub fn check_word_mute_bare( +fn check_word_mute_impl( texts: &[String], muted_words: &[String], muted_patterns: &[String], @@ -35,7 +35,7 @@ pub async fn check_word_mute( if muted_words.is_empty() && muted_patterns.is_empty() { Ok(false) } else { - Ok(check_word_mute_bare( + Ok(check_word_mute_impl( &all_texts(note).await?, muted_words, muted_patterns, diff --git a/packages/backend-rs/src/misc/is_quote.rs b/packages/backend-rs/src/misc/is_quote.rs new file mode 100644 index 0000000000..5e8ce056c0 --- /dev/null +++ b/packages/backend-rs/src/misc/is_quote.rs @@ -0,0 +1,9 @@ +use crate::model::entity::note; + +// https://github.com/napi-rs/napi-rs/issues/2060 +type Note = note::Model; + +#[crate::export] +pub fn is_quote(note: Note) -> bool { + note.renote_id.is_some() && (note.text.is_some() || note.has_poll || !note.file_ids.is_empty()) +} diff --git a/packages/backend-rs/src/misc/mod.rs b/packages/backend-rs/src/misc/mod.rs index a91dc4c862..9255237dd0 100644 --- a/packages/backend-rs/src/misc/mod.rs +++ b/packages/backend-rs/src/misc/mod.rs @@ -8,6 +8,7 @@ pub mod format_milliseconds; pub mod get_image_size; pub mod get_note_all_texts; pub mod get_note_summary; +pub mod is_quote; pub mod is_safe_url; pub mod latest_version; pub mod mastodon_id; diff --git a/packages/backend-rs/src/service/antenna.rs b/packages/backend-rs/src/service/antenna.rs index 582f2f541a..7d1a771e18 100644 --- a/packages/backend-rs/src/service/antenna.rs +++ b/packages/backend-rs/src/service/antenna.rs @@ -1,13 +1,19 @@ -use crate::database::{redis_conn, redis_key}; +use crate::database::cache; +use crate::database::{db_conn, redis_conn, redis_key}; use crate::federation::acct::Acct; use crate::misc::check_hit_antenna::{check_hit_antenna, AntennaCheckError}; use crate::model::entity::{antenna, note}; use crate::service::stream; use crate::util::id::{get_timestamp, InvalidIdErr}; use redis::{streams::StreamMaxlen, Commands, RedisError}; +use sea_orm::{DbErr, EntityTrait}; #[derive(thiserror::Error, Debug)] pub enum Error { + #[error("Database error: {0}")] + DbErr(#[from] DbErr), + #[error("Cache error: {0}")] + CacheErr(#[from] cache::Error), #[error("Redis error: {0}")] RedisErr(#[from] RedisError), #[error("Invalid ID: {0}")] @@ -22,15 +28,33 @@ pub enum Error { type Antenna = antenna::Model; type Note = note::Model; +// TODO?: it might be better to store this directly in memory +// (like fetch_meta) instead of Redis as it's used so much +async fn antennas() -> Result, Error> { + const CACHE_KEY: &str = "antennas"; + + Ok(cache::get::>(CACHE_KEY)?.unwrap_or({ + let antennas = antenna::Entity::find().all(db_conn().await?).await?; + cache::set(CACHE_KEY, &antennas, 5 * 60)?; + antennas + })) +} + #[crate::export] -pub async fn update_antenna_on_create_note( - antenna: &Antenna, +pub async fn update_antennas_on_new_note( note: Note, note_author: &Acct, + note_muted_users: Vec, ) -> Result<(), Error> { - if check_hit_antenna(antenna, note.clone(), note_author).await? { - add_note_to_antenna(&antenna.id, ¬e)?; + for antenna in antennas().await?.iter() { + if note_muted_users.contains(&antenna.user_id) { + continue; + } + if check_hit_antenna(antenna, note.clone(), note_author).await? { + add_note_to_antenna(&antenna.id, ¬e)?; + } } + Ok(()) } diff --git a/packages/backend/src/misc/antenna-cache.ts b/packages/backend/src/misc/antenna-cache.ts deleted file mode 100644 index 7f199c3967..0000000000 --- a/packages/backend/src/misc/antenna-cache.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { Antennas } from "@/models/index.js"; -import type { Antenna } from "@/models/entities/antenna.js"; -import { subscriber } from "@/db/redis.js"; - -let antennasFetched = false; -let antennas: Antenna[] = []; - -export async function getAntennas() { - if (!antennasFetched) { - antennas = await Antennas.find(); - antennasFetched = true; - } - - return antennas; -} - -subscriber.on("message", async (_, data) => { - const obj = JSON.parse(data); - - if (obj.channel === "internal") { - const { type, body } = obj.message; - switch (type) { - case "antennaCreated": - antennas.push(body); - break; - case "antennaUpdated": - antennas[antennas.findIndex((a) => a.id === body.id)] = body; - break; - case "antennaDeleted": - antennas = antennas.filter((a) => a.id !== body.id); - break; - default: - break; - } - } -}); diff --git a/packages/backend/src/services/note/create.ts b/packages/backend/src/services/note/create.ts index 32c9647a14..b56a36fd94 100644 --- a/packages/backend/src/services/note/create.ts +++ b/packages/backend/src/services/note/create.ts @@ -43,17 +43,17 @@ import { Poll } from "@/models/entities/poll.js"; import { createNotification } from "@/services/create-notification.js"; import { isDuplicateKeyValueError } from "@/misc/is-duplicate-key-value-error.js"; import { - updateAntennaOnCreateNote, + updateAntennasOnNewNote, checkWordMute, genId, genIdAt, + isQuote, isSilencedServer, } from "backend-rs"; import { countSameRenotes } from "@/misc/count-same-renotes.js"; import { deliverToRelays, getCachedRelays } from "../relay.js"; import type { Channel } from "@/models/entities/channel.js"; import { normalizeForSearch } from "@/misc/normalize-for-search.js"; -import { getAntennas } from "@/misc/antenna-cache.js"; import { endedPollNotificationQueue } from "@/queue/queues.js"; import { webhookDeliver } from "@/queue/index.js"; import { Cache } from "@/misc/cache.js"; @@ -370,8 +370,10 @@ export default async ( // Increment notes count (user) incNotesCountOfUser(user); - // Word mute - hardMutesCache + // Word mutes & antenna + const thisNoteIsMutedBy: string[] = []; + + await hardMutesCache .fetch(null, () => UserProfiles.find({ where: { @@ -380,12 +382,13 @@ export default async ( select: ["userId", "mutedWords", "mutedPatterns"], }), ) - .then((us) => { + .then(async (us) => { for (const u of us) { if (u.userId === user.id) return; - checkWordMute(note, u.mutedWords, u.mutedPatterns).then( + await checkWordMute(note, u.mutedWords, u.mutedPatterns).then( (shouldMute: boolean) => { if (shouldMute) { + thisNoteIsMutedBy.push(u.userId); MutedNotes.insert({ id: genId(), userId: u.userId, @@ -398,12 +401,13 @@ export default async ( } }); - // Antenna - for (const antenna of await getAntennas()) { - await updateAntennaOnCreateNote( - toRustObject(antenna), - toRustObject(note), + // type errors will be resolved by https://github.com/napi-rs/napi-rs/pull/2054 + const _note = toRustObject(note) + if (note.renoteId == null || isQuote(_note)) { + await updateAntennasOnNewNote( + _note, user, + thisNoteIsMutedBy, ); }