From 83c15b1026558d6e6b7ba243eb03a1b917f5bf27 Mon Sep 17 00:00:00 2001
From: naskya <m@naskya.net>
Date: Fri, 12 Apr 2024 15:43:17 +0900
Subject: [PATCH] refactor (backend): port checkWordMute to backend-rs
Co-authored-by: sup39 <dev@sup39.dev>
---
packages/backend-rs/index.d.ts | 9 ++
packages/backend-rs/index.js | 3 +-
packages/backend-rs/src/database/error.rs | 9 --
packages/backend-rs/src/database/mod.rs | 9 +-
.../backend-rs/src/misc/check_word_mute.rs | 107 ++++++++++++++++++
packages/backend-rs/src/misc/mod.rs | 1 +
packages/backend-rs/src/model/error.rs | 6 +-
.../backend/src/misc/check-hit-antenna.ts | 5 +-
packages/backend/src/misc/check-word-mute.ts | 76 -------------
.../api/stream/channels/global-timeline.ts | 4 +-
.../api/stream/channels/home-timeline.ts | 4 +-
.../api/stream/channels/hybrid-timeline.ts | 4 +-
.../api/stream/channels/local-timeline.ts | 4 +-
.../stream/channels/recommended-timeline.ts | 4 +-
packages/backend/src/services/note/create.ts | 4 +-
15 files changed, 138 insertions(+), 111 deletions(-)
delete mode 100644 packages/backend-rs/src/database/error.rs
create mode 100644 packages/backend-rs/src/misc/check_word_mute.rs
delete mode 100644 packages/backend/src/misc/check-word-mute.ts
diff --git a/packages/backend-rs/index.d.ts b/packages/backend-rs/index.d.ts
index 3168aa7105..6bc51dc7c4 100644
--- a/packages/backend-rs/index.d.ts
+++ b/packages/backend-rs/index.d.ts
@@ -118,6 +118,15 @@ export interface Acct {
}
export function stringToAcct(acct: string): Acct
export function acctToString(acct: Acct): string
+export interface NoteLike {
+ fileIds: Array<string>
+ userId: string | null
+ text: string | null
+ cw: string | null
+ renoteId: string | null
+ replyId: string | null
+}
+export function checkWordMute(note: NoteLike, mutedWordLists: Array<Array<string>>, mutedPatterns: Array<string>): Promise<boolean>
export function nyaify(text: string, lang?: string | undefined | null): string
export interface AbuseUserReport {
id: string
diff --git a/packages/backend-rs/index.js b/packages/backend-rs/index.js
index 800baabeb2..868e632d85 100644
--- a/packages/backend-rs/index.js
+++ b/packages/backend-rs/index.js
@@ -310,11 +310,12 @@ if (!nativeBinding) {
throw new Error(`Failed to load native binding`)
}
-const { readServerConfig, stringToAcct, acctToString, nyaify, AntennaSrcEnum, MutedNoteReasonEnum, NoteVisibilityEnum, NotificationTypeEnum, PageVisibilityEnum, PollNotevisibilityEnum, RelayStatusEnum, UserEmojimodpermEnum, UserProfileFfvisibilityEnum, UserProfileMutingnotificationtypesEnum, initIdGenerator, getTimestamp, genId, secureRndstr, IdConvertType, convertId } = nativeBinding
+const { readServerConfig, stringToAcct, acctToString, checkWordMute, nyaify, AntennaSrcEnum, MutedNoteReasonEnum, NoteVisibilityEnum, NotificationTypeEnum, PageVisibilityEnum, PollNotevisibilityEnum, RelayStatusEnum, UserEmojimodpermEnum, UserProfileFfvisibilityEnum, UserProfileMutingnotificationtypesEnum, initIdGenerator, getTimestamp, genId, secureRndstr, IdConvertType, convertId } = nativeBinding
module.exports.readServerConfig = readServerConfig
module.exports.stringToAcct = stringToAcct
module.exports.acctToString = acctToString
+module.exports.checkWordMute = checkWordMute
module.exports.nyaify = nyaify
module.exports.AntennaSrcEnum = AntennaSrcEnum
module.exports.MutedNoteReasonEnum = MutedNoteReasonEnum
diff --git a/packages/backend-rs/src/database/error.rs b/packages/backend-rs/src/database/error.rs
deleted file mode 100644
index babdd68318..0000000000
--- a/packages/backend-rs/src/database/error.rs
+++ /dev/null
@@ -1,9 +0,0 @@
-use sea_orm::error::DbErr;
-
-#[derive(thiserror::Error, Debug, PartialEq, Eq)]
-pub enum Error {
- #[error("The database connections have not been initialized yet")]
- Uninitialized,
- #[error("ORM error: {0}")]
- OrmError(#[from] DbErr),
-}
diff --git a/packages/backend-rs/src/database/mod.rs b/packages/backend-rs/src/database/mod.rs
index eb4935f6e9..f598e35cc7 100644
--- a/packages/backend-rs/src/database/mod.rs
+++ b/packages/backend-rs/src/database/mod.rs
@@ -1,12 +1,9 @@
-pub mod error;
-
use crate::config::server::SERVER_CONFIG;
-use error::Error;
-use sea_orm::{Database, DbConn};
+use sea_orm::{Database, DbConn, DbErr};
static DB_CONN: once_cell::sync::OnceCell<DbConn> = once_cell::sync::OnceCell::new();
-async fn init_database() -> Result<&'static DbConn, Error> {
+async fn init_database() -> Result<&'static DbConn, DbErr> {
let database_uri = format!(
"postgres://{}:{}@{}:{}/{}",
SERVER_CONFIG.db.user,
@@ -19,7 +16,7 @@ async fn init_database() -> Result<&'static DbConn, Error> {
Ok(DB_CONN.get_or_init(move || conn))
}
-pub async fn db_conn() -> Result<&'static DbConn, Error> {
+pub async fn db_conn() -> Result<&'static DbConn, DbErr> {
match DB_CONN.get() {
Some(conn) => Ok(conn),
None => init_database().await,
diff --git a/packages/backend-rs/src/misc/check_word_mute.rs b/packages/backend-rs/src/misc/check_word_mute.rs
new file mode 100644
index 0000000000..3a761a61b0
--- /dev/null
+++ b/packages/backend-rs/src/misc/check_word_mute.rs
@@ -0,0 +1,107 @@
+use crate::database::db_conn;
+use crate::model::entity::{drive_file, note};
+use once_cell::sync::Lazy;
+use regex::Regex;
+use sea_orm::{prelude::*, QuerySelect};
+
+#[crate::export(object)]
+pub struct NoteLike {
+ pub file_ids: Vec<String>,
+ pub user_id: Option<String>,
+ pub text: Option<String>,
+ pub cw: Option<String>,
+ pub renote_id: Option<String>,
+ pub reply_id: Option<String>,
+}
+
+async fn all_texts(note: NoteLike) -> Result<Vec<String>, DbErr> {
+ let db = db_conn().await?;
+
+ let mut texts: Vec<String> = vec![];
+
+ if let Some(text) = note.text {
+ texts.push(text);
+ }
+ if let Some(cw) = note.cw {
+ texts.push(cw);
+ }
+
+ texts.extend(
+ drive_file::Entity::find()
+ .select_only()
+ .column(drive_file::Column::Comment)
+ .filter(drive_file::Column::Id.is_in(note.file_ids))
+ .into_tuple::<String>()
+ .all(db)
+ .await?,
+ );
+
+ if let Some(renote_id) = note.renote_id {
+ if let Some((text, cw)) = note::Entity::find_by_id(renote_id)
+ .select_only()
+ .columns([note::Column::Text, note::Column::Cw])
+ .into_tuple::<(String, String)>()
+ .one(db)
+ .await?
+ {
+ texts.push(text);
+ texts.push(cw);
+ }
+ }
+
+ if let Some(reply_id) = note.reply_id {
+ if let Some((text, cw)) = note::Entity::find_by_id(reply_id)
+ .select_only()
+ .columns([note::Column::Text, note::Column::Cw])
+ .into_tuple::<(String, String)>()
+ .one(db)
+ .await?
+ {
+ texts.push(text);
+ texts.push(cw);
+ }
+ }
+
+ Ok(texts)
+}
+
+fn convert_regex(js_regex: &str) -> String {
+ static RE: Lazy<Regex> = Lazy::new(|| Regex::new(r"^/(.+)/(.*)$").unwrap());
+ RE.replace(js_regex, "(?$2)$1").to_string()
+}
+
+fn check_word_mute_impl(
+ texts: &[String],
+ muted_word_lists: &[Vec<String>],
+ muted_patterns: &[String],
+) -> bool {
+ muted_word_lists.iter().any(|muted_word_list| {
+ texts.iter().any(|text| {
+ let text_lower = text.to_lowercase();
+ muted_word_list
+ .iter()
+ .all(|muted_word| text_lower.contains(&muted_word.to_lowercase()))
+ })
+ }) || muted_patterns.iter().any(|muted_pattern| {
+ Regex::new(convert_regex(muted_pattern).as_str())
+ .map(|re| texts.iter().any(|text| re.is_match(text)))
+ .unwrap_or(false)
+ })
+}
+
+#[crate::export]
+pub async fn check_word_mute(
+ note: NoteLike,
+ muted_word_lists: Vec<Vec<String>>,
+ muted_patterns: Vec<String>,
+) -> Result<bool, DbErr> {
+ if muted_word_lists.is_empty() && muted_patterns.is_empty() {
+ Ok(false)
+ } else {
+ Ok(check_word_mute_impl(
+ &all_texts(note).await?,
+ &muted_word_lists,
+ &muted_patterns,
+ ))
+ }
+}
diff --git a/packages/backend-rs/src/misc/mod.rs b/packages/backend-rs/src/misc/mod.rs
index 02cdbfb404..ba404ca0d0 100644
--- a/packages/backend-rs/src/misc/mod.rs
+++ b/packages/backend-rs/src/misc/mod.rs
@@ -1,2 +1,3 @@
pub mod acct;
+pub mod check_word_mute;
pub mod nyaify;
diff --git a/packages/backend-rs/src/model/error.rs b/packages/backend-rs/src/model/error.rs
index 25a74ce10f..b3d9f6eda5 100644
--- a/packages/backend-rs/src/model/error.rs
+++ b/packages/backend-rs/src/model/error.rs
@@ -2,10 +2,8 @@
pub enum Error {
#[error("Failed to parse string: {0}")]
ParseError(#[from] parse_display::ParseError),
- #[error("Failed to get database connection: {0}")]
- DbConnError(#[from] crate::database::error::Error),
- #[error("Database operation error: {0}")]
- DbOperationError(#[from] sea_orm::DbErr),
+ #[error("Database error: {0}")]
+ DbError(#[from] sea_orm::DbErr),
#[error("Requested entity not found")]
NotFound,
}
diff --git a/packages/backend/src/misc/check-hit-antenna.ts b/packages/backend/src/misc/check-hit-antenna.ts
index a2090934b6..5b795a2c21 100644
--- a/packages/backend/src/misc/check-hit-antenna.ts
+++ b/packages/backend/src/misc/check-hit-antenna.ts
@@ -4,8 +4,7 @@ import type { User } from "@/models/entities/user.js";
import type { UserProfile } from "@/models/entities/user-profile.js";
import { Blockings, Followings, UserProfiles } from "@/models/index.js";
import { getFullApAccount } from "@/misc/convert-host.js";
-import { stringToAcct } from "backend-rs";
-import { getWordHardMute } from "@/misc/check-word-mute.js";
+import { checkWordMute, stringToAcct } from "backend-rs";
import type { Packed } from "@/misc/schema.js";
import { Cache } from "@/misc/cache.js";
@@ -124,7 +123,7 @@ export async function checkHitAntenna(
mutes.mutedWords != null &&
mutes.mutedPatterns != null &&
antenna.userId !== note.userId &&
- (await getWordHardMute(note, mutes.mutedWords, mutes.mutedPatterns))
+ (await checkWordMute(note, mutes.mutedWords, mutes.mutedPatterns))
)
return false;
diff --git a/packages/backend/src/misc/check-word-mute.ts b/packages/backend/src/misc/check-word-mute.ts
deleted file mode 100644
index f07f2a0fe5..0000000000
--- a/packages/backend/src/misc/check-word-mute.ts
+++ /dev/null
@@ -1,76 +0,0 @@
-import RE2 from "re2";
-import type { Note } from "@/models/entities/note.js";
-
-type NoteLike = {
- userId: Note["userId"];
- text: Note["text"];
- files?: Note["files"];
- cw?: Note["cw"];
- reply?: NoteLike | null;
- renote?: NoteLike | null;
-};
-
-function checkWordMute(
- note: NoteLike | null | undefined,
- mutedWords: string[][],
- mutedPatterns: string[],
-): boolean {
- if (note == null) return false;
-
- let text = `${note.cw ?? ""} ${note.text ?? ""}`;
- if (note.files != null)
- text += ` ${note.files.map((f) => f.comment ?? "").join(" ")}`;
- text = text.trim();
-
- if (text === "") return false;
-
- for (const mutedWord of mutedWords) {
- // Clean up
- const keywords = mutedWord.filter((keyword) => keyword !== "");
-
- if (
- keywords.length > 0 &&
- keywords.every((keyword) =>
- text.toLowerCase().includes(keyword.toLowerCase()),
- )
- )
- return true;
- }
-
- for (const mutedPattern of mutedPatterns) {
- // represents RegExp
- const regexp = mutedPattern.match(/^\/(.+)\/(.*)$/);
-
- // This should never happen due to input sanitisation.
- if (!regexp) {
- console.warn(`Found invalid regex in word mutes: ${mutedPattern}`);
- continue;
- }
-
- try {
- if (new RE2(regexp[1], regexp[2]).test(text)) return true;
- } catch (err) {
- // This should never happen due to input sanitisation.
- }
- }
-
- return false;
-}
-
-export async function getWordHardMute(
- note: NoteLike | null,
- mutedWords: string[][],
- mutedPatterns: string[],
-): Promise<boolean> {
- if (note == null || mutedWords == null || mutedPatterns == null) return false;
-
- if (mutedWords.length > 0) {
- return (
- checkWordMute(note, mutedWords, mutedPatterns) ||
- checkWordMute(note.reply, mutedWords, mutedPatterns) ||
- checkWordMute(note.renote, mutedWords, mutedPatterns)
- );
- }
-
- return false;
-}
diff --git a/packages/backend/src/server/api/stream/channels/global-timeline.ts b/packages/backend/src/server/api/stream/channels/global-timeline.ts
index 79d2fe90ec..97295af57a 100644
--- a/packages/backend/src/server/api/stream/channels/global-timeline.ts
+++ b/packages/backend/src/server/api/stream/channels/global-timeline.ts
@@ -1,6 +1,6 @@
import Channel from "../channel.js";
import { fetchMeta } from "@/misc/fetch-meta.js";
-import { getWordHardMute } from "@/misc/check-word-mute.js";
+import { checkWordMute } from "backend-rs";
import { isInstanceMuted } from "@/misc/is-instance-muted.js";
import { isUserRelated } from "@/misc/is-user-related.js";
import type { Packed } from "@/misc/schema.js";
@@ -72,7 +72,7 @@ export default class extends Channel {
if (
this.userProfile &&
this.user?.id !== note.userId &&
- (await getWordHardMute(
+ (await checkWordMute(
note,
this.userProfile.mutedWords,
this.userProfile.mutedPatterns,
diff --git a/packages/backend/src/server/api/stream/channels/home-timeline.ts b/packages/backend/src/server/api/stream/channels/home-timeline.ts
index 8f23946259..8103fe9b69 100644
--- a/packages/backend/src/server/api/stream/channels/home-timeline.ts
+++ b/packages/backend/src/server/api/stream/channels/home-timeline.ts
@@ -1,5 +1,5 @@
import Channel from "../channel.js";
-import { getWordHardMute } from "@/misc/check-word-mute.js";
+import { checkWordMute } from "backend-rs";
import { isUserRelated } from "@/misc/is-user-related.js";
import { isInstanceMuted } from "@/misc/is-instance-muted.js";
import type { Packed } from "@/misc/schema.js";
@@ -69,7 +69,7 @@ export default class extends Channel {
if (
this.userProfile &&
this.user?.id !== note.userId &&
- (await getWordHardMute(
+ (await checkWordMute(
note,
this.userProfile.mutedWords,
this.userProfile.mutedPatterns,
diff --git a/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts b/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts
index 7f5c662b8c..9052e7c2a5 100644
--- a/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts
+++ b/packages/backend/src/server/api/stream/channels/hybrid-timeline.ts
@@ -1,6 +1,6 @@
import Channel from "../channel.js";
import { fetchMeta } from "@/misc/fetch-meta.js";
-import { getWordHardMute } from "@/misc/check-word-mute.js";
+import { checkWordMute } from "backend-rs";
import { isUserRelated } from "@/misc/is-user-related.js";
import { isInstanceMuted } from "@/misc/is-instance-muted.js";
import type { Packed } from "@/misc/schema.js";
@@ -86,7 +86,7 @@ export default class extends Channel {
if (
this.userProfile &&
this.user?.id !== note.userId &&
- (await getWordHardMute(
+ (await checkWordMute(
note,
this.userProfile.mutedWords,
this.userProfile.mutedPatterns,
diff --git a/packages/backend/src/server/api/stream/channels/local-timeline.ts b/packages/backend/src/server/api/stream/channels/local-timeline.ts
index 1df87dbfc8..bd31c94f9d 100644
--- a/packages/backend/src/server/api/stream/channels/local-timeline.ts
+++ b/packages/backend/src/server/api/stream/channels/local-timeline.ts
@@ -1,6 +1,6 @@
import Channel from "../channel.js";
import { fetchMeta } from "@/misc/fetch-meta.js";
-import { getWordHardMute } from "@/misc/check-word-mute.js";
+import { checkWordMute } from "backend-rs";
import { isUserRelated } from "@/misc/is-user-related.js";
import type { Packed } from "@/misc/schema.js";
@@ -64,7 +64,7 @@ export default class extends Channel {
if (
this.userProfile &&
this.user?.id !== note.userId &&
- (await getWordHardMute(
+ (await checkWordMute(
note,
this.userProfile.mutedWords,
this.userProfile.mutedPatterns,
diff --git a/packages/backend/src/server/api/stream/channels/recommended-timeline.ts b/packages/backend/src/server/api/stream/channels/recommended-timeline.ts
index a9da732f89..26c3cbfc68 100644
--- a/packages/backend/src/server/api/stream/channels/recommended-timeline.ts
+++ b/packages/backend/src/server/api/stream/channels/recommended-timeline.ts
@@ -1,6 +1,6 @@
import Channel from "../channel.js";
import { fetchMeta } from "@/misc/fetch-meta.js";
-import { getWordHardMute } from "@/misc/check-word-mute.js";
+import { checkWordMute } from "backend-rs";
import { isUserRelated } from "@/misc/is-user-related.js";
import { isInstanceMuted } from "@/misc/is-instance-muted.js";
import type { Packed } from "@/misc/schema.js";
@@ -84,7 +84,7 @@ export default class extends Channel {
if (
this.userProfile &&
this.user?.id !== note.userId &&
- (await getWordHardMute(
+ (await checkWordMute(
note,
this.userProfile.mutedWords,
this.userProfile.mutedPatterns,
diff --git a/packages/backend/src/services/note/create.ts b/packages/backend/src/services/note/create.ts
index 54e53c6c96..0a4ddc517f 100644
--- a/packages/backend/src/services/note/create.ts
+++ b/packages/backend/src/services/note/create.ts
@@ -44,7 +44,7 @@ import { Poll } from "@/models/entities/poll.js";
import { createNotification } from "@/services/create-notification.js";
import { isDuplicateKeyValueError } from "@/misc/is-duplicate-key-value-error.js";
import { checkHitAntenna } from "@/misc/check-hit-antenna.js";
-import { getWordHardMute } from "@/misc/check-word-mute.js";
+import { checkWordMute } from "backend-rs";
import { addNoteToAntenna } from "@/services/add-note-to-antenna.js";
import { countSameRenotes } from "@/misc/count-same-renotes.js";
import { deliverToRelays, getCachedRelays } from "../relay.js";
@@ -380,7 +380,7 @@ export default async (
.then((us) => {
for (const u of us) {
if (u.userId === user.id) return;
- getWordHardMute(note, u.mutedWords, u.mutedPatterns).then(
+ checkWordMute(note, u.mutedWords, u.mutedPatterns).then(
(shouldMute: boolean) => {
if (shouldMute) {
MutedNotes.insert({