refactor (backend): separate muted words and muted patterns

Co-authored-by: sup39 <dev@sup39.dev>
This commit is contained in:
naskya 2024-03-01 22:52:21 +09:00
parent b30e68c98c
commit d70d0f4232
No known key found for this signature in database
GPG key ID: 712D413B3A9FED5C
21 changed files with 319 additions and 97 deletions

View file

@ -5,6 +5,7 @@ Critical security updates are indicated by the :warning: icon.
## Unreleased
- Introduce new full-text search engine and post search filters
- Refactoring
## v20240301

View file

@ -1,6 +1,7 @@
BEGIN;
DELETE FROM "migrations" WHERE name IN (
'SeparateHardMuteWordsAndPatterns1706413792769',
'IndexAltTextAndCw1708872574733',
'Pgroonga1698420787202',
'ChangeDefaultConfigs1709251460718',
@ -14,6 +15,10 @@ DELETE FROM "migrations" WHERE name IN (
'RemoveNativeUtilsMigration1705877093218'
);
-- separate-hard-mute-words-and-patterns
UPDATE "user_profile" SET "mutedWords" = "mutedWords" || array_to_json("mutedPatterns")::jsonb;
ALTER TABLE "user_profile" DROP "mutedPatterns";
-- index-alt-text-and-cw
DROP INDEX "IDX_f4f7b93d05958527300d79ac82";
DROP INDEX "IDX_8e3bbbeb3df04d1a8105da4c8f";

View file

@ -1429,6 +1429,7 @@ _menuDisplay:
hide: "Hide"
_wordMute:
muteWords: "Muted words"
mutePatterns: "Muted patterns"
muteLangs: "Muted Languages"
muteWordsDescription: "Separate with spaces for an AND condition or with line breaks
for an OR condition."

View file

@ -1220,7 +1220,8 @@ _menuDisplay:
hide: "隠す"
_wordMute:
muteWords: "ミュートするワード"
muteLangs: "ミュートされた言語"
mutePatterns: "ミュートするパターン"
muteLangs: "ミュートする言語"
muteWordsDescription: "スペースで区切るとAND指定になり、改行で区切るとOR指定になります。"
muteWordsDescription2: "キーワードをスラッシュで囲むと正規表現になります。"
muteLangsDescription: "OR 条件の場合はスペースまたは改行で区切ります。"

View file

@ -71,6 +71,8 @@ pub struct Model {
pub prevent_ai_learning: bool,
#[sea_orm(column_name = "isIndexable")]
pub is_indexable: bool,
#[sea_orm(column_name = "mutedPatterns")]
pub muted_patterns: Vec<String>,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]

View file

@ -0,0 +1,32 @@
export class SeparateHardMuteWordsAndPatterns1706413792769 {
name = "SeparateHardMuteWordsAndPatterns1706413792769";
async up(queryRunner) {
await queryRunner.query(
`ALTER TABLE "user_profile" ADD "mutedPatterns" text[] DEFAULT '{}'`,
);
await queryRunner.query(`
UPDATE "user_profile" SET
"mutedPatterns" = ARRAY(
SELECT jsonb_array_elements_text(jsonb_path_query_array(
"mutedWords",
'$ ? (@.type() == "string")'
))
),
"mutedWords" = jsonb_path_query_array(
"mutedWords",
'$ ? (@.type() == "array")'
)
`);
await queryRunner.query(
`ALTER TABLE "user_profile" ALTER "mutedPatterns" SET NOT NULL`,
);
}
async down(queryRunner) {
await queryRunner.query(
`UPDATE "user_profile" SET "mutedWords" = "mutedWords" || array_to_json("mutedPatterns")::jsonb`,
);
await queryRunner.query(`ALTER TABLE "user_profile" DROP "mutedPatterns"`);
}
}

View file

@ -1,6 +1,7 @@
import type { Antenna } from "@/models/entities/antenna.js";
import type { Note } from "@/models/entities/note.js";
import type { User } from "@/models/entities/user.js";
import type { UserProfile } from "@/models/entities/user-profile.js";
import { Blockings, UserProfiles } from "@/models/index.js";
import { getFullApAccount } from "@/misc/convert-host.js";
import * as Acct from "@/misc/acct.js";
@ -9,7 +10,11 @@ import { Cache } from "@/misc/cache.js";
import { getWordHardMute } from "@/misc/check-word-mute.js";
const blockingCache = new Cache<User["id"][]>("blocking", 60 * 5);
const mutedWordsCache = new Cache<string[][] | undefined>("mutedWords", 60 * 5);
const hardMutesCache = new Cache<{
userId: UserProfile["userId"];
mutedWords: UserProfile["mutedWords"];
mutedPatterns: UserProfile["mutedPatterns"];
}>("hardMutes", 60 * 5);
export async function checkHitAntenna(
antenna: Antenna,
@ -89,12 +94,24 @@ export async function checkHitAntenna(
);
if (blockings.includes(antenna.userId)) return false;
const mutedWords = await mutedWordsCache.fetch(antenna.userId, () =>
UserProfiles.findOneBy({ userId: antenna.userId }).then(
(profile) => profile?.mutedWords,
),
const mutes = await hardMutesCache.fetch(antenna.userId, () =>
UserProfiles.findOneByOrFail({
userId: antenna.userId,
}).then((profile) => {
return {
userId: antenna.userId,
mutedWords: profile.mutedWords,
mutedPatterns: profile.mutedPatterns,
};
}),
);
if (await getWordHardMute(note, antenna.userId, mutedWords)) return false;
if (
mutes.mutedWords != null &&
mutes.mutedPatterns != null &&
antenna.userId !== note.userId &&
(await getWordHardMute(note, mutes.mutedWords, mutes.mutedPatterns))
)
return false;
// TODO: eval expression

View file

@ -6,11 +6,14 @@ type NoteLike = {
text: Note["text"];
files?: Note["files"];
cw?: Note["cw"];
reply?: NoteLike | null;
renote?: NoteLike | null;
};
function checkWordMute(
note: NoteLike,
mutedWords: Array<string | string[]>,
note: NoteLike | null | undefined,
mutedWords: string[][],
mutedPatterns: string[],
): boolean {
if (note == null) return false;
@ -21,33 +24,33 @@ function checkWordMute(
if (text === "") return false;
for (const mutePattern of mutedWords) {
if (Array.isArray(mutePattern)) {
// Clean up
const keywords = mutePattern.filter((keyword) => keyword !== "");
for (const mutedWord of mutedWords) {
// Clean up
const keywords = mutedWord.filter((keyword) => keyword !== "");
if (
keywords.length > 0 &&
keywords.every((keyword) =>
text.toLowerCase().includes(keyword.toLowerCase()),
)
if (
keywords.length > 0 &&
keywords.every((keyword) =>
text.toLowerCase().includes(keyword.toLowerCase()),
)
return true;
} else {
// represents RegExp
const regexp = mutePattern.match(/^\/(.+)\/(.*)$/);
)
return true;
}
for (const mutedPattern of mutedPatterns) {
// represents RegExp
const regexp = mutedPattern.match(/^\/(.+)\/(.*)$/);
// This should never happen due to input sanitisation.
if (!regexp) {
console.warn(`Found invalid regex in word mutes: ${mutedPattern}`);
continue;
}
try {
if (new RE2(regexp[1], regexp[2]).test(text)) return true;
} catch (err) {
// This should never happen due to input sanitisation.
if (!regexp) {
console.warn(`Found invalid regex in word mutes: ${mutePattern}`);
continue;
}
try {
if (new RE2(regexp[1], regexp[2]).test(text)) return true;
} catch (err) {
// This should never happen due to input sanitisation.
}
}
}
@ -55,17 +58,17 @@ function checkWordMute(
}
export async function getWordHardMute(
note: NoteLike,
meId: string | null | undefined,
mutedWords?: Array<string | string[]>,
note: NoteLike | null,
mutedWords: string[][],
mutedPatterns: string[],
): Promise<boolean> {
if (note.userId === meId || mutedWords == null) return false;
if (note == null || mutedWords == null || mutedPatterns == null) return false;
if (mutedWords.length > 0) {
return (
checkWordMute(note, mutedWords) ||
checkWordMute(note.reply, mutedWords) ||
checkWordMute(note.renote, mutedWords)
checkWordMute(note, mutedWords, mutedPatterns) ||
checkWordMute(note.reply, mutedWords, mutedPatterns) ||
checkWordMute(note.renote, mutedWords, mutedPatterns)
);
}

View file

@ -217,6 +217,12 @@ export class UserProfile {
})
public mutedWords: string[][];
@Column("text", {
array: true,
nullable: false,
})
public mutedPatterns: string[];
@Column("jsonb", {
default: [],
comment: "List of instances muted by the user.",

View file

@ -544,13 +544,13 @@ export const UserRepository = db.getRepository(User).extend({
? {
avatarId: user.avatarId,
bannerId: user.bannerId,
injectFeaturedNote: profile!.injectFeaturedNote,
receiveAnnouncementEmail: profile!.receiveAnnouncementEmail,
alwaysMarkNsfw: profile!.alwaysMarkNsfw,
carefulBot: profile!.carefulBot,
autoAcceptFollowed: profile!.autoAcceptFollowed,
noCrawle: profile!.noCrawle,
preventAiLearning: profile!.preventAiLearning,
injectFeaturedNote: profile?.injectFeaturedNote,
receiveAnnouncementEmail: profile?.receiveAnnouncementEmail,
alwaysMarkNsfw: profile?.alwaysMarkNsfw,
carefulBot: profile?.carefulBot,
autoAcceptFollowed: profile?.autoAcceptFollowed,
noCrawle: profile?.noCrawle,
preventAiLearning: profile?.preventAiLearning,
isExplorable: user.isExplorable,
isDeleted: user.isDeleted,
hideOnlineStatus: user.hideOnlineStatus,
@ -571,17 +571,18 @@ export const UserRepository = db.getRepository(User).extend({
hasUnreadNotification: this.getHasUnreadNotification(user.id),
hasPendingReceivedFollowRequest:
this.getHasPendingReceivedFollowRequest(user.id),
mutedWords: profile!.mutedWords,
mutedInstances: profile!.mutedInstances,
mutingNotificationTypes: profile!.mutingNotificationTypes,
emailNotificationTypes: profile!.emailNotificationTypes,
mutedWords: profile?.mutedWords,
mutedPatterns: profile?.mutedPatterns,
mutedInstances: profile?.mutedInstances,
mutingNotificationTypes: profile?.mutingNotificationTypes,
emailNotificationTypes: profile?.emailNotificationTypes,
}
: {}),
...(opts.includeSecrets
? {
email: profile!.email,
emailVerified: profile!.emailVerified,
email: profile?.email,
emailVerified: profile?.emailVerified,
securityKeysList: UserSecurityKeys.find({
where: {
userId: user.id,

View file

@ -468,6 +468,16 @@ export const packedMeDetailedOnlySchema = {
},
},
},
mutedPatterns: {
type: "array",
nullable: false,
optional: false,
items: {
type: "string",
nullable: false,
optional: false,
},
},
mutedInstances: {
type: "array",
nullable: true,

View file

@ -60,6 +60,7 @@ export default define(meta, paramDef, async (ps, me) => {
injectFeaturedNote: profile.injectFeaturedNote,
receiveAnnouncementEmail: profile.receiveAnnouncementEmail,
mutedWords: profile.mutedWords,
mutedPatterns: profile.mutedPatterns,
mutedInstances: profile.mutedInstances,
mutingNotificationTypes: profile.mutingNotificationTypes,
isModerator: user.isModerator,

View file

@ -120,6 +120,7 @@ export const paramDef = {
ffVisibility: { type: "string", enum: ["public", "followers", "private"] },
pinnedPageId: { type: "string", format: "misskey:id", nullable: true },
mutedWords: { type: "array" },
mutedPatterns: { type: "array", items: { type: "string" } },
mutedInstances: {
type: "array",
items: {
@ -159,23 +160,52 @@ export default define(meta, paramDef, async (ps, _user, token) => {
profileUpdates.ffVisibility = ps.ffVisibility;
if (ps.avatarId !== undefined) updates.avatarId = ps.avatarId;
if (ps.bannerId !== undefined) updates.bannerId = ps.bannerId;
if (ps.mutedPatterns !== undefined) {
for (const item of ps.mutedPatterns) {
const regexp = item.match(/^\/(.+)\/(.*)$/);
if (!regexp) throw new ApiError(meta.errors.invalidRegexp);
try {
new RegExp(regexp[1], regexp[2]);
} catch (err) {
throw new ApiError(meta.errors.invalidRegexp);
}
profileUpdates.mutedPatterns = profileUpdates.mutedPatterns ?? [];
profileUpdates.mutedPatterns.push(item);
}
}
if (ps.mutedWords !== undefined) {
// validate regular expression syntax
ps.mutedWords
.filter((x) => !Array.isArray(x))
.forEach((x) => {
const regexp = x.match(/^\/(.+)\/(.*)$/);
if (!regexp) throw new ApiError(meta.errors.invalidRegexp);
// for backward compatibility
for (const item of ps.mutedWords) {
if (Array.isArray(item)) continue;
try {
new RE2(regexp[1], regexp[2]);
} catch (err) {
throw new ApiError(meta.errors.invalidRegexp);
}
});
const regexp = item.match(/^\/(.+)\/(.*)$/);
if (!regexp) throw new ApiError(meta.errors.invalidRegexp);
profileUpdates.mutedWords = ps.mutedWords;
profileUpdates.enableWordMute = ps.mutedWords.length > 0;
try {
new RegExp(regexp[1], regexp[2]);
} catch (err) {
throw new ApiError(meta.errors.invalidRegexp);
}
profileUpdates.mutedPatterns = profileUpdates.mutedPatterns ?? [];
profileUpdates.mutedPatterns.push(item);
}
profileUpdates.mutedWords = ps.mutedWords.filter((item) =>
Array.isArray(item),
);
}
if (
profileUpdates.mutedWords !== undefined ||
profileUpdates.mutedPatterns !== undefined
) {
profileUpdates.enableWordMute =
(profileUpdates.mutedWords != null &&
profileUpdates.mutedWords.length > 0) ||
(profileUpdates.mutedPatterns != null &&
profileUpdates.mutedPatterns.length > 0);
}
if (ps.mutedInstances !== undefined)
profileUpdates.mutedInstances = ps.mutedInstances;

View file

@ -69,7 +69,12 @@ export default class extends Channel {
// そのためレコードが存在するかのチェックでは不十分なので、改めてgetWordHardMuteを呼んでいる
if (
this.userProfile &&
(await getWordHardMute(note, this.user?.id, this.userProfile.mutedWords))
this.user?.id !== note.userId &&
(await getWordHardMute(
note,
this.userProfile.mutedWords,
this.userProfile.mutedPatterns,
))
)
return;

View file

@ -68,7 +68,12 @@ export default class extends Channel {
// そのためレコードが存在するかのチェックでは不十分なので、改めてgetWordHardMuteを呼んでいる
if (
this.userProfile &&
(await getWordHardMute(note, this.user?.id, this.userProfile.mutedWords))
this.user?.id !== note.userId &&
(await getWordHardMute(
note,
this.userProfile.mutedWords,
this.userProfile.mutedPatterns,
))
)
return;

View file

@ -85,7 +85,12 @@ export default class extends Channel {
// そのためレコードが存在するかのチェックでは不十分なので、改めてgetWordHardMuteを呼んでいる
if (
this.userProfile &&
(await getWordHardMute(note, this.user?.id, this.userProfile.mutedWords))
this.user?.id !== note.userId &&
(await getWordHardMute(
note,
this.userProfile.mutedWords,
this.userProfile.mutedPatterns,
))
)
return;

View file

@ -61,7 +61,12 @@ export default class extends Channel {
// そのためレコードが存在するかのチェックでは不十分なので、改めてgetWordHardMuteを呼んでいる
if (
this.userProfile &&
(await getWordHardMute(note, this.user?.id, this.userProfile.mutedWords))
this.user?.id !== note.userId &&
(await getWordHardMute(
note,
this.userProfile.mutedWords,
this.userProfile.mutedPatterns,
))
)
return;

View file

@ -83,7 +83,12 @@ export default class extends Channel {
// そのためレコードが存在するかのチェックでは不十分なので、改めてgetWordHardMuteを呼んでいる
if (
this.userProfile &&
(await getWordHardMute(note, this.user?.id, this.userProfile.mutedWords))
this.user?.id !== note.userId &&
(await getWordHardMute(
note,
this.userProfile.mutedWords,
this.userProfile.mutedPatterns,
))
)
return;

View file

@ -65,9 +65,13 @@ import { inspect } from "node:util";
const logger = new Logger("create-note");
const mutedWordsCache = new Cache<
{ userId: UserProfile["userId"]; mutedWords: UserProfile["mutedWords"] }[]
>("mutedWords", 60 * 5);
const hardMutesCache = new Cache<
{
userId: UserProfile["userId"];
mutedWords: UserProfile["mutedWords"];
mutedPatterns: UserProfile["mutedPatterns"];
}[]
>("hardMutes", 60 * 5);
type NotificationType = "reply" | "renote" | "quote" | "mention";
@ -357,27 +361,30 @@ export default async (
incNotesCountOfUser(user);
// Word mute
mutedWordsCache
hardMutesCache
.fetch(null, () =>
UserProfiles.find({
where: {
enableWordMute: true,
},
select: ["userId", "mutedWords"],
select: ["userId", "mutedWords", "mutedPatterns"],
}),
)
.then((us) => {
for (const u of us) {
getWordHardMute(data, u.userId, u.mutedWords).then((shouldMute) => {
if (shouldMute) {
MutedNotes.insert({
id: genId(),
userId: u.userId,
noteId: note.id,
reason: "word",
});
}
});
if (u.userId === user.id) return;
getWordHardMute(note, u.mutedWords, u.mutedPatterns).then(
(shouldMute: boolean) => {
if (shouldMute) {
MutedNotes.insert({
id: genId(),
userId: u.userId,
noteId: note.id,
reason: "word",
});
}
},
);
}
});

View file

@ -36,11 +36,15 @@
>
<FormTextarea v-model="hardMutedWords" class="_formBlock">
<span>{{ i18n.ts._wordMute.muteWords }}</span>
<template #caption
>{{ i18n.ts._wordMute.muteWordsDescription }}<br />{{
i18n.ts._wordMute.muteWordsDescription2
}}</template
>
<template #caption>{{
i18n.ts._wordMute.muteWordsDescription
}}</template>
</FormTextarea>
<FormTextarea v-model="hardMutedPatterns" class="_formBlock">
<span>{{ i18n.ts._wordMute.mutePatterns }}</span>
<template #caption>{{
i18n.ts._wordMute.muteWordsDescription2
}}</template>
</FormTextarea>
<MkKeyValue
v-if="hardWordMutedNotesCount != null"
@ -90,6 +94,7 @@ const tab = ref("soft");
const softMutedWords = ref(render(defaultStore.state.mutedWords));
const softMutedLangs = ref(render(defaultStore.state.mutedLangs));
const hardMutedWords = ref(render($i!.mutedWords));
const hardMutedPatterns = ref($i!.mutedPatterns.join("\n"));
const hardWordMutedNotesCount = ref(null);
const changed = ref(false);
@ -109,8 +114,12 @@ watch(hardMutedWords, () => {
changed.value = true;
});
watch(hardMutedPatterns, () => {
changed.value = true;
});
async function save() {
const parseMutes = (mutes, tab) => {
const parseSoftMutes = (mutes, tab) => {
// split into lines, remove empty lines and unnecessary whitespace
const lines = mutes
.trim()
@ -151,11 +160,80 @@ async function save() {
return lines;
};
let softMutes, softMLangs, hardMutes;
const parseMutedWords = (mutes) => {
// split into lines, remove empty lines and unnecessary whitespace
return mutes
.trim()
.split("\n")
.map((line) => line.trim())
.filter((line) => line !== "")
.map((line) => line.split(" "))
.filter((line) => line.length > 0);
};
const parseMutedPatterns = (mutes, tab) => {
// split into lines, remove empty lines and unnecessary whitespace
const lines = mutes
.trim()
.split("\n")
.map((line) => line.trim())
.filter((line) => line !== "");
// check each line if it is a RegExp or not
for (let i = 0; i < lines.length; i++) {
const line = lines[i];
const regexp = line.match(/^\/(.+)\/(.*)$/);
if (regexp) {
// check that the RegExp is valid
try {
new RegExp(regexp[1], regexp[2]);
// note that regex lines will not be split by spaces!
} catch (err: any) {
// invalid syntax: do not save, do not reset changed flag
os.alert({
type: "error",
title: i18n.ts.regexpError,
text:
i18n.t("regexpErrorDescription", {
tab,
line: i + 1,
}) +
"\n" +
err.toString(),
});
// re-throw error so these invalid settings are not saved
throw err;
}
} else {
// invalid syntax: do not save, do not reset changed flag
os.alert({
type: "error",
title: i18n.ts.regexpError,
text: i18n.t("regexpErrorDescription", {
tab,
line: i + 1,
}),
});
// re-throw error so these invalid settings are not saved
throw new Error("Invalid regular expression");
}
}
return lines;
};
let softMutes, softMLangs, hardMWords, hardMPatterns;
try {
softMutes = parseMutes(softMutedWords.value, i18n.ts._wordMute.soft);
softMLangs = parseMutes(softMutedLangs.value, i18n.ts._wordMute.lang);
hardMutes = parseMutes(hardMutedWords.value, i18n.ts._wordMute.hard);
softMutes = parseSoftMutes(
softMutedWords.value,
i18n.ts._wordMute.soft,
);
softMLangs = parseMutedWords(softMutedLangs.value);
hardMWords = parseMutedWords(hardMutedWords.value);
hardMPatterns = parseMutedPatterns(
hardMutedPatterns.value,
i18n.ts._wordMute.hard,
);
} catch (err) {
// already displayed error message in parseMutes
return;
@ -164,7 +242,8 @@ async function save() {
defaultStore.set("mutedWords", softMutes);
defaultStore.set("mutedLangs", softMLangs);
await os.api("i/update", {
mutedWords: hardMutes,
mutedWords: hardMWords,
mutedPatterns: hardMPatterns,
});
changed.value = false;

View file

@ -107,6 +107,7 @@ export type MeDetailed = UserDetailed & {
isDeleted: boolean;
isExplorable: boolean;
mutedWords: string[][];
mutedPatterns: string[];
mutingNotificationTypes: string[];
noCrawle: boolean;
preventAiLearning: boolean;