Merge branch 'feat/warn-detected-language' into 'develop'

feat: Automatically detect and warn to correct the language of post

Co-authored-by: Lhcfl <Lhcfl@outlook.com>

See merge request firefish/firefish!10704

locales/en-US.yml

@@ -2227,3 +2227,5 @@ moreUrlsDescription: "Enter the pages you want to pin to the help menu in the lo
   left corner using this notation:\n\"Display name\": https://example.com/"
 messagingUnencryptedInfo: "Chats on Firefish are not end-to-end encrypted. Don't share
   any sensitive infomation over Firefish."
+autocorrectNoteLanguage: "Show a warning if the post language does not match the auto-detected result"
+incorrectLanguageWarning: "It looks like your post is in {detected}, but you selected {current}.\nWould you like to set the language to {detected} instead?"

locales/zh-CN.yml

@@ -2056,3 +2056,5 @@ searchRangeDescription: "如果您要过滤时间段，请按以下格式输入
 messagingUnencryptedInfo: "Firefish 上的聊天没有经过端到端加密，请不要在聊天中分享您的敏感信息。"
 noAltTextWarning: 有些附件没有描述。您是否忘记写描述了？
 showNoAltTextWarning: 当您尝试发布没有描述的帖子附件时显示警告
+autocorrectNoteLanguage: 当帖子语言不符合自动检测的结果的时候显示警告
+incorrectLanguageWarning: "看上去您帖子使用的语言是{detected}，但您选择的语言是{current}。\n要改为以{detected}发帖吗？"

packages/client/src/components/MkPostForm.vue

@@ -329,6 +329,7 @@ import XCheatSheet from "@/components/MkCheatSheetDialog.vue";
 import preprocess from "@/scripts/preprocess";
 import { vibrate } from "@/scripts/vibrate";
 import { langmap } from "@/scripts/langmap";
+import { isSupportedLang, isSameLanguage, languageContains, parentLanguage } from "@/scripts/language-utils";
 import type { MenuItem } from "@/types/menu";
 import detectLanguage from "@/scripts/detect-language";
 import icon from "@/scripts/icon";
@@ -758,22 +759,14 @@ const language = ref<string | null>(
 		localStorage.getItem("lang")?.split("-")[0],
 );
 
-function filterLangmapByPrefix(
+function filterSubclassLanguages(
-	prefix: string,
+	langCode: string,
 ): { langCode: string; nativeName: string }[] {
-	let to_return = Object.entries(langmap)
+	return Object.entries(langmap)
-		.filter(([langCode, _]) => langCode.startsWith(prefix))
+		.filter(([lc, _]) => languageContains(langCode, lc))
 		.map(([langCode, v]) => {
 			return { langCode, nativeName: v.nativeName };
 		});
-
-	if (prefix === "zh")
-		to_return = to_return.concat([
-			{ langCode: "yue", nativeName: langmap.yue.nativeName },
-			{ langCode: "nan", nativeName: langmap.nan.nativeName },
-		]);
-
-	return to_return;
 }
 
 function setLanguage() {
@@ -785,7 +778,7 @@ function setLanguage() {
 			type: "label",
 			text: i18n.ts.suggested,
 		});
-		filterLangmapByPrefix(detectedLanguage).forEach((v) => {
+		for (const v of filterSubclassLanguages(detectedLanguage)) {
 			actions.push({
 				text: v.nativeName,
 				danger: false,
@@ -794,7 +787,7 @@ function setLanguage() {
 					language.value = v.langCode;
 				},
 			});
-		});
+		}
 		actions.push(null);
 	}
 
@@ -1019,7 +1012,42 @@ function deleteDraft() {
 	localStorage.setItem("drafts", JSON.stringify(draftData));
 }
 
+
+
 async function post() {
+	// For text that is too short, the false positive rate may be too high, so we don't show alarm.
+	if (defaultStore.state.autocorrectNoteLanguage && text.value.length > 10) {
+		const detectedLanguage: string = detectLanguage(text.value) ?? "";
+
+		const currentLanguageName: string | undefined | false =
+			language.value && langmap[language.value]?.nativeName;
+		const detectedLanguageName: string | undefined | false =
+			detectedLanguage !== "" && langmap[detectedLanguage]?.nativeName;
+
+		if (
+			currentLanguageName &&
+			detectedLanguageName &&
+			!isSameLanguage(detectedLanguage, language.value) &&
+			isSupportedLang(parentLanguage(language.value))
+		) {
+			// "canceled" means "post with detected language".
+			const { canceled } = await os.confirm({
+				type: "warning",
+				text: i18n.t("incorrectLanguageWarning", {
+					detected: detectedLanguageName,
+					current: currentLanguageName,
+				}),
+				okText: i18n.ts.no,
+				cancelText: i18n.ts.yes,
+				isPlaintext: true,
+			});
+
+			if (canceled) {
+				language.value = detectedLanguage;
+			}
+		}
+	}
+
 	if (
 		defaultStore.state.showNoAltTextWarning &&
 		files.value.some((f) => f.comment == null || f.comment.length === 0)

packages/client/src/i18n.ts

@@ -22,7 +22,7 @@ class I18n<T extends Record<string, any>> {
 
 			if (args) {
 				for (const [k, v] of Object.entries(args)) {
-					str = str.replace(`{${k}}`, v.toString());
+					str = str.replaceAll(`{${k}}`, v.toString());
 				}
 			}
 			return str;

packages/client/src/pages/settings/general.vue

@@ -124,6 +124,9 @@
 			<FormSwitch v-model="showNoAltTextWarning" class="_formBlock">{{
 				i18n.ts.showNoAltTextWarning
 			}}</FormSwitch>
+			<FormSwitch v-model="autocorrectNoteLanguage" class="_formBlock">{{
+				i18n.ts.autocorrectNoteLanguage
+			}}</FormSwitch>
 
 			<FormSelect v-model="serverDisconnectedBehavior" class="_formBlock">
 				<template #label>{{ i18n.ts.whenServerDisconnected }}</template>
@@ -530,6 +533,9 @@ const pullToRefreshThreshold = computed(
 const showNoAltTextWarning = computed(
 	defaultStore.makeGetterSetter("showNoAltTextWarning"),
 );
+const autocorrectNoteLanguage = computed(
+	defaultStore.makeGetterSetter("autocorrectNoteLanguage"),
+);
 
 // This feature (along with injectPromo) is currently disabled
 // function onChangeInjectFeaturedNote(v) {

packages/client/src/pages/settings/preferences-backups.vue

@@ -125,6 +125,7 @@ const defaultStoreSaveKeys: (keyof (typeof defaultStore)["state"])[] = [
 	"enablePullToRefresh",
 	"pullToRefreshThreshold",
 	"showNoAltTextWarning",
+	"autocorrectNoteLanguage",
 ];
 const coldDeviceStorageSaveKeys: (keyof typeof ColdDeviceStorage.default)[] = [
 	"lightTheme",

packages/client/src/scripts/langmap.ts

@@ -380,3 +380,71 @@ export const iso639Regional = {
 };
 
 export const langmap = Object.assign({}, langmapNoRegion, iso639Regional);
+
+/**
+* @see https://github.com/komodojp/tinyld/blob/develop/docs/langs.md
+*/
+export const supportedLangs: Record<string, boolean> = {
+ af: true, afr: true,
+ am: true, amh: true,
+ ber: true,
+ rn: true, run: true,
+ my: true, mya: true,
+ id: true, ind: true,
+ km: true, khm: true,
+ tl: true, tgl: true,
+ th: true, tha: true,
+ vi: true, vie: true,
+ zh: true, cmn: true,
+ ja: true, jpn: true,
+ ko: true, kor: true,
+ bn: true, ben: true,
+ gu: true, guj: true,
+ hi: true, hin: true,
+ kn: true, kan: true,
+ ta: true, tam: true,
+ te: true, tel: true,
+ ur: true, urd: true,
+ cs: true, ces: true,
+ el: true, ell: true,
+ la: true, lat: true,
+ mk: true, mkd: true,
+ sr: true, srp: true,
+ sk: true, slk: true,
+ be: true, bel: true,
+ bg: true, bul: true,
+ et: true, est: true,
+ hu: true, hun: true,
+ lv: true, lvs: true,
+ lt: true, lit: true,
+ pl: true, pol: true,
+ ro: true, ron: true,
+ ru: true, rus: true,
+ uk: true, ukr: true,
+ da: true, dan: true,
+ fi: true, fin: true,
+ is: true, isl: true,
+ no: true, nob: true,
+ sv: true, swe: true,
+ nl: true, nld: true,
+ en: true, eng: true,
+ fr: true, fra: true,
+ de: true, deu: true,
+ ga: true, gle: true,
+ it: true, ita: true,
+ pt: true, por: true,
+ es: true, spa: true,
+ ar: true, ara: true,
+ hy: true, hye: true,
+ he: true, heb: true,
+ kk: true, kaz: true,
+ mn: true, mon: true,
+ fa: true, pes: true,
+ tt: true, tat: true,
+ tr: true, tur: true,
+ tk: true, tuk: true,
+ yi: true, yid: true,
+ eo: true, epo: true,
+ tlh: true,
+ vo: true, vol: true,
+}

packages/client/src/scripts/language-utils.ts

@@ -0,0 +1,37 @@
+import { supportedLangs } from "@/scripts/langmap"
+
+export function isSupportedLang(langCode: string | null) {
+	if (!langCode) return false;
+	return supportedLangs[langCode] ?? false;
+}
+
+/**
+ * Compare two language codes to determine whether they are decisively different
+ * @returns false if they are close enough
+ */
+export function isSameLanguage(langCode1: string | null, langCode2: string | null) {
+	return (
+		languageContains(langCode1, langCode2) ||
+		languageContains(langCode2, langCode1)
+	);
+}
+
+/**
+ * Returns true if langCode1 contains langCode2
+ */
+export function languageContains(langCode1: string | null, langCode2: string | null) {
+	if (!langCode1 || !langCode2) return false;
+
+	return parentLanguage(langCode2) === langCode1;
+}
+
+export function parentLanguage(langCode: string | null) {
+	if (!langCode) return null;
+	if (["zh-hant", "zh-hans", "yue", "nan"].includes(langCode)) {
+		return "zh";
+	}
+	if (["nb", "nn"].includes(langCode)) {
+		return "no";
+	}
+	return langCode;
+}

packages/client/src/store.ts

@@ -432,6 +432,10 @@ export const defaultStore = markRaw(
 			where: "account",
 			default: true,
 		},
+		autocorrectNoteLanguage: {
+			where: "account",
+			default: true,
+		},
 	}),
 );