From 872ea8adb0b7a77e0caa62b408f5d1d5e92064b3 Mon Sep 17 00:00:00 2001 From: Lhcfl Date: Wed, 20 Mar 2024 01:08:14 +0800 Subject: [PATCH 1/7] feat: Automatically detect and warn to correct the language of post --- locales/en-US.yml | 2 ++ locales/zh-CN.yml | 2 ++ packages/client/src/components/MkPostForm.vue | 31 +++++++++++++++++++ packages/client/src/i18n.ts | 2 +- .../client/src/pages/settings/general.vue | 6 ++++ .../pages/settings/preferences-backups.vue | 1 + packages/client/src/store.ts | 4 +++ 7 files changed, 47 insertions(+), 1 deletion(-) diff --git a/locales/en-US.yml b/locales/en-US.yml index f5b597e3c9..7a7d911e27 100644 --- a/locales/en-US.yml +++ b/locales/en-US.yml @@ -2227,3 +2227,5 @@ moreUrlsDescription: "Enter the pages you want to pin to the help menu in the lo left corner using this notation:\n\"Display name\": https://example.com/" messagingUnencryptedInfo: "Chats on Firefish are not end-to-end encrypted. Don't share any sensitive infomation over Firefish." +autocorrectNoteLanguage: Automatically detect and warn to correct the language of your post. +incorrectLanguageWarning: "It looks like your post is in {detected}, but the language you selected is {current}.\nWould you like to post in {detected} instead?" diff --git a/locales/zh-CN.yml b/locales/zh-CN.yml index 085fbeb435..23ad51206c 100644 --- a/locales/zh-CN.yml +++ b/locales/zh-CN.yml @@ -2056,3 +2056,5 @@ searchRangeDescription: "如果您要过滤时间段，请按以下格式输入 messagingUnencryptedInfo: "Firefish 上的聊天没有经过端到端加密，请不要在聊天中分享您的敏感信息。" noAltTextWarning: 有些附件没有描述。您是否忘记写描述了？ showNoAltTextWarning: 当您尝试发布没有描述的帖子附件时显示警告 +autocorrectNoteLanguage: 自动检测和弹窗纠正您所使用的帖子语言 +incorrectLanguageWarning: "看上去您帖子使用的语言是{detected}，但您选择的语言是{current}。\n要改为以{detected}发帖吗？" diff --git a/packages/client/src/components/MkPostForm.vue b/packages/client/src/components/MkPostForm.vue index d3f0a1ee7e..0d0710bb3e 100644 --- a/packages/client/src/components/MkPostForm.vue +++ b/packages/client/src/components/MkPostForm.vue @@ -1020,6 +1020,37 @@ function deleteDraft() { } async function post() { + // For text that is too short, the false positive rate may be too high, so we don't show alarm. + if (defaultStore.state.autocorrectNoteLanguage && text.value.length > 10) { + const detectedLanguage: string = detectLanguage(text.value) ?? ""; + + const currentLanguageName : string | undefined | false = + language.value && + langmap[language.value]?.nativeName + const detectedLanguageName : string | undefined | false = + detectedLanguage !== "" && + detectedLanguage !== language.value && + langmap[detectedLanguage]?.nativeName; + + if (currentLanguageName && detectedLanguageName) { + // "canceled" means "post with detected language". + const { canceled } = await os.confirm({ + type: "warning", + text: i18n.t("incorrectLanguageWarning", { + detected: `${detectedLanguageName}`, + current: `${currentLanguageName}`, + }), + okText: i18n.ts.no, + cancelText: i18n.ts.yes, + isPlaintext: true, + }); + + if (canceled) { + language.value = detectedLanguage; + } + } + } + if ( defaultStore.state.showNoAltTextWarning && files.value.some((f) => f.comment == null || f.comment.length === 0) diff --git a/packages/client/src/i18n.ts b/packages/client/src/i18n.ts index 1b3fdc855d..6d352ba03e 100644 --- a/packages/client/src/i18n.ts +++ b/packages/client/src/i18n.ts @@ -22,7 +22,7 @@ class I18n> { if (args) { for (const [k, v] of Object.entries(args)) { - str = str.replace(`{${k}}`, v.toString()); + str = str.replaceAll(`{${k}}`, v.toString()); } } return str; diff --git a/packages/client/src/pages/settings/general.vue b/packages/client/src/pages/settings/general.vue index 2a7190bd6d..15e1172169 100644 --- a/packages/client/src/pages/settings/general.vue +++ b/packages/client/src/pages/settings/general.vue @@ -124,6 +124,9 @@ {{ i18n.ts.showNoAltTextWarning }} + {{ + i18n.ts.autocorrectNoteLanguage + }} @@ -530,6 +533,9 @@ const pullToRefreshThreshold = computed( const showNoAltTextWarning = computed( defaultStore.makeGetterSetter("showNoAltTextWarning"), ); +const autocorrectNoteLanguage = computed( + defaultStore.makeGetterSetter("autocorrectNoteLanguage"), +); // This feature (along with injectPromo) is currently disabled // function onChangeInjectFeaturedNote(v) { diff --git a/packages/client/src/pages/settings/preferences-backups.vue b/packages/client/src/pages/settings/preferences-backups.vue index 41e7dc0ad1..cd55895b48 100644 --- a/packages/client/src/pages/settings/preferences-backups.vue +++ b/packages/client/src/pages/settings/preferences-backups.vue @@ -125,6 +125,7 @@ const defaultStoreSaveKeys: (keyof (typeof defaultStore)["state"])[] = [ "enablePullToRefresh", "pullToRefreshThreshold", "showNoAltTextWarning", + "autocorrectNoteLanguage", ]; const coldDeviceStorageSaveKeys: (keyof typeof ColdDeviceStorage.default)[] = [ "lightTheme", diff --git a/packages/client/src/store.ts b/packages/client/src/store.ts index 6e7e4b8570..877ebc3e10 100644 --- a/packages/client/src/store.ts +++ b/packages/client/src/store.ts @@ -432,6 +432,10 @@ export const defaultStore = markRaw( where: "account", default: true, }, + autocorrectNoteLanguage: { + where: "account", + default: true, + } }), ); From b733948a5327ede281c130da1515d7960c4bfc0e Mon Sep 17 00:00:00 2001 From: Lhcfl Date: Wed, 20 Mar 2024 01:25:21 +0800 Subject: [PATCH 2/7] modify description of autocorrectNoteLanguage --- locales/en-US.yml | 2 +- locales/zh-CN.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/locales/en-US.yml b/locales/en-US.yml index 7a7d911e27..a149d7933a 100644 --- a/locales/en-US.yml +++ b/locales/en-US.yml @@ -2227,5 +2227,5 @@ moreUrlsDescription: "Enter the pages you want to pin to the help menu in the lo left corner using this notation:\n\"Display name\": https://example.com/" messagingUnencryptedInfo: "Chats on Firefish are not end-to-end encrypted. Don't share any sensitive infomation over Firefish." -autocorrectNoteLanguage: Automatically detect and warn to correct the language of your post. +autocorrectNoteLanguage: "Show a waring when the post language does not match the automatically detected result." incorrectLanguageWarning: "It looks like your post is in {detected}, but the language you selected is {current}.\nWould you like to post in {detected} instead?" diff --git a/locales/zh-CN.yml b/locales/zh-CN.yml index 23ad51206c..dee2cd3149 100644 --- a/locales/zh-CN.yml +++ b/locales/zh-CN.yml @@ -2056,5 +2056,5 @@ searchRangeDescription: "如果您要过滤时间段，请按以下格式输入 messagingUnencryptedInfo: "Firefish 上的聊天没有经过端到端加密，请不要在聊天中分享您的敏感信息。" noAltTextWarning: 有些附件没有描述。您是否忘记写描述了？ showNoAltTextWarning: 当您尝试发布没有描述的帖子附件时显示警告 -autocorrectNoteLanguage: 自动检测和弹窗纠正您所使用的帖子语言 +autocorrectNoteLanguage: 当帖子语言不符合自动检测的结果的时候显示警告 incorrectLanguageWarning: "看上去您帖子使用的语言是{detected}，但您选择的语言是{current}。\n要改为以{detected}发帖吗？" From c3a57a3dad88ce93c65867923dd4ba340890debf Mon Sep 17 00:00:00 2001 From: Lhcfl Date: Wed, 20 Mar 2024 02:07:55 +0800 Subject: [PATCH 3/7] fix: check if two languages are close enough --- packages/client/src/components/MkPostForm.vue | 50 ++++++++++++++----- packages/client/src/store.ts | 2 +- 2 files changed, 39 insertions(+), 13 deletions(-) diff --git a/packages/client/src/components/MkPostForm.vue b/packages/client/src/components/MkPostForm.vue index 0d0710bb3e..beee4262cd 100644 --- a/packages/client/src/components/MkPostForm.vue +++ b/packages/client/src/components/MkPostForm.vue @@ -1019,32 +1019,58 @@ function deleteDraft() { localStorage.setItem("drafts", JSON.stringify(draftData)); } +/** + * Compare two language codes to determine whether they are decisively different + * @returns false if they are close enough + */ +function isSameLanguage(langCode1: string | null, langCode2: string | null) { + if (!langCode1 || !langCode2) return false; + + // Sort them alphabetically + if (langCode1 > langCode2) { + [langCode1, langCode2] = [langCode2, langCode1]; + } + if (langCode2.startsWith(langCode1)) return true; + + const inSameSeries = (series: (string | null)[]) => + series.includes(langCode1) && series.includes(langCode2); + + if (inSameSeries(["zh", "zh-hant", "zh-hans", "yue", "nan"])) { + return true; + } + if (inSameSeries(["nb", "no", "nn"])) { + return true; + } + return false; +} + async function post() { // For text that is too short, the false positive rate may be too high, so we don't show alarm. if (defaultStore.state.autocorrectNoteLanguage && text.value.length > 10) { const detectedLanguage: string = detectLanguage(text.value) ?? ""; - const currentLanguageName : string | undefined | false = - language.value && - langmap[language.value]?.nativeName - const detectedLanguageName : string | undefined | false = - detectedLanguage !== "" && - detectedLanguage !== language.value && - langmap[detectedLanguage]?.nativeName; + const currentLanguageName: string | undefined | false = + language.value && langmap[language.value]?.nativeName; + const detectedLanguageName: string | undefined | false = + detectedLanguage !== "" && langmap[detectedLanguage]?.nativeName; - if (currentLanguageName && detectedLanguageName) { + if ( + currentLanguageName && + detectedLanguageName && + !isSameLanguage(detectedLanguage, language.value) + ) { // "canceled" means "post with detected language". const { canceled } = await os.confirm({ type: "warning", text: i18n.t("incorrectLanguageWarning", { - detected: `${detectedLanguageName}`, - current: `${currentLanguageName}`, - }), + detected: detectedLanguageName, + current: currentLanguageName, + }), okText: i18n.ts.no, cancelText: i18n.ts.yes, isPlaintext: true, }); - + if (canceled) { language.value = detectedLanguage; } diff --git a/packages/client/src/store.ts b/packages/client/src/store.ts index 877ebc3e10..cf17917477 100644 --- a/packages/client/src/store.ts +++ b/packages/client/src/store.ts @@ -435,7 +435,7 @@ export const defaultStore = markRaw( autocorrectNoteLanguage: { where: "account", default: true, - } + }, }), ); From 5dc313d6d1bf8f7b84cdd61c1696166940423e4e Mon Sep 17 00:00:00 2001 From: Lhcfl Date: Thu, 21 Mar 2024 17:52:48 +0800 Subject: [PATCH 4/7] dev: change filterLangmapByPrefix to filterSubclassLanguages --- packages/client/src/components/MkPostForm.vue | 45 +++++++++---------- 1 file changed, 20 insertions(+), 25 deletions(-) diff --git a/packages/client/src/components/MkPostForm.vue b/packages/client/src/components/MkPostForm.vue index beee4262cd..ca0c4e1bee 100644 --- a/packages/client/src/components/MkPostForm.vue +++ b/packages/client/src/components/MkPostForm.vue @@ -758,22 +758,14 @@ const language = ref( localStorage.getItem("lang")?.split("-")[0], ); -function filterLangmapByPrefix( - prefix: string, +function filterSubclassLanguages( + langCode: string, ): { langCode: string; nativeName: string }[] { - let to_return = Object.entries(langmap) - .filter(([langCode, _]) => langCode.startsWith(prefix)) + return Object.entries(langmap) + .filter(([lc, _]) => languageContains(langCode, lc)) .map(([langCode, v]) => { return { langCode, nativeName: v.nativeName }; }); - - if (prefix === "zh") - to_return = to_return.concat([ - { langCode: "yue", nativeName: langmap.yue.nativeName }, - { langCode: "nan", nativeName: langmap.nan.nativeName }, - ]); - - return to_return; } function setLanguage() { @@ -785,7 +777,7 @@ function setLanguage() { type: "label", text: i18n.ts.suggested, }); - filterLangmapByPrefix(detectedLanguage).forEach((v) => { + for (const v of filterSubclassLanguages(detectedLanguage)) { actions.push({ text: v.nativeName, danger: false, @@ -794,7 +786,7 @@ function setLanguage() { language.value = v.langCode; }, }); - }); + } actions.push(null); } @@ -1024,21 +1016,24 @@ function deleteDraft() { * @returns false if they are close enough */ function isSameLanguage(langCode1: string | null, langCode2: string | null) { + return languageContains(langCode1, langCode2) || languageContains(langCode2, langCode1); +} + +/** + * Returns true if langCode1 contains langCode2 + */ +function languageContains(langCode1: string | null, langCode2: string | null) { if (!langCode1 || !langCode2) return false; - // Sort them alphabetically - if (langCode1 > langCode2) { - [langCode1, langCode2] = [langCode2, langCode1]; - } - if (langCode2.startsWith(langCode1)) return true; - - const inSameSeries = (series: (string | null)[]) => - series.includes(langCode1) && series.includes(langCode2); - - if (inSameSeries(["zh", "zh-hant", "zh-hans", "yue", "nan"])) { + if (langCode1 === "zh" && + ["zh-hant", "zh-hans", "yue", "nan"].includes(langCode2) + ) { return true; } - if (inSameSeries(["nb", "no", "nn"])) { + + if (langCode1 === "no" && + ["nb", "nn"].includes(langCode2) + ) { return true; } return false; From 17a42f015aa2e69bd7e5d896a26c2d595327ab73 Mon Sep 17 00:00:00 2001 From: Lhcfl Date: Thu, 21 Mar 2024 18:00:32 +0800 Subject: [PATCH 5/7] chore: format --- packages/client/src/components/MkPostForm.vue | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/client/src/components/MkPostForm.vue b/packages/client/src/components/MkPostForm.vue index ca0c4e1bee..b4f949f682 100644 --- a/packages/client/src/components/MkPostForm.vue +++ b/packages/client/src/components/MkPostForm.vue @@ -1016,7 +1016,10 @@ function deleteDraft() { * @returns false if they are close enough */ function isSameLanguage(langCode1: string | null, langCode2: string | null) { - return languageContains(langCode1, langCode2) || languageContains(langCode2, langCode1); + return ( + languageContains(langCode1, langCode2) || + languageContains(langCode2, langCode1) + ); } /** @@ -1025,15 +1028,14 @@ function isSameLanguage(langCode1: string | null, langCode2: string | null) { function languageContains(langCode1: string | null, langCode2: string | null) { if (!langCode1 || !langCode2) return false; - if (langCode1 === "zh" && + if ( + langCode1 === "zh" && ["zh-hant", "zh-hans", "yue", "nan"].includes(langCode2) ) { return true; } - if (langCode1 === "no" && - ["nb", "nn"].includes(langCode2) - ) { + if (langCode1 === "no" && ["nb", "nn"].includes(langCode2)) { return true; } return false; From 95da62680ad2c311e8d1e522a4ec054b3ae7ab05 Mon Sep 17 00:00:00 2001 From: Lhcfl Date: Sun, 24 Mar 2024 11:00:49 +0800 Subject: [PATCH 6/7] add isSupportedLanguage --- packages/client/src/components/MkPostForm.vue | 32 +-------- packages/client/src/scripts/langmap.ts | 68 +++++++++++++++++++ packages/client/src/scripts/language-utils.ts | 37 ++++++++++ 3 files changed, 108 insertions(+), 29 deletions(-) create mode 100644 packages/client/src/scripts/language-utils.ts diff --git a/packages/client/src/components/MkPostForm.vue b/packages/client/src/components/MkPostForm.vue index b4f949f682..b3245fa815 100644 --- a/packages/client/src/components/MkPostForm.vue +++ b/packages/client/src/components/MkPostForm.vue @@ -329,6 +329,7 @@ import XCheatSheet from "@/components/MkCheatSheetDialog.vue"; import preprocess from "@/scripts/preprocess"; import { vibrate } from "@/scripts/vibrate"; import { langmap } from "@/scripts/langmap"; +import { isSupportedLang, isSameLanguage, languageContains, parentLanguage } from "@/scripts/language-utils"; import type { MenuItem } from "@/types/menu"; import detectLanguage from "@/scripts/detect-language"; import icon from "@/scripts/icon"; @@ -1011,35 +1012,7 @@ function deleteDraft() { localStorage.setItem("drafts", JSON.stringify(draftData)); } -/** - * Compare two language codes to determine whether they are decisively different - * @returns false if they are close enough - */ -function isSameLanguage(langCode1: string | null, langCode2: string | null) { - return ( - languageContains(langCode1, langCode2) || - languageContains(langCode2, langCode1) - ); -} -/** - * Returns true if langCode1 contains langCode2 - */ -function languageContains(langCode1: string | null, langCode2: string | null) { - if (!langCode1 || !langCode2) return false; - - if ( - langCode1 === "zh" && - ["zh-hant", "zh-hans", "yue", "nan"].includes(langCode2) - ) { - return true; - } - - if (langCode1 === "no" && ["nb", "nn"].includes(langCode2)) { - return true; - } - return false; -} async function post() { // For text that is too short, the false positive rate may be too high, so we don't show alarm. @@ -1054,7 +1027,8 @@ async function post() { if ( currentLanguageName && detectedLanguageName && - !isSameLanguage(detectedLanguage, language.value) + !isSameLanguage(detectedLanguage, language.value) && + isSupportedLang(parentLanguage(language.value)) ) { // "canceled" means "post with detected language". const { canceled } = await os.confirm({ diff --git a/packages/client/src/scripts/langmap.ts b/packages/client/src/scripts/langmap.ts index df3214e449..8dd12ac5b9 100644 --- a/packages/client/src/scripts/langmap.ts +++ b/packages/client/src/scripts/langmap.ts @@ -380,3 +380,71 @@ export const iso639Regional = { }; export const langmap = Object.assign({}, langmapNoRegion, iso639Regional); + +/** +* @see https://github.com/komodojp/tinyld/blob/develop/docs/langs.md +*/ +export const supportedLangs: Record = { + af: true, afr: true, + am: true, amh: true, + ber: true, + rn: true, run: true, + my: true, mya: true, + id: true, ind: true, + km: true, khm: true, + tl: true, tgl: true, + th: true, tha: true, + vi: true, vie: true, + zh: true, cmn: true, + ja: true, jpn: true, + ko: true, kor: true, + bn: true, ben: true, + gu: true, guj: true, + hi: true, hin: true, + kn: true, kan: true, + ta: true, tam: true, + te: true, tel: true, + ur: true, urd: true, + cs: true, ces: true, + el: true, ell: true, + la: true, lat: true, + mk: true, mkd: true, + sr: true, srp: true, + sk: true, slk: true, + be: true, bel: true, + bg: true, bul: true, + et: true, est: true, + hu: true, hun: true, + lv: true, lvs: true, + lt: true, lit: true, + pl: true, pol: true, + ro: true, ron: true, + ru: true, rus: true, + uk: true, ukr: true, + da: true, dan: true, + fi: true, fin: true, + is: true, isl: true, + no: true, nob: true, + sv: true, swe: true, + nl: true, nld: true, + en: true, eng: true, + fr: true, fra: true, + de: true, deu: true, + ga: true, gle: true, + it: true, ita: true, + pt: true, por: true, + es: true, spa: true, + ar: true, ara: true, + hy: true, hye: true, + he: true, heb: true, + kk: true, kaz: true, + mn: true, mon: true, + fa: true, pes: true, + tt: true, tat: true, + tr: true, tur: true, + tk: true, tuk: true, + yi: true, yid: true, + eo: true, epo: true, + tlh: true, + vo: true, vol: true, +} diff --git a/packages/client/src/scripts/language-utils.ts b/packages/client/src/scripts/language-utils.ts new file mode 100644 index 0000000000..32591e6eea --- /dev/null +++ b/packages/client/src/scripts/language-utils.ts @@ -0,0 +1,37 @@ +import { supportedLangs } from "@/scripts/langmap" + +export function isSupportedLang(langCode: string | null) { + if (!langCode) return false; + return supportedLangs[langCode] ?? false; +} + +/** + * Compare two language codes to determine whether they are decisively different + * @returns false if they are close enough + */ +export function isSameLanguage(langCode1: string | null, langCode2: string | null) { + return ( + languageContains(langCode1, langCode2) || + languageContains(langCode2, langCode1) + ); +} + +/** + * Returns true if langCode1 contains langCode2 + */ +export function languageContains(langCode1: string | null, langCode2: string | null) { + if (!langCode1 || !langCode2) return false; + + return parentLanguage(langCode2) === langCode1; +} + +export function parentLanguage(langCode: string | null) { + if (!langCode) return null; + if (["zh-hant", "zh-hans", "yue", "nan"].includes(langCode)) { + return "zh"; + } + if (["nb", "nn"].includes(langCode)) { + return "no"; + } + return langCode; +} From 5bc3e0861c609a3bbd32da25da86f48fe0c36e34 Mon Sep 17 00:00:00 2001 From: naskya Date: Sun, 24 Mar 2024 20:55:26 +0900 Subject: [PATCH 7/7] locale (minor): update en-US.yml --- locales/en-US.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/locales/en-US.yml b/locales/en-US.yml index a149d7933a..2a16417faa 100644 --- a/locales/en-US.yml +++ b/locales/en-US.yml @@ -2227,5 +2227,5 @@ moreUrlsDescription: "Enter the pages you want to pin to the help menu in the lo left corner using this notation:\n\"Display name\": https://example.com/" messagingUnencryptedInfo: "Chats on Firefish are not end-to-end encrypted. Don't share any sensitive infomation over Firefish." -autocorrectNoteLanguage: "Show a waring when the post language does not match the automatically detected result." -incorrectLanguageWarning: "It looks like your post is in {detected}, but the language you selected is {current}.\nWould you like to post in {detected} instead?" +autocorrectNoteLanguage: "Show a warning if the post language does not match the auto-detected result" +incorrectLanguageWarning: "It looks like your post is in {detected}, but you selected {current}.\nWould you like to set the language to {detected} instead?"