From be163498c0a8e7cba4538804fb382ee4978b4dec Mon Sep 17 00:00:00 2001 From: Essem Date: Thu, 5 Oct 2023 22:28:12 -0500 Subject: [PATCH] feat: :globe_with_meridians: Add language picker to post form This also refactors the langmap and makes many other small language-related changes to get it working. Fixes #9692 --- locales/en-US.yml | 1 + packages/backend/src/misc/langmap.ts | 460 ++++-------------- packages/backend/src/models/schema/note.ts | 7 + .../src/remote/activitypub/models/note.ts | 20 +- .../src/server/api/endpoints/i/update.ts | 2 +- .../src/server/api/endpoints/notes/create.ts | 7 +- .../src/server/api/endpoints/notes/edit.ts | 10 +- packages/backend/src/services/note/create.ts | 4 +- packages/client/src/components/MkPostForm.vue | 59 +++ .../src/components/MkPostFormDialog.vue | 1 + packages/client/src/scripts/langmap.ts | 460 ++++-------------- packages/firefish-js/src/consts.ts | 93 ++++ packages/firefish-js/src/index.ts | 1 + 13 files changed, 361 insertions(+), 764 deletions(-) diff --git a/locales/en-US.yml b/locales/en-US.yml index ae04dd25e7..792ac79d2e 100644 --- a/locales/en-US.yml +++ b/locales/en-US.yml @@ -1012,6 +1012,7 @@ speed: "Speed" slow: "Slow" fast: "Fast" sensitiveMediaDetection: "Detection of NSFW media" +autoDetect: "Auto detect" localOnly: "Local only" remoteOnly: "Remote only" failedToUpload: "Upload failed" diff --git a/packages/backend/src/misc/langmap.ts b/packages/backend/src/misc/langmap.ts index 106130d3c3..74e8598d56 100644 --- a/packages/backend/src/misc/langmap.ts +++ b/packages/backend/src/misc/langmap.ts @@ -1,372 +1,156 @@ // TODO: sharedに置いてフロントエンドのと統合したい -export const langmap = { - ach: { - nativeName: "Lwo", - }, - ady: { - nativeName: "Адыгэбзэ", - }, +export const iso639Langs1 = { af: { nativeName: "Afrikaans", }, - "af-NA": { - nativeName: "Afrikaans (Namibia)", - }, - "af-ZA": { - nativeName: "Afrikaans (South Africa)", - }, ak: { nativeName: "Tɕɥi", }, ar: { nativeName: "العربية", }, - "ar-AR": { - nativeName: "العربية", - }, - "ar-MA": { - nativeName: "العربية", - }, - "ar-SA": { - nativeName: "العربية (السعودية)", - }, - "ay-BO": { + ay: { nativeName: "Aymar aru", }, az: { nativeName: "Azərbaycan dili", }, - "az-AZ": { - nativeName: "Azərbaycan dili", - }, - "be-BY": { + be: { nativeName: "Беларуская", }, bg: { nativeName: "Български", }, - "bg-BG": { - nativeName: "Български", - }, bn: { nativeName: "বাংলা", }, - "bn-IN": { - nativeName: "বাংলা (ভারত)", - }, - "bn-BD": { - nativeName: "বাংলা(বাংলাদেশ)", - }, br: { nativeName: "Brezhoneg", }, - "bs-BA": { + bs: { nativeName: "Bosanski", }, ca: { nativeName: "Català", }, - "ca-ES": { - nativeName: "Català", - }, - cak: { - nativeName: "Maya Kaqchikel", - }, - "ck-US": { - nativeName: "ᏣᎳᎩ (tsalagi)", - }, cs: { nativeName: "Čeština", }, - "cs-CZ": { - nativeName: "Čeština", - }, cy: { nativeName: "Cymraeg", }, - "cy-GB": { - nativeName: "Cymraeg", - }, da: { nativeName: "Dansk", }, - "da-DK": { - nativeName: "Dansk", - }, de: { nativeName: "Deutsch", }, - "de-AT": { - nativeName: "Deutsch (Österreich)", - }, - "de-DE": { - nativeName: "Deutsch (Deutschland)", - }, - "de-CH": { - nativeName: "Deutsch (Schweiz)", - }, - dsb: { - nativeName: "Dolnoserbšćina", - }, el: { nativeName: "Ελληνικά", }, - "el-GR": { - nativeName: "Ελληνικά", - }, en: { nativeName: "English", }, - "en-GB": { - nativeName: "English (UK)", - }, - "en-AU": { - nativeName: "English (Australia)", - }, - "en-CA": { - nativeName: "English (Canada)", - }, - "en-IE": { - nativeName: "English (Ireland)", - }, - "en-IN": { - nativeName: "English (India)", - }, - "en-PI": { - nativeName: "English (Pirate)", - }, - "en-SG": { - nativeName: "English (Singapore)", - }, - "en-UD": { - nativeName: "English (Upside Down)", - }, - "en-US": { - nativeName: "English (US)", - }, - "en-ZA": { - nativeName: "English (South Africa)", - }, - "en@pirate": { - nativeName: "English (Pirate)", - }, eo: { nativeName: "Esperanto", }, - "eo-EO": { - nativeName: "Esperanto", - }, es: { nativeName: "Español", }, - "es-AR": { - nativeName: "Español (Argentine)", - }, - "es-419": { - nativeName: "Español (Latinoamérica)", - }, - "es-CL": { - nativeName: "Español (Chile)", - }, - "es-CO": { - nativeName: "Español (Colombia)", - }, - "es-EC": { - nativeName: "Español (Ecuador)", - }, - "es-ES": { - nativeName: "Español (España)", - }, - "es-LA": { - nativeName: "Español (Latinoamérica)", - }, - "es-NI": { - nativeName: "Español (Nicaragua)", - }, - "es-MX": { - nativeName: "Español (México)", - }, - "es-US": { - nativeName: "Español (Estados Unidos)", - }, - "es-VE": { - nativeName: "Español (Venezuela)", - }, et: { nativeName: "eesti keel", }, - "et-EE": { - nativeName: "Eesti (Estonia)", - }, eu: { nativeName: "Euskara", }, - "eu-ES": { - nativeName: "Euskara", - }, fa: { nativeName: "فارسی", }, - "fa-IR": { - nativeName: "فارسی", - }, - "fb-LT": { - nativeName: "Leet Speak", - }, ff: { nativeName: "Fulah", }, fi: { nativeName: "Suomi", }, - "fi-FI": { - nativeName: "Suomi", - }, fo: { nativeName: "Føroyskt", }, - "fo-FO": { - nativeName: "Føroyskt (Færeyjar)", - }, fr: { nativeName: "Français", }, - "fr-CA": { - nativeName: "Français (Canada)", - }, - "fr-FR": { - nativeName: "Français (France)", - }, - "fr-BE": { - nativeName: "Français (Belgique)", - }, - "fr-CH": { - nativeName: "Français (Suisse)", - }, - "fy-NL": { + fy: { nativeName: "Frysk", }, ga: { nativeName: "Gaeilge", }, - "ga-IE": { - nativeName: "Gaeilge", - }, gd: { nativeName: "Gàidhlig", }, gl: { nativeName: "Galego", }, - "gl-ES": { - nativeName: "Galego", - }, - "gn-PY": { + gn: { nativeName: "Avañe'ẽ", }, - "gu-IN": { + gu: { nativeName: "ગુજરાતી", }, gv: { nativeName: "Gaelg", }, - "gx-GR": { - nativeName: "Ἑλληνική ἀρχαία", - }, he: { nativeName: "עברית‏", }, - "he-IL": { - nativeName: "עברית‏", - }, hi: { nativeName: "हिन्दी", }, - "hi-IN": { - nativeName: "हिन्दी", - }, hr: { nativeName: "Hrvatski", }, - "hr-HR": { - nativeName: "Hrvatski", - }, - hsb: { - nativeName: "Hornjoserbšćina", - }, ht: { nativeName: "Kreyòl", }, hu: { nativeName: "Magyar", }, - "hu-HU": { - nativeName: "Magyar", - }, hy: { nativeName: "Հայերեն", }, - "hy-AM": { - nativeName: "Հայերեն (Հայաստան)", - }, id: { nativeName: "Bahasa Indonesia", }, - "id-ID": { - nativeName: "Bahasa Indonesia", - }, is: { nativeName: "Íslenska", }, - "is-IS": { - nativeName: "Íslenska (Iceland)", - }, it: { nativeName: "Italiano", }, - "it-IT": { - nativeName: "Italiano", - }, ja: { nativeName: "日本語", }, - "ja-JP": { - nativeName: "日本語 (日本)", - }, - "jv-ID": { + jv: { nativeName: "Basa Jawa", }, - "ka-GE": { + ka: { nativeName: "ქართული", }, - "kk-KZ": { + kk: { nativeName: "Қазақша", }, - km: { - nativeName: "ភាសាខ្មែរ", - }, kl: { nativeName: "kalaallisut", }, - "km-KH": { + km: { nativeName: "ភាសាខ្មែរ", }, - kab: { - nativeName: "Taqbaylit", - }, kn: { nativeName: "ಕನ್ನಡ", }, - "kn-IN": { - nativeName: "ಕನ್ನಡ (India)", - }, ko: { nativeName: "한국어", }, - "ko-KR": { - nativeName: "한국어 (한국)", - }, - "ku-TR": { + ku: { nativeName: "Kurdî", }, kw: { @@ -375,66 +159,39 @@ export const langmap = { la: { nativeName: "Latin", }, - "la-VA": { - nativeName: "Latin", - }, lb: { nativeName: "Lëtzebuergesch", }, - "li-NL": { + li: { nativeName: "Lèmbörgs", }, lt: { nativeName: "Lietuvių", }, - "lt-LT": { - nativeName: "Lietuvių", - }, lv: { nativeName: "Latviešu", }, - "lv-LV": { - nativeName: "Latviešu", - }, - mai: { - nativeName: "मैथिली, মৈথিলী", - }, - "mg-MG": { + mg: { nativeName: "Malagasy", }, mk: { nativeName: "Македонски", }, - "mk-MK": { - nativeName: "Македонски (Македонски)", - }, ml: { nativeName: "മലയാളം", }, - "ml-IN": { - nativeName: "മലയാളം", - }, - "mn-MN": { + mn: { nativeName: "Монгол", }, mr: { nativeName: "मराठी", }, - "mr-IN": { - nativeName: "मराठी", - }, ms: { nativeName: "Bahasa Melayu", }, - "ms-MY": { - nativeName: "Bahasa Melayu", - }, mt: { nativeName: "Malti", }, - "mt-MT": { - nativeName: "Malti", - }, my: { nativeName: "ဗမာစကာ", }, @@ -444,223 +201,182 @@ export const langmap = { nb: { nativeName: "Norsk (bokmål)", }, - "nb-NO": { - nativeName: "Norsk (bokmål)", - }, ne: { nativeName: "नेपाली", }, - "ne-NP": { - nativeName: "नेपाली", - }, nl: { nativeName: "Nederlands", }, - "nl-BE": { - nativeName: "Nederlands (België)", - }, - "nl-NL": { - nativeName: "Nederlands (Nederland)", - }, - "nn-NO": { + nn: { nativeName: "Norsk (nynorsk)", }, oc: { nativeName: "Occitan", }, - "or-IN": { + or: { nativeName: "ଓଡ଼ିଆ", }, pa: { nativeName: "ਪੰਜਾਬੀ", }, - "pa-IN": { - nativeName: "ਪੰਜਾਬੀ (ਭਾਰਤ ਨੂੰ)", - }, pl: { nativeName: "Polski", }, - "pl-PL": { - nativeName: "Polski", - }, - "ps-AF": { + ps: { nativeName: "پښتو", }, pt: { nativeName: "Português", }, - "pt-BR": { - nativeName: "Português (Brasil)", - }, - "pt-PT": { - nativeName: "Português (Portugal)", - }, - "qu-PE": { + qu: { nativeName: "Qhichwa", }, - "rm-CH": { + rm: { nativeName: "Rumantsch", }, ro: { nativeName: "Română", }, - "ro-RO": { - nativeName: "Română", - }, ru: { nativeName: "Русский", }, - "ru-RU": { - nativeName: "Русский", - }, - "sa-IN": { + sa: { nativeName: "संस्कृतम्", }, - "se-NO": { + se: { nativeName: "Davvisámegiella", }, sh: { nativeName: "српскохрватски", }, - "si-LK": { + si: { nativeName: "සිංහල", }, sk: { nativeName: "Slovenčina", }, - "sk-SK": { - nativeName: "Slovenčina (Slovakia)", - }, sl: { nativeName: "Slovenščina", }, - "sl-SI": { - nativeName: "Slovenščina", - }, - "so-SO": { + so: { nativeName: "Soomaaliga", }, sq: { nativeName: "Shqip", }, - "sq-AL": { - nativeName: "Shqip", - }, sr: { nativeName: "Српски", }, - "sr-RS": { - nativeName: "Српски (Serbia)", - }, su: { nativeName: "Basa Sunda", }, sv: { nativeName: "Svenska", }, - "sv-SE": { - nativeName: "Svenska", - }, sw: { nativeName: "Kiswahili", }, - "sw-KE": { - nativeName: "Kiswahili", - }, ta: { nativeName: "தமிழ்", }, - "ta-IN": { - nativeName: "தமிழ்", - }, te: { nativeName: "తెలుగు", }, - "te-IN": { - nativeName: "తెలుగు", - }, tg: { nativeName: "забо́ни тоҷикӣ́", }, - "tg-TJ": { - nativeName: "тоҷикӣ", - }, th: { nativeName: "ภาษาไทย", }, - "th-TH": { - nativeName: "ภาษาไทย (ประเทศไทย)", - }, - fil: { - nativeName: "Filipino", - }, - tlh: { - nativeName: "tlhIngan-Hol", - }, tr: { nativeName: "Türkçe", }, - "tr-TR": { - nativeName: "Türkçe", - }, - "tt-RU": { + tt: { nativeName: "татарча", }, uk: { nativeName: "Українська", }, - "uk-UA": { - nativeName: "Українська", - }, ur: { nativeName: "اردو", }, - "ur-PK": { - nativeName: "اردو", - }, uz: { nativeName: "O'zbek", }, - "uz-UZ": { - nativeName: "O'zbek", - }, vi: { nativeName: "Tiếng Việt", }, - "vi-VN": { - nativeName: "Tiếng Việt", - }, - "xh-ZA": { + xh: { nativeName: "isiXhosa", }, yi: { nativeName: "ייִדיש", }, - "yi-DE": { - nativeName: "ייִדיש (German)", - }, zh: { nativeName: "中文", }, - "zh-Hans": { - nativeName: "中文简体", - }, - "zh-Hant": { - nativeName: "中文繁體", - }, - "zh-CN": { - nativeName: "中文（中国大陆）", - }, - "zh-HK": { - nativeName: "中文（香港）", - }, - "zh-SG": { - nativeName: "中文（新加坡）", - }, - "zh-TW": { - nativeName: "中文（台灣）", - }, - "zu-ZA": { + zu: { nativeName: "isiZulu", }, }; + +export const iso639Langs3 = { + ach: { + nativeName: "Lwo", + }, + ady: { + nativeName: "Адыгэбзэ", + }, + cak: { + nativeName: "Maya Kaqchikel", + }, + chr: { + nativeName: "ᏣᎳᎩ (tsalagi)", + }, + dsb: { + nativeName: "Dolnoserbšćina", + }, + fil: { + nativeName: "Filipino", + }, + hsb: { + nativeName: "Hornjoserbšćina", + }, + kab: { + nativeName: "Taqbaylit", + }, + mai: { + nativeName: "मैथिली, মৈথিলী", + }, + tlh: { + nativeName: "tlhIngan-Hol", + }, + tok: { + nativeName: "Toki Pona", + }, +}; + +export const langmapNoRegion = Object.assign({}, iso639Langs1, iso639Langs3); + +export const iso639Regional = { + "zh-hans": { + nativeName: "中文简体", + }, + "zh-hant": { + nativeName: "中文繁體", + }, + "zh-cn": { + nativeName: "中文（中国大陆）", + }, + "zh-hk": { + nativeName: "中文（香港）", + }, + "zh-sg": { + nativeName: "中文（新加坡）", + }, + "zh-tw": { + nativeName: "中文（台灣）", + }, +}; + +export const langmap = Object.assign({}, langmapNoRegion, iso639Regional); diff --git a/packages/backend/src/models/schema/note.ts b/packages/backend/src/models/schema/note.ts index e17f054e8e..7dcdbc9b03 100644 --- a/packages/backend/src/models/schema/note.ts +++ b/packages/backend/src/models/schema/note.ts @@ -1,3 +1,5 @@ +import { langmap } from "@/misc/langmap.js"; + export const packedNoteSchema = { type: "object", properties: { @@ -19,6 +21,11 @@ export const packedNoteSchema = { optional: false, nullable: true, }, + lang: { + type: "string", + enum: [...Object.keys(langmap)], + nullable: true, + }, cw: { type: "string", optional: true, diff --git a/packages/backend/src/remote/activitypub/models/note.ts b/packages/backend/src/remote/activitypub/models/note.ts index b2354bba61..552a19a951 100644 --- a/packages/backend/src/remote/activitypub/models/note.ts +++ b/packages/backend/src/remote/activitypub/models/note.ts @@ -313,16 +313,13 @@ export async function createNote( ) { text = note.source.content; if (note.contentMap != null) { - const key = Object.keys(note.contentMap)[0]; - lang = Object.keys(langmap).includes(key) - ? key.trim().split("-")[0].split("@")[0] - : null; + const key = Object.keys(note.contentMap)[0].toLowerCase(); + lang = Object.keys(langmap).includes(key) ? key : null; } } else if (note.contentMap != null) { const entry = Object.entries(note.contentMap)[0]; - lang = Object.keys(langmap).includes(entry[0]) - ? entry[0].trim().split("-")[0].split("@")[0] - : null; + const key = entry[0].toLowerCase(); + lang = Object.keys(langmap).includes(key) ? key : null; text = htmlToMfm(entry[1], note.tag); } else if (typeof note.content === "string") { text = htmlToMfm(note.content, note.tag); @@ -588,15 +585,12 @@ export async function updateNote(value: string | IObject, resolver?: Resolver) { text = post.source.content; if (post.contentMap != null) { const key = Object.keys(post.contentMap)[0]; - lang = Object.keys(langmap).includes(key) - ? key.trim().split("-")[0].split("@")[0] - : null; + lang = Object.keys(langmap).includes(key) ? key : null; } } else if (post.contentMap != null) { const entry = Object.entries(post.contentMap)[0]; - lang = Object.keys(langmap).includes(entry[0]) - ? entry[0].trim().split("-")[0].split("@")[0] - : null; + const key = entry[0].toLowerCase(); + lang = Object.keys(langmap).includes(key) ? key : null; text = htmlToMfm(entry[1], post.tag); } else if (typeof post.content === "string") { text = htmlToMfm(post.content, post.tag); diff --git a/packages/backend/src/server/api/endpoints/i/update.ts b/packages/backend/src/server/api/endpoints/i/update.ts index 0037839b5c..a464bd344a 100644 --- a/packages/backend/src/server/api/endpoints/i/update.ts +++ b/packages/backend/src/server/api/endpoints/i/update.ts @@ -91,7 +91,7 @@ export const paramDef = { birthday: { ...Users.birthdaySchema, nullable: true }, lang: { type: "string", - enum: [null, ...Object.keys(langmap)], + enum: Object.keys(langmap), nullable: true, }, avatarId: { type: "string", format: "misskey:id", nullable: true }, diff --git a/packages/backend/src/server/api/endpoints/notes/create.ts b/packages/backend/src/server/api/endpoints/notes/create.ts index 150356811d..f3fc2ac28b 100644 --- a/packages/backend/src/server/api/endpoints/notes/create.ts +++ b/packages/backend/src/server/api/endpoints/notes/create.ts @@ -17,6 +17,7 @@ import { ApiError } from "../../error.js"; import define from "../../define.js"; import { HOUR } from "@/const.js"; import { getNote } from "../../common/getters.js"; +import { langmap } from "@/misc/langmap.js"; export const meta = { tags: ["notes"], @@ -108,7 +109,11 @@ export const paramDef = { }, }, text: { type: "string", maxLength: MAX_NOTE_TEXT_LENGTH, nullable: true }, - lang: { type: "string", nullable: true, maxLength: 10 }, + lang: { + type: "string", + enum: Object.keys(langmap), + nullable: true, + }, cw: { type: "string", nullable: true, maxLength: 100 }, localOnly: { type: "boolean", default: false }, noExtractMentions: { type: "boolean", default: false }, diff --git a/packages/backend/src/server/api/endpoints/notes/edit.ts b/packages/backend/src/server/api/endpoints/notes/edit.ts index e6d39c5249..404e7e35f1 100644 --- a/packages/backend/src/server/api/endpoints/notes/edit.ts +++ b/packages/backend/src/server/api/endpoints/notes/edit.ts @@ -171,7 +171,11 @@ export const paramDef = { }, }, text: { type: "string", maxLength: MAX_NOTE_TEXT_LENGTH, nullable: true }, - lang: { type: "string", nullable: true, maxLength: 10 }, + lang: { + type: "string", + enum: Object.keys(langmap), + nullable: true, + }, cw: { type: "string", nullable: true, maxLength: 250 }, localOnly: { type: "boolean", default: false }, noExtractMentions: { type: "boolean", default: false }, @@ -379,9 +383,9 @@ export default define(meta, paramDef, async (ps, user) => { } if (ps.lang) { - if (!Object.keys(langmap).includes(ps.lang.trim())) + if (!Object.keys(langmap).includes(ps.lang.toLowerCase())) throw new Error("invalid param"); - ps.lang = ps.lang.trim().split("-")[0].split("@")[0]; + ps.lang = ps.lang.toLowerCase(); } else if (ps.text) { ps.lang = detectLanguage(ps.text); } else { diff --git a/packages/backend/src/services/note/create.ts b/packages/backend/src/services/note/create.ts index 632cb960e2..ae677cb8ac 100644 --- a/packages/backend/src/services/note/create.ts +++ b/packages/backend/src/services/note/create.ts @@ -280,9 +280,9 @@ export default async ( } if (data.lang) { - if (!Object.keys(langmap).includes(data.lang.trim())) + if (!Object.keys(langmap).includes(data.lang.toLowerCase())) throw new Error("invalid param"); - data.lang = data.lang.trim().split("-")[0].split("@")[0]; + data.lang = data.lang.toLowerCase(); } else if (data.text) { data.lang = detectLanguage(data.text); } else { diff --git a/packages/client/src/components/MkPostForm.vue b/packages/client/src/components/MkPostForm.vue index b71dac6989..9022df1abe 100644 --- a/packages/client/src/components/MkPostForm.vue +++ b/packages/client/src/components/MkPostForm.vue @@ -51,6 +51,14 @@ > +