From 2dcc46a9e3aa4f1e0840506edacf43e319e74239 Mon Sep 17 00:00:00 2001 From: naskya Date: Fri, 17 Nov 2023 17:50:20 +0900 Subject: [PATCH] chore: remove language detection from backend --- packages/backend/package.json | 1 - packages/backend/src/misc/detect-language.ts | 11 ----------- .../backend/src/remote/activitypub/renderer/note.ts | 6 ++---- .../backend/src/server/api/endpoints/notes/edit.ts | 3 --- packages/backend/src/services/note/create.ts | 3 --- pnpm-lock.yaml | 4 +--- 6 files changed, 3 insertions(+), 25 deletions(-) delete mode 100644 packages/backend/src/misc/detect-language.ts diff --git a/packages/backend/package.json b/packages/backend/package.json index 1b880121a0..5166f909a3 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -130,7 +130,6 @@ "tar-stream": "^3.1.6", "tesseract.js": "^5.0.3", "tinycolor2": "1.6.0", - "tinyld": "^1.3.4", "tmp": "0.2.1", "twemoji-parser": "14.0.0", "typeorm": "0.3.17", diff --git a/packages/backend/src/misc/detect-language.ts b/packages/backend/src/misc/detect-language.ts deleted file mode 100644 index 6147247dee..0000000000 --- a/packages/backend/src/misc/detect-language.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { detect } from "tinyld"; -import * as mfm from "mfm-js"; - -export default function detectLanguage(text: string): string { - const nodes = mfm.parse(text); - const filtered = mfm.extract(nodes, (node) => { - return node.type === "text" || node.type === "quote"; - }); - const purified = mfm.toString(filtered); - return detect(purified); -} diff --git a/packages/backend/src/remote/activitypub/renderer/note.ts b/packages/backend/src/remote/activitypub/renderer/note.ts index b840e2d6f1..f3bce41aa7 100644 --- a/packages/backend/src/remote/activitypub/renderer/note.ts +++ b/packages/backend/src/remote/activitypub/renderer/note.ts @@ -6,7 +6,6 @@ import { DriveFiles, Notes, Users, Emojis, Polls } from "@/models/index.js"; import type { Emoji } from "@/models/entities/emoji.js"; import type { Poll } from "@/models/entities/poll.js"; import toHtml from "@/remote/activitypub/misc/get-note-html.js"; -import detectLanguage from "@/misc/detect-language.js"; import renderEmoji from "./emoji.js"; import renderMention from "./mention.js"; import renderHashtag from "./hashtag.js"; @@ -115,10 +114,9 @@ export default async function renderNote( }), ); - const lang = note.lang ?? detectLanguage(text); - const contentMap = lang + const contentMap = note.lang ? { - [lang]: content, + [note.lang]: content, } : null; diff --git a/packages/backend/src/server/api/endpoints/notes/edit.ts b/packages/backend/src/server/api/endpoints/notes/edit.ts index a05580aa21..fdb6ede385 100644 --- a/packages/backend/src/server/api/endpoints/notes/edit.ts +++ b/packages/backend/src/server/api/endpoints/notes/edit.ts @@ -35,7 +35,6 @@ import renderUpdate from "@/remote/activitypub/renderer/update.js"; import { deliverToRelays } from "@/services/relay.js"; // import { deliverQuestionUpdate } from "@/services/note/polls/update.js"; import { langmap } from "@/misc/langmap.js"; -import detectLanguage from "@/misc/detect-language.js"; export const meta = { tags: ["notes"], @@ -385,8 +384,6 @@ export default define(meta, paramDef, async (ps, user) => { if (!Object.keys(langmap).includes(ps.lang.toLowerCase())) throw new Error("invalid param"); ps.lang = ps.lang.toLowerCase(); - } else if (ps.text) { - ps.lang = detectLanguage(ps.text); } else { ps.lang = null; } diff --git a/packages/backend/src/services/note/create.ts b/packages/backend/src/services/note/create.ts index f9c6acba18..f0bbf3a16e 100644 --- a/packages/backend/src/services/note/create.ts +++ b/packages/backend/src/services/note/create.ts @@ -68,7 +68,6 @@ import meilisearch from "@/db/meilisearch.js"; import { redisClient } from "@/db/redis.js"; import { Mutex } from "redis-semaphore"; import { langmap } from "@/misc/langmap.js"; -import detectLanguage from "@/misc/detect-language.js"; const mutedWordsCache = new Cache< { userId: UserProfile["userId"]; mutedWords: UserProfile["mutedWords"] }[] @@ -284,8 +283,6 @@ export default async ( if (!Object.keys(langmap).includes(data.lang.toLowerCase())) throw new Error("invalid param"); data.lang = data.lang.toLowerCase(); - } else if (data.text) { - data.lang = detectLanguage(data.text); } else { data.lang = null; } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a4d35b87ab..bac760acda 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -408,9 +408,6 @@ importers: tinycolor2: specifier: 1.6.0 version: 1.6.0 - tinyld: - specifier: ^1.3.4 - version: 1.3.4 tmp: specifier: 0.2.1 version: 0.2.1 @@ -17257,6 +17254,7 @@ packages: resolution: {integrity: sha512-u26CNoaInA4XpDU+8s/6Cq8xHc2T5M4fXB3ICfXPokUQoLzmPgSZU02TAkFwFMJCWTjk53gtkS8pETTreZwCqw==} engines: {node: '>= 12.10.0', npm: '>= 6.12.0', yarn: '>= 1.20.0'} hasBin: true + dev: true /titleize@3.0.0: resolution: {integrity: sha512-KxVu8EYHDPBdUYdKZdKtU2aj2XfEx9AfjXxE/Aj0vT06w2icA09Vus1rh6eSu1y01akYg6BjIK/hxyLJINoMLQ==}