From 2414cf3ec7c1bd2515856f8875f73b79b508b987 Mon Sep 17 00:00:00 2001 From: naskya Date: Sat, 2 Mar 2024 00:27:21 +0900 Subject: [PATCH] refactor (backend): separate translate function into another file, use post language info for translations, use deepl-node package --- packages/backend/package.json | 1 + packages/backend/src/misc/langmap.ts | 1 + packages/backend/src/misc/translate.ts | 88 +++++++++++++++ .../server/api/endpoints/notes/translate.ts | 102 ++---------------- pnpm-lock.yaml | 29 +++++ 5 files changed, 126 insertions(+), 95 deletions(-) create mode 100644 packages/backend/src/misc/translate.ts diff --git a/packages/backend/package.json b/packages/backend/package.json index b51fecfa08..0956cf5071 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -53,6 +53,7 @@ "date-fns": "3.3.1", "decompress": "^4.2.1", "deep-email-validator": "0.1.21", + "deepl-node": "1.12.0", "escape-regexp": "0.0.1", "feed": "4.2.2", "file-type": "19.0.0", diff --git a/packages/backend/src/misc/langmap.ts b/packages/backend/src/misc/langmap.ts index df3214e449..2506a36151 100644 --- a/packages/backend/src/misc/langmap.ts +++ b/packages/backend/src/misc/langmap.ts @@ -380,3 +380,4 @@ export const iso639Regional = { }; export const langmap = Object.assign({}, langmapNoRegion, iso639Regional); +export type PostLanguage = keyof typeof langmap; diff --git a/packages/backend/src/misc/translate.ts b/packages/backend/src/misc/translate.ts new file mode 100644 index 0000000000..8592d639db --- /dev/null +++ b/packages/backend/src/misc/translate.ts @@ -0,0 +1,88 @@ +import fetch from "node-fetch"; +import { Converter } from "opencc-js"; +import { getAgentByUrl } from "@/misc/fetch.js"; +import { fetchMeta } from "@/misc/fetch-meta.js"; +import type { PostLanguage } from "@/misc/langmap"; +import * as deepl from "deepl-node"; + +function convertChinese(convert: boolean, src: string) { + if (!convert) return src; + const converter = Converter({ from: "cn", to: "twp" }); + return converter(src); +} + +function stem(lang: PostLanguage): string { + let toReturn = lang as string; + if (toReturn.includes("-")) toReturn = toReturn.split("-")[0]; + if (toReturn.includes("_")) toReturn = toReturn.split("_")[0]; + return toReturn; +} + +export async function translate( + text: string, + from: PostLanguage | null, + to: PostLanguage, +) { + const instance = await fetchMeta(); + + if (instance.deeplAuthKey == null && instance.libreTranslateApiUrl == null) { + throw Error("No translator is set up on this server."); + } + + const source = from == null ? null : stem(from); + const target = stem(to); + + if (instance.libreTranslateApiUrl != null) { + const jsonBody = { + q: text, + source: source ?? "auto", + target, + format: "text", + api_key: instance.libreTranslateApiKey ?? "", + }; + + const url = new URL(instance.libreTranslateApiUrl); + if (url.pathname.endsWith("/")) { + url.pathname = url.pathname.slice(0, -1); + } + if (!url.pathname.endsWith("/translate")) { + url.pathname += "/translate"; + } + const res = await fetch(url.toString(), { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify(jsonBody), + agent: getAgentByUrl, + }); + + const json = (await res.json()) as { + detectedLanguage?: { + confidence: number; + language: string; + }; + translatedText: string; + }; + + return { + sourceLang: source ?? json.detectedLanguage?.language, + text: convertChinese( + ["zh-hant", "zh-TW"].includes(to), + json.translatedText, + ), + }; + } + + const deeplTranslator = new deepl.Translator(instance.deeplAuthKey ?? ""); + const result = await deeplTranslator.translateText( + text, + source as deepl.SourceLanguageCode | null, + (target === "en" ? to : target) as deepl.TargetLanguageCode, + ); + + return { + sourceLang: source ?? result.detectedSourceLang, + text: convertChinese(["zh-hant", "zh-TW"].includes(to), result.text), + }; +} diff --git a/packages/backend/src/server/api/endpoints/notes/translate.ts b/packages/backend/src/server/api/endpoints/notes/translate.ts index d1de399936..094b7fed33 100644 --- a/packages/backend/src/server/api/endpoints/notes/translate.ts +++ b/packages/backend/src/server/api/endpoints/notes/translate.ts @@ -1,11 +1,7 @@ -import { URLSearchParams } from "node:url"; -import fetch from "node-fetch"; -import config from "@/config/index.js"; -import { Converter } from "opencc-js"; -import { getAgentByUrl } from "@/misc/fetch.js"; -import { fetchMeta } from "@/misc/fetch-meta.js"; import { ApiError } from "@/server/api/error.js"; import { getNote } from "@/server/api/common/getters.js"; +import { translate } from "@/misc/translate.js"; +import type { PostLanguage } from "@/misc/langmap.js"; import define from "@/server/api/define.js"; export const meta = { @@ -38,13 +34,6 @@ export const paramDef = { required: ["noteId", "targetLang"], } as const; -function convertChinese(convert: boolean, src: string) { - if (!convert) return src; - - const converter = Converter({ from: "cn", to: "twp" }); - return converter(src); -} - export default define(meta, paramDef, async (ps, user) => { const note = await getNote(ps.noteId, user).catch((err) => { if (err.id === "9725d0ce-ba28-4dde-95a7-2cbb2c15de24") @@ -56,86 +45,9 @@ export default define(meta, paramDef, async (ps, user) => { return 204; } - const instance = await fetchMeta(); - - if (instance.deeplAuthKey == null && instance.libreTranslateApiUrl == null) { - return 204; // TODO: 良い感じのエラー返す - } - - let targetLang = ps.targetLang; - if (targetLang.includes("-")) targetLang = targetLang.split("-")[0]; - if (targetLang.includes("_")) targetLang = targetLang.split("_")[0]; - - if (instance.libreTranslateApiUrl != null) { - const jsonBody = { - q: note.text, - source: "auto", - target: targetLang, - format: "text", - api_key: instance.libreTranslateApiKey ?? "", - }; - - const url = new URL(instance.libreTranslateApiUrl); - if (url.pathname.endsWith("/")) { - url.pathname = url.pathname.slice(0, -1); - } - if (!url.pathname.endsWith("/translate")) { - url.pathname += "/translate"; - } - const res = await fetch(url.toString(), { - method: "POST", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify(jsonBody), - agent: getAgentByUrl, - }); - - const json = (await res.json()) as { - detectedLanguage?: { - confidence: number; - language: string; - }; - translatedText: string; - }; - - return { - sourceLang: json.detectedLanguage?.language, - text: convertChinese(ps.targetLang === "zh-TW", json.translatedText), - }; - } - - const params = new URLSearchParams(); - params.append("auth_key", instance.deeplAuthKey ?? ""); - params.append("text", note.text); - params.append("target_lang", targetLang); - - const endpoint = instance.deeplIsPro - ? "https://api.deepl.com/v2/translate" - : "https://api-free.deepl.com/v2/translate"; - - const res = await fetch(endpoint, { - method: "POST", - headers: { - "Content-Type": "application/x-www-form-urlencoded", - "User-Agent": config.userAgent, - Accept: "application/json, */*", - }, - body: params, - // TODO - //timeout: 10000, - agent: getAgentByUrl, - }); - - const json = (await res.json()) as { - translations: { - detected_source_language: string; - text: string; - }[]; - }; - - return { - sourceLang: json.translations[0].detected_source_language, - text: convertChinese(ps.targetLang === "zh-TW", json.translations[0].text), - }; + return translate( + note.text, + note.lang as PostLanguage | null, + ps.targetLang as PostLanguage, + ); }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d811635bed..e132ae0c1f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -150,6 +150,9 @@ importers: deep-email-validator: specifier: 0.1.21 version: 0.1.21 + deepl-node: + specifier: 1.12.0 + version: 1.12.0 escape-regexp: specifier: 0.0.1 version: 0.0.1 @@ -7430,6 +7433,18 @@ packages: resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==} dev: true + /deepl-node@1.12.0: + resolution: {integrity: sha512-c/8x1R0dXPL7NSDdQ94lYPou/A+I6cbo6b7gFb/28HbjcHnKB4RtWXWLgdv7n51GEXL7OE2eoRZQcAu4ZI+vGg==} + engines: {node: '>=12.0'} + dependencies: + '@types/node': 20.11.21 + axios: 1.6.7 + form-data: 3.0.1 + loglevel: 1.9.1 + transitivePeerDependencies: + - debug + dev: false + /deepmerge@4.3.1: resolution: {integrity: sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==} engines: {node: '>=0.10.0'} @@ -9416,6 +9431,15 @@ packages: engines: {node: '>= 18'} dev: false + /form-data@3.0.1: + resolution: {integrity: sha512-RHkBKtLWUVwd7SqRIvCZMEvAMoGUp0XU+seQiZejj0COz3RI3hWP4sCv3gZWWLjJTd7rGwcsF5eKZGii0r/hbg==} + engines: {node: '>= 6'} + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + mime-types: 2.1.35 + dev: false + /form-data@4.0.0: resolution: {integrity: sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==} engines: {node: '>= 6'} @@ -12254,6 +12278,11 @@ packages: is-unicode-supported: 0.1.0 dev: true + /loglevel@1.9.1: + resolution: {integrity: sha512-hP3I3kCrDIMuRwAwHltphhDM1r8i55H33GgqjXbrisuJhF4kRhW1dNuxsRklp4bXl8DSdLaNLuiL4A/LWRfxvg==} + engines: {node: '>= 0.6.0'} + dev: false + /lowercase-keys@2.0.0: resolution: {integrity: sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA==} engines: {node: '>=8'}