HoovesSocial/hippofish
2
0
Fork 0
Code Issues 3 Pull requests 1 Projects Releases Packages Wiki Activity Actions

chore: remove language detection from backend

Browse source
This commit is contained in:
naskya 2023-11-17 17:50:20 +09:00
parent 636cbee66b
commit 2dcc46a9e3
No known key found for this signature in database
GPG key ID: 164DFF24E2D40139
6 changed files with 3 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
packages/backend/package.json
View file

@ -130,7 +130,6 @@
"tar-stream": "^3.1.6", "tar-stream": "^3.1.6",
"tesseract.js": "^5.0.3", "tesseract.js": "^5.0.3",
"tinycolor2": "1.6.0", "tinycolor2": "1.6.0",
"tinyld": "^1.3.4",
"tmp": "0.2.1", "tmp": "0.2.1",
"twemoji-parser": "14.0.0", "twemoji-parser": "14.0.0",
"typeorm": "0.3.17", "typeorm": "0.3.17",

11
packages/backend/src/misc/detect-language.ts
View file

@ -1,11 +0,0 @@
import { detect } from "tinyld";
import * as mfm from "mfm-js";
export default function detectLanguage(text: string): string {
const nodes = mfm.parse(text);
const filtered = mfm.extract(nodes, (node) => {
return node.type === "text" || node.type === "quote";
});
const purified = mfm.toString(filtered);
return detect(purified);
}

6
packages/backend/src/remote/activitypub/renderer/note.ts
View file

@ -6,7 +6,6 @@ import { DriveFiles, Notes, Users, Emojis, Polls } from "@/models/index.js";
import type { Emoji } from "@/models/entities/emoji.js"; import type { Emoji } from "@/models/entities/emoji.js";
import type { Poll } from "@/models/entities/poll.js"; import type { Poll } from "@/models/entities/poll.js";
import toHtml from "@/remote/activitypub/misc/get-note-html.js"; import toHtml from "@/remote/activitypub/misc/get-note-html.js";
import detectLanguage from "@/misc/detect-language.js";
import renderEmoji from "./emoji.js"; import renderEmoji from "./emoji.js";
import renderMention from "./mention.js"; import renderMention from "./mention.js";
import renderHashtag from "./hashtag.js"; import renderHashtag from "./hashtag.js";
@ -115,10 +114,9 @@ export default async function renderNote(
}), }),
); );
const lang = note.lang ?? detectLanguage(text); const contentMap = note.lang
const contentMap = lang
? { ? {
[lang]: content, [note.lang]: content,
} }
: null; : null;

3
packages/backend/src/server/api/endpoints/notes/edit.ts
View file

@ -35,7 +35,6 @@ import renderUpdate from "@/remote/activitypub/renderer/update.js";
import { deliverToRelays } from "@/services/relay.js"; import { deliverToRelays } from "@/services/relay.js";
// import { deliverQuestionUpdate } from "@/services/note/polls/update.js"; // import { deliverQuestionUpdate } from "@/services/note/polls/update.js";
import { langmap } from "@/misc/langmap.js"; import { langmap } from "@/misc/langmap.js";
import detectLanguage from "@/misc/detect-language.js";
export const meta = { export const meta = {
tags: ["notes"], tags: ["notes"],
@ -385,8 +384,6 @@ export default define(meta, paramDef, async (ps, user) => {
if (!Object.keys(langmap).includes(ps.lang.toLowerCase())) if (!Object.keys(langmap).includes(ps.lang.toLowerCase()))
throw new Error("invalid param"); throw new Error("invalid param");
ps.lang = ps.lang.toLowerCase(); ps.lang = ps.lang.toLowerCase();
} else if (ps.text) {
ps.lang = detectLanguage(ps.text);
} else { } else {
ps.lang = null; ps.lang = null;
} }

3
packages/backend/src/services/note/create.ts
View file

@ -68,7 +68,6 @@ import meilisearch from "@/db/meilisearch.js";
import { redisClient } from "@/db/redis.js"; import { redisClient } from "@/db/redis.js";
import { Mutex } from "redis-semaphore"; import { Mutex } from "redis-semaphore";
import { langmap } from "@/misc/langmap.js"; import { langmap } from "@/misc/langmap.js";
import detectLanguage from "@/misc/detect-language.js";
const mutedWordsCache = new Cache< const mutedWordsCache = new Cache<
{ userId: UserProfile["userId"]; mutedWords: UserProfile["mutedWords"] }[] { userId: UserProfile["userId"]; mutedWords: UserProfile["mutedWords"] }[]
@ -284,8 +283,6 @@ export default async (
if (!Object.keys(langmap).includes(data.lang.toLowerCase())) if (!Object.keys(langmap).includes(data.lang.toLowerCase()))
throw new Error("invalid param"); throw new Error("invalid param");
data.lang = data.lang.toLowerCase(); data.lang = data.lang.toLowerCase();
} else if (data.text) {
data.lang = detectLanguage(data.text);
} else { } else {
data.lang = null; data.lang = null;
} }

4
pnpm-lock.yaml
View file

@ -408,9 +408,6 @@ importers:
tinycolor2: tinycolor2:
specifier: 1.6.0 specifier: 1.6.0
version: 1.6.0 version: 1.6.0
tinyld:
specifier: ^1.3.4
version: 1.3.4
tmp: tmp:
specifier: 0.2.1 specifier: 0.2.1
version: 0.2.1 version: 0.2.1
@ -17257,6 +17254,7 @@ packages:
resolution: {integrity: sha512-u26CNoaInA4XpDU+8s/6Cq8xHc2T5M4fXB3ICfXPokUQoLzmPgSZU02TAkFwFMJCWTjk53gtkS8pETTreZwCqw==} resolution: {integrity: sha512-u26CNoaInA4XpDU+8s/6Cq8xHc2T5M4fXB3ICfXPokUQoLzmPgSZU02TAkFwFMJCWTjk53gtkS8pETTreZwCqw==}
engines: {node: '>= 12.10.0', npm: '>= 6.12.0', yarn: '>= 1.20.0'} engines: {node: '>= 12.10.0', npm: '>= 6.12.0', yarn: '>= 1.20.0'}
hasBin: true hasBin: true
dev: true
/titleize@3.0.0: /titleize@3.0.0:
resolution: {integrity: sha512-KxVu8EYHDPBdUYdKZdKtU2aj2XfEx9AfjXxE/Aj0vT06w2icA09Vus1rh6eSu1y01akYg6BjIK/hxyLJINoMLQ==} resolution: {integrity: sha512-KxVu8EYHDPBdUYdKZdKtU2aj2XfEx9AfjXxE/Aj0vT06w2icA09Vus1rh6eSu1y01akYg6BjIK/hxyLJINoMLQ==}