Merge pull request 'enhance: emoji width and height' (#10155) from nmkj/calckey:emoji-size into develop

Reviewed-on: https://codeberg.org/calckey/calckey/pulls/10155

packages/backend/migration/1684494870830-EmojiSize.js

@@ -0,0 +1,19 @@
+export class EmojiSize1684494870830 {
+	name = "EmojiSize1684494870830";
+
+	async up(queryRunner) {
+		await queryRunner.query(`ALTER TABLE "emoji" ADD "width" integer`);
+		await queryRunner.query(
+			`COMMENT ON COLUMN "emoji"."width" IS 'Image width'`,
+		);
+		await queryRunner.query(`ALTER TABLE "emoji" ADD "height" integer`);
+		await queryRunner.query(
+			`COMMENT ON COLUMN "emoji"."height" IS 'Image height'`,
+		);
+	}
+
+	async down(queryRunner) {
+		await queryRunner.query(`ALTER TABLE "emoji" DROP COLUMN "height"`);
+		await queryRunner.query(`ALTER TABLE "emoji" DROP COLUMN "width"`);
+	}
+}

packages/backend/package.json

@@ -41,6 +41,7 @@
 		"ajv": "8.11.2",
 		"archiver": "5.3.1",
 		"argon2": "^0.30.3",
+		"async-mutex": "^0.4.0",
 		"autobind-decorator": "2.4.0",
 		"autolinker": "4.0.0",
 		"autwh": "0.1.0",
@@ -161,6 +162,7 @@
 		"@types/node-fetch": "3.0.3",
 		"@types/nodemailer": "6.4.7",
 		"@types/oauth": "0.9.1",
+		"@types/probe-image-size": "^7.2.0",
 		"@types/pug": "2.0.6",
 		"@types/punycode": "2.1.0",
 		"@types/qrcode": "1.5.0",

packages/backend/src/@types/probe-image-size.d.ts

@@ -1,37 +0,0 @@
-declare module "probe-image-size" {
-	import type { ReadStream } from "node:fs";
-
-	type ProbeOptions = {
-		retries: 1;
-		timeout: 30000;
-	};
-
-	type ProbeResult = {
-		width: number;
-		height: number;
-		length?: number;
-		type: string;
-		mime: string;
-		wUnits: "in" | "mm" | "cm" | "pt" | "pc" | "px" | "em" | "ex";
-		hUnits: "in" | "mm" | "cm" | "pt" | "pc" | "px" | "em" | "ex";
-		url?: string;
-	};
-
-	function probeImageSize(
-		src: string | ReadStream,
-		options?: ProbeOptions,
-	): Promise<ProbeResult>;
-	function probeImageSize(
-		src: string | ReadStream,
-		callback: (err: Error | null, result?: ProbeResult) => void,
-	): void;
-	function probeImageSize(
-		src: string | ReadStream,
-		options: ProbeOptions,
-		callback: (err: Error | null, result?: ProbeResult) => void,
-	): void;
-
-	namespace probeImageSize {} // Hack
-
-	export = probeImageSize;
-}

packages/backend/src/misc/emoji-meta.ts

@@ -0,0 +1,50 @@
+import probeImageSize from "probe-image-size";
+import { Mutex, withTimeout } from "async-mutex";
+
+import { FILE_TYPE_BROWSERSAFE } from "@/const.js";
+import Logger from "@/services/logger.js";
+import { Cache } from "./cache.js";
+
+export type Size = {
+	width: number;
+	height: number;
+};
+
+const cache = new Cache<boolean>(1000 * 60 * 10); // once every 10 minutes for the same url
+const mutex = withTimeout(new Mutex(), 1000);
+
+export async function getEmojiSize(url: string): Promise<Size> {
+	const logger = new Logger("emoji");
+
+	await mutex.runExclusive(() => {
+		const attempted = cache.get(url);
+		if (!attempted) {
+			cache.set(url, true);
+		} else {
+			logger.warn(`Attempt limit exceeded: ${url}`);
+			throw new Error("Too many attempts");
+		}
+	});
+
+	try {
+		logger.info(`Retrieving emoji size from ${url}`);
+		const { width, height, mime } = await probeImageSize(url, {
+			timeout: 5000,
+		});
+		if (!(mime.startsWith("image/") && FILE_TYPE_BROWSERSAFE.includes(mime))) {
+			throw new Error("Unsupported image type");
+		}
+		return { width, height };
+	} catch (e) {
+		throw new Error(`Unable to retrieve metadata: ${e}`);
+	}
+}
+
+export function getNormalSize(
+	{ width, height }: Size,
+	orientation?: number,
+): Size {
+	return (orientation || 0) >= 5
+		? { width: height, height: width }
+		: { width, height };
+}

packages/backend/src/misc/get-file-info.ts

@@ -5,9 +5,9 @@ import * as stream from "node:stream";
 import * as util from "node:util";
 import { FSWatcher } from "chokidar";
 import { fileTypeFromFile } from "file-type";
+import probeImageSize from "probe-image-size";
 import FFmpeg from "fluent-ffmpeg";
 import isSvg from "is-svg";
-import probeImageSize from "probe-image-size";
 import { type predictionType } from "nsfwjs";
 import sharp from "sharp";
 import { encode } from "blurhash";

packages/backend/src/misc/populate-emojis.ts

@@ -16,6 +16,8 @@ const cache = new Cache<Emoji | null>(1000 * 60 * 60 * 12);
 type PopulatedEmoji = {
 	name: string;
 	url: string;
+	width: number | null;
+	height: number | null;
 };
 
 function normalizeHost(
@@ -68,7 +70,13 @@ export async function populateEmoji(
 			host: host ?? IsNull(),
 		})) || null;
 
-	const emoji = await cache.fetch(`${name} ${host}`, queryOrNull);
+	const cacheKey = `${name} ${host}`;
+	let emoji = await cache.fetch(cacheKey, queryOrNull);
+
+	if (emoji && !(emoji.width && emoji.height)) {
+		emoji = await queryOrNull();
+		cache.set(cacheKey, emoji);
+	}
 
 	if (emoji == null) return null;
 
@@ -83,6 +91,8 @@ export async function populateEmoji(
 	return {
 		name: emojiName,
 		url,
+		width: emoji.width,
+		height: emoji.height,
 	};
 }
 

packages/backend/src/models/entities/emoji.ts

@@ -46,6 +46,7 @@ export class Emoji {
 	public uri: string | null;
 
 	// publicUrlの方のtypeが入る
+	// (mime)
 	@Column('varchar', {
 		length: 64, nullable: true,
 	})
@@ -60,4 +61,14 @@ export class Emoji {
 		length: 1024, nullable: true,
 	})
 	public license: string | null;
+
+	@Column('integer', {
+		nullable: true, comment: 'Image width',
+	})
+	public width: number | null;
+
+	@Column('integer', {
+		nullable: true, comment: "Image height",
+	})
+	public height: number | null;
 }

packages/backend/src/models/repositories/emoji.ts

@@ -16,6 +16,8 @@ export const EmojiRepository = db.getRepository(Emoji).extend({
 			// || emoji.originalUrl してるのは後方互換性のため
 			url: emoji.publicUrl || emoji.originalUrl,
 			license: emoji.license,
+			width: emoji.width,
+			height: emoji.height,
 		};
 	},
 

packages/backend/src/models/schema/emoji.ts

@@ -45,5 +45,15 @@ export const packedEmojiSchema = {
 			optional: false,
 			nullable: true,
 		},
+		width: {
+			type: "number",
+			optional: false,
+			nullable: true,
+		},
+		height: {
+			type: "number",
+			optional: false,
+			nullable: true,
+		},
 	},
 } as const;

packages/backend/src/queue/index.ts

@@ -567,6 +567,12 @@ export default function () {
 		},
 	);
 
+	systemQueue.add(
+		"setLocalEmojiSizes",
+		{},
+		{ removeOnComplete: true, removeOnFail: true },
+	);
+
 	processSystemQueue(systemQueue);
 }
 

packages/backend/src/queue/processors/db/import-custom-emojis.ts

@@ -10,6 +10,7 @@ import type { DbUserImportJobData } from "@/queue/types.js";
 import { addFile } from "@/services/drive/add-file.js";
 import { genId } from "@/misc/gen-id.js";
 import { db } from "@/db/postgre.js";
+import probeImageSize from "probe-image-size";
 
 const logger = queueLogger.createSubLogger("import-custom-emojis");
 
@@ -66,7 +67,10 @@ export async function importCustomEmojis(
 				name: record.fileName,
 				force: true,
 			});
-			const emoji = await Emojis.insert({
+			const file = fs.createReadStream(emojiPath);
+			const size = await probeImageSize(file);
+			file.destroy();
+			await Emojis.insert({
 				id: genId(),
 				updatedAt: new Date(),
 				name: emojiInfo.name,
@@ -77,6 +81,8 @@ export async function importCustomEmojis(
 				publicUrl: driveFile.webpublicUrl ?? driveFile.url,
 				type: driveFile.webpublicType ?? driveFile.type,
 				license: emojiInfo.license,
+				width: size.width || null,
+				height: size.height || null,
 			}).then((x) => Emojis.findOneByOrFail(x.identifiers[0]));
 		}
 

packages/backend/src/queue/processors/system/index.ts

@@ -4,6 +4,7 @@ import { resyncCharts } from "./resync-charts.js";
 import { cleanCharts } from "./clean-charts.js";
 import { checkExpiredMutings } from "./check-expired-mutings.js";
 import { clean } from "./clean.js";
+import { setLocalEmojiSizes } from "./local-emoji-size.js";
 
 const jobs = {
 	tickCharts,
@@ -11,6 +12,7 @@ const jobs = {
 	cleanCharts,
 	checkExpiredMutings,
 	clean,
+	setLocalEmojiSizes,
 } as Record<
 	string,
 	| Bull.ProcessCallbackFunction<Record<string, unknown>>

packages/backend/src/queue/processors/system/local-emoji-size.ts

@@ -0,0 +1,42 @@
+import type Bull from "bull";
+import { IsNull } from "typeorm";
+import { Emojis } from "@/models/index.js";
+
+import { queueLogger } from "../../logger.js";
+import { getEmojiSize } from "@/misc/emoji-meta.js";
+
+const logger = queueLogger.createSubLogger("local-emoji-size");
+
+export async function setLocalEmojiSizes(
+	_job: Bull.Job<Record<string, unknown>>,
+	done: any,
+): Promise<void> {
+	logger.info("Setting sizes of local emojis...");
+
+	const emojis = await Emojis.findBy([
+		{ host: IsNull(), width: IsNull(), height: IsNull() },
+	]);
+	logger.info(`${emojis.length} emojis need to be fetched.`);
+
+	for (let i = 0; i < emojis.length; i++) {
+		try {
+			const size = await getEmojiSize(emojis[i].publicUrl);
+			await Emojis.update(emojis[i].id, {
+				width: size.width || null,
+				height: size.height || null,
+			});
+		} catch (e) {
+			logger.error(
+				`Unable to set emoji size (${i + 1}/${emojis.length}): ${e}`,
+			);
+			/* skip if any error happens */
+		} finally {
+			// wait for 1sec so that this would not overwhelm the object storage.
+			await new Promise((resolve) => setTimeout(resolve, 1000));
+			if (i % 10 === 9) logger.succ(`fetched ${i + 1}/${emojis.length} emojis`);
+		}
+	}
+
+	logger.succ("Done.");
+	done();
+}

packages/backend/src/remote/activitypub/models/note.ts

@@ -52,6 +52,7 @@ import { UserProfiles } from "@/models/index.js";
 import { In } from "typeorm";
 import { DB_MAX_IMAGE_COMMENT_LENGTH } from "@/misc/hard-limits.js";
 import { truncate } from "@/misc/truncate.js";
+import { type Size, getEmojiSize } from "@/misc/emoji-meta.js";
 
 const logger = apLogger;
 
@@ -472,8 +473,15 @@ export async function extractEmojis(
 					(tag.updated != null &&
 						exists.updatedAt != null &&
 						new Date(tag.updated) > exists.updatedAt) ||
-					tag.icon!.url !== exists.originalUrl
+					tag.icon!.url !== exists.originalUrl ||
+					!(exists.width && exists.height)
 				) {
+					let size: Size = { width: 0, height: 0 };
+					try {
+						size = await getEmojiSize(tag.icon!.url);
+					} catch {
+						/* skip if any error happens */
+					}
 					await Emojis.update(
 						{
 							host,
@@ -484,6 +492,8 @@ export async function extractEmojis(
 							originalUrl: tag.icon!.url,
 							publicUrl: tag.icon!.url,
 							updatedAt: new Date(),
+							width: size.width || null,
+							height: size.height || null,
 						},
 					);
 
@@ -498,6 +508,12 @@ export async function extractEmojis(
 
 			logger.info(`register emoji host=${host}, name=${name}`);
 
+			let size: Size = { width: 0, height: 0 };
+			try {
+				size = await getEmojiSize(tag.icon!.url);
+			} catch {
+				/* skip if any error happens */
+			}
 			return await Emojis.insert({
 				id: genId(),
 				host,
@@ -507,6 +523,8 @@ export async function extractEmojis(
 				publicUrl: tag.icon!.url,
 				updatedAt: new Date(),
 				aliases: [],
+				width: size.width || null,
+				height: size.height || null,
 			} as Partial<Emoji>).then((x) =>
 				Emojis.findOneByOrFail(x.identifiers[0]),
 			);

packages/backend/src/server/api/endpoints/admin/emoji/add.ts

@@ -6,6 +6,7 @@ import { ApiError } from "../../../error.js";
 import rndstr from "rndstr";
 import { publishBroadcastStream } from "@/services/stream.js";
 import { db } from "@/db/postgre.js";
+import { type Size, getEmojiSize } from "@/misc/emoji-meta.js";
 
 export const meta = {
 	tags: ["admin"],
@@ -39,6 +40,13 @@ export default define(meta, paramDef, async (ps, me) => {
 		? file.name.split(".")[0]
 		: `_${rndstr("a-z0-9", 8)}_`;
 
+	let size: Size = { width: 0, height: 0 };
+	try {
+		size = await getEmojiSize(file.url);
+	} catch {
+		/* skip if any error happens */
+	}
+
 	const emoji = await Emojis.insert({
 		id: genId(),
 		updatedAt: new Date(),
@@ -50,6 +58,8 @@ export default define(meta, paramDef, async (ps, me) => {
 		publicUrl: file.webpublicUrl ?? file.url,
 		type: file.webpublicType ?? file.type,
 		license: null,
+		width: size.width || null,
+		height: size.height || null,
 	}).then((x) => Emojis.findOneByOrFail(x.identifiers[0]));
 
 	await db.queryResultCache!.remove(["meta_emojis"]);

packages/backend/src/server/api/endpoints/admin/emoji/copy.ts

@@ -6,6 +6,7 @@ import type { DriveFile } from "@/models/entities/drive-file.js";
 import { uploadFromUrl } from "@/services/drive/upload-from-url.js";
 import { publishBroadcastStream } from "@/services/stream.js";
 import { db } from "@/db/postgre.js";
+import { type Size, getEmojiSize } from "@/misc/emoji-meta.js";
 
 export const meta = {
 	tags: ["admin"],
@@ -64,6 +65,13 @@ export default define(meta, paramDef, async (ps, me) => {
 		throw new ApiError();
 	}
 
+	let size: Size = { width: 0, height: 0 };
+	try {
+		size = await getEmojiSize(driveFile.url);
+	} catch {
+		/* skip if any error happens */
+	}
+
 	const copied = await Emojis.insert({
 		id: genId(),
 		updatedAt: new Date(),
@@ -74,6 +82,8 @@ export default define(meta, paramDef, async (ps, me) => {
 		publicUrl: driveFile.webpublicUrl ?? driveFile.url,
 		type: driveFile.webpublicType ?? driveFile.type,
 		license: emoji.license,
+		width: size.width || null,
+		height: size.height || null,
 	}).then((x) => Emojis.findOneByOrFail(x.identifiers[0]));
 
 	await db.queryResultCache!.remove(["meta_emojis"]);

packages/backend/src/server/api/endpoints/admin/emoji/list-remote.ts

@@ -60,6 +60,16 @@ export const meta = {
 					optional: false,
 					nullable: true,
 				},
+				width: {
+					type: "number",
+					optional: false,
+					nullable: true,
+				},
+				height: {
+					type: "number",
+					optional: false,
+					nullable: true,
+				},
 			},
 		},
 	},

packages/backend/src/server/api/endpoints/admin/emoji/list.ts

@@ -60,6 +60,16 @@ export const meta = {
 					optional: false,
 					nullable: true,
 				},
+				width: {
+					type: "number",
+					optional: false,
+					nullable: true,
+				},
+				height: {
+					type: "number",
+					optional: false,
+					nullable: true,
+				},
 			},
 		},
 	},

pnpm-lock.yaml

@@ -131,6 +131,9 @@ importers:
       argon2:
         specifier: ^0.30.3
         version: 0.30.3
+      async-mutex:
+        specifier: ^0.4.0
+        version: 0.4.0
       autobind-decorator:
         specifier: 2.4.0
         version: 2.4.0
@@ -493,6 +496,9 @@ importers:
       '@types/oauth':
         specifier: 0.9.1
         version: 0.9.1
+      '@types/probe-image-size':
+        specifier: ^7.2.0
+        version: 7.2.0
       '@types/pug':
         specifier: 2.0.6
         version: 2.0.6
@@ -3409,6 +3415,12 @@ packages:
     resolution: {integrity: sha512-Z61JK7DKDtdKTWwLeElSEBcWGRLY8g95ic5FoQqI9CMx0ns/Ghep3B4DfcEimiKMvtamNVULVNKEsiwV3aQmXw==}
     dev: true
 
+  /@types/needle@3.2.0:
+    resolution: {integrity: sha512-6XzvzEyJ2ozFNfPajFmqH9JOt0Hp+9TawaYpJT59iIP/zR0U37cfWCRwosyIeEBBZBi021Osq4jGAD3AOju5fg==}
+    dependencies:
+      '@types/node': 18.11.18
+    dev: true
+
   /@types/node-fetch@2.6.2:
     resolution: {integrity: sha512-DHqhlq5jeESLy19TYhLakJ07kNumXWjcDdxXsLUMJZ6ue8VZJj4kLPQVE/2mdHh3xZziNF1xppu5lwmS53HR+A==}
     dependencies:
@@ -3464,6 +3476,13 @@ packages:
     resolution: {integrity: sha512-KufADq8uQqo1pYKVIYzfKbJfBAc0sOeXqGbFaSpv8MRmC/zXgowNZmFcbngndGk922QDmOASEXUZCaY48gs4cg==}
     dev: true
 
+  /@types/probe-image-size@7.2.0:
+    resolution: {integrity: sha512-R5H3vw62gHNHrn+JGZbKejb+Z2D/6E5UNVlhCzIaBBLroMQMOFqy5Pap2gM+ZZHdqBtVU0/cx/M6to+mOJcoew==}
+    dependencies:
+      '@types/needle': 3.2.0
+      '@types/node': 18.11.18
+    dev: true
+
   /@types/pug@2.0.6:
     resolution: {integrity: sha512-SnHmG9wN1UVmagJOnyo/qkk0Z7gejYxOYYmaAwr5u2yFYfsupN3sg10kyzN8Hep/2zbHxCnsumxOoRIRMBwKCg==}
     dev: true
@@ -4466,6 +4485,12 @@ packages:
       stream-exhaust: 1.0.2
     dev: true
 
+  /async-mutex@0.4.0:
+    resolution: {integrity: sha512-eJFZ1YhRR8UN8eBLoNzcDPcy/jqjsg6I1AP+KvWQX80BqOSW1oJPJXDylPUEeMr2ZQvHgnQ//Lp6f3RQ1zI7HA==}
+    dependencies:
+      tslib: 2.4.1
+    dev: false
+
   /async-settle@1.0.0:
     resolution: {integrity: sha512-VPXfB4Vk49z1LHHodrEQ6Xf7W4gg1w0dAPROHngx7qgDjqmIQ+fXmwgGXTW/ITLai0YLSvWepJOP9EVpMnEAcw==}
     engines: {node: '>= 0.10'}