HoovesSocial/hippofish
2
0
Fork 0
Code Issues 3 Pull requests 1 Projects Releases Packages Wiki Activity Actions

refactor (backend): port emoji-meta to backend-rs

Browse source
This commit is contained in:
naskya 2024-04-25 19:06:16 +09:00
parent b9c3dfbd3d
commit b12d7e4c63
No known key found for this signature in database
GPG key ID: 712D413B3A9FED5C
13 changed files with 1333 additions and 87 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1095
Cargo.lock generated
View file

File diff suppressed because it is too large Load diff

3
Cargo.toml
View file

@ -17,6 +17,8 @@ convert_case = "0.6.0"
cuid2 = "0.1.2"
emojis = "0.6.1"
idna = "0.5.0"
image = "0.25.1"
nom-exif = "1.2.0"
once_cell = "1.19.0"
pretty_assertions = "1.4.0"
proc-macro2 = "1.0.79"
@ -24,6 +26,7 @@ quote = "1.0.36"
rand = "0.8.5"
redis = "0.25.3"
regex = "1.10.4"
reqwest = "0.12.4"
rmp-serde = "1.2.0"
sea-orm = "0.12.15"
serde = "1.0.197"

3
packages/backend-rs/Cargo.toml
View file

@ -24,10 +24,13 @@ chrono = { workspace = true }
cuid2 = { workspace = true }
emojis = { workspace = true }
idna = { workspace = true }
image = { workspace = true }
nom-exif = { workspace = true }
once_cell = { workspace = true }
rand = { workspace = true }
redis = { workspace = true }
regex = { workspace = true }
reqwest = { workspace = true, features = ["blocking"] }
rmp-serde = { workspace = true }
sea-orm = { workspace = true, features = ["sqlx-postgres", "runtime-tokio-rustls"] }
serde = { workspace = true, features = ["derive"] }

5
packages/backend-rs/index.d.ts vendored
View file

@ -248,6 +248,11 @@ export function sqlLikeEscape(src: string): string
export function safeForSql(src: string): boolean
/** Convert milliseconds to a human readable string */
export function formatMilliseconds(milliseconds: number): string
export interface ImageSize {
width: number
height: number
}
export function getImageSizeFromUrl(url: string): Promise<ImageSize>
/** TODO: handle name collisions better */
export interface NoteLikeForGetNoteSummary {
fileIds: Array<string>

3
packages/backend-rs/index.js
View file

@ -310,7 +310,7 @@ if (!nativeBinding) {
throw new Error(`Failed to load native binding`)
}
const { SECOND, MINUTE, HOUR, DAY, USER_ONLINE_THRESHOLD, USER_ACTIVE_THRESHOLD, FILE_TYPE_BROWSERSAFE, loadEnv, loadConfig, stringToAcct, acctToString, addNoteToAntenna, isBlockedServer, isSilencedServer, isAllowedServer, checkWordMute, getFullApAccount, isSelfHost, isSameOrigin, extractHost, toPuny, isUnicodeEmoji, sqlLikeEscape, safeForSql, formatMilliseconds, getNoteSummary, toMastodonId, fromMastodonId, fetchMeta, metaToPugArgs, nyaify, hashPassword, verifyPassword, isOldPasswordAlgorithm, decodeReaction, countReactions, toDbReaction, removeOldAttestationChallenges, AntennaSrcEnum, DriveFileUsageHintEnum, MutedNoteReasonEnum, NoteVisibilityEnum, NotificationTypeEnum, PageVisibilityEnum, PollNotevisibilityEnum, RelayStatusEnum, UserEmojimodpermEnum, UserProfileFfvisibilityEnum, UserProfileMutingnotificationtypesEnum, initializeRustLogger, watchNote, unwatchNote, ChatEvent, publishToChatStream, getTimestamp, genId, genIdAt, secureRndstr } = nativeBinding
const { SECOND, MINUTE, HOUR, DAY, USER_ONLINE_THRESHOLD, USER_ACTIVE_THRESHOLD, FILE_TYPE_BROWSERSAFE, loadEnv, loadConfig, stringToAcct, acctToString, addNoteToAntenna, isBlockedServer, isSilencedServer, isAllowedServer, checkWordMute, getFullApAccount, isSelfHost, isSameOrigin, extractHost, toPuny, isUnicodeEmoji, sqlLikeEscape, safeForSql, formatMilliseconds, getImageSizeFromUrl, getNoteSummary, toMastodonId, fromMastodonId, fetchMeta, metaToPugArgs, nyaify, hashPassword, verifyPassword, isOldPasswordAlgorithm, decodeReaction, countReactions, toDbReaction, removeOldAttestationChallenges, AntennaSrcEnum, DriveFileUsageHintEnum, MutedNoteReasonEnum, NoteVisibilityEnum, NotificationTypeEnum, PageVisibilityEnum, PollNotevisibilityEnum, RelayStatusEnum, UserEmojimodpermEnum, UserProfileFfvisibilityEnum, UserProfileMutingnotificationtypesEnum, initializeRustLogger, watchNote, unwatchNote, ChatEvent, publishToChatStream, getTimestamp, genId, genIdAt, secureRndstr } = nativeBinding
module.exports.SECOND = SECOND
module.exports.MINUTE = MINUTE
@ -337,6 +337,7 @@ module.exports.isUnicodeEmoji = isUnicodeEmoji
module.exports.sqlLikeEscape = sqlLikeEscape
module.exports.safeForSql = safeForSql
module.exports.formatMilliseconds = formatMilliseconds
module.exports.getImageSizeFromUrl = getImageSizeFromUrl
module.exports.getNoteSummary = getNoteSummary
module.exports.toMastodonId = toMastodonId
module.exports.fromMastodonId = fromMastodonId

206
packages/backend-rs/src/misc/get_image_size.rs Normal file
View file

@ -0,0 +1,206 @@
use crate::misc::redis_cache::{get_cache, set_cache, CacheError};
use image::{io::Reader, ImageError, ImageFormat};
use nom_exif::{parse_jpeg_exif, EntryValue, ExifTag};
use once_cell::sync::OnceCell;
use std::io::Cursor;
use std::time::Duration;
use tokio::sync::Mutex;
#[derive(thiserror::Error, Debug)]
pub enum Error {
#[error("Redis cache error: {0}")]
CacheErr(#[from] CacheError),
#[error("Reqewst error: {0}")]
ReqewstErr(#[from] reqwest::Error),
#[error("Image decoding error: {0}")]
ImageErr(#[from] ImageError),
#[error("Image decoding error: {0}")]
IoErr(#[from] std::io::Error),
#[error("Exif extraction error: {0}")]
ExifErr(#[from] nom_exif::Error),
#[error("Emoji meta attempt limit exceeded: {0}")]
TooManyAttempts(String),
#[error("Unsupported image type: {0}")]
UnsupportedImageErr(String),
}
const BROWSER_SAFE_IMAGE_TYPES: [ImageFormat; 8] = [
ImageFormat::Png,
ImageFormat::Jpeg,
ImageFormat::Gif,
ImageFormat::WebP,
ImageFormat::Tiff,
ImageFormat::Bmp,
ImageFormat::Ico,
ImageFormat::Avif,
];
static CLIENT: OnceCell<reqwest::Client> = OnceCell::new();
fn client() -> Result<reqwest::Client, reqwest::Error> {
CLIENT
.get_or_try_init(|| {
reqwest::Client::builder()
.timeout(Duration::from_secs(5))
.build()
})
.cloned()
}
static MTX_GUARD: Mutex<()> = Mutex::const_new(());
#[derive(Debug, PartialEq)]
#[crate::export(object)]
pub struct ImageSize {
pub width: u32,
pub height: u32,
}
#[crate::export]
pub async fn get_image_size_from_url(url: &str) -> Result<ImageSize, Error> {
let attempted: bool;
{
let _ = MTX_GUARD.lock().await;
let key = format!("fetchImage:{}", url);
attempted = get_cache::<bool>(&key)?.is_some();
if !attempted {
set_cache(&key, &true, 10 * 60)?;
}
}
if attempted {
tracing::warn!("attempt limit exceeded: {}", url);
return Err(Error::TooManyAttempts(url.to_string()));
}
tracing::info!("retrieving image size from {}", url);
let image_bytes = client()?.get(url).send().await?.bytes().await?;
let reader = Reader::new(Cursor::new(&image_bytes)).with_guessed_format()?;
let format = reader.format();
if format.is_none() || !BROWSER_SAFE_IMAGE_TYPES.contains(&format.unwrap()) {
return Err(Error::UnsupportedImageErr(format!("{:?}", format)));
}
let size = reader.into_dimensions()?;
let res = ImageSize {
width: size.0,
height: size.1,
};
if format.unwrap() != ImageFormat::Jpeg {
return Ok(res);
}
// handle jpeg orientation
// https://magnushoff.com/articles/jpeg-orientation/
let exif = parse_jpeg_exif(&*image_bytes)?;
if exif.is_none() {
return Ok(res);
}
let orientation = exif.unwrap().get_value(&ExifTag::Orientation)?;
let rotated =
orientation.is_some() && matches!(orientation.unwrap(), EntryValue::U32(v) if v >= 5);
if !rotated {
return Ok(res);
}
Ok(ImageSize {
width: size.1,
height: size.0,
})
}
#[cfg(test)]
mod unit_test {
use super::{get_image_size_from_url, ImageSize};
use crate::misc::redis_cache::delete_cache;
use pretty_assertions::assert_eq;
#[tokio::test]
async fn test_get_image_size() {
let png_url_1 = "https://firefish.dev/firefish/firefish/-/raw/5891a90f71a8b9d5ea99c683ade7e485c685d642/packages/backend/assets/splash.png";
let png_url_2 = "https://firefish.dev/firefish/firefish/-/raw/5891a90f71a8b9d5ea99c683ade7e485c685d642/packages/backend/assets/notification-badges/at.png";
let png_url_3 = "https://firefish.dev/firefish/firefish/-/raw/5891a90f71a8b9d5ea99c683ade7e485c685d642/packages/backend/assets/api-doc.png";
let rotated_jpeg_url = "https://firefish.dev/firefish/firefish/-/raw/5891a90f71a8b9d5ea99c683ade7e485c685d642/packages/backend/test/resources/rotate.jpg";
let webp_url_1 = "https://firefish.dev/firefish/firefish/-/raw/5891a90f71a8b9d5ea99c683ade7e485c685d642/custom/assets/badges/error.webp";
let webp_url_2 = "https://firefish.dev/firefish/firefish/-/raw/5891a90f71a8b9d5ea99c683ade7e485c685d642/packages/backend/assets/screenshots/1.webp";
let ico_url = "https://firefish.dev/firefish/firefish/-/raw/5891a90f71a8b9d5ea99c683ade7e485c685d642/packages/backend/assets/favicon.ico";
let mp3_url = "https://firefish.dev/firefish/firefish/-/blob/5891a90f71a8b9d5ea99c683ade7e485c685d642/packages/backend/assets/sounds/aisha/1.mp3";
// Delete caches in case you run this test multiple times
// (should be disabled in CI tasks)
delete_cache(&format!("fetchImage:{}", png_url_1)).unwrap();
delete_cache(&format!("fetchImage:{}", png_url_2)).unwrap();
delete_cache(&format!("fetchImage:{}", png_url_3)).unwrap();
delete_cache(&format!("fetchImage:{}", rotated_jpeg_url)).unwrap();
delete_cache(&format!("fetchImage:{}", webp_url_1)).unwrap();
delete_cache(&format!("fetchImage:{}", webp_url_2)).unwrap();
delete_cache(&format!("fetchImage:{}", ico_url)).unwrap();
delete_cache(&format!("fetchImage:{}", mp3_url)).unwrap();
let png_size_1 = ImageSize {
width: 1024,
height: 1024,
};
let png_size_2 = ImageSize {
width: 96,
height: 96,
};
let png_size_3 = ImageSize {
width: 1024,
height: 354,
};
let rotated_jpeg_size = ImageSize {
width: 256,
height: 512,
};
let webp_size_1 = ImageSize {
width: 256,
height: 256,
};
let webp_size_2 = ImageSize {
width: 1080,
height: 2340,
};
let ico_size = ImageSize {
width: 256,
height: 256,
};
assert_eq!(
png_size_1,
get_image_size_from_url(png_url_1).await.unwrap()
);
assert_eq!(
png_size_2,
get_image_size_from_url(png_url_2).await.unwrap()
);
assert_eq!(
png_size_3,
get_image_size_from_url(png_url_3).await.unwrap()
);
assert_eq!(
rotated_jpeg_size,
get_image_size_from_url(rotated_jpeg_url).await.unwrap()
);
assert_eq!(
webp_size_1,
get_image_size_from_url(webp_url_1).await.unwrap()
);
assert_eq!(
webp_size_2,
get_image_size_from_url(webp_url_2).await.unwrap()
);
assert_eq!(ico_size, get_image_size_from_url(ico_url).await.unwrap());
assert!(get_image_size_from_url(mp3_url).await.is_err());
}
}

1
packages/backend-rs/src/misc/mod.rs
View file

@ -6,6 +6,7 @@ pub mod convert_host;
pub mod emoji;
pub mod escape_sql;
pub mod format_milliseconds;
pub mod get_image_size;
pub mod get_note_summary;
pub mod mastodon_id;
pub mod meta;

20
packages/backend-rs/src/misc/redis_cache.rs
View file

@ -3,7 +3,7 @@ use redis::{Commands, RedisError};
use serde::{Deserialize, Serialize};
#[derive(thiserror::Error, Debug)]
pub enum Error {
pub enum CacheError {
#[error("Redis error: {0}")]
RedisError(#[from] RedisError),
#[error("Data serialization error: {0}")]
@ -12,27 +12,37 @@ pub enum Error {
DeserializeError(#[from] rmp_serde::decode::Error),
}
fn prefix_key(key: &str) -> String {
redis_key(format!("cache:{}", key))
}
pub fn set_cache<V: for<'a> Deserialize<'a> + Serialize>(
key: &str,
value: &V,
expire_seconds: u64,
) -> Result<(), Error> {
) -> Result<(), CacheError> {
redis_conn()?.set_ex(
redis_key(key),
prefix_key(key),
rmp_serde::encode::to_vec(&value)?,
expire_seconds,
)?;
Ok(())
}
pub fn get_cache<V: for<'a> Deserialize<'a> + Serialize>(key: &str) -> Result<Option<V>, Error> {
let serialized_value: Option<Vec<u8>> = redis_conn()?.get(redis_key(key))?;
pub fn get_cache<V: for<'a> Deserialize<'a> + Serialize>(
key: &str,
) -> Result<Option<V>, CacheError> {
let serialized_value: Option<Vec<u8>> = redis_conn()?.get(prefix_key(key))?;
Ok(match serialized_value {
Some(v) => Some(rmp_serde::from_slice::<V>(v.as_ref())?),
None => None,
})
}
pub fn delete_cache(key: &str) -> Result<(), CacheError> {
Ok(redis_conn()?.del(prefix_key(key))?)
}
#[cfg(test)]
mod unit_test {
use super::{get_cache, set_cache};

59
packages/backend/src/misc/emoji-meta.ts
View file

@ -1,59 +0,0 @@
import probeImageSize from "probe-image-size";
import { Mutex } from "redis-semaphore";
import { FILE_TYPE_BROWSERSAFE } from "backend-rs";
import Logger from "@/services/logger.js";
import { Cache } from "./cache.js";
import { redisClient } from "@/db/redis.js";
import { inspect } from "node:util";
export type Size = {
width: number;
height: number;
};
const cache = new Cache<boolean>("emojiMeta", 60 * 10); // once every 10 minutes for the same url
const logger = new Logger("emoji");
export async function getEmojiSize(url: string): Promise<Size> {
let attempted = true;
const lock = new Mutex(redisClient, "getEmojiSize");
await lock.acquire();
try {
attempted = (await cache.get(url)) === true;
if (!attempted) {
await cache.set(url, true);
}
} finally {
await lock.release();
}
if (attempted) {
logger.warn(`Attempt limit exceeded: ${url}`);
throw new Error("Too many attempts");
}
try {
logger.debug(`Retrieving emoji size from ${url}`);
const { width, height, mime } = await probeImageSize(url, {
timeout: 5000,
});
if (!(mime.startsWith("image/") && FILE_TYPE_BROWSERSAFE.includes(mime))) {
throw new Error("Unsupported image type");
}
return { width, height };
} catch (e) {
throw new Error(`Unable to retrieve metadata:\n${inspect(e)}`);
}
}
export function getNormalSize(
{ width, height }: Size,
orientation?: number,
): Size {
return (orientation || 0) >= 5
? { width: height, height: width }
: { width, height };
}

4
packages/backend/src/queue/processors/system/local-emoji-size.ts
View file

@ -3,7 +3,7 @@ import { IsNull } from "typeorm";
import { Emojis } from "@/models/index.js";
import { queueLogger } from "../../logger.js";
import { getEmojiSize } from "@/misc/emoji-meta.js";
import { getImageSizeFromUrl } from "backend-rs";
import { inspect } from "node:util";
const logger = queueLogger.createSubLogger("local-emoji-size");
@ -21,7 +21,7 @@ export async function setLocalEmojiSizes(
for (let i = 0; i < emojis.length; i++) {
try {
const size = await getEmojiSize(emojis[i].publicUrl);
const size = await getImageSizeFromUrl(emojis[i].publicUrl);
await Emojis.update(emojis[i].id, {
width: size.width || null,
height: size.height || null,

11
packages/backend/src/remote/activitypub/models/note.ts
View file

@ -13,7 +13,7 @@ import { extractPollFromQuestion } from "./question.js";
import vote from "@/services/note/polls/vote.js";
import { apLogger } from "../logger.js";
import type { DriveFile } from "@/models/entities/drive-file.js";
import { extractHost, isSameOrigin, toPuny } from "backend-rs";
import { type ImageSize, extractHost, getImageSizeFromUrl, isSameOrigin, toPuny } from "backend-rs";
import {
Emojis,
Polls,
@ -46,7 +46,6 @@ import { UserProfiles } from "@/models/index.js";
import { In } from "typeorm";
import { config } from "@/config.js";
import { truncate } from "@/misc/truncate.js";
import { type Size, getEmojiSize } from "@/misc/emoji-meta.js";
import { langmap } from "@/misc/langmap.js";
import { inspect } from "node:util";
@ -488,9 +487,9 @@ export async function extractEmojis(
tag.icon!.url !== exists.originalUrl ||
!(exists.width && exists.height)
) {
let size: Size = { width: 0, height: 0 };
let size: ImageSize = { width: 0, height: 0 };
try {
size = await getEmojiSize(tag.icon!.url);
size = await getImageSizeFromUrl(tag.icon!.url);
} catch {
/* skip if any error happens */
}
@ -520,9 +519,9 @@ export async function extractEmojis(
apLogger.info(`register emoji host=${host}, name=${name}`);
let size: Size = { width: 0, height: 0 };
let size: ImageSize = { width: 0, height: 0 };
try {
size = await getEmojiSize(tag.icon!.url);
size = await getImageSizeFromUrl(tag.icon!.url);
} catch {
/* skip if any error happens */
}

5
packages/backend/src/server/api/endpoints/admin/emoji/add.ts
View file

@ -1,12 +1,11 @@
import define from "@/server/api/define.js";
import { Emojis, DriveFiles } from "@/models/index.js";
import { genId } from "backend-rs";
import { genId, getImageSizeFromUrl } from "backend-rs";
import { insertModerationLog } from "@/services/insert-moderation-log.js";
import { ApiError } from "@/server/api/error.js";
import rndstr from "rndstr";
import { publishBroadcastStream } from "@/services/stream.js";
import { db } from "@/db/postgre.js";
import { getEmojiSize } from "@/misc/emoji-meta.js";
export const meta = {
tags: ["admin", "emoji"],
@ -49,7 +48,7 @@ export default define(meta, paramDef, async (ps, me) => {
? file.name.split(".")[0]
: `_${rndstr("a-z0-9", 8)}_`;
const size = await getEmojiSize(file.url);
const size = await getImageSizeFromUrl(file.url);
const emoji = await Emojis.insert({
id: genId(),

5
packages/backend/src/server/api/endpoints/admin/emoji/copy.ts
View file

@ -1,12 +1,11 @@
import define from "@/server/api/define.js";
import { Emojis } from "@/models/index.js";
import { genId } from "backend-rs";
import { genId, getImageSizeFromUrl } from "backend-rs";
import { ApiError } from "@/server/api/error.js";
import type { DriveFile } from "@/models/entities/drive-file.js";
import { uploadFromUrl } from "@/services/drive/upload-from-url.js";
import { publishBroadcastStream } from "@/services/stream.js";
import { db } from "@/db/postgre.js";
import { getEmojiSize } from "@/misc/emoji-meta.js";
export const meta = {
tags: ["admin", "emoji"],
@ -76,7 +75,7 @@ export default define(meta, paramDef, async (ps, me) => {
throw new ApiError();
}
const size = await getEmojiSize(driveFile.url);
const size = await getImageSizeFromUrl(driveFile.url);
const copied = await Emojis.insert({
id: genId(),