Merge branch 'develop' into redis

This commit is contained in:
naskya 2024-04-21 22:25:50 +09:00
commit 2fa0ca355d
No known key found for this signature in database
GPG key ID: 712D413B3A9FED5C
21 changed files with 157 additions and 29 deletions

View file

@ -5,6 +5,10 @@ Critical security updates are indicated by the :warning: icon.
- Server administrators should check [notice-for-admins.md](./notice-for-admins.md) as well.
- Third-party client/bot developers may want to check [api-change.md](./api-change.md) as well.
## [v20240421](https://firefish.dev/firefish/firefish/-/merge_requests/10756/commits)
- Fix bugs
## [v20240413](https://firefish.dev/firefish/firefish/-/merge_requests/10741/commits)
- Add "Media" tab to user page

View file

@ -1,6 +1,7 @@
BEGIN;
DELETE FROM "migrations" WHERE name IN (
'AddDriveFileUsage1713451569342',
'ConvertCwVarcharToText1713225866247',
'FixChatFileConstraint1712855579316',
'DropTimeZone1712425488543',
@ -23,7 +24,11 @@ DELETE FROM "migrations" WHERE name IN (
'RemoveNativeUtilsMigration1705877093218'
);
--convert-cw-varchar-to-text
-- AddDriveFileUsage
ALTER TABLE "drive_file" DROP COLUMN "usageHint";
DROP TYPE "drive_file_usage_hint_enum";
-- convert-cw-varchar-to-text
DROP INDEX "IDX_8e3bbbeb3df04d1a8105da4c8f";
ALTER TABLE "note" ALTER COLUMN "cw" TYPE character varying(512);
CREATE INDEX "IDX_8e3bbbeb3df04d1a8105da4c8f" ON "note" USING "pgroonga" ("cw" pgroonga_varchar_full_text_search_ops_v2);

View file

@ -1,6 +1,6 @@
{
"name": "firefish",
"version": "20240413",
"version": "20240421",
"repository": {
"type": "git",
"url": "https://firefish.dev/firefish/firefish.git"

View file

@ -413,6 +413,7 @@ export interface DriveFile {
webpublicType: string | null
requestHeaders: Json | null
requestIp: string | null
usageHint: DriveFileUsageHintEnum | null
}
export interface DriveFolder {
id: string
@ -845,6 +846,10 @@ export enum AntennaSrcEnum {
List = 'list',
Users = 'users'
}
export enum DriveFileUsageHintEnum {
UserAvatar = 'userAvatar',
UserBanner = 'userBanner'
}
export enum MutedNoteReasonEnum {
Manual = 'manual',
Other = 'other',

View file

@ -310,7 +310,7 @@ if (!nativeBinding) {
throw new Error(`Failed to load native binding`)
}
const { loadEnv, loadConfig, stringToAcct, acctToString, checkWordMute, getFullApAccount, isSelfHost, isSameOrigin, extractHost, toPuny, isUnicodeEmoji, sqlLikeEscape, safeForSql, formatMilliseconds, getNoteSummary, toMastodonId, fromMastodonId, fetchMeta, metaToPugArgs, nyaify, hashPassword, verifyPassword, isOldPasswordAlgorithm, decodeReaction, countReactions, toDbReaction, AntennaSrcEnum, MutedNoteReasonEnum, NoteVisibilityEnum, NotificationTypeEnum, PageVisibilityEnum, PollNotevisibilityEnum, RelayStatusEnum, UserEmojimodpermEnum, UserProfileFfvisibilityEnum, UserProfileMutingnotificationtypesEnum, addNoteToAntenna, initIdGenerator, getTimestamp, genId, secureRndstr } = nativeBinding
const { loadEnv, loadConfig, stringToAcct, acctToString, checkWordMute, getFullApAccount, isSelfHost, isSameOrigin, extractHost, toPuny, isUnicodeEmoji, sqlLikeEscape, safeForSql, formatMilliseconds, getNoteSummary, toMastodonId, fromMastodonId, fetchMeta, metaToPugArgs, nyaify, hashPassword, verifyPassword, isOldPasswordAlgorithm, decodeReaction, countReactions, toDbReaction, AntennaSrcEnum, DriveFileUsageHintEnum, MutedNoteReasonEnum, NoteVisibilityEnum, NotificationTypeEnum, PageVisibilityEnum, PollNotevisibilityEnum, RelayStatusEnum, UserEmojimodpermEnum, UserProfileFfvisibilityEnum, UserProfileMutingnotificationtypesEnum, addNoteToAntenna, initIdGenerator, getTimestamp, genId, secureRndstr } = nativeBinding
module.exports.loadEnv = loadEnv
module.exports.loadConfig = loadConfig
@ -339,6 +339,7 @@ module.exports.decodeReaction = decodeReaction
module.exports.countReactions = countReactions
module.exports.toDbReaction = toDbReaction
module.exports.AntennaSrcEnum = AntennaSrcEnum
module.exports.DriveFileUsageHintEnum = DriveFileUsageHintEnum
module.exports.MutedNoteReasonEnum = MutedNoteReasonEnum
module.exports.NoteVisibilityEnum = NoteVisibilityEnum
module.exports.NotificationTypeEnum = NotificationTypeEnum

View file

@ -1,5 +1,6 @@
//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.15
use super::sea_orm_active_enums::DriveFileUsageHintEnum;
use sea_orm::entity::prelude::*;
#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, serde::Serialize, serde::Deserialize)]
@ -53,6 +54,8 @@ pub struct Model {
pub request_headers: Option<Json>,
#[sea_orm(column_name = "requestIp")]
pub request_ip: Option<String>,
#[sea_orm(column_name = "usageHint")]
pub usage_hint: Option<DriveFileUsageHintEnum>,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]

View file

@ -29,6 +29,23 @@ pub enum AntennaSrcEnum {
#[serde(rename_all = "camelCase")]
#[cfg_attr(not(feature = "napi"), derive(Clone))]
#[cfg_attr(feature = "napi", napi_derive::napi(string_enum = "camelCase"))]
#[sea_orm(
rs_type = "String",
db_type = "Enum",
enum_name = "drive_file_usage_hint_enum"
)]
pub enum DriveFileUsageHintEnum {
#[sea_orm(string_value = "userAvatar")]
UserAvatar,
#[sea_orm(string_value = "userBanner")]
UserBanner,
}
#[derive(
Debug, PartialEq, Eq, EnumIter, DeriveActiveEnum, serde::Serialize, serde::Deserialize,
)]
#[serde(rename_all = "camelCase")]
#[cfg_attr(not(feature = "napi"), derive(Clone))]
#[cfg_attr(feature = "napi", napi_derive::napi(string_enum = "camelCase"))]
#[sea_orm(
rs_type = "String",
db_type = "Enum",

View file

@ -0,0 +1,17 @@
import type { MigrationInterface, QueryRunner } from "typeorm";
export class AddDriveFileUsage1713451569342 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(
`CREATE TYPE drive_file_usage_hint_enum AS ENUM ('userAvatar', 'userBanner')`,
);
await queryRunner.query(
`ALTER TABLE "drive_file" ADD "usageHint" drive_file_usage_hint_enum DEFAULT NULL`,
);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`ALTER TABLE "drive_file" DROP COLUMN "usageHint"`);
await queryRunner.query(`DROP TYPE drive_file_usage_hint_enum`);
}
}

View file

@ -16,6 +16,8 @@ import { DriveFolder } from "./drive-folder.js";
import { DB_MAX_IMAGE_COMMENT_LENGTH } from "@/misc/hard-limits.js";
import { NoteFile } from "./note-file.js";
export type DriveFileUsageHint = "userAvatar" | "userBanner" | null;
@Entity()
@Index(["userId", "folderId", "id"])
export class DriveFile {
@ -177,6 +179,14 @@ export class DriveFile {
})
public isSensitive: boolean;
// Hint for what this file is used for
@Column({
type: "enum",
enum: ["userAvatar", "userBanner"],
nullable: true,
})
public usageHint: DriveFileUsageHint;
/**
* ()URLへの直リンクか否か
*/

View file

@ -152,6 +152,7 @@ export const DriveFileRepository = db.getRepository(DriveFile).extend({
md5: file.md5,
size: file.size,
isSensitive: file.isSensitive,
usageHint: file.usageHint,
blurhash: file.blurhash,
properties: opts.self ? file.properties : this.getPublicProperties(file),
url: opts.self ? file.url : this.getPublicUrl(file, false),
@ -193,6 +194,7 @@ export const DriveFileRepository = db.getRepository(DriveFile).extend({
md5: file.md5,
size: file.size,
isSensitive: file.isSensitive,
usageHint: file.usageHint,
blurhash: file.blurhash,
properties: opts.self ? file.properties : this.getPublicProperties(file),
url: opts.self ? file.url : this.getPublicUrl(file, false),

View file

@ -44,6 +44,12 @@ export const packedDriveFileSchema = {
optional: false,
nullable: false,
},
usageHint: {
type: "string",
optional: false,
nullable: true,
enum: ["userAvatar", "userBanner"],
},
blurhash: {
type: "string",
optional: false,

View file

@ -34,7 +34,7 @@ export function initialize<T>(name: string, limitPerSec = -1) {
function apBackoff(attemptsMade: number, err: Error) {
const baseDelay = 60 * 1000; // 1min
const maxBackoff = 8 * 60 * 60 * 1000; // 8hours
let backoff = (Math.pow(2, attemptsMade) - 1) * baseDelay;
let backoff = (2 ** attemptsMade - 1) * baseDelay;
backoff = Math.min(backoff, maxBackoff);
backoff += Math.round(backoff * Math.random() * 0.2);
return backoff;

View file

@ -3,7 +3,10 @@ import type { CacheableRemoteUser } from "@/models/entities/user.js";
import Resolver from "../resolver.js";
import { fetchMeta } from "backend-rs";
import { apLogger } from "../logger.js";
import type { DriveFile } from "@/models/entities/drive-file.js";
import type {
DriveFile,
DriveFileUsageHint,
} from "@/models/entities/drive-file.js";
import { DriveFiles } from "@/models/index.js";
import { truncate } from "@/misc/truncate.js";
import { DB_MAX_IMAGE_COMMENT_LENGTH } from "@/misc/hard-limits.js";
@ -16,6 +19,7 @@ const logger = apLogger;
export async function createImage(
actor: CacheableRemoteUser,
value: any,
usage: DriveFileUsageHint,
): Promise<DriveFile> {
// Skip if author is frozen.
if (actor.isSuspended) {
@ -43,6 +47,7 @@ export async function createImage(
sensitive: image.sensitive,
isLink: !instance.cacheRemoteFiles,
comment: truncate(image.name, DB_MAX_IMAGE_COMMENT_LENGTH),
usageHint: usage,
});
if (file.isLink) {
@ -73,9 +78,10 @@ export async function createImage(
export async function resolveImage(
actor: CacheableRemoteUser,
value: any,
usage: DriveFileUsageHint,
): Promise<DriveFile> {
// TODO
// Fetch from remote server and register
return await createImage(actor, value);
return await createImage(actor, value, usage);
}

View file

@ -213,7 +213,8 @@ export async function createNote(
? (
await Promise.all(
note.attachment.map(
(x) => limit(() => resolveImage(actor, x)) as Promise<DriveFile>,
(x) =>
limit(() => resolveImage(actor, x, null)) as Promise<DriveFile>,
),
)
).filter((image) => image != null)
@ -616,7 +617,7 @@ export async function updateNote(value: string | IObject, resolver?: Resolver) {
fileList.map(
(x) =>
limit(async () => {
const file = await resolveImage(actor, x);
const file = await resolveImage(actor, x, null);
const update: Partial<DriveFile> = {};
const altText = truncate(x.name, DB_MAX_IMAGE_COMMENT_LENGTH);

View file

@ -10,6 +10,7 @@ import {
Followings,
UserProfiles,
UserPublickeys,
DriveFiles,
} from "@/models/index.js";
import type { IRemoteUser, CacheableUser } from "@/models/entities/user.js";
import { User } from "@/models/entities/user.js";
@ -362,10 +363,14 @@ export async function createPerson(
//#region Fetch avatar and header image
const [avatar, banner] = await Promise.all(
[person.icon, person.image].map((img) =>
[person.icon, person.image].map((img, index) =>
img == null
? Promise.resolve(null)
: resolveImage(user!, img).catch(() => null),
: resolveImage(
user,
img,
index === 0 ? "userAvatar" : index === 1 ? "userBanner" : null,
).catch(() => null),
),
);
@ -438,10 +443,14 @@ export async function updatePerson(
// Fetch avatar and header image
const [avatar, banner] = await Promise.all(
[person.icon, person.image].map((img) =>
[person.icon, person.image].map((img, index) =>
img == null
? Promise.resolve(null)
: resolveImage(user, img).catch(() => null),
: resolveImage(
user,
img,
index === 0 ? "userAvatar" : index === 1 ? "userBanner" : null,
).catch(() => null),
),
);
@ -561,10 +570,14 @@ export async function updatePerson(
} as Partial<User>;
if (avatar) {
if (user?.avatarId)
await DriveFiles.update(user.avatarId, { usageHint: null });
updates.avatarId = avatar.id;
}
if (banner) {
if (user?.bannerId)
await DriveFiles.update(user.bannerId, { usageHint: null });
updates.bannerId = banner.id;
}

View file

@ -13,6 +13,7 @@ import { normalizeForSearch } from "@/misc/normalize-for-search.js";
import { verifyLink } from "@/services/fetch-rel-me.js";
import { ApiError } from "@/server/api/error.js";
import define from "@/server/api/define.js";
import { DriveFile } from "@/models/entities/drive-file";
export const meta = {
tags: ["account"],
@ -241,8 +242,9 @@ export default define(meta, paramDef, async (ps, _user, token) => {
if (ps.emailNotificationTypes !== undefined)
profileUpdates.emailNotificationTypes = ps.emailNotificationTypes;
let avatar: DriveFile | null = null;
if (ps.avatarId) {
const avatar = await DriveFiles.findOneBy({ id: ps.avatarId });
avatar = await DriveFiles.findOneBy({ id: ps.avatarId });
if (avatar == null || avatar.userId !== user.id)
throw new ApiError(meta.errors.noSuchAvatar);
@ -250,8 +252,9 @@ export default define(meta, paramDef, async (ps, _user, token) => {
throw new ApiError(meta.errors.avatarNotAnImage);
}
let banner: DriveFile | null = null;
if (ps.bannerId) {
const banner = await DriveFiles.findOneBy({ id: ps.bannerId });
banner = await DriveFiles.findOneBy({ id: ps.bannerId });
if (banner == null || banner.userId !== user.id)
throw new ApiError(meta.errors.noSuchBanner);
@ -328,6 +331,20 @@ export default define(meta, paramDef, async (ps, _user, token) => {
updateUsertags(user, tags);
//#endregion
// Update old/new avatar usage hints
if (avatar) {
if (user.avatarId)
await DriveFiles.update(user.avatarId, { usageHint: null });
await DriveFiles.update(avatar.id, { usageHint: "userAvatar" });
}
// Update old/new banner usage hints
if (banner) {
if (user.bannerId)
await DriveFiles.update(user.bannerId, { usageHint: null });
await DriveFiles.update(banner.id, { usageHint: "userBanner" });
}
if (Object.keys(updates).length > 0) await Users.update(user.id, updates);
if (Object.keys(profileUpdates).length > 0)
await UserProfiles.update(user.id, profileUpdates);

View file

@ -54,6 +54,10 @@ app.use(async (ctx, next) => {
const url = decodeURI(ctx.path);
if (url === bullBoardPath || url.startsWith(`${bullBoardPath}/`)) {
if (!url.startsWith(`${bullBoardPath}/static/`)) {
ctx.set("Cache-Control", "private, max-age=0, must-revalidate");
}
const token = ctx.cookies.get("token");
if (token == null) {
ctx.status = 401;

View file

@ -16,6 +16,7 @@ import {
UserProfiles,
} from "@/models/index.js";
import { DriveFile } from "@/models/entities/drive-file.js";
import type { DriveFileUsageHint } from "@/models/entities/drive-file.js";
import type { IRemoteUser, User } from "@/models/entities/user.js";
import { genId } from "backend-rs";
import { isDuplicateKeyValueError } from "@/misc/is-duplicate-key-value-error.js";
@ -65,6 +66,7 @@ function urlPathJoin(
* @param type Content-Type for original
* @param hash Hash for original
* @param size Size for original
* @param usage Optional usage hint for file (f.e. "userAvatar")
*/
async function save(
file: DriveFile,
@ -73,6 +75,7 @@ async function save(
type: string,
hash: string,
size: number,
usage: DriveFileUsageHint = null,
): Promise<DriveFile> {
// thunbnail, webpublic を必要なら生成
const alts = await generateAlts(path, type, !file.uri);
@ -161,6 +164,7 @@ async function save(
file.md5 = hash;
file.size = size;
file.storedInternal = false;
file.usageHint = usage ?? null;
return await DriveFiles.insert(file).then((x) =>
DriveFiles.findOneByOrFail(x.identifiers[0]),
@ -204,6 +208,7 @@ async function save(
file.type = type;
file.md5 = hash;
file.size = size;
file.usageHint = usage ?? null;
return await DriveFiles.insert(file).then((x) =>
DriveFiles.findOneByOrFail(x.identifiers[0]),
@ -450,6 +455,9 @@ type AddFileArgs = {
requestIp?: string | null;
requestHeaders?: Record<string, string> | null;
/** Whether this file has a known use case, like user avatar or instance icon */
usageHint?: DriveFileUsageHint;
};
/**
@ -469,6 +477,7 @@ export async function addFile({
sensitive = null,
requestIp = null,
requestHeaders = null,
usageHint = null,
}: AddFileArgs): Promise<DriveFile> {
const info = await getFileInfo(path);
logger.info(`${JSON.stringify(info)}`);
@ -581,6 +590,7 @@ export async function addFile({
file.isLink = isLink;
file.requestIp = requestIp;
file.requestHeaders = requestHeaders;
file.usageHint = usageHint;
file.isSensitive = user
? Users.isLocalUser(user) &&
(instance!.markLocalFilesNsfwByDefault || profile!.alwaysMarkNsfw)
@ -639,6 +649,7 @@ export async function addFile({
info.type.mime,
info.md5,
info.size,
usageHint,
);
}

View file

@ -3,7 +3,10 @@ import type { User } from "@/models/entities/user.js";
import { createTemp } from "@/misc/create-temp.js";
import { downloadUrl, isPrivateIp } from "@/misc/download-url.js";
import type { DriveFolder } from "@/models/entities/drive-folder.js";
import type { DriveFile } from "@/models/entities/drive-file.js";
import type {
DriveFile,
DriveFileUsageHint,
} from "@/models/entities/drive-file.js";
import { DriveFiles } from "@/models/index.js";
import { driveLogger } from "./logger.js";
import { addFile } from "./add-file.js";
@ -13,7 +16,11 @@ const logger = driveLogger.createSubLogger("downloader");
type Args = {
url: string;
user: { id: User["id"]; host: User["host"] } | null;
user: {
id: User["id"];
host: User["host"];
driveCapacityOverrideMb: User["driveCapacityOverrideMb"];
} | null;
folderId?: DriveFolder["id"] | null;
uri?: string | null;
sensitive?: boolean;
@ -22,6 +29,7 @@ type Args = {
comment?: string | null;
requestIp?: string | null;
requestHeaders?: Record<string, string> | null;
usageHint?: DriveFileUsageHint;
};
export async function uploadFromUrl({
@ -35,6 +43,7 @@ export async function uploadFromUrl({
comment = null,
requestIp = null,
requestHeaders = null,
usageHint = null,
}: Args): Promise<DriveFile> {
const parsedUrl = new URL(url);
if (
@ -75,9 +84,10 @@ export async function uploadFromUrl({
sensitive,
requestIp,
requestHeaders,
usageHint,
});
logger.succ(`Got: ${driveFile.id}`);
return driveFile!;
return driveFile;
} catch (e) {
logger.error(`Failed to create drive file:\n${inspect(e)}`);
throw e;

View file

@ -28,9 +28,9 @@ export default class Logger {
if (config.syslog) {
this.syslogClient = new SyslogPro.RFC5424({
applacationName: "Firefish",
applicationName: "Firefish",
timestamp: true,
encludeStructuredData: true,
includeStructuredData: true,
color: true,
extendedColor: true,
server: {
@ -144,12 +144,12 @@ export default class Logger {
}
}
// Used when the process can't continue (fatal error)
public error(
x: string | Error,
data?: Record<string, any> | null,
important = false,
): void {
// 実行を継続できない状況で使う
if (x instanceof Error) {
data = data || {};
data.e = x;
@ -166,30 +166,30 @@ export default class Logger {
}
}
// Used when the process can continue but some action should be taken
public warn(
message: string,
data?: Record<string, any> | null,
important = false,
): void {
// 実行を継続できるが改善すべき状況で使う
this.log("warning", message, data, important);
}
// Used when something is successful
public succ(
message: string,
data?: Record<string, any> | null,
important = false,
): void {
// 何かに成功した状況で使う
this.log("success", message, data, important);
}
// Used for debugging (information necessary for developers but unnecessary for users)
public debug(
message: string,
data?: Record<string, any> | null,
important = false,
): void {
// Used for debugging (information necessary for developers but unnecessary for users)
// Fixed if statement is ignored when logLevel includes debug
if (
config.logLevel?.includes("debug") ||
@ -200,12 +200,12 @@ export default class Logger {
}
}
// Other generic logs
public info(
message: string,
data?: Record<string, any> | null,
important = false,
): void {
// それ以外
this.log("info", message, data, important);
}
}

View file

@ -1,11 +1,7 @@
import { publishMainStream } from "@/services/stream.js";
import type { Note } from "@/models/entities/note.js";
import type { User } from "@/models/entities/user.js";
import {
NoteUnreads,
Followings,
ChannelFollowings,
} from "@/models/index.js";
import { NoteUnreads, Followings, ChannelFollowings } from "@/models/index.js";
import { Not, IsNull, In } from "typeorm";
import type { Channel } from "@/models/entities/channel.js";
import { readNotificationByQuery } from "@/server/api/common/read-notification.js";