HoovesSocial/hippofish
2
0
Fork 0
Code Issues 3 Pull requests 1 Projects Releases Packages Wiki Activity Actions

refactor: 🎨 rome

Browse source
This commit is contained in:
ThatOneCalculator 2023-01-12 20:40:33 -08:00
parent 54d3acafb3
commit 6b00abf05c
No known key found for this signature in database
GPG key ID: 8703CACD01000000
940 changed files with 33578 additions and 24085 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
CALCKEY.md
View file

@ -106,6 +106,8 @@
- New post style
- Admins set default reaction emoji
- Allows custom emoji
- Fix lint errors
- Use Rome instead of ESLint
- MissV: [fix Misskey Forkbomb](https://code.vtopia.live/Vtopia/MissV/commit/40b23c070bd4adbb3188c73546c6c625138fb3c1)
- [Make showing ads optional](https://github.com/misskey-dev/misskey/pull/8996)
- [Tapping avatar in mobile opens account modal](https://github.com/misskey-dev/misskey/pull/9056)

1
package.json
View file

@ -59,6 +59,7 @@
"cross-env": "7.0.3",
"cypress": "10.11.0",
"install-peers": "^1.0.4",
"rome": "^11.0.0",
"start-server-and-test": "1.15.2",
"typescript": "4.9.4",
"vue-eslint-parser": "^9.1.0"

2
packages/backend/package.json
View file

@ -10,7 +10,7 @@
"revertmigration": "typeorm migration:revert -d ormconfig.js",
"build": "pnpm swc src -d built -D",
"watch": "pnpm swc src -d built -D -w",
"lint": "eslint --quiet \"src/**/*.ts\"",
"lint": "pnpm eslint --quiet \"src/**/*.ts\"",
"mocha": "cross-env NODE_ENV=test TS_NODE_FILES=true TS_NODE_TRANSPILE_ONLY=true TS_NODE_PROJECT=\"./test/tsconfig.json\" mocha",
"test": "pnpm run mocha"
},

9
packages/backend/src/@types/hcaptcha.d.ts vendored
View file

@ -1,11 +1,14 @@
declare module 'hcaptcha' {
declare module "hcaptcha" {
interface IVerifyResponse {
success: boolean;
challenge_ts: string;
hostname: string;
credit?: boolean;
'error-codes'?: unknown[];
"error-codes"?: unknown[];
}
export function verify(secret: string, token: string): Promise<IVerifyResponse>;
export function verify(
secret: string,
token: string,
): Promise<IVerifyResponse>;
}

43
packages/backend/src/@types/http-signature.d.ts vendored
View file

@ -1,5 +1,5 @@
declare module '@peertube/http-signature' {
import { IncomingMessage, ClientRequest } from 'node:http';
declare module "@peertube/http-signature" {
import type { IncomingMessage, ClientRequest } from "node:http";
interface ISignature {
keyId: string;
@ -28,8 +28,8 @@ declare module '@peertube/http-signature' {
}
type RequestSignerConstructorOptions =
IRequestSignerConstructorOptionsFromProperties |
IRequestSignerConstructorOptionsFromFunction;
| IRequestSignerConstructorOptionsFromProperties
| IRequestSignerConstructorOptionsFromFunction;
interface IRequestSignerConstructorOptionsFromProperties {
keyId: string;
@ -59,11 +59,23 @@ declare module '@peertube/http-signature' {
httpVersion?: string;
}
export function parse(request: IncomingMessage, options?: IParseRequestOptions): IParsedSignature;
export function parseRequest(request: IncomingMessage, options?: IParseRequestOptions): IParsedSignature;
export function parse(
request: IncomingMessage,
options?: IParseRequestOptions,
): IParsedSignature;
export function parseRequest(
request: IncomingMessage,
options?: IParseRequestOptions,
): IParsedSignature;
export function sign(request: ClientRequest, options: ISignRequestOptions): boolean;
export function signRequest(request: ClientRequest, options: ISignRequestOptions): boolean;
export function sign(
request: ClientRequest,
options: ISignRequestOptions,
): boolean;
export function signRequest(
request: ClientRequest,
options: ISignRequestOptions,
): boolean;
export function createSigner(): RequestSigner;
export function isSigner(obj: any): obj is RequestSigner;
@ -71,7 +83,16 @@ declare module '@peertube/http-signature' {
export function sshKeyFingerprint(key: string): string;
export function pemToRsaSSHKey(pem: string, comment: string): string;
export function verify(parsedSignature: IParsedSignature, pubkey: string | Buffer): boolean;
export function verifySignature(parsedSignature: IParsedSignature, pubkey: string | Buffer): boolean;
export function verifyHMAC(parsedSignature: IParsedSignature, secret: string): boolean;
export function verify(
parsedSignature: IParsedSignature,
pubkey: string | Buffer,
): boolean;
export function verifySignature(
parsedSignature: IParsedSignature,
pubkey: string | Buffer,
): boolean;
export function verifyHMAC(
parsedSignature: IParsedSignature,
secret: string,
): boolean;
}

4
packages/backend/src/@types/koa-json-body.d.ts vendored
View file

@ -1,5 +1,5 @@
declare module 'koa-json-body' {
import { Middleware } from 'koa';
declare module "koa-json-body" {
import type { Middleware } from "koa";
interface IKoaJsonBodyOptions {
strict: boolean;

4
packages/backend/src/@types/koa-slow.d.ts vendored
View file

@ -1,5 +1,5 @@
declare module 'koa-slow' {
import { Middleware } from 'koa';
declare module "koa-slow" {
import type { Middleware } from "koa";
interface ISlowOptions {
url?: RegExp;

7
packages/backend/src/@types/os-utils.d.ts vendored
View file

@ -1,4 +1,4 @@
declare module 'os-utils' {
declare module "os-utils" {
type FreeCommandCallback = (usedmem: number) => void;
type HarddriveCallback = (total: number, free: number, used: number) => void;
@ -20,7 +20,10 @@ declare module 'os-utils' {
export function harddrive(callback: HarddriveCallback): void;
export function getProcesses(callback: GetProcessesCallback): void;
export function getProcesses(nProcess: number, callback: GetProcessesCallback): void;
export function getProcesses(
nProcess: number,
callback: GetProcessesCallback,
): void;
export function allLoadavg(): string;
export function loadavg(_time?: number): number;

2
packages/backend/src/@types/package.json.d.ts vendored
View file

@ -1,4 +1,4 @@
declare module '*/package.json' {
declare module "*/package.json" {
interface IRepository {
type: string;
url: string;

24
packages/backend/src/@types/probe-image-size.d.ts vendored
View file

@ -1,5 +1,5 @@
declare module 'probe-image-size' {
import { ReadStream } from 'node:fs';
declare module "probe-image-size" {
import type { ReadStream } from "node:fs";
type ProbeOptions = {
retries: 1;
@ -12,14 +12,24 @@ declare module 'probe-image-size' {
length?: number;
type: string;
mime: string;
wUnits: 'in' | 'mm' | 'cm' | 'pt' | 'pc' | 'px' | 'em' | 'ex';
hUnits: 'in' | 'mm' | 'cm' | 'pt' | 'pc' | 'px' | 'em' | 'ex';
wUnits: "in" | "mm" | "cm" | "pt" | "pc" | "px" | "em" | "ex";
hUnits: "in" | "mm" | "cm" | "pt" | "pc" | "px" | "em" | "ex";
url?: string;
};
function probeImageSize(src: string | ReadStream, options?: ProbeOptions): Promise<ProbeResult>;
function probeImageSize(src: string | ReadStream, callback: (err: Error | null, result?: ProbeResult) => void): void;
function probeImageSize(src: string | ReadStream, options: ProbeOptions, callback: (err: Error | null, result?: ProbeResult) => void): void;
function probeImageSize(
src: string | ReadStream,
options?: ProbeOptions,
): Promise<ProbeResult>;
function probeImageSize(
src: string | ReadStream,
callback: (err: Error | null, result?: ProbeResult) => void,
): void;
function probeImageSize(
src: string | ReadStream,
options: ProbeOptions,
callback: (err: Error | null, result?: ProbeResult) => void,
): void;
namespace probeImageSize {} // Hack

81
packages/backend/src/boot/index.ts
View file

@ -1,79 +1,78 @@
import cluster from 'node:cluster';
import chalk from 'chalk';
import Xev from 'xev';
import cluster from "node:cluster";
import chalk from "chalk";
import Xev from "xev";
import Logger from '@/services/logger.js';
import { envOption } from '../env.js';
import Logger from "@/services/logger.js";
import { envOption } from "../env.js";
// for typeorm
import 'reflect-metadata';
import { masterMain } from './master.js';
import { workerMain } from './worker.js';
import "reflect-metadata";
import { masterMain } from "./master.js";
import { workerMain } from "./worker.js";
const logger = new Logger('core', 'cyan');
const clusterLogger = logger.createSubLogger('cluster', 'orange', false);
const logger = new Logger("core", "cyan");
const clusterLogger = logger.createSubLogger("cluster", "orange", false);
const ev = new Xev();
/**
* Init process
*/
export default async function() {
process.title = `Calckey (${cluster.isPrimary ? 'master' : 'worker'})`;
if (cluster.isPrimary || envOption.disableClustering) {
export default async function () {
process.title = `Calckey (${cluster.isPrimary ? "master" : "worker"})`;
if (cluster.isPrimary || envOption.disableClustering) {
await masterMain();
if (cluster.isPrimary) {
ev.mount();
ev.mount();
}
}
if (cluster.isWorker || envOption.disableClustering) {
}
if (cluster.isWorker || envOption.disableClustering) {
await workerMain();
}
// For when Calckey is started in a child process during unit testing.
// Otherwise, process.send cannot be used, so start it.
if (process.send) {
process.send('ok');
}
}
// For when Calckey is started in a child process during unit testing.
// Otherwise, process.send cannot be used, so start it.
if (process.send) {
process.send("ok");
}
}
//#region Events
// Listen new workers
cluster.on('fork', worker => {
clusterLogger.debug(`Process forked: [${worker.id}]`);
cluster.on("fork", (worker) => {
clusterLogger.debug(`Process forked: [${worker.id}]`);
});
// Listen online workers
cluster.on('online', worker => {
clusterLogger.debug(`Process is now online: [${worker.id}]`);
cluster.on("online", (worker) => {
clusterLogger.debug(`Process is now online: [${worker.id}]`);
});
// Listen for dying workers
cluster.on('exit', worker => {
// Replace the dead worker,
// we're not sentimental
clusterLogger.error(chalk.red(`[${worker.id}] died :(`));
cluster.fork();
cluster.on("exit", (worker) => {
// Replace the dead worker,
// we're not sentimental
clusterLogger.error(chalk.red(`[${worker.id}] died :(`));
cluster.fork();
});
// Display detail of unhandled promise rejection
if (!envOption.quiet) {
process.on('unhandledRejection', console.dir);
process.on("unhandledRejection", console.dir);
}
// Display detail of uncaught exception
process.on('uncaughtException', err => {
try {
process.on("uncaughtException", (err) => {
try {
logger.error(err);
} catch { }
} catch {}
});
// Dying away...
process.on('exit', code => {
logger.info(`The process is going to exit with code ${code}`);
process.on("exit", (code) => {
logger.info(`The process is going to exit with code ${code}`);
});
//#endregion

140
packages/backend/src/boot/master.ts
View file

@ -1,50 +1,64 @@
import * as fs from 'node:fs';
import { fileURLToPath } from 'node:url';
import { dirname } from 'node:path';
import * as os from 'node:os';
import cluster from 'node:cluster';
import chalk from 'chalk';
import chalkTemplate from 'chalk-template';
import semver from 'semver';
import * as fs from "node:fs";
import { fileURLToPath } from "node:url";
import { dirname } from "node:path";
import * as os from "node:os";
import cluster from "node:cluster";
import chalk from "chalk";
import chalkTemplate from "chalk-template";
import semver from "semver";
import Logger from '@/services/logger.js';
import loadConfig from '@/config/load.js';
import { Config } from '@/config/types.js';
import { lessThan } from '@/prelude/array.js';
import { envOption } from '../env.js';
import { showMachineInfo } from '@/misc/show-machine-info.js';
import { db, initDb } from '../db/postgre.js';
import Logger from "@/services/logger.js";
import loadConfig from "@/config/load.js";
import type { Config } from "@/config/types.js";
import { lessThan } from "@/prelude/array.js";
import { envOption } from "../env.js";
import { showMachineInfo } from "@/misc/show-machine-info.js";
import { db, initDb } from "../db/postgre.js";
const _filename = fileURLToPath(import.meta.url);
const _dirname = dirname(_filename);
const meta = JSON.parse(fs.readFileSync(`${_dirname}/../../../../built/meta.json`, 'utf-8'));
const meta = JSON.parse(
fs.readFileSync(`${_dirname}/../../../../built/meta.json`, "utf-8"),
);
const logger = new Logger('core', 'cyan');
const bootLogger = logger.createSubLogger('boot', 'magenta', false);
const logger = new Logger("core", "cyan");
const bootLogger = logger.createSubLogger("boot", "magenta", false);
const themeColor = chalk.hex('#31748f');
const themeColor = chalk.hex("#31748f");
function greet() {
if (!envOption.quiet) {
//#region Calckey logo
const v = `v${meta.version}`;
console.log(themeColor(' ___ _ _ '));
console.log(themeColor(' / __\\__ _| | ___| | _____ _ _ '));
console.log(themeColor(' / / / _` | |/ __| |/ / _ \ | | |'));
console.log(themeColor('/ /__| (_| | | (__| < __/ |_| |'));
console.log(themeColor('\\____/\\__,_|_|\\___|_|\\_\\___|\\__, |'));
console.log(themeColor(' (___/ '));
console.log(themeColor(" ___ _ _ "));
console.log(themeColor(" / __\\__ _| | ___| | _____ _ _ "));
console.log(themeColor(" / / / _` | |/ __| |/ / _ | | |"));
console.log(themeColor("/ /__| (_| | | (__| < __/ |_| |"));
console.log(themeColor("\\____/\\__,_|_|\\___|_|\\_\\___|\\__, |"));
console.log(themeColor(" (___/ "));
//#endregion
console.log(' Calckey is an open-source decentralized microblogging platform.');
console.log(chalk.rgb(255, 136, 0)(' If you like Calckey, please consider starring or contributing to the repo. https://codeberg.org/calckey/calckey'));
console.log(
" Calckey is an open-source decentralized microblogging platform.",
);
console.log(
chalk.rgb(
255,
136,
0,
)(
" If you like Calckey, please consider starring or contributing to the repo. https://codeberg.org/calckey/calckey",
),
);
console.log('');
console.log(chalkTemplate`--- ${os.hostname()} {gray (PID: ${process.pid.toString()})} ---`);
console.log("");
console.log(
chalkTemplate`--- ${os.hostname()} {gray (PID: ${process.pid.toString()})} ---`,
);
}
bootLogger.info('Welcome to Calckey!');
bootLogger.info("Welcome to Calckey!");
bootLogger.info(`Calckey v${meta.version}`, null, true);
}
@ -63,42 +77,50 @@ export async function masterMain() {
config = loadConfigBoot();
await connectDb();
} catch (e) {
bootLogger.error('Fatal error occurred during initialization', null, true);
bootLogger.error("Fatal error occurred during initialization", null, true);
process.exit(1);
}
bootLogger.succ('Calckey initialized');
bootLogger.succ("Calckey initialized");
if (!envOption.disableClustering) {
await spawnWorkers(config.clusterLimit);
}
bootLogger.succ(`Now listening on port ${config.port} on ${config.url}`, null, true);
bootLogger.succ(
`Now listening on port ${config.port} on ${config.url}`,
null,
true,
);
if (!envOption.noDaemons) {
import('../daemons/server-stats.js').then(x => x.default());
import('../daemons/queue-stats.js').then(x => x.default());
import('../daemons/janitor.js').then(x => x.default());
import("../daemons/server-stats.js").then((x) => x.default());
import("../daemons/queue-stats.js").then((x) => x.default());
import("../daemons/janitor.js").then((x) => x.default());
}
}
function showEnvironment(): void {
const env = process.env.NODE_ENV;
const logger = bootLogger.createSubLogger('env');
logger.info(typeof env === 'undefined' ? 'NODE_ENV is not set' : `NODE_ENV: ${env}`);
const logger = bootLogger.createSubLogger("env");
logger.info(
typeof env === "undefined" ? "NODE_ENV is not set" : `NODE_ENV: ${env}`,
);
if (env !== 'production') {
logger.warn('The environment is not in production mode.');
logger.warn('DO NOT USE FOR PRODUCTION PURPOSE!', null, true);
if (env !== "production") {
logger.warn("The environment is not in production mode.");
logger.warn("DO NOT USE FOR PRODUCTION PURPOSE!", null, true);
}
}
function showNodejsVersion(): void {
const nodejsLogger = bootLogger.createSubLogger('nodejs');
const nodejsLogger = bootLogger.createSubLogger("nodejs");
nodejsLogger.info(`Version ${process.version} detected.`);
const minVersion = fs.readFileSync(`${_dirname}/../../../../.node-version`, 'utf-8').trim();
const minVersion = fs
.readFileSync(`${_dirname}/../../../../.node-version`, "utf-8")
.trim();
if (semver.lt(process.version, minVersion)) {
nodejsLogger.error(`At least Node.js ${minVersion} required!`);
process.exit(1);
@ -106,14 +128,14 @@ function showNodejsVersion(): void {
}
function loadConfigBoot(): Config {
const configLogger = bootLogger.createSubLogger('config');
const configLogger = bootLogger.createSubLogger("config");
let config;
try {
config = loadConfig();
} catch (exception) {
if (exception.code === 'ENOENT') {
configLogger.error('Configuration file not found', null, true);
if (exception.code === "ENOENT") {
configLogger.error("Configuration file not found", null, true);
process.exit(1);
} else if (e instanceof Error) {
configLogger.error(e.message);
@ -122,22 +144,24 @@ function loadConfigBoot(): Config {
throw exception;
}
configLogger.succ('Loaded');
configLogger.succ("Loaded");
return config;
}
async function connectDb(): Promise<void> {
const dbLogger = bootLogger.createSubLogger('db');
const dbLogger = bootLogger.createSubLogger("db");
// Try to connect to DB
try {
dbLogger.info('Connecting...');
dbLogger.info("Connecting...");
await initDb();
const v = await db.query('SHOW server_version').then(x => x[0].server_version);
const v = await db
.query("SHOW server_version")
.then((x) => x[0].server_version);
dbLogger.succ(`Connected: v${v}`);
} catch (e) {
dbLogger.error('Cannot connect', null, true);
dbLogger.error("Cannot connect", null, true);
dbLogger.error(e);
process.exit(1);
}
@ -145,20 +169,20 @@ async function connectDb(): Promise<void> {
async function spawnWorkers(limit: number = 1) {
const workers = Math.min(limit, os.cpus().length);
bootLogger.info(`Starting ${workers} worker${workers === 1 ? '' : 's'}...`);
bootLogger.info(`Starting ${workers} worker${workers === 1 ? "" : "s"}...`);
await Promise.all([...Array(workers)].map(spawnWorker));
bootLogger.succ('All workers started');
bootLogger.succ("All workers started");
}
function spawnWorker(): Promise<void> {
return new Promise(res => {
return new Promise((res) => {
const worker = cluster.fork();
worker.on('message', message => {
if (message === 'listenFailed') {
bootLogger.error(`The server Listen failed due to the previous error.`);
worker.on("message", (message) => {
if (message === "listenFailed") {
bootLogger.error("The server Listen failed due to the previous error.");
process.exit(1);
}
if (message !== 'ready') return;
if (message !== "ready") return;
res();
});
});

10
packages/backend/src/boot/worker.ts
View file

@ -1,5 +1,5 @@
import cluster from 'node:cluster';
import { initDb } from '../db/postgre.js';
import cluster from "node:cluster";
import { initDb } from "../db/postgre.js";
/**
* Init worker process
@ -8,13 +8,13 @@ export async function workerMain() {
await initDb();
// start server
await import('../server/index.js').then(x => x.default());
await import("../server/index.js").then((x) => x.default());
// start job queue
import('../queue/index.js').then(x => x.default());
import("../queue/index.js").then((x) => x.default());
if (cluster.isWorker) {
// Send a 'ready' message to parent process
process.send!('ready');
process.send!("ready");
}
}

2
packages/backend/src/config/index.ts
View file

@ -1,3 +1,3 @@
import load from './load.js';
import load from "./load.js";
export default load();

36
packages/backend/src/config/load.ts
View file

@ -2,11 +2,11 @@
* Config loader
*/
import * as fs from 'node:fs';
import { fileURLToPath } from 'node:url';
import { dirname } from 'node:path';
import * as yaml from 'js-yaml';
import type { Source, Mixin } from './types.js';
import * as fs from "node:fs";
import { fileURLToPath } from "node:url";
import { dirname } from "node:path";
import * as yaml from "js-yaml";
import type { Source, Mixin } from "./types.js";
const _filename = fileURLToPath(import.meta.url);
const _dirname = dirname(_filename);
@ -19,14 +19,20 @@ const dir = `${_dirname}/../../../../.config`;
/**
* Path of configuration file
*/
const path = process.env.NODE_ENV === 'test'
? `${dir}/test.yml`
: `${dir}/default.yml`;
const path =
process.env.NODE_ENV === "test" ? `${dir}/test.yml` : `${dir}/default.yml`;
export default function load() {
const meta = JSON.parse(fs.readFileSync(`${_dirname}/../../../../built/meta.json`, 'utf-8'));
const clientManifest = JSON.parse(fs.readFileSync(`${_dirname}/../../../../built/_client_dist_/manifest.json`, 'utf-8'));
const config = yaml.load(fs.readFileSync(path, 'utf-8')) as Source;
const meta = JSON.parse(
fs.readFileSync(`${_dirname}/../../../../built/meta.json`, "utf-8"),
);
const clientManifest = JSON.parse(
fs.readFileSync(
`${_dirname}/../../../../built/_client_dist_/manifest.json`,
"utf-8",
),
);
const config = yaml.load(fs.readFileSync(path, "utf-8")) as Source;
const mixin = {} as Mixin;
@ -34,19 +40,19 @@ export default function load() {
config.url = url.origin;
config.port = config.port || parseInt(process.env.PORT || '', 10);
config.port = config.port || parseInt(process.env.PORT || "", 10);
mixin.version = meta.version;
mixin.host = url.host;
mixin.hostname = url.hostname;
mixin.scheme = url.protocol.replace(/:$/, '');
mixin.wsScheme = mixin.scheme.replace('http', 'ws');
mixin.scheme = url.protocol.replace(/:$/, "");
mixin.wsScheme = mixin.scheme.replace("http", "ws");
mixin.wsUrl = `${mixin.wsScheme}://${mixin.host}`;
mixin.apiUrl = `${mixin.scheme}://${mixin.host}/api`;
mixin.authUrl = `${mixin.scheme}://${mixin.host}/auth`;
mixin.driveUrl = `${mixin.scheme}://${mixin.host}/files`;
mixin.userAgent = `Calckey/${meta.version} (${config.url})`;
mixin.clientEntry = clientManifest['src/init.ts'];
mixin.clientEntry = clientManifest["src/init.ts"];
if (!config.redis.prefix) config.redis.prefix = mixin.host;

3
packages/backend/src/config/types.ts
View file

@ -47,7 +47,7 @@ export type Source = {
id: string;
outgoingAddressFamily?: 'ipv4' | 'ipv6' | 'dual';
outgoingAddressFamily?: "ipv4" | "ipv6" | "dual";
deliverJobConcurrency?: number;
inboxJobConcurrency?: number;
@ -81,7 +81,6 @@ export type Source = {
user?: string;
pass?: string;
useImplicitSslTls?: boolean;
};
objectStorage: {
managed?: boolean;

57
packages/backend/src/const.ts
View file

@ -1,6 +1,7 @@
import config from '@/config/index.js';
import config from "@/config/index.js";
export const MAX_NOTE_TEXT_LENGTH = config.maxNoteLength != null ? config.maxNoteLength : 3000;
export const MAX_NOTE_TEXT_LENGTH =
config.maxNoteLength != null ? config.maxNoteLength : 3000;
export const SECOND = 1000;
export const SEC = 1000;
@ -17,39 +18,39 @@ export const USER_ACTIVE_THRESHOLD = 1000 * 60 * 60 * 24 * 3; // 3days
// SVGはXSSを生むので許可しない
export const FILE_TYPE_BROWSERSAFE = [
// Images
'image/png',
'image/gif',
'image/jpeg',
'image/webp',
'image/apng',
'image/bmp',
'image/tiff',
'image/x-icon',
"image/png",
"image/gif",
"image/jpeg",
"image/webp",
"image/apng",
"image/bmp",
"image/tiff",
"image/x-icon",
// OggS
'audio/opus',
'video/ogg',
'audio/ogg',
'application/ogg',
"audio/opus",
"video/ogg",
"audio/ogg",
"application/ogg",
// ISO/IEC base media file format
'video/quicktime',
'video/mp4',
'audio/mp4',
'video/x-m4v',
'audio/x-m4a',
'video/3gpp',
'video/3gpp2',
"video/quicktime",
"video/mp4",
"audio/mp4",
"video/x-m4v",
"audio/x-m4a",
"video/3gpp",
"video/3gpp2",
'video/mpeg',
'audio/mpeg',
"video/mpeg",
"audio/mpeg",
'video/webm',
'audio/webm',
"video/webm",
"audio/webm",
'audio/aac',
'audio/x-flac',
'audio/vnd.wave',
"audio/aac",
"audio/x-flac",
"audio/vnd.wave",
];
/*
https://github.com/sindresorhus/file-type/blob/main/supported.js

6
packages/backend/src/daemons/janitor.ts
View file

@ -1,13 +1,13 @@
// TODO: 消したい
const interval = 30 * 60 * 1000;
import { AttestationChallenges } from '@/models/index.js';
import { LessThan } from 'typeorm';
import { AttestationChallenges } from "@/models/index.js";
import { LessThan } from "typeorm";
/**
* Clean up database occasionally
*/
export default function() {
export default function () {
async function tick() {
await AttestationChallenges.delete({
createdAt: LessThan(new Date(new Date().getTime() - 5 * 60 * 1000)),

14
packages/backend/src/daemons/queue-stats.ts
View file

@ -1,5 +1,5 @@
import Xev from 'xev';
import { deliverQueue, inboxQueue } from '../queue/queues.js';
import Xev from "xev";
import { deliverQueue, inboxQueue } from "../queue/queues.js";
const ev = new Xev();
@ -8,21 +8,21 @@ const interval = 10000;
/**
* Report queue stats regularly
*/
export default function() {
export default function () {
const log = [] as any[];
ev.on('requestQueueStatsLog', x => {
ev.on("requestQueueStatsLog", (x) => {
ev.emit(`queueStatsLog:${x.id}`, log.slice(0, x.length || 50));
});
let activeDeliverJobs = 0;
let activeInboxJobs = 0;
deliverQueue.on('global:active', () => {
deliverQueue.on("global:active", () => {
activeDeliverJobs++;
});
inboxQueue.on('global:active', () => {
inboxQueue.on("global:active", () => {
activeInboxJobs++;
});
@ -45,7 +45,7 @@ export default function() {
},
};
ev.emit('queueStats', stats);
ev.emit("queueStats", stats);
log.unshift(stats);
if (log.length > 200) log.pop();

12
packages/backend/src/daemons/server-stats.ts
View file

@ -1,6 +1,6 @@
import si from 'systeminformation';
import Xev from 'xev';
import * as osUtils from 'os-utils';
import si from "systeminformation";
import Xev from "xev";
import * as osUtils from "os-utils";
const ev = new Xev();
@ -12,10 +12,10 @@ const round = (num: number) => Math.round(num * 10) / 10;
/**
* Report server stats regularly
*/
export default function() {
export default function () {
const log = [] as any[];
ev.on('requestServerStatsLog', x => {
ev.on("requestServerStatsLog", (x) => {
ev.emit(`serverStatsLog:${x.id}`, log.slice(0, x.length || 50));
});
@ -40,7 +40,7 @@ export default function() {
w: round(Math.max(0, fsStats.wIO_sec ?? 0)),
},
};
ev.emit('serverStats', stats);
ev.emit("serverStats", stats);
log.unshift(stats);
if (log.length > 200) log.pop();
}

59
packages/backend/src/db/elasticsearch.ts
View file

@ -1,12 +1,12 @@
import * as elasticsearch from '@elastic/elasticsearch';
import config from '@/config/index.js';
import * as elasticsearch from "@elastic/elasticsearch";
import config from "@/config/index.js";
const index = {
settings: {
analysis: {
analyzer: {
ngram: {
tokenizer: 'ngram',
tokenizer: "ngram",
},
},
},
@ -14,16 +14,16 @@ const index = {
mappings: {
properties: {
text: {
type: 'text',
type: "text",
index: true,
analyzer: 'ngram',
analyzer: "ngram",
},
userId: {
type: 'keyword',
type: "keyword",
index: true,
},
userHost: {
type: 'keyword',
type: "keyword",
index: true,
},
},
@ -31,26 +31,35 @@ const index = {
};
// Init ElasticSearch connection
const client = config.elasticsearch ? new elasticsearch.Client({
node: `${config.elasticsearch.ssl ? 'https://' : 'http://'}${config.elasticsearch.host}:${config.elasticsearch.port}`,
auth: (config.elasticsearch.user && config.elasticsearch.pass) ? {
username: config.elasticsearch.user,
password: config.elasticsearch.pass,
} : undefined,
pingTimeout: 30000,
}) : null;
const client = config.elasticsearch
? new elasticsearch.Client({
node: `${config.elasticsearch.ssl ? "https://" : "http://"}${
config.elasticsearch.host
}:${config.elasticsearch.port}`,
auth:
config.elasticsearch.user && config.elasticsearch.pass
? {
username: config.elasticsearch.user,
password: config.elasticsearch.pass,
}
: undefined,
pingTimeout: 30000,
})
: null;
if (client) {
client.indices.exists({
index: config.elasticsearch.index || 'misskey_note',
}).then(exist => {
if (!exist.body) {
client.indices.create({
index: config.elasticsearch.index || 'misskey_note',
body: index,
});
}
});
client.indices
.exists({
index: config.elasticsearch.index || "misskey_note",
})
.then((exist) => {
if (!exist.body) {
client.indices.create({
index: config.elasticsearch.index || "misskey_note",
body: index,
});
}
});
}
export default client;

4
packages/backend/src/db/logger.ts
View file

@ -1,3 +1,3 @@
import Logger from '@/services/logger.js';
import Logger from "@/services/logger.js";
export const dbLogger = new Logger('db');
export const dbLogger = new Logger("db");

186
packages/backend/src/db/postgre.ts
View file

@ -1,87 +1,89 @@
// https://github.com/typeorm/typeorm/issues/2400
import pg from 'pg';
import pg from "pg";
pg.types.setTypeParser(20, Number);
import { Logger, DataSource } from 'typeorm';
import * as highlight from 'cli-highlight';
import config from '@/config/index.js';
import type { Logger } from "typeorm";
import { DataSource } from "typeorm";
import * as highlight from "cli-highlight";
import config from "@/config/index.js";
import { User } from '@/models/entities/user.js';
import { DriveFile } from '@/models/entities/drive-file.js';
import { DriveFolder } from '@/models/entities/drive-folder.js';
import { AccessToken } from '@/models/entities/access-token.js';
import { App } from '@/models/entities/app.js';
import { PollVote } from '@/models/entities/poll-vote.js';
import { Note } from '@/models/entities/note.js';
import { NoteReaction } from '@/models/entities/note-reaction.js';
import { NoteWatching } from '@/models/entities/note-watching.js';
import { NoteThreadMuting } from '@/models/entities/note-thread-muting.js';
import { NoteUnread } from '@/models/entities/note-unread.js';
import { Notification } from '@/models/entities/notification.js';
import { Meta } from '@/models/entities/meta.js';
import { Following } from '@/models/entities/following.js';
import { Instance } from '@/models/entities/instance.js';
import { Muting } from '@/models/entities/muting.js';
import { SwSubscription } from '@/models/entities/sw-subscription.js';
import { Blocking } from '@/models/entities/blocking.js';
import { UserList } from '@/models/entities/user-list.js';
import { UserListJoining } from '@/models/entities/user-list-joining.js';
import { UserGroup } from '@/models/entities/user-group.js';
import { UserGroupJoining } from '@/models/entities/user-group-joining.js';
import { UserGroupInvitation } from '@/models/entities/user-group-invitation.js';
import { Hashtag } from '@/models/entities/hashtag.js';
import { NoteFavorite } from '@/models/entities/note-favorite.js';
import { AbuseUserReport } from '@/models/entities/abuse-user-report.js';
import { RegistrationTicket } from '@/models/entities/registration-tickets.js';
import { MessagingMessage } from '@/models/entities/messaging-message.js';
import { Signin } from '@/models/entities/signin.js';
import { AuthSession } from '@/models/entities/auth-session.js';
import { FollowRequest } from '@/models/entities/follow-request.js';
import { Emoji } from '@/models/entities/emoji.js';
import { UserNotePining } from '@/models/entities/user-note-pining.js';
import { Poll } from '@/models/entities/poll.js';
import { UserKeypair } from '@/models/entities/user-keypair.js';
import { UserPublickey } from '@/models/entities/user-publickey.js';
import { UserProfile } from '@/models/entities/user-profile.js';
import { UserSecurityKey } from '@/models/entities/user-security-key.js';
import { AttestationChallenge } from '@/models/entities/attestation-challenge.js';
import { Page } from '@/models/entities/page.js';
import { PageLike } from '@/models/entities/page-like.js';
import { GalleryPost } from '@/models/entities/gallery-post.js';
import { GalleryLike } from '@/models/entities/gallery-like.js';
import { ModerationLog } from '@/models/entities/moderation-log.js';
import { UsedUsername } from '@/models/entities/used-username.js';
import { Announcement } from '@/models/entities/announcement.js';
import { AnnouncementRead } from '@/models/entities/announcement-read.js';
import { Clip } from '@/models/entities/clip.js';
import { ClipNote } from '@/models/entities/clip-note.js';
import { Antenna } from '@/models/entities/antenna.js';
import { AntennaNote } from '@/models/entities/antenna-note.js';
import { PromoNote } from '@/models/entities/promo-note.js';
import { PromoRead } from '@/models/entities/promo-read.js';
import { Relay } from '@/models/entities/relay.js';
import { MutedNote } from '@/models/entities/muted-note.js';
import { Channel } from '@/models/entities/channel.js';
import { ChannelFollowing } from '@/models/entities/channel-following.js';
import { ChannelNotePining } from '@/models/entities/channel-note-pining.js';
import { RegistryItem } from '@/models/entities/registry-item.js';
import { Ad } from '@/models/entities/ad.js';
import { PasswordResetRequest } from '@/models/entities/password-reset-request.js';
import { UserPending } from '@/models/entities/user-pending.js';
import { Webhook } from '@/models/entities/webhook.js';
import { UserIp } from '@/models/entities/user-ip.js';
import { User } from "@/models/entities/user.js";
import { DriveFile } from "@/models/entities/drive-file.js";
import { DriveFolder } from "@/models/entities/drive-folder.js";
import { AccessToken } from "@/models/entities/access-token.js";
import { App } from "@/models/entities/app.js";
import { PollVote } from "@/models/entities/poll-vote.js";
import { Note } from "@/models/entities/note.js";
import { NoteReaction } from "@/models/entities/note-reaction.js";
import { NoteWatching } from "@/models/entities/note-watching.js";
import { NoteThreadMuting } from "@/models/entities/note-thread-muting.js";
import { NoteUnread } from "@/models/entities/note-unread.js";
import { Notification } from "@/models/entities/notification.js";
import { Meta } from "@/models/entities/meta.js";
import { Following } from "@/models/entities/following.js";
import { Instance } from "@/models/entities/instance.js";
import { Muting } from "@/models/entities/muting.js";
import { SwSubscription } from "@/models/entities/sw-subscription.js";
import { Blocking } from "@/models/entities/blocking.js";
import { UserList } from "@/models/entities/user-list.js";
import { UserListJoining } from "@/models/entities/user-list-joining.js";
import { UserGroup } from "@/models/entities/user-group.js";
import { UserGroupJoining } from "@/models/entities/user-group-joining.js";
import { UserGroupInvitation } from "@/models/entities/user-group-invitation.js";
import { Hashtag } from "@/models/entities/hashtag.js";
import { NoteFavorite } from "@/models/entities/note-favorite.js";
import { AbuseUserReport } from "@/models/entities/abuse-user-report.js";
import { RegistrationTicket } from "@/models/entities/registration-tickets.js";
import { MessagingMessage } from "@/models/entities/messaging-message.js";
import { Signin } from "@/models/entities/signin.js";
import { AuthSession } from "@/models/entities/auth-session.js";
import { FollowRequest } from "@/models/entities/follow-request.js";
import { Emoji } from "@/models/entities/emoji.js";
import { UserNotePining } from "@/models/entities/user-note-pining.js";
import { Poll } from "@/models/entities/poll.js";
import { UserKeypair } from "@/models/entities/user-keypair.js";
import { UserPublickey } from "@/models/entities/user-publickey.js";
import { UserProfile } from "@/models/entities/user-profile.js";
import { UserSecurityKey } from "@/models/entities/user-security-key.js";
import { AttestationChallenge } from "@/models/entities/attestation-challenge.js";
import { Page } from "@/models/entities/page.js";
import { PageLike } from "@/models/entities/page-like.js";
import { GalleryPost } from "@/models/entities/gallery-post.js";
import { GalleryLike } from "@/models/entities/gallery-like.js";
import { ModerationLog } from "@/models/entities/moderation-log.js";
import { UsedUsername } from "@/models/entities/used-username.js";
import { Announcement } from "@/models/entities/announcement.js";
import { AnnouncementRead } from "@/models/entities/announcement-read.js";
import { Clip } from "@/models/entities/clip.js";
import { ClipNote } from "@/models/entities/clip-note.js";
import { Antenna } from "@/models/entities/antenna.js";
import { AntennaNote } from "@/models/entities/antenna-note.js";
import { PromoNote } from "@/models/entities/promo-note.js";
import { PromoRead } from "@/models/entities/promo-read.js";
import { Relay } from "@/models/entities/relay.js";
import { MutedNote } from "@/models/entities/muted-note.js";
import { Channel } from "@/models/entities/channel.js";
import { ChannelFollowing } from "@/models/entities/channel-following.js";
import { ChannelNotePining } from "@/models/entities/channel-note-pining.js";
import { RegistryItem } from "@/models/entities/registry-item.js";
import { Ad } from "@/models/entities/ad.js";
import { PasswordResetRequest } from "@/models/entities/password-reset-request.js";
import { UserPending } from "@/models/entities/user-pending.js";
import { Webhook } from "@/models/entities/webhook.js";
import { UserIp } from "@/models/entities/user-ip.js";
import { entities as charts } from '@/services/chart/entities.js';
import { envOption } from '../env.js';
import { dbLogger } from './logger.js';
import { redisClient } from './redis.js';
import { entities as charts } from "@/services/chart/entities.js";
import { envOption } from "../env.js";
import { dbLogger } from "./logger.js";
import { redisClient } from "./redis.js";
const sqlLogger = dbLogger.createSubLogger('sql', 'gray', false);
const sqlLogger = dbLogger.createSubLogger("sql", "gray", false);
class MyCustomLogger implements Logger {
private highlight(sql: string) {
return highlight.highlight(sql, {
language: 'sql', ignoreIllegals: true,
language: "sql",
ignoreIllegals: true,
});
}
@ -178,10 +180,10 @@ export const entities = [
...charts,
];
const log = process.env.NODE_ENV !== 'production';
const log = process.env.NODE_ENV !== "production";
export const db = new DataSource({
type: 'postgres',
type: "postgres",
host: config.db.host,
port: config.db.port,
username: config.db.user,
@ -191,24 +193,26 @@ export const db = new DataSource({
statement_timeout: 1000 * 10,
...config.db.extra,
},
synchronize: process.env.NODE_ENV === 'test',
dropSchema: process.env.NODE_ENV === 'test',
cache: !config.db.disableCache ? {
type: 'ioredis',
options: {
host: config.redis.host,
port: config.redis.port,
family: config.redis.family == null ? 0 : config.redis.family,
password: config.redis.pass,
keyPrefix: `${config.redis.prefix}:query:`,
db: config.redis.db || 0,
},
} : false,
synchronize: process.env.NODE_ENV === "test",
dropSchema: process.env.NODE_ENV === "test",
cache: !config.db.disableCache
? {
type: "ioredis",
options: {
host: config.redis.host,
port: config.redis.port,
family: config.redis.family == null ? 0 : config.redis.family,
password: config.redis.pass,
keyPrefix: `${config.redis.prefix}:query:`,
db: config.redis.db || 0,
},
}
: false,
logging: log,
logger: log ? new MyCustomLogger() : undefined,
maxQueryExecutionTime: 300,
entities: entities,
migrations: ['../../migration/*.js'],
migrations: ["../../migration/*.js"],
});
export async function initDb(force = false) {
@ -247,7 +251,7 @@ export async function resetDb() {
if (i === 3) {
throw e;
} else {
await new Promise(resolve => setTimeout(resolve, 1000));
await new Promise((resolve) => setTimeout(resolve, 1000));
continue;
}
}

4
packages/backend/src/db/redis.ts
View file

@ -1,5 +1,5 @@
import Redis from 'ioredis';
import config from '@/config/index.js';
import Redis from "ioredis";
import config from "@/config/index.js";
export function createConnection() {
return new Redis({

13
packages/backend/src/env.ts
View file

@ -10,11 +10,16 @@ const envOption = {
};
for (const key of Object.keys(envOption) as (keyof typeof envOption)[]) {
if (process.env['MK_' + key.replace(/[A-Z]/g, letter => `_${letter}`).toUpperCase()]) envOption[key] = true;
if (
process.env[
`MK_${key.replace(/[A-Z]/g, (letter) => `_${letter}`).toUpperCase()}`
]
)
envOption[key] = true;
}
if (process.env.NODE_ENV === 'test') envOption.disableClustering = true;
if (process.env.NODE_ENV === 'test') envOption.quiet = true;
if (process.env.NODE_ENV === 'test') envOption.noDaemons = true;
if (process.env.NODE_ENV === "test") envOption.disableClustering = true;
if (process.env.NODE_ENV === "test") envOption.quiet = true;
if (process.env.NODE_ENV === "test") envOption.noDaemons = true;
export { envOption };

6
packages/backend/src/index.ts
View file

@ -2,12 +2,12 @@
* Misskey Entry Point!
*/
import { EventEmitter } from 'node:events';
import boot from './boot/index.js';
import { EventEmitter } from "node:events";
import boot from "./boot/index.js";
Error.stackTraceLimit = Infinity;
EventEmitter.defaultMaxListeners = 128;
boot().catch(err => {
boot().catch((err) => {
console.error(err);
});

154
packages/backend/src/mfm/from-html.ts
View file

@ -1,6 +1,6 @@
import { URL } from 'node:url';
import * as parse5 from 'parse5';
import * as TreeAdapter from '../../node_modules/parse5/dist/tree-adapters/default.js';
import { URL } from "node:url";
import * as parse5 from "parse5";
import * as TreeAdapter from "../../node_modules/parse5/dist/tree-adapters/default.js";
const treeAdapter = TreeAdapter.defaultTreeAdapter;
@ -9,11 +9,11 @@ const urlRegexFull = /^https?:\/\/[\w\/:%#@$&?!()\[\]~.,=+\-]+$/;
export function fromHtml(html: string, hashtagNames?: string[]): string {
// some AP servers like Pixelfed use br tags as well as newlines
html = html.replace(/<br\s?\/?>\r?\n/gi, '\n');
html = html.replace(/<br\s?\/?>\r?\n/gi, "\n");
const dom = parse5.parseFragment(html);
let text = '';
let text = "";
for (const n of dom.childNodes) {
analyze(n);
@ -23,14 +23,14 @@ export function fromHtml(html: string, hashtagNames?: string[]): string {
function getText(node: TreeAdapter.Node): string {
if (treeAdapter.isTextNode(node)) return node.value;
if (!treeAdapter.isElementNode(node)) return '';
if (node.nodeName === 'br') return '\n';
if (!treeAdapter.isElementNode(node)) return "";
if (node.nodeName === "br") return "\n";
if (node.childNodes) {
return node.childNodes.map(n => getText(n)).join('');
return node.childNodes.map((n) => getText(n)).join("");
}
return '';
return "";
}
function appendChildren(childNodes: TreeAdapter.ChildNode[]): void {
@ -51,42 +51,46 @@ export function fromHtml(html: string, hashtagNames?: string[]): string {
if (!treeAdapter.isElementNode(node)) return;
switch (node.nodeName) {
case 'br': {
text += '\n';
case "br": {
text += "\n";
break;
}
case 'a':
{
case "a": {
const txt = getText(node);
const rel = node.attrs.find(x => x.name === 'rel');
const href = node.attrs.find(x => x.name === 'href');
const rel = node.attrs.find((x) => x.name === "rel");
const href = node.attrs.find((x) => x.name === "href");
// ハッシュタグ
if (hashtagNames && href && hashtagNames.map(x => x.toLowerCase()).includes(txt.toLowerCase())) {
if (
hashtagNames &&
href &&
hashtagNames.map((x) => x.toLowerCase()).includes(txt.toLowerCase())
) {
text += txt;
// メンション
} else if (txt.startsWith('@') && !(rel && rel.value.match(/^me /))) {
const part = txt.split('@');
// メンション
} else if (txt.startsWith("@") && !(rel?.value.match(/^me /))) {
const part = txt.split("@");
if (part.length === 2 && href) {
//#region ホスト名部分が省略されているので復元する
const acct = `${txt}@${(new URL(href.value)).hostname}`;
const acct = `${txt}@${new URL(href.value).hostname}`;
text += acct;
//#endregion
} else if (part.length === 3) {
text += txt;
}
// その他
// その他
} else {
const generateLink = () => {
if (!href && !txt) {
return '';
if (!(href || txt)) {
return "";
}
if (!href) {
return txt;
}
if (!txt || txt === href.value) { // #6383: Missing text node
if (!txt || txt === href.value) {
// #6383: Missing text node
if (href.value.match(urlRegexFull)) {
return href.value;
} else {
@ -94,7 +98,7 @@ export function fromHtml(html: string, hashtagNames?: string[]): string {
}
}
if (href.value.match(urlRegex) && !href.value.match(urlRegexFull)) {
return `[${txt}](<${href.value}>)`; // #6846
return `[${txt}](<${href.value}>)`; // #6846
} else {
return `[${txt}](${href.value})`;
}
@ -105,55 +109,53 @@ export function fromHtml(html: string, hashtagNames?: string[]): string {
break;
}
case 'h1':
{
text += '【';
case "h1": {
text += "【";
appendChildren(node.childNodes);
text += '】\n';
text += "】\n";
break;
}
case 'b':
case 'strong':
{
text += '**';
case "b":
case "strong": {
text += "**";
appendChildren(node.childNodes);
text += '**';
text += "**";
break;
}
case 'small':
{
text += '<small>';
case "small": {
text += "<small>";
appendChildren(node.childNodes);
text += '</small>';
text += "</small>";
break;
}
case 's':
case 'del':
{
text += '~~';
case "s":
case "del": {
text += "~~";
appendChildren(node.childNodes);
text += '~~';
text += "~~";
break;
}
case 'i':
case 'em':
{
text += '<i>';
case "i":
case "em": {
text += "<i>";
appendChildren(node.childNodes);
text += '</i>';
text += "</i>";
break;
}
// block code (<pre><code>)
case 'pre': {
if (node.childNodes.length === 1 && node.childNodes[0].nodeName === 'code') {
text += '\n```\n';
case "pre": {
if (
node.childNodes.length === 1 &&
node.childNodes[0].nodeName === "code"
) {
text += "\n```\n";
text += getText(node.childNodes[0]);
text += '\n```\n';
text += "\n```\n";
} else {
appendChildren(node.childNodes);
}
@ -161,50 +163,48 @@ export function fromHtml(html: string, hashtagNames?: string[]): string {
}
// inline code (<code>)
case 'code': {
text += '`';
case "code": {
text += "`";
appendChildren(node.childNodes);
text += '`';
text += "`";
break;
}
case 'blockquote': {
case "blockquote": {
const t = getText(node);
if (t) {
text += '\n> ';
text += t.split('\n').join('\n> ');
text += "\n> ";
text += t.split("\n").join("\n> ");
}
break;
}
case 'p':
case 'h2':
case 'h3':
case 'h4':
case 'h5':
case 'h6':
{
text += '\n\n';
case "p":
case "h2":
case "h3":
case "h4":
case "h5":
case "h6": {
text += "\n\n";
appendChildren(node.childNodes);
break;
}
// other block elements
case 'div':
case 'header':
case 'footer':
case 'article':
case 'li':
case 'dt':
case 'dd':
{
text += '\n';
case "div":
case "header":
case "footer":
case "article":
case "li":
case "dt":
case "dd": {
text += "\n";
appendChildren(node.childNodes);
break;
}
default: // includes inline elements
{
default: {
// includes inline elements
appendChildren(node.childNodes);
break;
}

85
packages/backend/src/mfm/to-html.ts
View file

@ -1,65 +1,71 @@
import { JSDOM } from 'jsdom';
import * as mfm from 'mfm-js';
import config from '@/config/index.js';
import { intersperse } from '@/prelude/array.js';
import { IMentionedRemoteUsers } from '@/models/entities/note.js';
import { JSDOM } from "jsdom";
import type * as mfm from "mfm-js";
import config from "@/config/index.js";
import { intersperse } from "@/prelude/array.js";
import type { IMentionedRemoteUsers } from "@/models/entities/note.js";
export function toHtml(nodes: mfm.MfmNode[] | null, mentionedRemoteUsers: IMentionedRemoteUsers = []) {
export function toHtml(
nodes: mfm.MfmNode[] | null,
mentionedRemoteUsers: IMentionedRemoteUsers = [],
) {
if (nodes == null) {
return null;
}
const { window } = new JSDOM('');
const { window } = new JSDOM("");
const doc = window.document;
function appendChildren(children: mfm.MfmNode[], targetElement: any): void {
if (children) {
for (const child of children.map(x => (handlers as any)[x.type](x))) targetElement.appendChild(child);
for (const child of children.map((x) => (handlers as any)[x.type](x)))
targetElement.appendChild(child);
}
}
const handlers: { [K in mfm.MfmNode['type']]: (node: mfm.NodeType<K>) => any } = {
const handlers: {
[K in mfm.MfmNode["type"]]: (node: mfm.NodeType<K>) => any;
} = {
bold(node) {
const el = doc.createElement('b');
const el = doc.createElement("b");
appendChildren(node.children, el);
return el;
},
small(node) {
const el = doc.createElement('small');
const el = doc.createElement("small");
appendChildren(node.children, el);
return el;
},
strike(node) {
const el = doc.createElement('del');
const el = doc.createElement("del");
appendChildren(node.children, el);
return el;
},
italic(node) {
const el = doc.createElement('i');
const el = doc.createElement("i");
appendChildren(node.children, el);
return el;
},
fn(node) {
const el = doc.createElement('i');
const el = doc.createElement("i");
appendChildren(node.children, el);
return el;
},
blockCode(node) {
const pre = doc.createElement('pre');
const inner = doc.createElement('code');
const pre = doc.createElement("pre");
const inner = doc.createElement("code");
inner.textContent = node.props.code;
pre.appendChild(inner);
return pre;
},
center(node) {
const el = doc.createElement('div');
const el = doc.createElement("div");
appendChildren(node.children, el);
return el;
},
@ -73,81 +79,90 @@ export function toHtml(nodes: mfm.MfmNode[] | null, mentionedRemoteUsers: IMenti
},
hashtag(node) {
const a = doc.createElement('a');
const a = doc.createElement("a");
a.href = `${config.url}/tags/${node.props.hashtag}`;
a.textContent = `#${node.props.hashtag}`;
a.setAttribute('rel', 'tag');
a.setAttribute("rel", "tag");
return a;
},
inlineCode(node) {
const el = doc.createElement('code');
const el = doc.createElement("code");
el.textContent = node.props.code;
return el;
},
mathInline(node) {
const el = doc.createElement('code');
const el = doc.createElement("code");
el.textContent = node.props.formula;
return el;
},
mathBlock(node) {
const el = doc.createElement('code');
const el = doc.createElement("code");
el.textContent = node.props.formula;
return el;
},
link(node) {
const a = doc.createElement('a');
const a = doc.createElement("a");
a.href = node.props.url;
appendChildren(node.children, a);
return a;
},
mention(node) {
const a = doc.createElement('a');
const a = doc.createElement("a");
const { username, host, acct } = node.props;
const remoteUserInfo = mentionedRemoteUsers.find(remoteUser => remoteUser.username === username && remoteUser.host === host);
a.href = remoteUserInfo ? (remoteUserInfo.url ? remoteUserInfo.url : remoteUserInfo.uri) : `${config.url}/${acct}`;
a.className = 'u-url mention';
const remoteUserInfo = mentionedRemoteUsers.find(
(remoteUser) =>
remoteUser.username === username && remoteUser.host === host,
);
a.href = remoteUserInfo
? remoteUserInfo.url
? remoteUserInfo.url
: remoteUserInfo.uri
: `${config.url}/${acct}`;
a.className = "u-url mention";
a.textContent = acct;
return a;
},
quote(node) {
const el = doc.createElement('blockquote');
const el = doc.createElement("blockquote");
appendChildren(node.children, el);
return el;
},
text(node) {
const el = doc.createElement('span');
const nodes = node.props.text.split(/\r\n|\r|\n/).map(x => doc.createTextNode(x));
const el = doc.createElement("span");
const nodes = node.props.text
.split(/\r\n|\r|\n/)
.map((x) => doc.createTextNode(x));
for (const x of intersperse<FIXME | 'br'>('br', nodes)) {
el.appendChild(x === 'br' ? doc.createElement('br') : x);
for (const x of intersperse<FIXME | "br">("br", nodes)) {
el.appendChild(x === "br" ? doc.createElement("br") : x);
}
return el;
},
url(node) {
const a = doc.createElement('a');
const a = doc.createElement("a");
a.href = node.props.url;
a.textContent = node.props.url;
return a;
},
search(node) {
const a = doc.createElement('a');
const a = doc.createElement("a");
a.href = `https://search.annoyingorange.xyz/search?q=${node.props.query}`;
a.textContent = node.props.content;
return a;
},
plain(node) {
const el = doc.createElement('span');
const el = doc.createElement("span");
appendChildren(node.children, el);
return el;
},

4
packages/backend/src/misc/acct.ts
View file

@ -4,8 +4,8 @@ export type Acct = {
};
export function parse(acct: string): Acct {
if (acct.startsWith('@')) acct = acct.substr(1);
const split = acct.split('@', 2);
if (acct.startsWith("@")) acct = acct.substr(1);
const split = acct.split("@", 2);
return { username: split[0], host: split[1] || null };
}

20
packages/backend/src/misc/antenna-cache.ts
View file

@ -1,6 +1,6 @@
import { Antennas } from '@/models/index.js';
import { Antenna } from '@/models/entities/antenna.js';
import { subscriber } from '@/db/redis.js';
import { Antennas } from "@/models/index.js";
import type { Antenna } from "@/models/entities/antenna.js";
import { subscriber } from "@/db/redis.js";
let antennasFetched = false;
let antennas: Antenna[] = [];
@ -14,20 +14,20 @@ export async function getAntennas() {
return antennas;
}
subscriber.on('message', async (_, data) => {
subscriber.on("message", async (_, data) => {
const obj = JSON.parse(data);
if (obj.channel === 'internal') {
if (obj.channel === "internal") {
const { type, body } = obj.message;
switch (type) {
case 'antennaCreated':
case "antennaCreated":
antennas.push(body);
break;
case 'antennaUpdated':
antennas[antennas.findIndex(a => a.id === body.id)] = body;
case "antennaUpdated":
antennas[antennas.findIndex((a) => a.id === body.id)] = body;
break;
case 'antennaDeleted':
antennas = antennas.filter(a => a.id !== body.id);
case "antennaDeleted":
antennas = antennas.filter((a) => a.id !== body.id);
break;
default:
break;

64
packages/backend/src/misc/api-permissions.ts
View file

@ -1,35 +1,35 @@
export const kinds = [
'read:account',
'write:account',
'read:blocks',
'write:blocks',
'read:drive',
'write:drive',
'read:favorites',
'write:favorites',
'read:following',
'write:following',
'read:messaging',
'write:messaging',
'read:mutes',
'write:mutes',
'write:notes',
'read:notifications',
'write:notifications',
'read:reactions',
'write:reactions',
'write:votes',
'read:pages',
'write:pages',
'write:page-likes',
'read:page-likes',
'read:user-groups',
'write:user-groups',
'read:channels',
'write:channels',
'read:gallery',
'write:gallery',
'read:gallery-likes',
'write:gallery-likes',
"read:account",
"write:account",
"read:blocks",
"write:blocks",
"read:drive",
"write:drive",
"read:favorites",
"write:favorites",
"read:following",
"write:following",
"read:messaging",
"write:messaging",
"read:mutes",
"write:mutes",
"write:notes",
"read:notifications",
"write:notifications",
"read:reactions",
"write:reactions",
"write:votes",
"read:pages",
"write:pages",
"write:page-likes",
"read:page-likes",
"read:user-groups",
"write:user-groups",
"read:channels",
"write:channels",
"read:gallery",
"write:gallery",
"read:gallery-likes",
"write:gallery-likes",
];
// IF YOU ADD KINDS(PERMISSIONS), YOU MUST ADD TRANSLATIONS (under _permissions).

16
packages/backend/src/misc/app-lock.ts
View file

@ -1,16 +1,15 @@
import { redisClient } from '../db/redis.js';
import { promisify } from 'node:util';
import redisLock from 'redis-lock';
import { redisClient } from "../db/redis.js";
import { promisify } from "node:util";
import redisLock from "redis-lock";
/**
* Retry delay (ms) for lock acquisition
*/
const retryDelay = 100;
const lock: (key: string, timeout?: number) => Promise<() => void>
= redisClient
const lock: (key: string, timeout?: number) => Promise<() => void> = redisClient
? promisify(redisLock(redisClient, retryDelay))
: async () => () => { };
: async () => () => {};
/**
* Get AP Object lock
@ -22,7 +21,10 @@ export function getApLock(uri: string, timeout = 30 * 1000) {
return lock(`ap-object:${uri}`, timeout);
}
export function getFetchInstanceMetadataLock(host: string, timeout = 30 * 1000) {
export function getFetchInstanceMetadataLock(
host: string,
timeout = 30 * 1000,
) {
return lock(`instance:${host}`, timeout);
}

21
packages/backend/src/misc/before-shutdown.ts
View file

@ -1,6 +1,6 @@
// https://gist.github.com/nfantone/1eaa803772025df69d07f4dbf5df7e58
'use strict';
"use strict";
/**
* @callback BeforeShutdownListener
@ -11,7 +11,7 @@
* System signals the app will listen to initiate shutdown.
* @const {string[]}
*/
const SHUTDOWN_SIGNALS = ['SIGINT', 'SIGTERM'];
const SHUTDOWN_SIGNALS = ["SIGINT", "SIGTERM"];
/**
* Time in milliseconds to wait before forcing shutdown.
@ -31,7 +31,10 @@ const shutdownListeners: ((signalOrEvent: string) => void)[] = [];
* @param {string[]} signals System signals to listen to.
* @param {function(string)} fn Function to execute on shutdown.
*/
const processOnce = (signals: string[], fn: (signalOrEvent: string) => void) => {
const processOnce = (
signals: string[],
fn: (signalOrEvent: string) => void,
) => {
for (const sig of signals) {
process.once(sig, fn);
}
@ -44,7 +47,9 @@ const processOnce = (signals: string[], fn: (signalOrEvent: string) => void) =>
const forceExitAfter = (timeout: number) => () => {
setTimeout(() => {
// Force shutdown after timeout
console.warn(`Could not close resources gracefully after ${timeout}ms: forcing shutdown`);
console.warn(
`Could not close resources gracefully after ${timeout}ms: forcing shutdown`,
);
return process.exit(1);
}, timeout).unref();
};
@ -56,7 +61,7 @@ const forceExitAfter = (timeout: number) => () => {
* @param {string} signalOrEvent The exit signal or event name received on the process.
*/
async function shutdownHandler(signalOrEvent: string) {
if (process.env.NODE_ENV === 'test') return process.exit(0);
if (process.env.NODE_ENV === "test") return process.exit(0);
console.warn(`Shutting down: received [${signalOrEvent}] signal`);
@ -65,7 +70,11 @@ async function shutdownHandler(signalOrEvent: string) {
await listener(signalOrEvent);
} catch (err) {
if (err instanceof Error) {
console.warn(`A shutdown handler failed before completing with: ${err.message || err}`);
console.warn(
`A shutdown handler failed before completing with: ${
err.message || err
}`,
);
}
}
}

18
packages/backend/src/misc/cache.ts
View file

@ -1,8 +1,8 @@
export class Cache<T> {
public cache: Map<string | null, { date: number; value: T; }>;
public cache: Map<string | null, { date: number; value: T }>;
private lifetime: number;
constructor(lifetime: Cache<never>['lifetime']) {
constructor(lifetime: Cache<never>["lifetime"]) {
this.cache = new Map();
this.lifetime = lifetime;
}
@ -17,7 +17,7 @@ export class Cache<T> {
public get(key: string | null): T | undefined {
const cached = this.cache.get(key);
if (cached == null) return undefined;
if ((Date.now() - cached.date) > this.lifetime) {
if (Date.now() - cached.date > this.lifetime) {
this.cache.delete(key);
return undefined;
}
@ -32,7 +32,11 @@ export class Cache<T> {
* キャッシュがあればそれを返し、無ければfetcherを呼び出して結果をキャッシュ&返します
* optional: キャッシュが存在してもvalidatorでfalseを返すとキャッシュ無効扱いにします
*/
public async fetch(key: string | null, fetcher: () => Promise<T>, validator?: (cachedValue: T) => boolean): Promise<T> {
public async fetch(
key: string | null,
fetcher: () => Promise<T>,
validator?: (cachedValue: T) => boolean,
): Promise<T> {
const cachedValue = this.get(key);
if (cachedValue !== undefined) {
if (validator) {
@ -56,7 +60,11 @@ export class Cache<T> {
* キャッシュがあればそれを返し、無ければfetcherを呼び出して結果をキャッシュ&返します
* optional: キャッシュが存在してもvalidatorでfalseを返すとキャッシュ無効扱いにします
*/
public async fetchMaybe(key: string | null, fetcher: () => Promise<T | undefined>, validator?: (cachedValue: T) => boolean): Promise<T | undefined> {
public async fetchMaybe(
key: string | null,
fetcher: () => Promise<T | undefined>,
validator?: (cachedValue: T) => boolean,
): Promise<T | undefined> {
const cachedValue = this.get(key);
if (cachedValue !== undefined) {
if (validator) {

44
packages/backend/src/misc/captcha.ts
View file

@ -1,51 +1,67 @@
import fetch from 'node-fetch';
import { URLSearchParams } from 'node:url';
import { getAgentByUrl } from './fetch.js';
import config from '@/config/index.js';
import fetch from "node-fetch";
import { URLSearchParams } from "node:url";
import { getAgentByUrl } from "./fetch.js";
import config from "@/config/index.js";
export async function verifyRecaptcha(secret: string, response: string) {
const result = await getCaptchaResponse('https://www.recaptcha.net/recaptcha/api/siteverify', secret, response).catch(e => {
const result = await getCaptchaResponse(
"https://www.recaptcha.net/recaptcha/api/siteverify",
secret,
response,
).catch((e) => {
throw new Error(`recaptcha-request-failed: ${e.message}`);
});
if (result.success !== true) {
const errorCodes = result['error-codes'] ? result['error-codes']?.join(', ') : '';
const errorCodes = result["error-codes"]
? result["error-codes"]?.join(", ")
: "";
throw new Error(`recaptcha-failed: ${errorCodes}`);
}
}
export async function verifyHcaptcha(secret: string, response: string) {
const result = await getCaptchaResponse('https://hcaptcha.com/siteverify', secret, response).catch(e => {
const result = await getCaptchaResponse(
"https://hcaptcha.com/siteverify",
secret,
response,
).catch((e) => {
throw new Error(`hcaptcha-request-failed: ${e.message}`);
});
if (result.success !== true) {
const errorCodes = result['error-codes'] ? result['error-codes']?.join(', ') : '';
const errorCodes = result["error-codes"]
? result["error-codes"]?.join(", ")
: "";
throw new Error(`hcaptcha-failed: ${errorCodes}`);
}
}
type CaptchaResponse = {
success: boolean;
'error-codes'?: string[];
"error-codes"?: string[];
};
async function getCaptchaResponse(url: string, secret: string, response: string): Promise<CaptchaResponse> {
async function getCaptchaResponse(
url: string,
secret: string,
response: string,
): Promise<CaptchaResponse> {
const params = new URLSearchParams({
secret,
response,
});
const res = await fetch(url, {
method: 'POST',
method: "POST",
body: params,
headers: {
'User-Agent': config.userAgent,
"User-Agent": config.userAgent,
},
// TODO
//timeout: 10 * 1000,
agent: getAgentByUrl,
}).catch(e => {
}).catch((e) => {
throw new Error(`${e.message || e}`);
});
@ -53,5 +69,5 @@ async function getCaptchaResponse(url: string, secret: string, response: string)
throw new Error(`${res.status}`);
}
return await res.json() as CaptchaResponse;
return (await res.json()) as CaptchaResponse;
}

117
packages/backend/src/misc/check-hit-antenna.ts
View file

@ -1,90 +1,121 @@
import { Antenna } from '@/models/entities/antenna.js';
import { Note } from '@/models/entities/note.js';
import { User } from '@/models/entities/user.js';
import { UserListJoinings, UserGroupJoinings, Blockings } from '@/models/index.js';
import { getFullApAccount } from './convert-host.js';
import * as Acct from '@/misc/acct.js';
import { Packed } from './schema.js';
import { Cache } from './cache.js';
import type { Antenna } from "@/models/entities/antenna.js";
import type { Note } from "@/models/entities/note.js";
import type { User } from "@/models/entities/user.js";
import {
UserListJoinings,
UserGroupJoinings,
Blockings,
} from "@/models/index.js";
import { getFullApAccount } from "./convert-host.js";
import * as Acct from "@/misc/acct.js";
import type { Packed } from "./schema.js";
import { Cache } from "./cache.js";
const blockingCache = new Cache<User['id'][]>(1000 * 60 * 5);
const blockingCache = new Cache<User["id"][]>(1000 * 60 * 5);
// NOTE: フォローしているユーザーのノート、リストのユーザーのノート、グループのユーザーのノート指定はパフォーマンス上の理由で無効になっている
/**
* noteUserFollowers / antennaUserFollowing はどちらか一方が指定されていればよい
*/
export async function checkHitAntenna(antenna: Antenna, note: (Note | Packed<'Note'>), noteUser: { id: User['id']; username: string; host: string | null; }, noteUserFollowers?: User['id'][], antennaUserFollowing?: User['id'][]): Promise<boolean> {
if (note.visibility === 'specified') return false;
export async function checkHitAntenna(
antenna: Antenna,
note: Note | Packed<"Note">,
noteUser: { id: User["id"]; username: string; host: string | null },
noteUserFollowers?: User["id"][],
antennaUserFollowing?: User["id"][],
): Promise<boolean> {
if (note.visibility === "specified") return false;
// アンテナ作成者がノート作成者にブロックされていたらスキップ
const blockings = await blockingCache.fetch(noteUser.id, () => Blockings.findBy({ blockerId: noteUser.id }).then(res => res.map(x => x.blockeeId)));
if (blockings.some(blocking => blocking === antenna.userId)) return false;
const blockings = await blockingCache.fetch(noteUser.id, () =>
Blockings.findBy({ blockerId: noteUser.id }).then((res) =>
res.map((x) => x.blockeeId),
),
);
if (blockings.some((blocking) => blocking === antenna.userId)) return false;
if (note.visibility === 'followers') {
if (noteUserFollowers && !noteUserFollowers.includes(antenna.userId)) return false;
if (antennaUserFollowing && !antennaUserFollowing.includes(note.userId)) return false;
if (note.visibility === "followers") {
if (noteUserFollowers && !noteUserFollowers.includes(antenna.userId))
return false;
if (antennaUserFollowing && !antennaUserFollowing.includes(note.userId))
return false;
}
if (!antenna.withReplies && note.replyId != null) return false;
if (antenna.src === 'home') {
if (noteUserFollowers && !noteUserFollowers.includes(antenna.userId)) return false;
if (antennaUserFollowing && !antennaUserFollowing.includes(note.userId)) return false;
} else if (antenna.src === 'list') {
const listUsers = (await UserListJoinings.findBy({
userListId: antenna.userListId!,
})).map(x => x.userId);
if (antenna.src === "home") {
if (noteUserFollowers && !noteUserFollowers.includes(antenna.userId))
return false;
if (antennaUserFollowing && !antennaUserFollowing.includes(note.userId))
return false;
} else if (antenna.src === "list") {
const listUsers = (
await UserListJoinings.findBy({
userListId: antenna.userListId!,
})
).map((x) => x.userId);
if (!listUsers.includes(note.userId)) return false;
} else if (antenna.src === 'group') {
const joining = await UserGroupJoinings.findOneByOrFail({ id: antenna.userGroupJoiningId! });
} else if (antenna.src === "group") {
const joining = await UserGroupJoinings.findOneByOrFail({
id: antenna.userGroupJoiningId!,
});
const groupUsers = (await UserGroupJoinings.findBy({
userGroupId: joining.userGroupId,
})).map(x => x.userId);
const groupUsers = (
await UserGroupJoinings.findBy({
userGroupId: joining.userGroupId,
})
).map((x) => x.userId);
if (!groupUsers.includes(note.userId)) return false;
} else if (antenna.src === 'users') {
const accts = antenna.users.map(x => {
} else if (antenna.src === "users") {
const accts = antenna.users.map((x) => {
const { username, host } = Acct.parse(x);
return getFullApAccount(username, host).toLowerCase();
});
if (!accts.includes(getFullApAccount(noteUser.username, noteUser.host).toLowerCase())) return false;
if (
!accts.includes(
getFullApAccount(noteUser.username, noteUser.host).toLowerCase(),
)
)
return false;
}
const keywords = antenna.keywords
// Clean up
.map(xs => xs.filter(x => x !== ''))
.filter(xs => xs.length > 0);
.map((xs) => xs.filter((x) => x !== ""))
.filter((xs) => xs.length > 0);
if (keywords.length > 0) {
if (note.text == null) return false;
const matched = keywords.some(and =>
and.every(keyword =>
const matched = keywords.some((and) =>
and.every((keyword) =>
antenna.caseSensitive
? note.text!.includes(keyword)
: note.text!.toLowerCase().includes(keyword.toLowerCase())
));
: note.text!.toLowerCase().includes(keyword.toLowerCase()),
),
);
if (!matched) return false;
}
const excludeKeywords = antenna.excludeKeywords
// Clean up
.map(xs => xs.filter(x => x !== ''))
.filter(xs => xs.length > 0);
.map((xs) => xs.filter((x) => x !== ""))
.filter((xs) => xs.length > 0);
if (excludeKeywords.length > 0) {
if (note.text == null) return false;
const matched = excludeKeywords.some(and =>
and.every(keyword =>
const matched = excludeKeywords.some((and) =>
and.every((keyword) =>
antenna.caseSensitive
? note.text!.includes(keyword)
: note.text!.toLowerCase().includes(keyword.toLowerCase())
));
: note.text!.toLowerCase().includes(keyword.toLowerCase()),
),
);
if (matched) return false;
}

28
packages/backend/src/misc/check-word-mute.ts
View file

@ -1,28 +1,32 @@
import RE2 from 're2';
import { Note } from '@/models/entities/note.js';
import { User } from '@/models/entities/user.js';
import RE2 from "re2";
import type { Note } from "@/models/entities/note.js";
import type { User } from "@/models/entities/user.js";
type NoteLike = {
userId: Note['userId'];
text: Note['text'];
userId: Note["userId"];
text: Note["text"];
};
type UserLike = {
id: User['id'];
id: User["id"];
};
export async function checkWordMute(note: NoteLike, me: UserLike | null | undefined, mutedWords: Array<string | string[]>): Promise<boolean> {
export async function checkWordMute(
note: NoteLike,
me: UserLike | null | undefined,
mutedWords: Array<string | string[]>,
): Promise<boolean> {
// 自分自身
if (me && (note.userId === me.id)) return false;
if (me && note.userId === me.id) return false;
if (mutedWords.length > 0) {
const text = ((note.cw ?? '') + '\n' + (note.text ?? '')).trim();
const text = ((note.cw ?? "") + "\n" + (note.text ?? "")).trim();
if (text === '') return false;
if (text === "") return false;
const matched = mutedWords.some(filter => {
const matched = mutedWords.some((filter) => {
if (Array.isArray(filter)) {
return filter.every(keyword => text.includes(keyword));
return filter.every((keyword) => text.includes(keyword));
} else {
// represents RegExp
const regexp = filter.match(/^\/(.+)\/(.*)$/);

10
packages/backend/src/misc/clone.ts
View file

@ -1,10 +1,16 @@
// structredCloneが遅いため
// SEE: http://var.blog.jp/archives/86038606.html
type Cloneable = string | number | boolean | null | { [key: string]: Cloneable } | Cloneable[];
type Cloneable =
| string
| number
| boolean
| null
| { [key: string]: Cloneable }
| Cloneable[];
export function deepClone<T extends Cloneable>(x: T): T {
if (typeof x === 'object') {
if (typeof x === "object") {
if (x === null) return x;
if (Array.isArray(x)) return x.map(deepClone) as T;
const obj = {} as Record<string, Cloneable>;

9
packages/backend/src/misc/content-disposition.ts
View file

@ -1,6 +1,9 @@
import cd from 'content-disposition';
import cd from "content-disposition";
export function contentDisposition(type: 'inline' | 'attachment', filename: string): string {
const fallback = filename.replace(/[^\w.-]/g, '_');
export function contentDisposition(
type: "inline" | "attachment",
filename: string,
): string {
const fallback = filename.replace(/[^\w.-]/g, "_");
return cd(filename, { type, fallback });
}

10
packages/backend/src/misc/convert-host.ts
View file

@ -1,9 +1,11 @@
import { URL } from 'node:url';
import config from '@/config/index.js';
import { toASCII } from 'punycode';
import { URL } from "node:url";
import config from "@/config/index.js";
import { toASCII } from "punycode";
export function getFullApAccount(username: string, host: string | null) {
return host ? `${username}@${toPuny(host)}` : `${username}@${toPuny(config.host)}`;
return host
? `${username}@${toPuny(host)}`
: `${username}@${toPuny(config.host)}`;
}
export function isSelfHost(host: string) {

16
packages/backend/src/misc/count-same-renotes.ts
View file

@ -1,14 +1,18 @@
import { Notes } from '@/models/index.js';
import { Notes } from "@/models/index.js";
export async function countSameRenotes(userId: string, renoteId: string, excludeNoteId: string | undefined): Promise<number> {
export async function countSameRenotes(
userId: string,
renoteId: string,
excludeNoteId: string | undefined,
): Promise<number> {
// 指定したユーザーの指定したノートのリノートがいくつあるか数える
const query = Notes.createQueryBuilder('note')
.where('note.userId = :userId', { userId })
.andWhere('note.renoteId = :renoteId', { renoteId });
const query = Notes.createQueryBuilder("note")
.where("note.userId = :userId", { userId })
.andWhere("note.renoteId = :renoteId", { renoteId });
// 指定した投稿を除く
if (excludeNoteId) {
query.andWhere('note.id != :excludeNoteId', { excludeNoteId });
query.andWhere("note.id != :excludeNoteId", { excludeNoteId });
}
return await query.getCount();

4
packages/backend/src/misc/create-temp.ts
View file

@ -1,4 +1,4 @@
import * as tmp from 'tmp';
import * as tmp from "tmp";
export function createTemp(): Promise<[string, () => void]> {
return new Promise<[string, () => void]>((res, rej) => {
@ -18,7 +18,7 @@ export function createTempDir(): Promise<[string, () => void]> {
(e, path, cleanup) => {
if (e) return rej(e);
res([path, cleanup]);
}
},
);
});
}

6
packages/backend/src/misc/detect-url-mime.ts
View file

@ -1,6 +1,6 @@
import { createTemp } from './create-temp.js';
import { downloadUrl } from './download-url.js';
import { detectType } from './get-file-info.js';
import { createTemp } from "./create-temp.js";
import { downloadUrl } from "./download-url.js";
import { detectType } from "./get-file-info.js";
export async function detectUrlMime(url: string) {
const [path, cleanup] = await createTemp();

14
packages/backend/src/misc/download-text-file.ts
View file

@ -1,10 +1,10 @@
import * as fs from 'node:fs';
import * as util from 'node:util';
import Logger from '@/services/logger.js';
import { createTemp } from './create-temp.js';
import { downloadUrl } from './download-url.js';
import * as fs from "node:fs";
import * as util from "node:util";
import Logger from "@/services/logger.js";
import { createTemp } from "./create-temp.js";
import { downloadUrl } from "./download-url.js";
const logger = new Logger('download-text-file');
const logger = new Logger("download-text-file");
export async function downloadTextFile(url: string): Promise<string> {
// Create temp file
@ -16,7 +16,7 @@ export async function downloadTextFile(url: string): Promise<string> {
// write content at URL to temp file
await downloadUrl(url, path);
const text = await util.promisify(fs.readFile)(path, 'utf8');
const text = await util.promisify(fs.readFile)(path, "utf8");
return text;
} finally {

116
packages/backend/src/misc/download-url.ts
View file

@ -1,18 +1,18 @@
import * as fs from 'node:fs';
import * as stream from 'node:stream';
import * as util from 'node:util';
import got, * as Got from 'got';
import { httpAgent, httpsAgent, StatusError } from './fetch.js';
import config from '@/config/index.js';
import chalk from 'chalk';
import Logger from '@/services/logger.js';
import IPCIDR from 'ip-cidr';
import PrivateIp from 'private-ip';
import * as fs from "node:fs";
import * as stream from "node:stream";
import * as util from "node:util";
import got, * as Got from "got";
import { httpAgent, httpsAgent, StatusError } from "./fetch.js";
import config from "@/config/index.js";
import chalk from "chalk";
import Logger from "@/services/logger.js";
import IPCIDR from "ip-cidr";
import PrivateIp from "private-ip";
const pipeline = util.promisify(stream.pipeline);
export async function downloadUrl(url: string, path: string): Promise<void> {
const logger = new Logger('download');
const logger = new Logger("download");
logger.info(`Downloading ${chalk.cyan(url)} ...`);
@ -20,55 +20,69 @@ export async function downloadUrl(url: string, path: string): Promise<void> {
const operationTimeout = 60 * 1000;
const maxSize = config.maxFileSize || 262144000;
const req = got.stream(url, {
headers: {
'User-Agent': config.userAgent,
},
timeout: {
lookup: timeout,
connect: timeout,
secureConnect: timeout,
socket: timeout, // read timeout
response: timeout,
send: timeout,
request: operationTimeout, // whole operation timeout
},
agent: {
http: httpAgent,
https: httpsAgent,
},
http2: false, // default
retry: {
limit: 0,
},
}).on('response', (res: Got.Response) => {
if ((process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'test') && !config.proxy && res.ip) {
if (isPrivateIp(res.ip)) {
logger.warn(`Blocked address: ${res.ip}`);
req.destroy();
const req = got
.stream(url, {
headers: {
"User-Agent": config.userAgent,
},
timeout: {
lookup: timeout,
connect: timeout,
secureConnect: timeout,
socket: timeout, // read timeout
response: timeout,
send: timeout,
request: operationTimeout, // whole operation timeout
},
agent: {
http: httpAgent,
https: httpsAgent,
},
http2: false, // default
retry: {
limit: 0,
},
})
.on("response", (res: Got.Response) => {
if (
(process.env.NODE_ENV === "production" ||
process.env.NODE_ENV === "test") &&
!config.proxy &&
res.ip
) {
if (isPrivateIp(res.ip)) {
logger.warn(`Blocked address: ${res.ip}`);
req.destroy();
}
}
}
const contentLength = res.headers['content-length'];
if (contentLength != null) {
const size = Number(contentLength);
if (size > maxSize) {
logger.warn(`maxSize exceeded (${size} > ${maxSize}) on response`);
const contentLength = res.headers["content-length"];
if (contentLength != null) {
const size = Number(contentLength);
if (size > maxSize) {
logger.warn(`maxSize exceeded (${size} > ${maxSize}) on response`);
req.destroy();
}
}
})
.on("downloadProgress", (progress: Got.Progress) => {
if (progress.transferred > maxSize) {
logger.warn(
`maxSize exceeded (${progress.transferred} > ${maxSize}) on downloadProgress`,
);
req.destroy();
}
}
}).on('downloadProgress', (progress: Got.Progress) => {
if (progress.transferred > maxSize) {
logger.warn(`maxSize exceeded (${progress.transferred} > ${maxSize}) on downloadProgress`);
req.destroy();
}
});
});
try {
await pipeline(req, fs.createWriteStream(path));
} catch (e) {
if (e instanceof Got.HTTPError) {
throw new StatusError(`${e.response.statusCode} ${e.response.statusMessage}`, e.response.statusCode, e.response.statusMessage);
throw new StatusError(
`${e.response.statusCode} ${e.response.statusMessage}`,
e.response.statusCode,
e.response.statusMessage,
);
} else {
throw e;
}

2
packages/backend/src/misc/emoji-regex.ts
View file

@ -1,4 +1,4 @@
import twemoji from 'twemoji-parser/dist/lib/regex.js';
import twemoji from "twemoji-parser/dist/lib/regex.js";
const twemojiRegex = twemoji.default;
export const emojiRegex = new RegExp(`(${twemojiRegex.source})`);

8
packages/backend/src/misc/extract-custom-emojis-from-mfm.ts
View file

@ -1,10 +1,10 @@
import * as mfm from 'mfm-js';
import { unique } from '@/prelude/array.js';
import * as mfm from "mfm-js";
import { unique } from "@/prelude/array.js";
export function extractCustomEmojisFromMfm(nodes: mfm.MfmNode[]): string[] {
const emojiNodes = mfm.extract(nodes, (node) => {
return (node.type === 'emojiCode' && node.props.name.length <= 100);
return node.type === "emojiCode" && node.props.name.length <= 100;
});
return unique(emojiNodes.map(x => x.props.name));
return unique(emojiNodes.map((x) => x.props.name));
}

8
packages/backend/src/misc/extract-hashtags.ts
View file

@ -1,9 +1,9 @@
import * as mfm from 'mfm-js';
import { unique } from '@/prelude/array.js';
import * as mfm from "mfm-js";
import { unique } from "@/prelude/array.js";
export function extractHashtags(nodes: mfm.MfmNode[]): string[] {
const hashtagNodes = mfm.extract(nodes, (node) => node.type === 'hashtag');
const hashtags = unique(hashtagNodes.map(x => x.props.hashtag));
const hashtagNodes = mfm.extract(nodes, (node) => node.type === "hashtag");
const hashtags = unique(hashtagNodes.map((x) => x.props.hashtag));
return hashtags;
}

10
packages/backend/src/misc/extract-mentions.ts
View file

@ -1,11 +1,13 @@
// test is located in test/extract-mentions
import * as mfm from 'mfm-js';
import * as mfm from "mfm-js";
export function extractMentions(nodes: mfm.MfmNode[]): mfm.MfmMention['props'][] {
export function extractMentions(
nodes: mfm.MfmNode[],
): mfm.MfmMention["props"][] {
// TODO: 重複を削除
const mentionNodes = mfm.extract(nodes, (node) => node.type === 'mention');
const mentions = mentionNodes.map(x => x.props);
const mentionNodes = mfm.extract(nodes, (node) => node.type === "mention");
const mentions = mentionNodes.map((x) => x.props);
return mentions;
}

18
packages/backend/src/misc/fetch-meta.ts
View file

@ -1,16 +1,16 @@
import { db } from '@/db/postgre.js';
import { Meta } from '@/models/entities/meta.js';
import { db } from "@/db/postgre.js";
import { Meta } from "@/models/entities/meta.js";
let cache: Meta;
export async function fetchMeta(noCache = false): Promise<Meta> {
if (!noCache && cache) return cache;
return await db.transaction(async transactionalEntityManager => {
return await db.transaction(async (transactionalEntityManager) => {
// New IDs are prioritized because multiple records may have been created due to past bugs.
const metas = await transactionalEntityManager.find(Meta, {
order: {
id: 'DESC',
id: "DESC",
},
});
@ -25,11 +25,13 @@ export async function fetchMeta(noCache = false): Promise<Meta> {
.upsert(
Meta,
{
id: 'x',
id: "x",
},
['id'],
["id"],
)
.then((x) => transactionalEntityManager.findOneByOrFail(Meta, x.identifiers[0]));
.then((x) =>
transactionalEntityManager.findOneByOrFail(Meta, x.identifiers[0]),
);
cache = saved;
return saved;
@ -38,7 +40,7 @@ export async function fetchMeta(noCache = false): Promise<Meta> {
}
setInterval(() => {
fetchMeta(true).then(meta => {
fetchMeta(true).then((meta) => {
cache = meta;
});
}, 1000 * 10);

10
packages/backend/src/misc/fetch-proxy-account.ts
View file

@ -1,9 +1,11 @@
import { fetchMeta } from './fetch-meta.js';
import { ILocalUser } from '@/models/entities/user.js';
import { Users } from '@/models/index.js';
import { fetchMeta } from "./fetch-meta.js";
import type { ILocalUser } from "@/models/entities/user.js";
import { Users } from "@/models/index.js";
export async function fetchProxyAccount(): Promise<ILocalUser | null> {
const meta = await fetchMeta();
if (meta.proxyAccountId == null) return null;
return await Users.findOneByOrFail({ id: meta.proxyAccountId }) as ILocalUser;
return (await Users.findOneByOrFail({
id: meta.proxyAccountId,
})) as ILocalUser;
}

114
packages/backend/src/misc/fetch.ts
View file

@ -1,40 +1,63 @@
import * as http from 'node:http';
import * as https from 'node:https';
import { URL } from 'node:url';
import CacheableLookup from 'cacheable-lookup';
import fetch from 'node-fetch';
import { HttpProxyAgent, HttpsProxyAgent } from 'hpagent';
import config from '@/config/index.js';
import * as http from "node:http";
import * as https from "node:https";
import type { URL } from "node:url";
import CacheableLookup from "cacheable-lookup";
import fetch from "node-fetch";
import { HttpProxyAgent, HttpsProxyAgent } from "hpagent";
import config from "@/config/index.js";
export async function getJson(url: string, accept = 'application/json, */*', timeout = 10000, headers?: Record<string, string>) {
export async function getJson(
url: string,
accept = "application/json, */*",
timeout = 10000,
headers?: Record<string, string>,
) {
const res = await getResponse({
url,
method: 'GET',
headers: Object.assign({
'User-Agent': config.userAgent,
Accept: accept,
}, headers || {}),
method: "GET",
headers: Object.assign(
{
"User-Agent": config.userAgent,
Accept: accept,
},
headers || {},
),
timeout,
});
return await res.json();
}
export async function getHtml(url: string, accept = 'text/html, */*', timeout = 10000, headers?: Record<string, string>) {
export async function getHtml(
url: string,
accept = "text/html, */*",
timeout = 10000,
headers?: Record<string, string>,
) {
const res = await getResponse({
url,
method: 'GET',
headers: Object.assign({
'User-Agent': config.userAgent,
Accept: accept,
}, headers || {}),
method: "GET",
headers: Object.assign(
{
"User-Agent": config.userAgent,
Accept: accept,
},
headers || {},
),
timeout,
});
return await res.text();
}
export async function getResponse(args: { url: string, method: string, body?: string, headers: Record<string, string>, timeout?: number, size?: number }) {
export async function getResponse(args: {
url: string;
method: string;
body?: string;
headers: Record<string, string>;
timeout?: number;
size?: number;
}) {
const timeout = args.timeout || 10 * 1000;
const controller = new AbortController();
@ -53,16 +76,20 @@ export async function getResponse(args: { url: string, method: string, body?: st
});
if (!res.ok) {
throw new StatusError(`${res.status} ${res.statusText}`, res.status, res.statusText);
throw new StatusError(
`${res.status} ${res.statusText}`,
res.status,
res.statusText,
);
}
return res;
}
const cache = new CacheableLookup({
maxTtl: 3600, // 1hours
errorTtl: 30, // 30secs
lookup: false, // nativeのdns.lookupにfallbackしない
maxTtl: 3600, // 1hours
errorTtl: 30, // 30secs
lookup: false, // nativeのdns.lookupにfallbackしない
});
/**
@ -90,13 +117,13 @@ const maxSockets = Math.max(256, config.deliverJobConcurrency || 128);
*/
export const httpAgent = config.proxy
? new HttpProxyAgent({
keepAlive: true,
keepAliveMsecs: 30 * 1000,
maxSockets,
maxFreeSockets: 256,
scheduling: 'lifo',
proxy: config.proxy,
})
keepAlive: true,
keepAliveMsecs: 30 * 1000,
maxSockets,
maxFreeSockets: 256,
scheduling: "lifo",
proxy: config.proxy,
})
: _http;
/**
@ -104,13 +131,13 @@ export const httpAgent = config.proxy
*/
export const httpsAgent = config.proxy
? new HttpsProxyAgent({
keepAlive: true,
keepAliveMsecs: 30 * 1000,
maxSockets,
maxFreeSockets: 256,
scheduling: 'lifo',
proxy: config.proxy,
})
keepAlive: true,
keepAliveMsecs: 30 * 1000,
maxSockets,
maxFreeSockets: 256,
scheduling: "lifo",
proxy: config.proxy,
})
: _https;
/**
@ -120,9 +147,9 @@ export const httpsAgent = config.proxy
*/
export function getAgentByUrl(url: URL, bypassProxy = false) {
if (bypassProxy || (config.proxyBypassHosts || []).includes(url.hostname)) {
return url.protocol === 'http:' ? _http : _https;
return url.protocol === "http:" ? _http : _https;
} else {
return url.protocol === 'http:' ? httpAgent : httpsAgent;
return url.protocol === "http:" ? httpAgent : httpsAgent;
}
}
@ -133,9 +160,12 @@ export class StatusError extends Error {
constructor(message: string, statusCode: number, statusMessage?: string) {
super(message);
this.name = 'StatusError';
this.name = "StatusError";
this.statusCode = statusCode;
this.statusMessage = statusMessage;
this.isClientError = typeof this.statusCode === 'number' && this.statusCode >= 400 && this.statusCode < 500;
this.isClientError =
typeof this.statusCode === "number" &&
this.statusCode >= 400 &&
this.statusCode < 500;
}
}

32
packages/backend/src/misc/gen-id.ts
View file

@ -1,21 +1,27 @@
import { ulid } from 'ulid';
import { genAid } from './id/aid.js';
import { genMeid } from './id/meid.js';
import { genMeidg } from './id/meidg.js';
import { genObjectId } from './id/object-id.js';
import config from '@/config/index.js';
import { ulid } from "ulid";
import { genAid } from "./id/aid.js";
import { genMeid } from "./id/meid.js";
import { genMeidg } from "./id/meidg.js";
import { genObjectId } from "./id/object-id.js";
import config from "@/config/index.js";
const metohd = config.id.toLowerCase();
export function genId(date?: Date): string {
if (!date || (date > new Date())) date = new Date();
if (!date || date > new Date()) date = new Date();
switch (metohd) {
case 'aid': return genAid(date);
case 'meid': return genMeid(date);
case 'meidg': return genMeidg(date);
case 'ulid': return ulid(date.getTime());
case 'objectid': return genObjectId(date);
default: throw new Error('unrecognized id generation method');
case "aid":
return genAid(date);
case "meid":
return genMeid(date);
case "meidg":
return genMeidg(date);
case "ulid":
return ulid(date.getTime());
case "objectid":
return genObjectId(date);
default:
throw new Error("unrecognized id generation method");
}
}

64
packages/backend/src/misc/gen-identicon.ts
View file

@ -3,37 +3,37 @@
* https://en.wikipedia.org/wiki/Identicon
*/
import { WriteStream } from 'node:fs';
import * as p from 'pureimage';
import gen from 'random-seed';
import type { WriteStream } from "node:fs";
import * as p from "pureimage";
import gen from "random-seed";
const size = 128; // px
const n = 5; // resolution
const margin = (size / 4);
const margin = size / 4;
const colors = [
['#FF512F', '#DD2476'],
['#FF61D2', '#FE9090'],
['#72FFB6', '#10D164'],
['#FD8451', '#FFBD6F'],
['#305170', '#6DFC6B'],
['#00C0FF', '#4218B8'],
['#009245', '#FCEE21'],
['#0100EC', '#FB36F4'],
['#FDABDD', '#374A5A'],
['#38A2D7', '#561139'],
['#121C84', '#8278DA'],
['#5761B2', '#1FC5A8'],
['#FFDB01', '#0E197D'],
['#FF3E9D', '#0E1F40'],
['#766eff', '#00d4ff'],
['#9bff6e', '#00d4ff'],
['#ff6e94', '#00d4ff'],
['#ffa96e', '#00d4ff'],
['#ffa96e', '#ff009d'],
['#ffdd6e', '#ff009d'],
["#FF512F", "#DD2476"],
["#FF61D2", "#FE9090"],
["#72FFB6", "#10D164"],
["#FD8451", "#FFBD6F"],
["#305170", "#6DFC6B"],
["#00C0FF", "#4218B8"],
["#009245", "#FCEE21"],
["#0100EC", "#FB36F4"],
["#FDABDD", "#374A5A"],
["#38A2D7", "#561139"],
["#121C84", "#8278DA"],
["#5761B2", "#1FC5A8"],
["#FFDB01", "#0E197D"],
["#FF3E9D", "#0E1F40"],
["#766eff", "#00d4ff"],
["#9bff6e", "#00d4ff"],
["#ff6e94", "#00d4ff"],
["#ffa96e", "#00d4ff"],
["#ffa96e", "#ff009d"],
["#ffdd6e", "#ff009d"],
];
const actualSize = size - (margin * 2);
const actualSize = size - margin * 2;
const cellSize = actualSize / n;
const sideN = Math.floor(n / 2);
@ -43,7 +43,7 @@ const sideN = Math.floor(n / 2);
export function genIdenticon(seed: string, stream: WriteStream): Promise<void> {
const rand = gen.create(seed);
const canvas = p.make(size, size, undefined);
const ctx = canvas.getContext('2d');
const ctx = canvas.getContext("2d");
const bgColors = colors[rand(colors.length)];
@ -55,7 +55,7 @@ export function genIdenticon(seed: string, stream: WriteStream): Promise<void> {
ctx.beginPath();
ctx.fillRect(0, 0, size, size);
ctx.fillStyle = '#ffffff';
ctx.fillStyle = "#ffffff";
// side bitmap (filled by false)
const side: boolean[][] = new Array(sideN);
@ -80,17 +80,17 @@ export function genIdenticon(seed: string, stream: WriteStream): Promise<void> {
// Draw
for (let x = 0; x < n; x++) {
for (let y = 0; y < n; y++) {
const isXCenter = x === ((n - 1) / 2);
const isXCenter = x === (n - 1) / 2;
if (isXCenter && !center[y]) continue;
const isLeftSide = x < ((n - 1) / 2);
const isLeftSide = x < (n - 1) / 2;
if (isLeftSide && !side[x][y]) continue;
const isRightSide = x > ((n - 1) / 2);
const isRightSide = x > (n - 1) / 2;
if (isRightSide && !side[sideN - (x - sideN)][y]) continue;
const actualX = margin + (cellSize * x);
const actualY = margin + (cellSize * y);
const actualX = margin + cellSize * x;
const actualY = margin + cellSize * y;
ctx.beginPath();
ctx.fillRect(actualX, actualY, cellSize, cellSize);
}

32
packages/backend/src/misc/gen-key-pair.ts
View file

@ -1,34 +1,40 @@
import * as crypto from 'node:crypto';
import * as util from 'node:util';
import * as crypto from "node:crypto";
import * as util from "node:util";
const generateKeyPair = util.promisify(crypto.generateKeyPair);
export async function genRsaKeyPair(modulusLength = 2048) {
return await generateKeyPair('rsa', {
return await generateKeyPair("rsa", {
modulusLength,
publicKeyEncoding: {
type: 'spki',
format: 'pem',
type: "spki",
format: "pem",
},
privateKeyEncoding: {
type: 'pkcs8',
format: 'pem',
type: "pkcs8",
format: "pem",
cipher: undefined,
passphrase: undefined,
},
});
}
export async function genEcKeyPair(namedCurve: 'prime256v1' | 'secp384r1' | 'secp521r1' | 'curve25519' = 'prime256v1') {
return await generateKeyPair('ec', {
export async function genEcKeyPair(
namedCurve:
| "prime256v1"
| "secp384r1"
| "secp521r1"
| "curve25519" = "prime256v1",
) {
return await generateKeyPair("ec", {
namedCurve,
publicKeyEncoding: {
type: 'spki',
format: 'pem',
type: "spki",
format: "pem",
},
privateKeyEncoding: {
type: 'pkcs8',
format: 'pem',
type: "pkcs8",
format: "pem",
cipher: undefined,
passphrase: undefined,
},

219
packages/backend/src/misc/get-file-info.ts
View file

@ -1,18 +1,18 @@
import * as fs from 'node:fs';
import * as crypto from 'node:crypto';
import { join } from 'node:path';
import * as stream from 'node:stream';
import * as util from 'node:util';
import { FSWatcher } from 'chokidar';
import { fileTypeFromFile } from 'file-type';
import FFmpeg from 'fluent-ffmpeg';
import isSvg from 'is-svg';
import probeImageSize from 'probe-image-size';
import { type predictionType } from 'nsfwjs';
import sharp from 'sharp';
import { encode } from 'blurhash';
import { detectSensitive } from '@/services/detect-sensitive.js';
import { createTempDir } from './create-temp.js';
import * as fs from "node:fs";
import * as crypto from "node:crypto";
import { join } from "node:path";
import * as stream from "node:stream";
import * as util from "node:util";
import { FSWatcher } from "chokidar";
import { fileTypeFromFile } from "file-type";
import FFmpeg from "fluent-ffmpeg";
import isSvg from "is-svg";
import probeImageSize from "probe-image-size";
import { type predictionType } from "nsfwjs";
import sharp from "sharp";
import { encode } from "blurhash";
import { detectSensitive } from "@/services/detect-sensitive.js";
import { createTempDir } from "./create-temp.js";
const pipeline = util.promisify(stream.pipeline);
@ -33,24 +33,27 @@ export type FileInfo = {
};
const TYPE_OCTET_STREAM = {
mime: 'application/octet-stream',
mime: "application/octet-stream",
ext: null,
};
const TYPE_SVG = {
mime: 'image/svg+xml',
ext: 'svg',
mime: "image/svg+xml",
ext: "svg",
};
/**
* Get file information
*/
export async function getFileInfo(path: string, opts: {
skipSensitiveDetection: boolean;
sensitiveThreshold?: number;
sensitiveThresholdForPorn?: number;
enableSensitiveMediaDetectionForVideos?: boolean;
}): Promise<FileInfo> {
export async function getFileInfo(
path: string,
opts: {
skipSensitiveDetection: boolean;
sensitiveThreshold?: number;
sensitiveThresholdForPorn?: number;
enableSensitiveMediaDetectionForVideos?: boolean;
},
): Promise<FileInfo> {
const warnings = [] as string[];
const size = await getFileSize(path);
@ -63,24 +66,37 @@ export async function getFileInfo(path: string, opts: {
let height: number | undefined;
let orientation: number | undefined;
if (['image/jpeg', 'image/gif', 'image/png', 'image/apng', 'image/webp', 'image/bmp', 'image/tiff', 'image/svg+xml', 'image/vnd.adobe.photoshop', 'image/avif'].includes(type.mime)) {
const imageSize = await detectImageSize(path).catch(e => {
if (
[
"image/jpeg",
"image/gif",
"image/png",
"image/apng",
"image/webp",
"image/bmp",
"image/tiff",
"image/svg+xml",
"image/vnd.adobe.photoshop",
"image/avif",
].includes(type.mime)
) {
const imageSize = await detectImageSize(path).catch((e) => {
warnings.push(`detectImageSize failed: ${e}`);
return undefined;
});
// うまく判定できない画像は octet-stream にする
if (!imageSize) {
warnings.push('cannot detect image dimensions');
warnings.push("cannot detect image dimensions");
type = TYPE_OCTET_STREAM;
} else if (imageSize.wUnits === 'px') {
} else if (imageSize.wUnits === "px") {
width = imageSize.width;
height = imageSize.height;
orientation = imageSize.orientation;
// 制限を超えている画像は octet-stream にする
if (imageSize.width > 16383 || imageSize.height > 16383) {
warnings.push('image dimensions exceeds limits');
warnings.push("image dimensions exceeds limits");
type = TYPE_OCTET_STREAM;
}
} else {
@ -90,8 +106,18 @@ export async function getFileInfo(path: string, opts: {
let blurhash: string | undefined;
if (['image/jpeg', 'image/gif', 'image/png', 'image/apng', 'image/webp', 'image/svg+xml', 'image/avif'].includes(type.mime)) {
blurhash = await getBlurhash(path).catch(e => {
if (
[
"image/jpeg",
"image/gif",
"image/png",
"image/apng",
"image/webp",
"image/svg+xml",
"image/avif",
].includes(type.mime)
) {
blurhash = await getBlurhash(path).catch((e) => {
warnings.push(`getBlurhash failed: ${e}`);
return undefined;
});
@ -107,11 +133,14 @@ export async function getFileInfo(path: string, opts: {
opts.sensitiveThreshold ?? 0.5,
opts.sensitiveThresholdForPorn ?? 0.75,
opts.enableSensitiveMediaDetectionForVideos ?? false,
).then(value => {
[sensitive, porn] = value;
}, error => {
warnings.push(`detectSensitivity failed: ${error}`);
});
).then(
(value) => {
[sensitive, porn] = value;
},
(error) => {
warnings.push(`detectSensitivity failed: ${error}`);
},
);
}
return {
@ -128,71 +157,100 @@ export async function getFileInfo(path: string, opts: {
};
}
async function detectSensitivity(source: string, mime: string, sensitiveThreshold: number, sensitiveThresholdForPorn: number, analyzeVideo: boolean): Promise<[sensitive: boolean, porn: boolean]> {
async function detectSensitivity(
source: string,
mime: string,
sensitiveThreshold: number,
sensitiveThresholdForPorn: number,
analyzeVideo: boolean,
): Promise<[sensitive: boolean, porn: boolean]> {
let sensitive = false;
let porn = false;
function judgePrediction(result: readonly predictionType[]): [sensitive: boolean, porn: boolean] {
function judgePrediction(
result: readonly predictionType[],
): [sensitive: boolean, porn: boolean] {
let sensitive = false;
let porn = false;
if ((result.find(x => x.className === 'Sexy')?.probability ?? 0) > sensitiveThreshold) sensitive = true;
if ((result.find(x => x.className === 'Hentai')?.probability ?? 0) > sensitiveThreshold) sensitive = true;
if ((result.find(x => x.className === 'Porn')?.probability ?? 0) > sensitiveThreshold) sensitive = true;
if (
(result.find((x) => x.className === "Sexy")?.probability ?? 0) >
sensitiveThreshold
)
sensitive = true;
if (
(result.find((x) => x.className === "Hentai")?.probability ?? 0) >
sensitiveThreshold
)
sensitive = true;
if (
(result.find((x) => x.className === "Porn")?.probability ?? 0) >
sensitiveThreshold
)
sensitive = true;
if ((result.find(x => x.className === 'Porn')?.probability ?? 0) > sensitiveThresholdForPorn) porn = true;
if (
(result.find((x) => x.className === "Porn")?.probability ?? 0) >
sensitiveThresholdForPorn
)
porn = true;
return [sensitive, porn];
}
if (['image/jpeg', 'image/png', 'image/webp'].includes(mime)) {
if (["image/jpeg", "image/png", "image/webp"].includes(mime)) {
const result = await detectSensitive(source);
if (result) {
[sensitive, porn] = judgePrediction(result);
}
} else if (analyzeVideo && (mime === 'image/apng' || mime.startsWith('video/'))) {
} else if (
analyzeVideo &&
(mime === "image/apng" || mime.startsWith("video/"))
) {
const [outDir, disposeOutDir] = await createTempDir();
try {
const command = FFmpeg()
.input(source)
.inputOptions([
'-skip_frame', 'nokey', // 可能ならキーフレームのみを取得してほしいとする（そうなるとは限らない）
'-lowres', '3', // 元の画質でデコードする必要はないので 1/8 画質でデコードしてもよいとする（そうなるとは限らない）
"-skip_frame",
"nokey", // 可能ならキーフレームのみを取得してほしいとする（そうなるとは限らない）
"-lowres",
"3", // 元の画質でデコードする必要はないので 1/8 画質でデコードしてもよいとする（そうなるとは限らない）
])
.noAudio()
.videoFilters([
{
filter: 'select', // フレームのフィルタリング
filter: "select", // フレームのフィルタリング
options: {
e: 'eq(pict_type,PICT_TYPE_I)', // I-Frame のみをフィルタする（VP9 とかはデコードしてみないとわからないっぽい）
e: "eq(pict_type,PICT_TYPE_I)", // I-Frame のみをフィルタする（VP9 とかはデコードしてみないとわからないっぽい）
},
},
{
filter: 'blackframe', // 暗いフレームの検出
filter: "blackframe", // 暗いフレームの検出
options: {
amount: '0', // 暗さに関わらず全てのフレームで測定値を取る
amount: "0", // 暗さに関わらず全てのフレームで測定値を取る
},
},
{
filter: 'metadata',
filter: "metadata",
options: {
mode: 'select', // フレーム選択モード
key: 'lavfi.blackframe.pblack', // フレームにおける暗部の百分率（前のフィルタからのメタデータを参照する）
value: '50',
function: 'less', // 50% 未満のフレームを選択する（50% 以上暗部があるフレームだと誤検知を招くかもしれないので）
mode: "select", // フレーム選択モード
key: "lavfi.blackframe.pblack", // フレームにおける暗部の百分率（前のフィルタからのメタデータを参照する）
value: "50",
function: "less", // 50% 未満のフレームを選択する（50% 以上暗部があるフレームだと誤検知を招くかもしれないので）
},
},
{
filter: 'scale',
filter: "scale",
options: {
w: 299,
h: 299,
},
},
])
.format('image2')
.output(join(outDir, '%d.png'))
.outputOptions(['-vsync', '0']); // 可変フレームレートにすることで穴埋めをさせない
.format("image2")
.output(join(outDir, "%d.png"))
.outputOptions(["-vsync", "0"]); // 可変フレームレートにすることで穴埋めをさせない
const results: ReturnType<typeof judgePrediction>[] = [];
let frameIndex = 0;
let targetIndex = 0;
@ -213,8 +271,12 @@ async function detectSensitivity(source: string, mime: string, sensitiveThreshol
fs.promises.unlink(path);
}
}
sensitive = results.filter(x => x[0]).length >= Math.ceil(results.length * sensitiveThreshold);
porn = results.filter(x => x[1]).length >= Math.ceil(results.length * sensitiveThresholdForPorn);
sensitive =
results.filter((x) => x[0]).length >=
Math.ceil(results.length * sensitiveThreshold);
porn =
results.filter((x) => x[1]).length >=
Math.ceil(results.length * sensitiveThresholdForPorn);
} finally {
disposeOutDir();
}
@ -223,35 +285,41 @@ async function detectSensitivity(source: string, mime: string, sensitiveThreshol
return [sensitive, porn];
}
async function* asyncIterateFrames(cwd: string, command: FFmpeg.FfmpegCommand): AsyncGenerator<string, void> {
async function* asyncIterateFrames(
cwd: string,
command: FFmpeg.FfmpegCommand,
): AsyncGenerator<string, void> {
const watcher = new FSWatcher({
cwd,
disableGlobbing: true,
});
let finished = false;
command.once('end', () => {
command.once("end", () => {
finished = true;
watcher.close();
});
command.run();
for (let i = 1; true; i++) { // eslint-disable-line @typescript-eslint/no-unnecessary-condition
for (let i = 1; true; i++) {
// eslint-disable-line @typescript-eslint/no-unnecessary-condition
const current = `${i}.png`;
const next = `${i + 1}.png`;
const framePath = join(cwd, current);
if (await exists(join(cwd, next))) {
yield framePath;
} else if (!finished) { // eslint-disable-line @typescript-eslint/no-unnecessary-condition
} else if (!finished) {
// eslint-disable-line @typescript-eslint/no-unnecessary-condition
watcher.add(next);
await new Promise<void>((resolve, reject) => {
watcher.on('add', function onAdd(path) {
if (path === next) { // 次フレームの書き出しが始まっているなら、現在フレームの書き出しは終わっている
watcher.on("add", function onAdd(path) {
if (path === next) {
// 次フレームの書き出しが始まっているなら、現在フレームの書き出しは終わっている
watcher.unwatch(current);
watcher.off('add', onAdd);
watcher.off("add", onAdd);
resolve();
}
});
command.once('end', resolve); // 全てのフレームを処理し終わったなら、最終フレームである現在フレームの書き出しは終わっている
command.once('error', reject);
command.once("end", resolve); // 全てのフレームを処理し終わったなら、最終フレームである現在フレームの書き出しは終わっている
command.once("error", reject);
});
yield framePath;
} else if (await exists(framePath)) {
@ -263,7 +331,10 @@ async function* asyncIterateFrames(cwd: string, command: FFmpeg.FfmpegCommand):
}
function exists(path: string): Promise<boolean> {
return fs.promises.access(path).then(() => true, () => false);
return fs.promises.access(path).then(
() => true,
() => false,
);
}
/**
@ -283,7 +354,7 @@ export async function detectType(path: string): Promise<{
if (type) {
// XMLはSVGかもしれない
if (type.mime === 'application/xml' && await checkSvg(path)) {
if (type.mime === "application/xml" && (await checkSvg(path))) {
return TYPE_SVG;
}
@ -327,7 +398,7 @@ export async function getFileSize(path: string): Promise<number> {
* Calculate MD5 hash
*/
async function calcHash(path: string): Promise<string> {
const hash = crypto.createHash('md5').setEncoding('hex');
const hash = crypto.createHash("md5").setEncoding("hex");
await pipeline(fs.createReadStream(path), hash);
return hash.read();
}
@ -356,7 +427,7 @@ function getBlurhash(path: string): Promise<string> {
sharp(path)
.raw()
.ensureAlpha()
.resize(64, 64, { fit: 'inside' })
.resize(64, 64, { fit: "inside" })
.toBuffer((err, buffer, { width, height }) => {
if (err) return reject(err);

4
packages/backend/src/misc/get-ip-hash.ts
View file

@ -1,9 +1,9 @@
import IPCIDR from 'ip-cidr';
import IPCIDR from "ip-cidr";
export function getIpHash(ip: string) {
// because a single person may control many IPv6 addresses,
// only a /64 subnet prefix of any IP will be taken into account.
// (this means for IPv4 the entire address is used)
const prefix = IPCIDR.createAddress(ip).mask(64);
return 'ip-' + BigInt('0b' + prefix).toString(36);
return `ip-${BigInt(`0b${prefix}`).toString(36)}`;
}

12
packages/backend/src/misc/get-note-summary.ts
View file

@ -1,21 +1,21 @@
import { Packed } from './schema.js';
import type { Packed } from "./schema.js";
/**
* 投稿を表す文字列を取得します。
* @param {*} note (packされた)投稿
*/
export const getNoteSummary = (note: Packed<'Note'>): string => {
export const getNoteSummary = (note: Packed<"Note">): string => {
if (note.deletedAt) {
return `❌`;
return "❌";
}
let summary = '';
let summary = "";
// 本文
if (note.cw != null) {
summary += note.cw;
} else {
summary += note.text ? note.text : '';
summary += note.text ? note.text : "";
}
// ファイルが添付されているとき
@ -25,7 +25,7 @@ export const getNoteSummary = (note: Packed<'Note'>): string => {
// 投票が添付されているとき
if (note.poll) {
summary += ` (📊)`;
summary += " (📊)";
}
/*

38
packages/backend/src/misc/get-reaction-emoji.ts
View file

@ -1,16 +1,28 @@
export default function(reaction: string): string {
export default function (reaction: string): string {
switch (reaction) {
case 'like': return '👍';
case 'love': return '❤️';
case 'laugh': return '😆';
case 'hmm': return '🤔';
case 'surprise': return '😮';
case 'congrats': return '🎉';
case 'angry': return '💢';
case 'confused': return '😥';
case 'rip': return '😇';
case 'pudding': return '🍮';
case 'star': return '⭐';
default: return reaction;
case "like":
return "👍";
case "love":
return "❤️";
case "laugh":
return "😆";
case "hmm":
return "🤔";
case "surprise":
return "😮";
case "congrats":
return "🎉";
case "angry":
return "💢";
case "confused":
return "😥";
case "rip":
return "😇";
case "pudding":
return "🍮";
case "star":
return "⭐";
default:
return reaction;
}
}

1
packages/backend/src/misc/hard-limits.ts
View file

@ -1,4 +1,3 @@
// If you change DB_* values, you must also change the DB schema.
/**

2
packages/backend/src/misc/i18n.ts
View file

@ -13,7 +13,7 @@ export class I18n<T extends Record<string, any>> {
// なるべくこのメソッド使うよりもlocale直接参照の方がvueのキャッシュ効いてパフォーマンスが良いかも
public t(key: string, args?: Record<string, any>): string {
try {
let str = key.split('.').reduce((o, i) => o[i], this.locale) as string;
let str = key.split(".").reduce((o, i) => o[i], this.locale) as string;
if (args) {
for (const [k, v] of Object.entries(args)) {

8
packages/backend/src/misc/id/aid.ts
View file

@ -1,7 +1,7 @@
// AID
// 長さ8の[2000年1月1日からの経過ミリ秒をbase36でエンコードしたもの] + 長さ2の[ノイズ文字列]
import * as crypto from 'node:crypto';
import * as crypto from "node:crypto";
const TIME2000 = 946684800000;
let counter = crypto.randomBytes(2).readUInt16LE(0);
@ -10,16 +10,16 @@ function getTime(time: number) {
time = time - TIME2000;
if (time < 0) time = 0;
return time.toString(36).padStart(8, '0');
return time.toString(36).padStart(8, "0");
}
function getNoise() {
return counter.toString(36).padStart(2, '0').slice(-2);
return counter.toString(36).padStart(2, "0").slice(-2);
}
export function genAid(date: Date): string {
const t = date.getTime();
if (isNaN(t)) throw 'Failed to create AID: Invalid Date';
if (isNaN(t)) throw "Failed to create AID: Invalid Date";
counter++;
return getTime(t) + getNoise();
}

4
packages/backend/src/misc/id/meid.ts
View file

@ -1,4 +1,4 @@
const CHARS = '0123456789abcdef';
const CHARS = "0123456789abcdef";
function getTime(time: number) {
if (time < 0) time = 0;
@ -12,7 +12,7 @@ function getTime(time: number) {
}
function getRandom() {
let str = '';
let str = "";
for (let i = 0; i < 12; i++) {
str += CHARS[Math.floor(Math.random() * CHARS.length)];

6
packages/backend/src/misc/id/meidg.ts
View file

@ -1,4 +1,4 @@
const CHARS = '0123456789abcdef';
const CHARS = "0123456789abcdef";
// 4bit Fixed hex value 'g'
// 44bit UNIX Time ms in Hex
@ -14,7 +14,7 @@ function getTime(time: number) {
}
function getRandom() {
let str = '';
let str = "";
for (let i = 0; i < 12; i++) {
str += CHARS[Math.floor(Math.random() * CHARS.length)];
@ -24,5 +24,5 @@ function getRandom() {
}
export function genMeidg(date: Date): string {
return 'g' + getTime(date.getTime()) + getRandom();
return `g${getTime(date.getTime())}${getRandom()}`;
}

4
packages/backend/src/misc/id/object-id.ts
View file

@ -1,4 +1,4 @@
const CHARS = '0123456789abcdef';
const CHARS = "0123456789abcdef";
function getTime(time: number) {
if (time < 0) time = 0;
@ -12,7 +12,7 @@ function getTime(time: number) {
}
function getRandom() {
let str = '';
let str = "";
for (let i = 0; i < 16; i++) {
str += CHARS[Math.floor(Math.random() * CHARS.length)];

2
packages/backend/src/misc/identifiable-error.ts
View file

@ -7,7 +7,7 @@ export class IdentifiableError extends Error {
constructor(id: string, message?: string) {
super(message);
this.message = message || '';
this.message = message || "";
this.id = id;
}
}

4
packages/backend/src/misc/is-duplicate-key-value-error.ts
View file

@ -1,3 +1,5 @@
export function isDuplicateKeyValueError(e: unknown | Error): boolean {
return (e as any).message && (e as Error).message.startsWith('duplicate key value');
return (
(e as Error).message?.startsWith("duplicate key value")
);
}

20
packages/backend/src/misc/is-instance-muted.ts
View file

@ -1,15 +1,21 @@
import { Packed } from './schema.js';
import type { Packed } from "./schema.js";
export function isInstanceMuted(note: Packed<'Note'>, mutedInstances: Set<string>): boolean {
if (mutedInstances.has(note?.user?.host ?? '')) return true;
if (mutedInstances.has(note?.reply?.user?.host ?? '')) return true;
if (mutedInstances.has(note?.renote?.user?.host ?? '')) return true;
export function isInstanceMuted(
note: Packed<"Note">,
mutedInstances: Set<string>,
): boolean {
if (mutedInstances.has(note?.user?.host ?? "")) return true;
if (mutedInstances.has(note?.reply?.user?.host ?? "")) return true;
if (mutedInstances.has(note?.renote?.user?.host ?? "")) return true;
return false;
}
export function isUserFromMutedInstance(notif: Packed<'Notification'>, mutedInstances: Set<string>): boolean {
if (mutedInstances.has(notif?.user?.host ?? '')) return true;
export function isUserFromMutedInstance(
notif: Packed<"Notification">,
mutedInstances: Set<string>,
): boolean {
if (mutedInstances.has(notif?.user?.host ?? "")) return true;
return false;
}

20
packages/backend/src/misc/is-mime-image.ts
View file

@ -1,8 +1,20 @@
import { FILE_TYPE_BROWSERSAFE } from '@/const.js';
import { FILE_TYPE_BROWSERSAFE } from "@/const.js";
const dictionary = {
'safe-file': FILE_TYPE_BROWSERSAFE,
'sharp-convertible-image': ['image/jpeg', 'image/png', 'image/gif', 'image/apng', 'image/vnd.mozilla.apng', 'image/webp', 'image/svg+xml', 'image/avif'],
"safe-file": FILE_TYPE_BROWSERSAFE,
"sharp-convertible-image": [
"image/jpeg",
"image/png",
"image/gif",
"image/apng",
"image/vnd.mozilla.apng",
"image/webp",
"image/svg+xml",
"image/avif",
],
};
export const isMimeImage = (mime: string, type: keyof typeof dictionary): boolean => dictionary[type].includes(mime);
export const isMimeImage = (
mime: string,
type: keyof typeof dictionary,
): boolean => dictionary[type].includes(mime);

11
packages/backend/src/misc/is-quote.ts
View file

@ -1,5 +1,10 @@
import { Note } from '@/models/entities/note.js';
import type { Note } from "@/models/entities/note.js";
export default function(note: Note): boolean {
return note.renoteId != null && (note.text != null || note.hasPoll || (note.fileIds != null && note.fileIds.length > 0));
export default function (note: Note): boolean {
return (
note.renoteId != null &&
(note.text != null ||
note.hasPoll ||
(note.fileIds != null && note.fileIds.length > 0))
);
}

5
packages/backend/src/misc/is-user-related.ts
View file

@ -1,7 +1,8 @@
export function isUserRelated(note: any, ids: Set<string>): boolean {
if (ids.has(note.userId)) return true; // note author is muted
if (note.mentions && note.mentions.some((user: string) => ids.has(user))) return true; // any of mentioned users are muted
if (note.reply && isUserRelated(note.reply, ids)) return true; // also check reply target
if (note.mentions?.some((user: string) => ids.has(user)))
return true; // any of mentioned users are muted
if (note.reply && isUserRelated(note.reply, ids)) return true; // also check reply target
if (note.renote && isUserRelated(note.renote, ids)) return true; // also check renote target
return false;
}

14
packages/backend/src/misc/keypair-store.ts
View file

@ -1,10 +1,12 @@
import { UserKeypairs } from '@/models/index.js';
import { User } from '@/models/entities/user.js';
import { UserKeypair } from '@/models/entities/user-keypair.js';
import { Cache } from './cache.js';
import { UserKeypairs } from "@/models/index.js";
import type { User } from "@/models/entities/user.js";
import type { UserKeypair } from "@/models/entities/user-keypair.js";
import { Cache } from "./cache.js";
const cache = new Cache<UserKeypair>(Infinity);
export async function getUserKeypair(userId: User['id']): Promise<UserKeypair> {
return await cache.fetch(userId, () => UserKeypairs.findOneByOrFail({ userId: userId }));
export async function getUserKeypair(userId: User["id"]): Promise<UserKeypair> {
return await cache.fetch(userId, () =>
UserKeypairs.findOneByOrFail({ userId: userId }),
);
}

884
packages/backend/src/misc/langmap.ts
View file

File diff suppressed because it is too large Load diff

2
packages/backend/src/misc/normalize-for-search.ts
View file

@ -2,5 +2,5 @@ export function normalizeForSearch(tag: string): string {
// ref.
// - https://analytics-note.xyz/programming/unicode-normalization-forms/
// - https://maku77.github.io/js/string/normalize.html
return tag.normalize('NFKC').toLowerCase();
return tag.normalize("NFKC").toLowerCase();
}

32
packages/backend/src/misc/nyaize.ts
View file

@ -1,15 +1,21 @@
export function nyaize(text: string): string {
return text
// ja-JP
.replace(/な/g, 'にゃ').replace(/ナ/g, 'ニャ').replace(/ﾅ/g, 'ﾆｬ')
// en-US
.replace(/(?<=n)a/gi, x => x === 'A' ? 'YA' : 'ya')
.replace(/(?<=morn)ing/gi, x => x === 'ING' ? 'YAN' : 'yan')
.replace(/(?<=every)one/gi, x => x === 'ONE' ? 'NYAN' : 'nyan')
// ko-KR
.replace(/[나-낳]/g, match => String.fromCharCode(
match.charCodeAt(0)! + '냐'.charCodeAt(0) - '나'.charCodeAt(0)
))
.replace(/(다$)|(다(?=\.))|(다(?= ))|(다(?=!))|(다(?=\?))/gm, '다냥')
.replace(/(야(?=\?))|(야$)|(야(?= ))/gm, '냥');
return (
text
// ja-JP
.replace(/な/g, "にゃ")
.replace(/ナ/g, "ニャ")
.replace(/ﾅ/g, "ﾆｬ")
// en-US
.replace(/(?<=n)a/gi, (x) => (x === "A" ? "YA" : "ya"))
.replace(/(?<=morn)ing/gi, (x) => (x === "ING" ? "YAN" : "yan"))
.replace(/(?<=every)one/gi, (x) => (x === "ONE" ? "NYAN" : "nyan"))
// ko-KR
.replace(/[나-낳]/g, (match) =>
String.fromCharCode(
match.charCodeAt(0)! + "냐".charCodeAt(0) - "나".charCodeAt(0),
),
)
.replace(/(다$)|(다(?=\.))|(다(?= ))|(다(?=!))|(다(?=\?))/gm, "다냥")
.replace(/(야(?=\?))|(야$)|(야(?= ))/gm, "냥")
);
}

122
packages/backend/src/misc/populate-emojis.ts
View file

@ -1,12 +1,12 @@
import { In, IsNull } from 'typeorm';
import { Emojis } from '@/models/index.js';
import { Emoji } from '@/models/entities/emoji.js';
import { Note } from '@/models/entities/note.js';
import { Cache } from './cache.js';
import { isSelfHost, toPunyNullable } from './convert-host.js';
import { decodeReaction } from './reaction-lib.js';
import config from '@/config/index.js';
import { query } from '@/prelude/url.js';
import { In, IsNull } from "typeorm";
import { Emojis } from "@/models/index.js";
import type { Emoji } from "@/models/entities/emoji.js";
import type { Note } from "@/models/entities/note.js";
import { Cache } from "./cache.js";
import { isSelfHost, toPunyNullable } from "./convert-host.js";
import { decodeReaction } from "./reaction-lib.js";
import config from "@/config/index.js";
import { query } from "@/prelude/url.js";
const cache = new Cache<Emoji | null>(1000 * 60 * 60 * 12);
@ -18,12 +18,19 @@ type PopulatedEmoji = {
url: string;
};
function normalizeHost(src: string | undefined, noteUserHost: string | null): string | null {
function normalizeHost(
src: string | undefined,
noteUserHost: string | null,
): string | null {
// クエリに使うホスト
let host = src === '.' ? null // .はローカルホスト (ここがマッチするのはリアクションのみ)
: src === undefined ? noteUserHost // ノートなどでホスト省略表記の場合はローカルホスト (ここがリアクションにマッチすることはない)
: isSelfHost(src) ? null // 自ホスト指定
: (src || noteUserHost); // 指定されたホスト || ノートなどの所有者のホスト (こっちがリアクションにマッチすることはない)
let host =
src === "."
? null // .はローカルホスト (ここがマッチするのはリアクションのみ)
: src === undefined
? noteUserHost // ノートなどでホスト省略表記の場合はローカルホスト (ここがリアクションにマッチすることはない)
: isSelfHost(src)
? null // 自ホスト指定
: src || noteUserHost; // 指定されたホスト || ノートなどの所有者のホスト (こっちがリアクションにマッチすることはない)
host = toPunyNullable(host);
@ -48,14 +55,18 @@ function parseEmojiStr(emojiName: string, noteUserHost: string | null) {
* @param noteUserHost ノートやユーザープロフィールの所有者のホスト
* @returns 絵文字情報, nullは未マッチを意味する
*/
export async function populateEmoji(emojiName: string, noteUserHost: string | null): Promise<PopulatedEmoji | null> {
export async function populateEmoji(
emojiName: string,
noteUserHost: string | null,
): Promise<PopulatedEmoji | null> {
const { name, host } = parseEmojiStr(emojiName, noteUserHost);
if (name == null) return null;
const queryOrNull = async () => (await Emojis.findOneBy({
name,
host: host ?? IsNull(),
})) || null;
const queryOrNull = async () =>
(await Emojis.findOneBy({
name,
host: host ?? IsNull(),
})) || null;
const emoji = await cache.fetch(`${name} ${host}`, queryOrNull);
@ -63,7 +74,11 @@ export async function populateEmoji(emojiName: string, noteUserHost: string | nu
const isLocal = emoji.host == null;
const emojiUrl = emoji.publicUrl || emoji.originalUrl; // || emoji.originalUrl してるのは後方互換性のため
const url = isLocal ? emojiUrl : `${config.url}/proxy/${encodeURIComponent((new URL(emojiUrl)).pathname)}?${query({ url: emojiUrl })}`;
const url = isLocal
? emojiUrl
: `${config.url}/proxy/${encodeURIComponent(
new URL(emojiUrl).pathname,
)}?${query({ url: emojiUrl })}`;
return {
name: emojiName,
@ -74,51 +89,76 @@ export async function populateEmoji(emojiName: string, noteUserHost: string | nu
/**
* 複数の添付用絵文字情報を解決する (キャシュ付き, 存在しないものは結果から除外される)
*/
export async function populateEmojis(emojiNames: string[], noteUserHost: string | null): Promise<PopulatedEmoji[]> {
const emojis = await Promise.all(emojiNames.map(x => populateEmoji(x, noteUserHost)));
export async function populateEmojis(
emojiNames: string[],
noteUserHost: string | null,
): Promise<PopulatedEmoji[]> {
const emojis = await Promise.all(
emojiNames.map((x) => populateEmoji(x, noteUserHost)),
);
return emojis.filter((x): x is PopulatedEmoji => x != null);
}
export function aggregateNoteEmojis(notes: Note[]) {
let emojis: { name: string | null; host: string | null; }[] = [];
let emojis: { name: string | null; host: string | null }[] = [];
for (const note of notes) {
emojis = emojis.concat(note.emojis
.map(e => parseEmojiStr(e, note.userHost)));
emojis = emojis.concat(
note.emojis.map((e) => parseEmojiStr(e, note.userHost)),
);
if (note.renote) {
emojis = emojis.concat(note.renote.emojis
.map(e => parseEmojiStr(e, note.renote!.userHost)));
emojis = emojis.concat(
note.renote.emojis.map((e) => parseEmojiStr(e, note.renote!.userHost)),
);
if (note.renote.user) {
emojis = emojis.concat(note.renote.user.emojis
.map(e => parseEmojiStr(e, note.renote!.userHost)));
emojis = emojis.concat(
note.renote.user.emojis.map((e) =>
parseEmojiStr(e, note.renote!.userHost),
),
);
}
}
const customReactions = Object.keys(note.reactions).map(x => decodeReaction(x)).filter(x => x.name != null) as typeof emojis;
const customReactions = Object.keys(note.reactions)
.map((x) => decodeReaction(x))
.filter((x) => x.name != null) as typeof emojis;
emojis = emojis.concat(customReactions);
if (note.user) {
emojis = emojis.concat(note.user.emojis
.map(e => parseEmojiStr(e, note.userHost)));
emojis = emojis.concat(
note.user.emojis.map((e) => parseEmojiStr(e, note.userHost)),
);
}
}
return emojis.filter(x => x.name != null) as { name: string; host: string | null; }[];
return emojis.filter((x) => x.name != null) as {
name: string;
host: string | null;
}[];
}
/**
* 与えられた絵文字のリストをデータベースから取得し、キャッシュに追加します
*/
export async function prefetchEmojis(emojis: { name: string; host: string | null; }[]): Promise<void> {
const notCachedEmojis = emojis.filter(emoji => cache.get(`${emoji.name} ${emoji.host}`) == null);
export async function prefetchEmojis(
emojis: { name: string; host: string | null }[],
): Promise<void> {
const notCachedEmojis = emojis.filter(
(emoji) => cache.get(`${emoji.name} ${emoji.host}`) == null,
);
const emojisQuery: any[] = [];
const hosts = new Set(notCachedEmojis.map(e => e.host));
const hosts = new Set(notCachedEmojis.map((e) => e.host));
for (const host of hosts) {
emojisQuery.push({
name: In(notCachedEmojis.filter(e => e.host === host).map(e => e.name)),
name: In(
notCachedEmojis.filter((e) => e.host === host).map((e) => e.name),
),
host: host ?? IsNull(),
});
}
const _emojis = emojisQuery.length > 0 ? await Emojis.find({
where: emojisQuery,
select: ['name', 'host', 'originalUrl', 'publicUrl'],
}) : [];
const _emojis =
emojisQuery.length > 0
? await Emojis.find({
where: emojisQuery,
select: ["name", "host", "originalUrl", "publicUrl"],
})
: [];
for (const emoji of _emojis) {
cache.set(`${emoji.name} ${emoji.host}`, emoji);
}

39
packages/backend/src/misc/reaction-lib.ts
View file

@ -1,22 +1,22 @@
/* eslint-disable key-spacing */
import { emojiRegex } from './emoji-regex.js';
import { fetchMeta } from './fetch-meta.js';
import { Emojis } from '@/models/index.js';
import { toPunyNullable } from './convert-host.js';
import { IsNull } from 'typeorm';
import { emojiRegex } from "./emoji-regex.js";
import { fetchMeta } from "./fetch-meta.js";
import { Emojis } from "@/models/index.js";
import { toPunyNullable } from "./convert-host.js";
import { IsNull } from "typeorm";
const legacies: Record<string, string> = {
'like': '👍',
'love': '❤️', // ここに記述する場合は異体字セレクタを入れない <- not that good because modern browsers just display it as the red heart so just convert it to it to not end up with two seperate reactions of "the same emoji" for the user
'laugh': '😆',
'hmm': '🤔',
'surprise': '😮',
'congrats': '🎉',
'angry': '💢',
'confused': '😥',
'rip': '😇',
'pudding': '🍮',
'star': '⭐',
like: "👍",
love: "❤️", // ここに記述する場合は異体字セレクタを入れない <- not that good because modern browsers just display it as the red heart so just convert it to it to not end up with two seperate reactions of "the same emoji" for the user
laugh: "😆",
hmm: "🤔",
surprise: "😮",
congrats: "🎉",
angry: "💢",
confused: "😥",
rip: "😇",
pudding: "🍮",
star: "⭐",
};
export async function getFallbackReaction(): Promise<string> {
@ -54,7 +54,10 @@ export function convertLegacyReactions(reactions: Record<string, number>) {
return _reactions2;
}
export async function toDbReaction(reaction?: string | null, reacterHost?: string | null): Promise<string> {
export async function toDbReaction(
reaction?: string | null,
reacterHost?: string | null,
): Promise<string> {
if (reaction == null) return await getFallbackReaction();
reacterHost = toPunyNullable(reacterHost);
@ -111,7 +114,7 @@ export function decodeReaction(str: string): DecodedReaction {
const host = custom[2] || null;
return {
reaction: `:${name}@${host || '.'}:`, // ローカル分は@以降を省略するのではなく.にする
reaction: `:${name}@${host || "."}:`, // ローカル分は@以降を省略するのではなく.にする
name,
host,
};

216
packages/backend/src/misc/schema.ts
View file

@ -6,29 +6,29 @@ import {
packedMeDetailedSchema,
packedUserDetailedSchema,
packedUserSchema,
} from '@/models/schema/user.js';
import { packedNoteSchema } from '@/models/schema/note.js';
import { packedUserListSchema } from '@/models/schema/user-list.js';
import { packedAppSchema } from '@/models/schema/app.js';
import { packedMessagingMessageSchema } from '@/models/schema/messaging-message.js';
import { packedNotificationSchema } from '@/models/schema/notification.js';
import { packedDriveFileSchema } from '@/models/schema/drive-file.js';
import { packedDriveFolderSchema } from '@/models/schema/drive-folder.js';
import { packedFollowingSchema } from '@/models/schema/following.js';
import { packedMutingSchema } from '@/models/schema/muting.js';
import { packedBlockingSchema } from '@/models/schema/blocking.js';
import { packedNoteReactionSchema } from '@/models/schema/note-reaction.js';
import { packedHashtagSchema } from '@/models/schema/hashtag.js';
import { packedPageSchema } from '@/models/schema/page.js';
import { packedUserGroupSchema } from '@/models/schema/user-group.js';
import { packedNoteFavoriteSchema } from '@/models/schema/note-favorite.js';
import { packedChannelSchema } from '@/models/schema/channel.js';
import { packedAntennaSchema } from '@/models/schema/antenna.js';
import { packedClipSchema } from '@/models/schema/clip.js';
import { packedFederationInstanceSchema } from '@/models/schema/federation-instance.js';
import { packedQueueCountSchema } from '@/models/schema/queue.js';
import { packedGalleryPostSchema } from '@/models/schema/gallery-post.js';
import { packedEmojiSchema } from '@/models/schema/emoji.js';
} from "@/models/schema/user.js";
import { packedNoteSchema } from "@/models/schema/note.js";
import { packedUserListSchema } from "@/models/schema/user-list.js";
import { packedAppSchema } from "@/models/schema/app.js";
import { packedMessagingMessageSchema } from "@/models/schema/messaging-message.js";
import { packedNotificationSchema } from "@/models/schema/notification.js";
import { packedDriveFileSchema } from "@/models/schema/drive-file.js";
import { packedDriveFolderSchema } from "@/models/schema/drive-folder.js";
import { packedFollowingSchema } from "@/models/schema/following.js";
import { packedMutingSchema } from "@/models/schema/muting.js";
import { packedBlockingSchema } from "@/models/schema/blocking.js";
import { packedNoteReactionSchema } from "@/models/schema/note-reaction.js";
import { packedHashtagSchema } from "@/models/schema/hashtag.js";
import { packedPageSchema } from "@/models/schema/page.js";
import { packedUserGroupSchema } from "@/models/schema/user-group.js";
import { packedNoteFavoriteSchema } from "@/models/schema/note-favorite.js";
import { packedChannelSchema } from "@/models/schema/channel.js";
import { packedAntennaSchema } from "@/models/schema/antenna.js";
import { packedClipSchema } from "@/models/schema/clip.js";
import { packedFederationInstanceSchema } from "@/models/schema/federation-instance.js";
import { packedQueueCountSchema } from "@/models/schema/queue.js";
import { packedGalleryPostSchema } from "@/models/schema/gallery-post.js";
import { packedEmojiSchema } from "@/models/schema/emoji.js";
export const refs = {
UserLite: packedUserLiteSchema,
@ -65,23 +65,37 @@ export const refs = {
export type Packed<x extends keyof typeof refs> = SchemaType<typeof refs[x]>;
type TypeStringef = 'null' | 'boolean' | 'integer' | 'number' | 'string' | 'array' | 'object' | 'any';
type StringDefToType<T extends TypeStringef> =
T extends 'null' ? null :
T extends 'boolean' ? boolean :
T extends 'integer' ? number :
T extends 'number' ? number :
T extends 'string' ? string | Date :
T extends 'array' ? ReadonlyArray<any> :
T extends 'object' ? Record<string, any> :
any;
type TypeStringef =
| "null"
| "boolean"
| "integer"
| "number"
| "string"
| "array"
| "object"
| "any";
type StringDefToType<T extends TypeStringef> = T extends "null"
? null
: T extends "boolean"
? boolean
: T extends "integer"
? number
: T extends "number"
? number
: T extends "string"
? string | Date
: T extends "array"
? ReadonlyArray<any>
: T extends "object"
? Record<string, any>
: any;
// https://swagger.io/specification/?sbsearch=optional#schema-object
type OfSchema = {
readonly anyOf?: ReadonlyArray<Schema>;
readonly oneOf?: ReadonlyArray<Schema>;
readonly allOf?: ReadonlyArray<Schema>;
}
};
export interface Schema extends OfSchema {
readonly type?: TypeStringef;
@ -89,13 +103,17 @@ export interface Schema extends OfSchema {
readonly optional?: boolean;
readonly items?: Schema;
readonly properties?: Obj;
readonly required?: ReadonlyArray<Extract<keyof NonNullable<this['properties']>, string>>;
readonly required?: ReadonlyArray<
Extract<keyof NonNullable<this["properties"]>, string>
>;
readonly description?: string;
readonly example?: any;
readonly format?: string;
readonly ref?: keyof typeof refs;
readonly enum?: ReadonlyArray<string>;
readonly default?: (this['type'] extends TypeStringef ? StringDefToType<this['type']> : any) | null;
readonly default?:
| (this["type"] extends TypeStringef ? StringDefToType<this["type"]> : any)
| null;
readonly maxLength?: number;
readonly minLength?: number;
readonly maximum?: number;
@ -104,12 +122,18 @@ export interface Schema extends OfSchema {
}
type RequiredPropertyNames<s extends Obj> = {
[K in keyof s]:
// K is not optional
s[K]['optional'] extends false ? K :
// K has default value
s[K]['default'] extends null | string | number | boolean | Record<string, unknown> ? K :
never
[K in keyof s]: // K is not optional
s[K]["optional"] extends false
? K
: // K has default value
s[K]["default"] extends
| null
| string
| number
| boolean
| Record<string, unknown>
? K
: never;
}[keyof s];
export type Obj = Record<string, Schema>;
@ -117,56 +141,80 @@ export type Obj = Record<string, Schema>;
// https://github.com/misskey-dev/misskey/issues/8535
// To avoid excessive stack depth error,
// deceive TypeScript with UnionToIntersection (or more precisely, `infer` expression within it).
export type ObjType<s extends Obj, RequiredProps extends keyof s> =
UnionToIntersection<
{ -readonly [R in RequiredPropertyNames<s>]-?: SchemaType<s[R]> } &
{ -readonly [R in RequiredProps]-?: SchemaType<s[R]> } &
{ -readonly [P in keyof s]?: SchemaType<s[P]> }
>;
export type ObjType<
s extends Obj,
RequiredProps extends keyof s,
> = UnionToIntersection<
{
-readonly [R in RequiredPropertyNames<s>]-?: SchemaType<s[R]>;
} & {
-readonly [R in RequiredProps]-?: SchemaType<s[R]>;
} & {
-readonly [P in keyof s]?: SchemaType<s[P]>;
}
>;
type NullOrUndefined<p extends Schema, T> =
| (p['nullable'] extends true ? null : never)
| (p['optional'] extends true ? undefined : never)
| (p["nullable"] extends true ? null : never)
| (p["optional"] extends true ? undefined : never)
| T;
// https://stackoverflow.com/questions/54938141/typescript-convert-union-to-intersection
// Get intersection from union
type UnionToIntersection<U> = (U extends any ? (k: U) => void : never) extends ((k: infer I) => void) ? I : never;
// Get intersection from union
type UnionToIntersection<U> = (U extends any ? (k: U) => void : never) extends (
k: infer I,
) => void
? I
: never;
// https://github.com/misskey-dev/misskey/pull/8144#discussion_r785287552
// To get union, we use `Foo extends any ? Hoge<Foo> : never`
type UnionSchemaType<a extends readonly any[], X extends Schema = a[number]> = X extends any ? SchemaType<X> : never;
type ArrayUnion<T> = T extends any ? Array<T> : never;
type UnionSchemaType<
a extends readonly any[],
X extends Schema = a[number],
> = X extends any ? SchemaType<X> : never;
type ArrayUnion<T> = T extends any ? Array<T> : never;
export type SchemaTypeDef<p extends Schema> =
p['type'] extends 'null' ? null :
p['type'] extends 'integer' ? number :
p['type'] extends 'number' ? number :
p['type'] extends 'string' ? (
p['enum'] extends readonly string[] ?
p['enum'][number] :
p['format'] extends 'date-time' ? string : // Dateにする？？
string
) :
p['type'] extends 'boolean' ? boolean :
p['type'] extends 'object' ? (
p['ref'] extends keyof typeof refs ? Packed<p['ref']> :
p['properties'] extends NonNullable<Obj> ? ObjType<p['properties'], NonNullable<p['required']>[number]> :
p['anyOf'] extends ReadonlyArray<Schema> ? UnionSchemaType<p['anyOf']> & Partial<UnionToIntersection<UnionSchemaType<p['anyOf']>>> :
p['allOf'] extends ReadonlyArray<Schema> ? UnionToIntersection<UnionSchemaType<p['allOf']>> :
any
) :
p['type'] extends 'array' ? (
p['items'] extends OfSchema ? (
p['items']['anyOf'] extends ReadonlyArray<Schema> ? UnionSchemaType<NonNullable<p['items']['anyOf']>>[] :
p['items']['oneOf'] extends ReadonlyArray<Schema> ? ArrayUnion<UnionSchemaType<NonNullable<p['items']['oneOf']>>> :
p['items']['allOf'] extends ReadonlyArray<Schema> ? UnionToIntersection<UnionSchemaType<NonNullable<p['items']['allOf']>>>[] :
never
) :
p['items'] extends NonNullable<Schema> ? SchemaTypeDef<p['items']>[] :
any[]
) :
p['oneOf'] extends ReadonlyArray<Schema> ? UnionSchemaType<p['oneOf']> :
any;
export type SchemaTypeDef<p extends Schema> = p["type"] extends "null"
? null
: p["type"] extends "integer"
? number
: p["type"] extends "number"
? number
: p["type"] extends "string"
? p["enum"] extends readonly string[]
? p["enum"][number]
: p["format"] extends "date-time"
? string
: // Dateにする？？
string
: p["type"] extends "boolean"
? boolean
: p["type"] extends "object"
? p["ref"] extends keyof typeof refs
? Packed<p["ref"]>
: p["properties"] extends NonNullable<Obj>
? ObjType<p["properties"], NonNullable<p["required"]>[number]>
: p["anyOf"] extends ReadonlyArray<Schema>
? UnionSchemaType<p["anyOf"]> &
Partial<UnionToIntersection<UnionSchemaType<p["anyOf"]>>>
: p["allOf"] extends ReadonlyArray<Schema>
? UnionToIntersection<UnionSchemaType<p["allOf"]>>
: any
: p["type"] extends "array"
? p["items"] extends OfSchema
? p["items"]["anyOf"] extends ReadonlyArray<Schema>
? UnionSchemaType<NonNullable<p["items"]["anyOf"]>>[]
: p["items"]["oneOf"] extends ReadonlyArray<Schema>
? ArrayUnion<UnionSchemaType<NonNullable<p["items"]["oneOf"]>>>
: p["items"]["allOf"] extends ReadonlyArray<Schema>
? UnionToIntersection<UnionSchemaType<NonNullable<p["items"]["allOf"]>>>[]
: never
: p["items"] extends NonNullable<Schema>
? SchemaTypeDef<p["items"]>[]
: any[]
: p["oneOf"] extends ReadonlyArray<Schema>
? UnionSchemaType<p["oneOf"]>
: any;
export type SchemaType<p extends Schema> = NullOrUndefined<p, SchemaTypeDef<p>>;

13
packages/backend/src/misc/secure-rndstr.ts
View file

@ -1,16 +1,19 @@
import * as crypto from 'node:crypto';
import * as crypto from "node:crypto";
const L_CHARS = '0123456789abcdefghijklmnopqrstuvwxyz';
const LU_CHARS = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ';
const L_CHARS = "0123456789abcdefghijklmnopqrstuvwxyz";
const LU_CHARS =
"0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
export function secureRndstr(length = 32, useLU = true): string {
const chars = useLU ? LU_CHARS : L_CHARS;
const chars_len = chars.length;
let str = '';
let str = "";
for (let i = 0; i < length; i++) {
let rand = Math.floor((crypto.randomBytes(1).readUInt8(0) / 0xFF) * chars_len);
let rand = Math.floor(
(crypto.randomBytes(1).readUInt8(0) / 0xff) * chars_len,
);
if (rand === chars_len) {
rand = chars_len - 1;
}

17
packages/backend/src/misc/should-block-instance.ts
View file

@ -1,6 +1,6 @@
import { fetchMeta } from '@/misc/fetch-meta.js';
import { Instance } from '@/models/entities/instance.js';
import { Meta } from '@/models/entities/meta.js';
import { fetchMeta } from "@/misc/fetch-meta.js";
import type { Instance } from "@/models/entities/instance.js";
import type { Meta } from "@/models/entities/meta.js";
/**
* Returns whether a specific host (punycoded) should be blocked.
@ -9,7 +9,12 @@ import { Meta } from '@/models/entities/meta.js';
* @param meta a resolved Meta table
* @returns whether the given host should be blocked
*/
export async function shouldBlockInstance(host: Instance['host'], meta?: Meta): Promise<boolean> {
const { blockedHosts } = meta ?? await fetchMeta();
return blockedHosts.some(blockedHost => host === blockedHost || host.endsWith('.' + blockedHost));
export async function shouldBlockInstance(
host: Instance["host"],
meta?: Meta,
): Promise<boolean> {
const { blockedHosts } = meta ?? (await fetchMeta());
return blockedHosts.some(
(blockedHost) => host === blockedHost || host.endsWith(`.${blockedHost}`),
);
}

14
packages/backend/src/misc/show-machine-info.ts
View file

@ -1,13 +1,17 @@
import * as os from 'node:os';
import sysUtils from 'systeminformation';
import Logger from '@/services/logger.js';
import * as os from "node:os";
import sysUtils from "systeminformation";
import type Logger from "@/services/logger.js";
export async function showMachineInfo(parentLogger: Logger) {
const logger = parentLogger.createSubLogger('machine');
const logger = parentLogger.createSubLogger("machine");
logger.debug(`Hostname: ${os.hostname()}`);
logger.debug(`Platform: ${process.platform} Arch: ${process.arch}`);
const mem = await sysUtils.mem();
const totalmem = (mem.total / 1024 / 1024 / 1024).toFixed(1);
const availmem = (mem.available / 1024 / 1024 / 1024).toFixed(1);
logger.debug(`CPU: ${os.cpus().length} core MEM: ${totalmem}GB (available: ${availmem}GB)`);
logger.debug(
`CPU: ${
os.cpus().length
} core MEM: ${totalmem}GB (available: ${availmem}GB)`,
);
}

46
packages/backend/src/misc/skipped-instances.ts
View file

@ -1,9 +1,9 @@
import { Brackets } from 'typeorm';
import { fetchMeta } from '@/misc/fetch-meta.js';
import { Instances } from '@/models/index.js';
import type { Instance } from '@/models/entities/instance.js';
import { DAY } from '@/const.js';
import { shouldBlockInstance } from './should-block-instance.js';
import { Brackets } from "typeorm";
import { fetchMeta } from "@/misc/fetch-meta.js";
import { Instances } from "@/models/index.js";
import type { Instance } from "@/models/entities/instance.js";
import { DAY } from "@/const.js";
import { shouldBlockInstance } from "./should-block-instance.js";
// Threshold from last contact after which an instance will be considered
// "dead" and should no longer get activities delivered to it.
@ -11,32 +11,38 @@ const deadThreshold = 7 * DAY;
/**
* Returns the subset of hosts which should be skipped.
*
*
* @param hosts array of punycoded instance hosts
* @returns array of punycoed instance hosts that should be skipped (subset of hosts parameter)
*/
export async function skippedInstances(hosts: Instance['host'][]): Promise<Instance['host'][]> {
export async function skippedInstances(
hosts: Instance["host"][],
): Promise<Instance["host"][]> {
// first check for blocked instances since that info may already be in memory
const meta = await fetchMeta();
const shouldSkip = await Promise.all(hosts.map(host => shouldBlockInstance(host, meta)));
const shouldSkip = await Promise.all(
hosts.map((host) => shouldBlockInstance(host, meta)),
);
const skipped = hosts.filter((_, i) => shouldSkip[i]);
// if possible return early and skip accessing the database
if (skipped.length === hosts.length) return hosts;
if (skipped.length === hosts.length) return hosts;
const deadTime = new Date(Date.now() - deadThreshold);
return skipped.concat(
await Instances.createQueryBuilder('instance')
.where('instance.host in (:...hosts)', {
await Instances.createQueryBuilder("instance")
.where("instance.host in (:...hosts)", {
// don't check hosts again that we already know are suspended
// also avoids adding duplicates to the list
hosts: hosts.filter(host => !skipped.includes(host)),
hosts: hosts.filter((host) => !skipped.includes(host)),
})
.andWhere(new Brackets(qb => { qb
.where('instance.isSuspended');
}))
.select('host')
.andWhere(
new Brackets((qb) => {
qb.where("instance.isSuspended");
}),
)
.select("host")
.getRawMany(),
);
}
@ -49,7 +55,9 @@ export async function skippedInstances(hosts: Instance['host'][]): Promise<Insta
* @param host punycoded instance host
* @returns whether the given host should be skipped
*/
export async function shouldSkipInstance(host: Instance['host']): Promise<boolean> {
export async function shouldSkipInstance(
host: Instance["host"],
): Promise<boolean> {
const skipped = await skippedInstances([host]);
return skipped.length > 0;
}

12
packages/backend/src/misc/truncate.ts
View file

@ -1,8 +1,14 @@
import { substring } from 'stringz';
import { substring } from "stringz";
export function truncate(input: string, size: number): string;
export function truncate(input: string | undefined, size: number): string | undefined;
export function truncate(input: string | undefined, size: number): string | undefined {
export function truncate(
input: string | undefined,
size: number,
): string | undefined;
export function truncate(
input: string | undefined,
size: number,
): string | undefined {
if (!input) {
return input;
} else {

22
packages/backend/src/misc/webhook-cache.ts
View file

@ -1,6 +1,6 @@
import { Webhooks } from '@/models/index.js';
import { Webhook } from '@/models/entities/webhook.js';
import { subscriber } from '@/db/redis.js';
import { Webhooks } from "@/models/index.js";
import type { Webhook } from "@/models/entities/webhook.js";
import { subscriber } from "@/db/redis.js";
let webhooksFetched = false;
let webhooks: Webhook[] = [];
@ -16,31 +16,31 @@ export async function getActiveWebhooks() {
return webhooks;
}
subscriber.on('message', async (_, data) => {
subscriber.on("message", async (_, data) => {
const obj = JSON.parse(data);
if (obj.channel === 'internal') {
if (obj.channel === "internal") {
const { type, body } = obj.message;
switch (type) {
case 'webhookCreated':
case "webhookCreated":
if (body.active) {
webhooks.push(body);
}
break;
case 'webhookUpdated':
case "webhookUpdated":
if (body.active) {
const i = webhooks.findIndex(a => a.id === body.id);
const i = webhooks.findIndex((a) => a.id === body.id);
if (i > -1) {
webhooks[i] = body;
} else {
webhooks.push(body);
}
} else {
webhooks = webhooks.filter(a => a.id !== body.id);
webhooks = webhooks.filter((a) => a.id !== body.id);
}
break;
case 'webhookDeleted':
webhooks = webhooks.filter(a => a.id !== body.id);
case "webhookDeleted":
webhooks = webhooks.filter((a) => a.id !== body.id);
break;
default:
break;

19
packages/backend/src/models/entities/abuse-user-report.ts
View file

@ -1,6 +1,13 @@
import { PrimaryColumn, Entity, Index, JoinColumn, Column, ManyToOne } from 'typeorm';
import { User } from './user.js';
import { id } from '../id.js';
import {
PrimaryColumn,
Entity,
Index,
JoinColumn,
Column,
ManyToOne,
} from "typeorm";
import { User } from "./user.js";
import { id } from "../id.js";
@Entity()
export class AbuseUserReport {
@ -15,7 +22,7 @@ export class AbuseUserReport {
@Index()
@Column(id())
public targetUserId: User['id'];
public targetUserId: User["id"];
@ManyToOne(type => User, {
onDelete: 'CASCADE',
@ -25,7 +32,7 @@ export class AbuseUserReport {
@Index()
@Column(id())
public reporterId: User['id'];
public reporterId: User["id"];
@ManyToOne(type => User, {
onDelete: 'CASCADE',
@ -37,7 +44,7 @@ export class AbuseUserReport {
...id(),
nullable: true,
})
public assigneeId: User['id'] | null;
public assigneeId: User["id"] | null;
@ManyToOne(type => User, {
onDelete: 'SET NULL',

19
packages/backend/src/models/entities/access-token.ts
View file

@ -1,7 +1,14 @@
import { Entity, PrimaryColumn, Index, Column, ManyToOne, JoinColumn } from 'typeorm';
import { User } from './user.js';
import { App } from './app.js';
import { id } from '../id.js';
import {
Entity,
PrimaryColumn,
Index,
Column,
ManyToOne,
JoinColumn,
} from "typeorm";
import { User } from "./user.js";
import { App } from "./app.js";
import { id } from "../id.js";
@Entity()
export class AccessToken {
@ -39,7 +46,7 @@ export class AccessToken {
@Index()
@Column(id())
public userId: User['id'];
public userId: User["id"];
@ManyToOne(type => User, {
onDelete: 'CASCADE',
@ -51,7 +58,7 @@ export class AccessToken {
...id(),
nullable: true,
})
public appId: App['id'] | null;
public appId: App["id"] | null;
@ManyToOne(type => App, {
onDelete: 'CASCADE',

4
packages/backend/src/models/entities/ad.ts
View file

@ -1,5 +1,5 @@
import { Entity, Index, Column, PrimaryColumn } from 'typeorm';
import { id } from '../id.js';
import { Entity, Index, Column, PrimaryColumn } from "typeorm";
import { id } from "../id.js";
@Entity()
export class Ad {

19
packages/backend/src/models/entities/announcement-read.ts
View file

@ -1,7 +1,14 @@
import { PrimaryColumn, Entity, Index, JoinColumn, Column, ManyToOne } from 'typeorm';
import { User } from './user.js';
import { Announcement } from './announcement.js';
import { id } from '../id.js';
import {
PrimaryColumn,
Entity,
Index,
JoinColumn,
Column,
ManyToOne,
} from "typeorm";
import { User } from "./user.js";
import { Announcement } from "./announcement.js";
import { id } from "../id.js";
@Entity()
@Index(['userId', 'announcementId'], { unique: true })
@ -16,7 +23,7 @@ export class AnnouncementRead {
@Index()
@Column(id())
public userId: User['id'];
public userId: User["id"];
@ManyToOne(type => User, {
onDelete: 'CASCADE',
@ -26,7 +33,7 @@ export class AnnouncementRead {
@Index()
@Column(id())
public announcementId: Announcement['id'];
public announcementId: Announcement["id"];
@ManyToOne(type => Announcement, {
onDelete: 'CASCADE',

4
packages/backend/src/models/entities/announcement.ts
View file

@ -1,5 +1,5 @@
import { Entity, Index, Column, PrimaryColumn } from 'typeorm';
import { id } from '../id.js';
import { Entity, Index, Column, PrimaryColumn } from "typeorm";
import { id } from "../id.js";
@Entity()
export class Announcement {

19
packages/backend/src/models/entities/antenna-note.ts
View file

@ -1,7 +1,14 @@
import { Entity, Index, JoinColumn, Column, ManyToOne, PrimaryColumn } from 'typeorm';
import { Note } from './note.js';
import { Antenna } from './antenna.js';
import { id } from '../id.js';
import {
Entity,
Index,
JoinColumn,
Column,
ManyToOne,
PrimaryColumn,
} from "typeorm";
import { Note } from "./note.js";
import { Antenna } from "./antenna.js";
import { id } from "../id.js";
@Entity()
@Index(['noteId', 'antennaId'], { unique: true })
@ -14,7 +21,7 @@ export class AntennaNote {
...id(),
comment: 'The note ID.',
})
public noteId: Note['id'];
public noteId: Note["id"];
@ManyToOne(type => Note, {
onDelete: 'CASCADE',
@ -27,7 +34,7 @@ export class AntennaNote {
...id(),
comment: 'The antenna ID.',
})
public antennaId: Antenna['id'];
public antennaId: Antenna["id"];
@ManyToOne(type => Antenna, {
onDelete: 'CASCADE',

25
packages/backend/src/models/entities/antenna.ts
View file

@ -1,8 +1,15 @@
import { PrimaryColumn, Entity, Index, JoinColumn, Column, ManyToOne } from 'typeorm';
import { User } from './user.js';
import { id } from '../id.js';
import { UserList } from './user-list.js';
import { UserGroupJoining } from './user-group-joining.js';
import {
PrimaryColumn,
Entity,
Index,
JoinColumn,
Column,
ManyToOne,
} from "typeorm";
import { User } from "./user.js";
import { id } from "../id.js";
import { UserList } from "./user-list.js";
import { UserGroupJoining } from "./user-group-joining.js";
@Entity()
export class Antenna {
@ -19,7 +26,7 @@ export class Antenna {
...id(),
comment: 'The owner ID.',
})
public userId: User['id'];
public userId: User["id"];
@ManyToOne(type => User, {
onDelete: 'CASCADE',
@ -34,13 +41,13 @@ export class Antenna {
public name: string;
@Column('enum', { enum: ['home', 'all', 'users', 'list', 'group'] })
public src: 'home' | 'all' | 'users' | 'list' | 'group';
public src: "home" | "all" | "users" | "list" | "group";
@Column({
...id(),
nullable: true,
})
public userListId: UserList['id'] | null;
public userListId: UserList["id"] | null;
@ManyToOne(type => UserList, {
onDelete: 'CASCADE',
@ -52,7 +59,7 @@ export class Antenna {
...id(),
nullable: true,
})
public userGroupJoiningId: UserGroupJoining['id'] | null;
public userGroupJoiningId: UserGroupJoining["id"] | null;
@ManyToOne(type => UserGroupJoining, {
onDelete: 'CASCADE',

8
packages/backend/src/models/entities/app.ts
View file

@ -1,6 +1,6 @@
import { Entity, PrimaryColumn, Column, Index, ManyToOne } from 'typeorm';
import { User } from './user.js';
import { id } from '../id.js';
import { Entity, PrimaryColumn, Column, Index, ManyToOne } from "typeorm";
import { User } from "./user.js";
import { id } from "../id.js";
@Entity()
export class App {
@ -19,7 +19,7 @@ export class App {
nullable: true,
comment: 'The owner ID.',
})
public userId: User['id'] | null;
public userId: User["id"] | null;
@ManyToOne(type => User, {
onDelete: 'SET NULL',

15
packages/backend/src/models/entities/attestation-challenge.ts
View file

@ -1,6 +1,13 @@
import { PrimaryColumn, Entity, JoinColumn, Column, ManyToOne, Index } from 'typeorm';
import { User } from './user.js';
import { id } from '../id.js';
import {
PrimaryColumn,
Entity,
JoinColumn,
Column,
ManyToOne,
Index,
} from "typeorm";
import { User } from "./user.js";
import { id } from "../id.js";
@Entity()
export class AttestationChallenge {
@ -9,7 +16,7 @@ export class AttestationChallenge {
@Index()
@PrimaryColumn(id())
public userId: User['id'];
public userId: User["id"];
@ManyToOne(type => User, {
onDelete: 'CASCADE',

19
packages/backend/src/models/entities/auth-session.ts
View file

@ -1,7 +1,14 @@
import { Entity, PrimaryColumn, Index, Column, ManyToOne, JoinColumn } from 'typeorm';
import { User } from './user.js';
import { App } from './app.js';
import { id } from '../id.js';
import {
Entity,
PrimaryColumn,
Index,
Column,
ManyToOne,
JoinColumn,
} from "typeorm";
import { User } from "./user.js";
import { App } from "./app.js";
import { id } from "../id.js";
@Entity()
export class AuthSession {
@ -23,7 +30,7 @@ export class AuthSession {
...id(),
nullable: true,
})
public userId: User['id'] | null;
public userId: User["id"] | null;
@ManyToOne(type => User, {
onDelete: 'CASCADE',
@ -33,7 +40,7 @@ export class AuthSession {
public user: User | null;
@Column(id())
public appId: App['id'];
public appId: App["id"];
@ManyToOne(type => App, {
onDelete: 'CASCADE',

17
packages/backend/src/models/entities/blocking.ts
View file

@ -1,6 +1,13 @@
import { PrimaryColumn, Entity, Index, JoinColumn, Column, ManyToOne } from 'typeorm';
import { User } from './user.js';
import { id } from '../id.js';
import {
PrimaryColumn,
Entity,
Index,
JoinColumn,
Column,
ManyToOne,
} from "typeorm";
import { User } from "./user.js";
import { id } from "../id.js";
@Entity()
@Index(['blockerId', 'blockeeId'], { unique: true })
@ -19,7 +26,7 @@ export class Blocking {
...id(),
comment: 'The blockee user ID.',
})
public blockeeId: User['id'];
public blockeeId: User["id"];
@ManyToOne(type => User, {
onDelete: 'CASCADE',
@ -32,7 +39,7 @@ export class Blocking {
...id(),
comment: 'The blocker user ID.',
})
public blockerId: User['id'];
public blockerId: User["id"];
@ManyToOne(type => User, {
onDelete: 'CASCADE',

19
packages/backend/src/models/entities/channel-following.ts
View file

@ -1,7 +1,14 @@
import { PrimaryColumn, Entity, Index, JoinColumn, Column, ManyToOne } from 'typeorm';
import { User } from './user.js';
import { id } from '../id.js';
import { Channel } from './channel.js';
import {
PrimaryColumn,
Entity,
Index,
JoinColumn,
Column,
ManyToOne,
} from "typeorm";
import { User } from "./user.js";
import { id } from "../id.js";
import { Channel } from "./channel.js";
@Entity()
@Index(['followerId', 'followeeId'], { unique: true })
@ -20,7 +27,7 @@ export class ChannelFollowing {
...id(),
comment: 'The followee channel ID.',
})
public followeeId: Channel['id'];
public followeeId: Channel["id"];
@ManyToOne(type => Channel, {
onDelete: 'CASCADE',
@ -33,7 +40,7 @@ export class ChannelFollowing {
...id(),
comment: 'The follower user ID.',
})
public followerId: User['id'];
public followerId: User["id"];
@ManyToOne(type => User, {
onDelete: 'CASCADE',

19
packages/backend/src/models/entities/channel-note-pining.ts
View file

@ -1,7 +1,14 @@
import { PrimaryColumn, Entity, Index, JoinColumn, Column, ManyToOne } from 'typeorm';
import { Note } from './note.js';
import { Channel } from './channel.js';
import { id } from '../id.js';
import {
PrimaryColumn,
Entity,
Index,
JoinColumn,
Column,
ManyToOne,
} from "typeorm";
import { Note } from "./note.js";
import { Channel } from "./channel.js";
import { id } from "../id.js";
@Entity()
@Index(['channelId', 'noteId'], { unique: true })
@ -16,7 +23,7 @@ export class ChannelNotePining {
@Index()
@Column(id())
public channelId: Channel['id'];
public channelId: Channel["id"];
@ManyToOne(type => Channel, {
onDelete: 'CASCADE',
@ -25,7 +32,7 @@ export class ChannelNotePining {
public channel: Channel | null;
@Column(id())
public noteId: Note['id'];
public noteId: Note["id"];
@ManyToOne(type => Note, {
onDelete: 'CASCADE',

19
packages/backend/src/models/entities/channel.ts
View file

@ -1,7 +1,14 @@
import { PrimaryColumn, Entity, Index, JoinColumn, Column, ManyToOne } from 'typeorm';
import { User } from './user.js';
import { id } from '../id.js';
import { DriveFile } from './drive-file.js';
import {
PrimaryColumn,
Entity,
Index,
JoinColumn,
Column,
ManyToOne,
} from "typeorm";
import { User } from "./user.js";
import { id } from "../id.js";
import { DriveFile } from "./drive-file.js";
@Entity()
export class Channel {
@ -26,7 +33,7 @@ export class Channel {
nullable: true,
comment: 'The owner ID.',
})
public userId: User['id'] | null;
public userId: User["id"] | null;
@ManyToOne(type => User, {
onDelete: 'SET NULL',
@ -51,7 +58,7 @@ export class Channel {
nullable: true,
comment: 'The ID of banner Channel.',
})
public bannerId: DriveFile['id'] | null;
public bannerId: DriveFile["id"] | null;
@ManyToOne(type => DriveFile, {
onDelete: 'SET NULL',

19
packages/backend/src/models/entities/clip-note.ts
View file

@ -1,7 +1,14 @@
import { Entity, Index, JoinColumn, Column, ManyToOne, PrimaryColumn } from 'typeorm';
import { Note } from './note.js';
import { Clip } from './clip.js';
import { id } from '../id.js';
import {
Entity,
Index,
JoinColumn,
Column,
ManyToOne,
PrimaryColumn,
} from "typeorm";
import { Note } from "./note.js";
import { Clip } from "./clip.js";
import { id } from "../id.js";
@Entity()
@Index(['noteId', 'clipId'], { unique: true })
@ -14,7 +21,7 @@ export class ClipNote {
...id(),
comment: 'The note ID.',
})
public noteId: Note['id'];
public noteId: Note["id"];
@ManyToOne(type => Note, {
onDelete: 'CASCADE',
@ -27,7 +34,7 @@ export class ClipNote {
...id(),
comment: 'The clip ID.',
})
public clipId: Clip['id'];
public clipId: Clip["id"];
@ManyToOne(type => Clip, {
onDelete: 'CASCADE',

15
packages/backend/src/models/entities/clip.ts
View file

@ -1,6 +1,13 @@
import { PrimaryColumn, Entity, Index, JoinColumn, Column, ManyToOne } from 'typeorm';
import { User } from './user.js';
import { id } from '../id.js';
import {
PrimaryColumn,
Entity,
Index,
JoinColumn,
Column,
ManyToOne,
} from "typeorm";
import { User } from "./user.js";
import { id } from "../id.js";
@Entity()
export class Clip {
@ -17,7 +24,7 @@ export class Clip {
...id(),
comment: 'The owner ID.',
})
public userId: User['id'];
public userId: User["id"];
@ManyToOne(type => User, {
onDelete: 'CASCADE',

Some files were not shown because too many files have changed in this diff Show more