Merge branch 'develop' into feat/scylladb

This commit is contained in:
Namekuji 2023-07-30 17:31:14 -04:00
commit 76b383dee1
No known key found for this signature in database
GPG key ID: 1D62332C07FBA532
18 changed files with 392 additions and 74 deletions

1
.gitignore vendored
View file

@ -1,6 +1,7 @@
# Visual Studio Code
/.vscode
!/.vscode/extensions.json
!/.vscode/launch.json
# Intelij-IDEA
/.idea

View file

@ -38,7 +38,7 @@ testCommit:
stage: build
script:
- pnpm i --frozen-lockfile
- pnpm run build
- pnpm run build:debug
- pnpm run migrate
only:
- main
@ -66,4 +66,4 @@ cache:
- node_modules/
- packages/*/node_modules/
- packages/backend/native-utils/node_modules/
- packages/backend/native-utils/.cargo/
- packages/backend/native-utils/target/

20
.vscode/launch.json vendored Normal file
View file

@ -0,0 +1,20 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"type": "node",
"request": "launch",
"name": "Launch PNPM Debug Script",
"skipFiles": [
"<node_internals>/**"
],
"runtimeExecutable": "pnpm",
"runtimeArgs": [
"run", "debug"
]
}
]
}

View file

@ -67,7 +67,7 @@ Closes #10313
- Fix: :bug: don't convert time since epoch for ratelimit
https://calckey.social/notes/9gkasnzglmi07rpa
https://firefish.social/notes/9gkasnzglmi07rpa
- Fix: :bug: fix tapping parent on mobile causing side effects
@ -1834,7 +1834,7 @@ Closes #9843
- Fix: :recycle: use locale for error
https://calckey.social/notes/9fippqiwhl287b5m
https://firefish.social/notes/9fippqiwhl287b5m
- Fix mfm-cheat-sheet styling

View file

@ -22,6 +22,8 @@
"dev": "pnpm node ./scripts/dev.js",
"dev:staging": "NODE_OPTIONS=--max_old_space_size=3072 NODE_ENV=development pnpm run build && pnpm run start",
"lint": "pnpm -r --parallel run lint",
"debug": "pnpm run build:debug && pnpm run start",
"build:debug": "pnpm -r --parallel run build:debug && pnpm run gulp",
"cy:open": "cypress open --browser --e2e --config-file=cypress.config.ts",
"cy:run": "cypress run",
"e2e": "start-server-and-test start:test http://localhost:61812 cy:run",

View file

@ -37,7 +37,7 @@
"build": "pnpm run build:napi && pnpm run build:migration",
"build:napi": "napi build --features napi --platform --release ./built/",
"build:migration": "cargo build --locked --release --manifest-path ./migration/Cargo.toml && cp ./target/release/migration ./built/migration",
"build:debug": "napi build --platform ./built/ && cargo build --manifest-path ./migration/Cargo.toml",
"build:debug": "napi build --features napi --platform ./built/ && cargo build --locked --manifest-path ./migration/Cargo.toml && cp -v ./target/debug/migration ./built/migration",
"prepublishOnly": "napi prepublish -t npm",
"test": "pnpm run cargo:test && pnpm run build:napi && ava",
"universal": "napi universal",

View file

@ -16,6 +16,7 @@
"revertmigration:scylla": "cargo run --release --locked --manifest-path ./native-utils/scylla-migration/Cargo.toml -- -d ./native-utils/scylla-migration/cql -c ../../.config/default.yml down",
"check:connect": "node ./check_connect.js",
"build": "pnpm swc src -d built -D",
"build:debug": "pnpm swc src -d built -s -D",
"watch": "pnpm swc src -d built -D -w",
"lint": "pnpm rome check --apply *",
"mocha": "cross-env NODE_ENV=test TS_NODE_FILES=true TS_NODE_TRANSPILE_ONLY=true TS_NODE_PROJECT=\"./test/tsconfig.json\" mocha",

View file

@ -0,0 +1,221 @@
import { Readable, ReadableOptions } from "node:stream";
import { Buffer, constants as BufferConstants } from "node:buffer";
import * as fs from "node:fs";
interface ByteRange {
start: bigint;
end: bigint;
size: bigint;
}
const BOUNDARY_CHARS =
"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
const BYTERANGE_SPEC_REGEX = /^bytes=(.+)$/;
const BYTERANGE_REGEX = /(\d*)-(\d*)/;
const BIGINT_0 = BigInt(0);
const BIGINT_1 = BigInt(1);
const BOUNDARY_SIZE = 40;
function extractRanges(
fileSize: bigint,
maxByteRanges: number,
rangeHeaderValue: string,
): ByteRange[] {
const ranges: ByteRange[] = [];
if (!rangeHeaderValue) return ranges;
const rangeSpecMatch = rangeHeaderValue.match(BYTERANGE_SPEC_REGEX);
if (!rangeSpecMatch) return [];
const rangeSpecs = rangeSpecMatch[1].split(",");
for (let i = 0; i < rangeSpecs.length; i = i + 1) {
const byteRange = rangeSpecs[i].match(BYTERANGE_REGEX);
if (!byteRange) return [];
let start: bigint;
let end: bigint;
let size: bigint;
if (byteRange[1]) {
start = BigInt(byteRange[1]);
}
if (byteRange[2]) {
end = BigInt(byteRange[2]);
}
if (start === undefined && end === undefined) {
/* some invalid range like bytes=- */
return [];
}
if (start === undefined) {
/* end-of-file range like -500 */
start = fileSize - end;
end = fileSize - BIGINT_1;
if (start < BIGINT_0) return []; /* range larger than file, return */
}
if (end === undefined) {
/* range like 0- */
end = fileSize - BIGINT_1;
}
if (start > end || end >= fileSize) {
/* return empty range to issue regular 200 */
return [];
}
size = end - start + BIGINT_1;
if (1 > maxByteRanges - ranges.length) return [];
ranges.push({
start: start,
end: end,
size: size,
});
}
return ranges;
}
function createBoundary(len: number): string {
let chars = [];
for (let i = 0; i < len; i = i + 1) {
chars[i] = BOUNDARY_CHARS.charAt(
Math.floor(Math.random() * BOUNDARY_CHARS.length),
);
}
return chars.join("");
}
class ByteRangeReadable extends Readable {
size: bigint; /* the total size in bytes */
boundary: string; /* boundary marker to use in multipart headers */
private fd: number;
private ranges: ByteRange[];
private index: number; /* index within ranges */
private position: bigint;
private end: bigint;
private contentType: string;
private fileSize: bigint;
private headers: Buffer[];
private trailer: Buffer;
static parseByteRanges(
fileSize: bigint,
maxByteRanges: number,
rangeHeaderValue?: string,
): ByteRange[] {
return extractRanges(fileSize, maxByteRanges, rangeHeaderValue);
}
private createPartHeader(range: ByteRange): Buffer {
return Buffer.from(
[
"",
`--${this.boundary}`,
`Content-Type: ${this.contentType}`,
`Content-Range: bytes ${range.start}-${range.end}/${this.fileSize}`,
"",
"",
].join("\r\n"),
);
}
constructor(
fd: number,
fileSize: bigint,
ranges: ByteRange[],
contentType: string,
opts?: ReadableOptions,
) {
super(opts);
if (ranges.length === 0) {
throw Error("this requires at least 1 byte range");
}
this.fd = fd;
this.ranges = ranges;
this.fileSize = fileSize;
this.contentType = contentType;
this.position = BIGINT_1;
this.end = BIGINT_0;
this.index = -1;
this.headers = [];
this.size = BIGINT_0;
if (this.ranges.length === 1) {
this.size = this.ranges[0].size;
} else {
this.boundary = createBoundary(BOUNDARY_SIZE);
this.ranges.forEach((r) => {
const header = this.createPartHeader(r);
this.headers.push(header);
this.size += BigInt(header.length) + r.size;
});
this.trailer = Buffer.from(`\r\n--${this.boundary}--\r\n`);
this.size += BigInt(this.trailer.length);
}
}
_read(n) {
if (this.index == this.ranges.length) {
this.push(null);
return;
}
if (this.position > this.end) {
/* move ahead to the next index */
this.index++;
if (this.index === this.ranges.length) {
if (this.trailer) {
this.push(this.trailer);
return;
}
this.push(null);
return;
}
this.position = this.ranges[this.index].start;
this.end = this.ranges[this.index].end;
if (this.ranges.length > 1) {
this.push(this.headers[this.index]);
return;
}
}
const max = this.end - this.position + BIGINT_1;
if (n > max) n = Number(max);
const buf = Buffer.alloc(n);
fs.read(this.fd, buf, 0, n, this.position, (err, bytesRead) => {
if (err) {
this.destroy(err);
return;
}
if (bytesRead == 0) {
/* something seems to have gone wrong? */
this.push(null);
return;
}
if (bytesRead > n) bytesRead = n;
this.position += BigInt(bytesRead);
this.push(buf.slice(0, bytesRead));
});
}
}
export { ByteRange, ByteRangeReadable };

View file

@ -14,6 +14,7 @@ import { detectType } from "@/misc/get-file-info.js";
import { convertToWebp } from "@/services/drive/image-processor.js";
import { GenerateVideoThumbnail } from "@/services/drive/generate-video-thumbnail.js";
import { StatusError } from "@/misc/fetch.js";
import { ByteRangeReadable } from "./byte-range-readable.js";
import { FILE_TYPE_BROWSERSAFE } from "@/const.js";
const _filename = fileURLToPath(import.meta.url);
@ -21,6 +22,8 @@ const _dirname = dirname(_filename);
const assets = `${_dirname}/../../server/file/assets/`;
const MAX_BYTE_RANGES = 10;
const commonReadableHandlerGenerator =
(ctx: Koa.Context) => (e: Error): void => {
serverLogger.error(e);
@ -122,31 +125,88 @@ export default async function (ctx: Koa.Context) {
return;
}
let contentType;
let filename;
let fileHandle;
if (isThumbnail || isWebpublic) {
const { mime, ext } = await detectType(InternalStorage.resolvePath(key));
const filename = rename(file.name, {
(contentType = FILE_TYPE_BROWSERSAFE.includes(mime)
? mime
: "application/octet-stream"),
(filename = rename(file.name, {
suffix: isThumbnail ? "-thumb" : "-web",
extname: ext ? `.${ext}` : undefined,
}).toString();
}).toString());
ctx.body = InternalStorage.read(key);
ctx.set(
"Content-Type",
FILE_TYPE_BROWSERSAFE.includes(mime) ? mime : "application/octet-stream",
);
ctx.set("Cache-Control", "max-age=31536000, immutable");
ctx.set("Content-Disposition", contentDisposition("inline", filename));
fileHandle = await InternalStorage.open(key, "r");
} else {
const readable = InternalStorage.read(file.accessKey!);
(contentType = FILE_TYPE_BROWSERSAFE.includes(file.type)
? file.type
: "application/octet-stream"),
(filename = file.name);
fileHandle = await InternalStorage.open(file.accessKey!, "r");
}
// We can let Koa evaluate conditionals by setting
// the status to 200, along with the lastModified
// and etag properties, then checking ctx.fresh.
// Additionally, Range is ignored if a conditional GET would
// result in a 304 response, so we can return early here.
ctx.status = 200;
ctx.etag = file.md5;
ctx.lastModified = file.createdAt;
// When doing a conditional request, we MUST return a "Cache-Control" header
// if a normal 200 response would have included.
ctx.set("Cache-Control", "max-age=31536000, immutable");
if (ctx.fresh) {
ctx.status = 304;
return;
}
ctx.length = file.size;
ctx.set("Content-Disposition", contentDisposition("inline", filename));
ctx.set("Content-Type", contentType);
const ranges = ByteRangeReadable.parseByteRanges(
BigInt(file.size),
MAX_BYTE_RANGES,
ctx.headers["range"],
);
const readable =
ranges.length === 0
? fileHandle.createReadStream()
: new ByteRangeReadable(
fileHandle.fd,
BigInt(file.size),
ranges,
contentType,
);
readable.on("error", commonReadableHandlerGenerator(ctx));
ctx.body = readable;
if (ranges.length === 0) {
ctx.set("Accept-Ranges", "bytes");
} else {
ctx.status = 206;
ctx.length = readable.size;
readable.on("close", async () => {
await fileHandle.close();
});
if (ranges.length === 1) {
ctx.set(
"Content-Range",
`bytes ${ranges[0].start}-${ranges[0].end}/${file.size}`,
);
} else {
ctx.set(
"Content-Type",
FILE_TYPE_BROWSERSAFE.includes(file.type)
? file.type
: "application/octet-stream",
`multipart/byteranges; boundary=${readable.boundary}`,
);
ctx.set("Cache-Control", "max-age=31536000, immutable");
ctx.set("Content-Disposition", contentDisposition("inline", file.name));
}
}
}

View file

@ -1,4 +1,5 @@
import * as fs from "node:fs";
import * as fsPromises from "node:fs/promises";
import * as Path from "node:path";
import { fileURLToPath } from "node:url";
import { dirname } from "node:path";
@ -13,6 +14,10 @@ export class InternalStorage {
public static resolvePath = (key: string) =>
Path.resolve(InternalStorage.path, key);
public static open(key: string, flags: string) {
return fsPromises.open(InternalStorage.resolvePath(key), flags);
}
public static read(key: string) {
return fs.createReadStream(InternalStorage.resolvePath(key));
}

View file

@ -6,10 +6,13 @@ async function getRelMeLinks(url: string): Promise<string[]> {
try {
const html = await getHtml(url);
const dom = new JSDOM(html);
const relMeLinks = [
...dom.window.document.querySelectorAll("a[rel='me']"),
...dom.window.document.querySelectorAll("link[rel='me']"),
].map((a) => (a as HTMLAnchorElement | HTMLLinkElement).href);
const allLinks = [...dom.window.document.querySelectorAll("a, link")];
const relMeLinks = allLinks
.filter((a) => {
const relAttribute = a.getAttribute("rel");
return relAttribute ? relAttribute.split(" ").includes("me") : false;
})
.map((a) => (a as HTMLAnchorElement | HTMLLinkElement).href);
return relMeLinks;
} catch {
return [];

View file

@ -4,6 +4,7 @@
"scripts": {
"watch": "pnpm vite build --watch --mode development",
"build": "pnpm vite build",
"build:debug": "pnpm run build",
"lint": "pnpm rome check **/*.ts --apply && pnpm run lint:vue",
"lint:vue": "pnpm paralint --ext .vue --fix '**/*.vue' --cache",
"format": "pnpm rome format * --write && pnpm prettier --write '**/*.{scss,vue}' --cache --cache-strategy metadata"

View file

@ -53,6 +53,7 @@
:aria-label="media.comment"
preload="none"
controls
playsinline
@contextmenu.stop
>
<source :src="media.url" :type="mediaType" />

View file

@ -127,8 +127,8 @@
</FormFolder>
<template #caption>{{
i18n.t("_profile.metadataDescription", {
a: "\<code\>\<a\>\</code\>",
l: "\<code\>\<a\>\</code\>",
a: "\<a\>",
l: "\<a\>",
rel: `rel="me" href="https://${host}/@${$i.username}"`,
})
}}</template>

View file

@ -6,6 +6,7 @@
"types": "./built/index.d.ts",
"scripts": {
"build": "pnpm swc src -d built -D",
"build:debug": "pnpm swc src -d built -s -D",
"render": "pnpm run build && pnpm run api && pnpm run api-prod && cp temp/firefish-js.api.json etc/ && pnpm run api-doc",
"tsd": "tsc && tsd",
"api": "pnpm api-extractor run --local --verbose",

View file

@ -5,6 +5,7 @@
"typings": "./lib/src/index.d.ts",
"scripts": {
"build": "tsc -p ./",
"build:debug": "pnpm run build",
"lint": "pnpm rome check **/*.ts --apply",
"format": "pnpm rome format --write src/**/*.ts",
"doc": "typedoc --out ../docs ./src",

View file

@ -3,6 +3,7 @@
"private": true,
"scripts": {
"build": "webpack",
"build:debug": "pnpm run build",
"watch": "pnpm swc src -d built -D -w",
"lint": "pnpm rome check **/*.ts --apply",
"format": "pnpm rome format * --write"

View file

@ -1,13 +1,13 @@
{
"patrons": [
"@atomicpoet@calckey.social",
"@atomicpoet@firefish.social",
"@shoq@mastodon.social",
"@pikadude@erisly.social",
"@sage@stop.voring.me",
"@sky@therian.club",
"@panos@electricrequiem.com",
"@redhunt07@www.foxyhole.io",
"@griff@calckey.social",
"@griff@firefish.social",
"@cafkafk@ck.cafkafk.com",
"@privateger@plasmatrap.com",
"@effye@toot.thoughtworks.com",
@ -17,56 +17,56 @@
"@topher@mastodon.online",
"@hanicef@stop.voring.me",
"@nmkj@calckey.jp",
"@unattributed@calckey.social",
"@unattributed@firefish.social",
"@cody@misskey.codingneko.com",
"@kate@blahaj.zone",
"@emtk@mkkey.net",
"@jovikowi@calckey.social",
"@padraig@calckey.social",
"@jovikowi@firefish.social",
"@padraig@firefish.social",
"@pancakes@cats.city",
"@theresmiling@calckey.social",
"@kristian@calckey.social",
"@theresmiling@firefish.social",
"@kristian@firefish.social",
"@jo@blahaj.zone",
"@narF@calckey.social",
"@narF@firefish.social",
"@AlderForrest@raining.anvil.top",
"@box464@calckey.social",
"@MariaTheMartian@calckey.social",
"@nisemikol@calckey.social",
"@box464@firefish.social",
"@MariaTheMartian@firefish.social",
"@nisemikol@firefish.social",
"@smallpatatas@blahaj.zone",
"@bayra@stop.voring.me",
"@frost@wolfdo.gg",
"@joebiden@fuckgov.org",
"@nyaa@calckey.social",
"@Dan@calckey.social",
"@dana@calckey.social",
"@Jdreben@calckey.social",
"@nyaa@firefish.social",
"@Dan@firefish.social",
"@dana@firefish.social",
"@Jdreben@firefish.social",
"@natalie@prismst.one",
"@KelsonV@wandering.shop",
"@breakfastmtn@calckey.social",
"@breakfastmtn@firefish.social",
"@richardazia@mastodon.social",
"@joestone@calckey.social",
"@aj@calckey.social",
"@joestone@firefish.social",
"@aj@firefish.social",
"@zepfanman@ramblingreaders.org",
"@kimby@stop.voring.me",
"@fyrfli@fyrfli.social",
"@riversidebryan@firefish.lgbt",
"@aRubes@sloth.run",
"@andreasdotorg@calckey.social",
"@andreasdotorg@firefish.social",
"@ozzy@calckey.online",
"@leni@windycity.style",
"@mhzmodels@calckey.art",
"@ReflexVE@calckey.social",
"@mark@calckey.social",
"@ReflexVE@firefish.social",
"@mark@firefish.social",
"@skyizwhite@himagine.club",
"@Uwu@calckey.social",
"@jGoose@calckey.social",
"@Uwu@firefish.social",
"@jGoose@firefish.social",
"@kunev@blewsky.social",
"@Simoto@electricrequiem.com",
"@Evoterra@calckey.social",
"@Evoterra@firefish.social",
"@LauraLangdon@procial.tchncs.de",
"@mho@social.heise.de",
"@richardazia@calckey.social",
"@blues653@calckey.social",
"@richardazia@firefish.social",
"@blues653@firefish.social",
"@rafale_blue@calc.04.si",
"@esm@lethallava.land",
"@vmstan@vmst.io",
@ -74,33 +74,33 @@
"@renere@distance.blue",
"@theking@kitsunes.club",
"@toof@fedi.toofie.net",
"@Punko@calckey.social",
"@joesbrat67@calckey.social",
"@arth@calckey.social",
"@Punko@firefish.social",
"@joesbrat67@firefish.social",
"@arth@firefish.social",
"@octofloofy@ck.octofloofy.ink",
"@pauliehedron@infosec.town",
"@soulthunk@lethallava.land",
"@bumble@ibe.social",
"@DarrenNevares@calckey.social",
"@irfan@calckey.social",
"@DarrenNevares@firefish.social",
"@irfan@firefish.social",
"@dvd@dvd.chat",
"@charlie2alpha@electricrequiem.com",
"@arndot@layer8.space",
"@ryan@c.ryanccn.dev",
"@lapastora_deprova@calckey.social",
"@rameez@calckey.social",
"@lapastora_deprova@firefish.social",
"@rameez@firefish.social",
"@dracoling@firetribe.org",
"@Space6host@calckey.social",
"@Space6host@firefish.social",
"@zakalwe@plasmatrap.com",
"@seasicksailor@calckey.social",
"@geerue@calckey.social",
"@seasicksailor@firefish.social",
"@geerue@firefish.social",
"@WXFanatic@m.ai6yr.org",
"@Hunkabilly@calckey.world",
"@samleegray@calckey.social",
"@samleegray@firefish.social",
"@schwarzewald@kodow.net",
"@Conatusprinciple@calckey.social",
"@Conatusprinciple@firefish.social",
"@183231bcb@firefish.lgbt",
"@wiase@calckey.social",
"@wiase@firefish.social",
"@leonieke@vitaulium.nl",
"@soulfire@wackywolf.xyz",
"@elbullazul@pub.elbullazul.com",
@ -110,8 +110,8 @@
"@hryggrbyr@ibe.social"
],
"sponsors": [
"@atomicpoet@calckey.social",
"@unattributed@calckey.social",
"@atomicpoet@firefish.social",
"@unattributed@firefish.social",
"@jtbennett@noc.social",
"\nInterkosmos Link"
]