Merge branch 'fix-ios-video' into 'develop'

fix iOS video playback (fixes #10517)

Closes #10517

See merge request firefish/firefish!10540

packages/backend/src/server/file/byte-range-readable.ts

@@ -0,0 +1,221 @@
+import { Readable, ReadableOptions } from "node:stream";
+import { Buffer, constants as BufferConstants } from "node:buffer";
+import * as fs from "node:fs";
+
+interface ByteRange {
+	start: bigint;
+	end: bigint;
+	size: bigint;
+}
+
+const BOUNDARY_CHARS =
+	"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
+const BYTERANGE_SPEC_REGEX = /^bytes=(.+)$/;
+const BYTERANGE_REGEX = /(\d*)-(\d*)/;
+
+const BIGINT_0 = BigInt(0);
+const BIGINT_1 = BigInt(1);
+const BOUNDARY_SIZE = 40;
+
+function extractRanges(
+	fileSize: bigint,
+	maxByteRanges: number,
+	rangeHeaderValue: string,
+): ByteRange[] {
+	const ranges: ByteRange[] = [];
+
+	if (!rangeHeaderValue) return ranges;
+
+	const rangeSpecMatch = rangeHeaderValue.match(BYTERANGE_SPEC_REGEX);
+	if (!rangeSpecMatch) return [];
+
+	const rangeSpecs = rangeSpecMatch[1].split(",");
+	for (let i = 0; i < rangeSpecs.length; i = i + 1) {
+		const byteRange = rangeSpecs[i].match(BYTERANGE_REGEX);
+		if (!byteRange) return [];
+
+		let start: bigint;
+		let end: bigint;
+		let size: bigint;
+
+		if (byteRange[1]) {
+			start = BigInt(byteRange[1]);
+		}
+
+		if (byteRange[2]) {
+			end = BigInt(byteRange[2]);
+		}
+
+		if (start === undefined && end === undefined) {
+			/* some invalid range like bytes=- */
+			return [];
+		}
+
+		if (start === undefined) {
+			/* end-of-file range like -500 */
+			start = fileSize - end;
+			end = fileSize - BIGINT_1;
+			if (start < BIGINT_0) return []; /* range larger than file, return */
+		}
+
+		if (end === undefined) {
+			/* range like 0- */
+			end = fileSize - BIGINT_1;
+		}
+
+		if (start > end || end >= fileSize) {
+			/* return empty range to issue regular 200 */
+			return [];
+		}
+		size = end - start + BIGINT_1;
+
+		if (1 > maxByteRanges - ranges.length) return [];
+
+		ranges.push({
+			start: start,
+			end: end,
+			size: size,
+		});
+	}
+
+	return ranges;
+}
+
+function createBoundary(len: number): string {
+	let chars = [];
+	for (let i = 0; i < len; i = i + 1) {
+		chars[i] = BOUNDARY_CHARS.charAt(
+			Math.floor(Math.random() * BOUNDARY_CHARS.length),
+		);
+	}
+	return chars.join("");
+}
+
+class ByteRangeReadable extends Readable {
+	size: bigint; /* the total size in bytes */
+	boundary: string; /* boundary marker to use in multipart headers */
+
+	private fd: number;
+	private ranges: ByteRange[];
+	private index: number; /* index within ranges */
+	private position: bigint;
+	private end: bigint;
+	private contentType: string;
+	private fileSize: bigint;
+	private headers: Buffer[];
+	private trailer: Buffer;
+
+	static parseByteRanges(
+		fileSize: bigint,
+		maxByteRanges: number,
+		rangeHeaderValue?: string,
+	): ByteRange[] {
+		return extractRanges(fileSize, maxByteRanges, rangeHeaderValue);
+	}
+
+	private createPartHeader(range: ByteRange): Buffer {
+		return Buffer.from(
+			[
+				"",
+				`--${this.boundary}`,
+				`Content-Type: ${this.contentType}`,
+				`Content-Range: bytes ${range.start}-${range.end}/${this.fileSize}`,
+				"",
+				"",
+			].join("\r\n"),
+		);
+	}
+
+	constructor(
+		fd: number,
+		fileSize: bigint,
+		ranges: ByteRange[],
+		contentType: string,
+		opts?: ReadableOptions,
+	) {
+		super(opts);
+
+		if (ranges.length === 0) {
+			throw Error("this requires at least 1 byte range");
+		}
+
+		this.fd = fd;
+		this.ranges = ranges;
+		this.fileSize = fileSize;
+		this.contentType = contentType;
+
+		this.position = BIGINT_1;
+		this.end = BIGINT_0;
+		this.index = -1;
+		this.headers = [];
+
+		this.size = BIGINT_0;
+
+		if (this.ranges.length === 1) {
+			this.size = this.ranges[0].size;
+		} else {
+			this.boundary = createBoundary(BOUNDARY_SIZE);
+			this.ranges.forEach((r) => {
+				const header = this.createPartHeader(r);
+				this.headers.push(header);
+
+				this.size += BigInt(header.length) + r.size;
+			});
+			this.trailer = Buffer.from(`\r\n--${this.boundary}--\r\n`);
+			this.size += BigInt(this.trailer.length);
+		}
+	}
+
+	_read(n) {
+		if (this.index == this.ranges.length) {
+			this.push(null);
+			return;
+		}
+
+		if (this.position > this.end) {
+			/* move ahead to the next index */
+			this.index++;
+
+			if (this.index === this.ranges.length) {
+				if (this.trailer) {
+					this.push(this.trailer);
+					return;
+				}
+				this.push(null);
+				return;
+			}
+
+			this.position = this.ranges[this.index].start;
+			this.end = this.ranges[this.index].end;
+
+			if (this.ranges.length > 1) {
+				this.push(this.headers[this.index]);
+				return;
+			}
+		}
+
+		const max = this.end - this.position + BIGINT_1;
+
+		if (n > max) n = Number(max);
+		const buf = Buffer.alloc(n);
+
+		fs.read(this.fd, buf, 0, n, this.position, (err, bytesRead) => {
+			if (err) {
+				this.destroy(err);
+				return;
+			}
+			if (bytesRead == 0) {
+				/* something seems to have gone wrong? */
+				this.push(null);
+				return;
+			}
+
+			if (bytesRead > n) bytesRead = n;
+
+			this.position += BigInt(bytesRead);
+			this.push(buf.slice(0, bytesRead));
+		});
+	}
+}
+
+export { ByteRange, ByteRangeReadable };

packages/backend/src/server/file/send-drive-file.ts

@@ -14,6 +14,7 @@ import { detectType } from "@/misc/get-file-info.js";
 import { convertToWebp } from "@/services/drive/image-processor.js";
 import { GenerateVideoThumbnail } from "@/services/drive/generate-video-thumbnail.js";
 import { StatusError } from "@/misc/fetch.js";
+import { ByteRangeReadable } from "./byte-range-readable.js";
 import { FILE_TYPE_BROWSERSAFE } from "@/const.js";
 
 const _filename = fileURLToPath(import.meta.url);
@@ -21,6 +22,8 @@ const _dirname = dirname(_filename);
 
 const assets = `${_dirname}/../../server/file/assets/`;
 
+const MAX_BYTE_RANGES = 10;
+
 const commonReadableHandlerGenerator =
 	(ctx: Koa.Context) => (e: Error): void => {
 		serverLogger.error(e);
@@ -122,31 +125,88 @@ export default async function (ctx: Koa.Context) {
 		return;
 	}
 
+	let contentType;
+	let filename;
+	let fileHandle;
+
 	if (isThumbnail || isWebpublic) {
 		const { mime, ext } = await detectType(InternalStorage.resolvePath(key));
-		const filename = rename(file.name, {
+		(contentType = FILE_TYPE_BROWSERSAFE.includes(mime)
-			suffix: isThumbnail ? "-thumb" : "-web",
+			? mime
-			extname: ext ? `.${ext}` : undefined,
+			: "application/octet-stream"),
-		}).toString();
+			(filename = rename(file.name, {
+				suffix: isThumbnail ? "-thumb" : "-web",
+				extname: ext ? `.${ext}` : undefined,
+			}).toString());
 
-		ctx.body = InternalStorage.read(key);
+		fileHandle = await InternalStorage.open(key, "r");
-		ctx.set(
-			"Content-Type",
-			FILE_TYPE_BROWSERSAFE.includes(mime) ? mime : "application/octet-stream",
-		);
-		ctx.set("Cache-Control", "max-age=31536000, immutable");
-		ctx.set("Content-Disposition", contentDisposition("inline", filename));
 	} else {
-		const readable = InternalStorage.read(file.accessKey!);
+		(contentType = FILE_TYPE_BROWSERSAFE.includes(file.type)
-		readable.on("error", commonReadableHandlerGenerator(ctx));
+			? file.type
-		ctx.body = readable;
+			: "application/octet-stream"),
-		ctx.set(
+			(filename = file.name);
-			"Content-Type",
+		fileHandle = await InternalStorage.open(file.accessKey!, "r");
-			FILE_TYPE_BROWSERSAFE.includes(file.type)
+	}
-				? file.type
+
-				: "application/octet-stream",
+	// We can let Koa evaluate conditionals by setting
-		);
+	// the status to 200, along with the lastModified
-		ctx.set("Cache-Control", "max-age=31536000, immutable");
+	// and etag properties, then checking ctx.fresh.
-		ctx.set("Content-Disposition", contentDisposition("inline", file.name));
+	// Additionally, Range is ignored if a conditional GET would
+	// result in a 304 response, so we can return early here.
+
+	ctx.status = 200;
+	ctx.etag = file.md5;
+	ctx.lastModified = file.createdAt;
+
+	// When doing a conditional request, we MUST return a "Cache-Control" header
+	// if a normal 200 response would have included.
+	ctx.set("Cache-Control", "max-age=31536000, immutable");
+
+	if (ctx.fresh) {
+		ctx.status = 304;
+		return;
+	}
+
+	ctx.length = file.size;
+	ctx.set("Content-Disposition", contentDisposition("inline", filename));
+	ctx.set("Content-Type", contentType);
+
+	const ranges = ByteRangeReadable.parseByteRanges(
+		BigInt(file.size),
+		MAX_BYTE_RANGES,
+		ctx.headers["range"],
+	);
+	const readable =
+		ranges.length === 0
+			? fileHandle.createReadStream()
+			: new ByteRangeReadable(
+					fileHandle.fd,
+					BigInt(file.size),
+					ranges,
+					contentType,
+			  );
+	readable.on("error", commonReadableHandlerGenerator(ctx));
+	ctx.body = readable;
+
+	if (ranges.length === 0) {
+		ctx.set("Accept-Ranges", "bytes");
+	} else {
+		ctx.status = 206;
+		ctx.length = readable.size;
+		readable.on("close", async () => {
+			await fileHandle.close();
+		});
+
+		if (ranges.length === 1) {
+			ctx.set(
+				"Content-Range",
+				`bytes ${ranges[0].start}-${ranges[0].end}/${file.size}`,
+			);
+		} else {
+			ctx.set(
+				"Content-Type",
+				`multipart/byteranges; boundary=${readable.boundary}`,
+			);
+		}
 	}
 }

packages/backend/src/services/drive/internal-storage.ts

@@ -1,4 +1,5 @@
 import * as fs from "node:fs";
+import * as fsPromises from "node:fs/promises";
 import * as Path from "node:path";
 import { fileURLToPath } from "node:url";
 import { dirname } from "node:path";
@@ -13,6 +14,10 @@ export class InternalStorage {
 	public static resolvePath = (key: string) =>
 		Path.resolve(InternalStorage.path, key);
 
+	public static open(key: string, flags: string) {
+		return fsPromises.open(InternalStorage.resolvePath(key), flags);
+	}
+
 	public static read(key: string) {
 		return fs.createReadStream(InternalStorage.resolvePath(key));
 	}

packages/client/src/components/MkMedia.vue

@@ -53,6 +53,7 @@
 					:aria-label="media.comment"
 					preload="none"
 					controls
+					playsinline
 					@contextmenu.stop
 				>
 					<source :src="media.url" :type="mediaType" />