/* * SPDX-FileCopyrightText: syuilo and misskey-project * SPDX-License-Identifier: AGPL-3.0-only */ import * as fs from 'node:fs'; import * as stream from 'node:stream/promises'; import { Inject, Injectable } from '@nestjs/common'; import ipaddr from 'ipaddr.js'; import chalk from 'chalk'; import got, * as Got from 'got'; import { parse } from 'content-disposition'; import { DI } from '@/di-symbols.js'; import type { Config } from '@/config.js'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { createTemp } from '@/misc/create-temp.js'; import { StatusError } from '@/misc/status-error.js'; import { LoggerService } from '@/core/LoggerService.js'; import type Logger from '@/logger.js'; import { bindThis } from '@/decorators.js'; @Injectable() export class DownloadService { private logger: Logger; constructor( @Inject(DI.config) private config: Config, private httpRequestService: HttpRequestService, private loggerService: LoggerService, ) { this.logger = this.loggerService.getLogger('download'); } @bindThis public async downloadUrl(url: string, path: string, options: { timeout?: number, operationTimeout?: number, maxSize?: number} = {} ): Promise<{ filename: string; }> { this.logger.info(`Downloading ${chalk.cyan(url)} to ${chalk.cyanBright(path)} ...`); const timeout = options.timeout ?? 30 * 1000; const operationTimeout = options.operationTimeout ?? 60 * 1000; const maxSize = options.maxSize ?? this.config.maxFileSize ?? 262144000; const urlObj = new URL(url); let filename = urlObj.pathname.split('/').pop() ?? 'untitled'; const req = got.stream(url, { headers: { 'User-Agent': this.config.userAgent, }, timeout: { lookup: timeout, connect: timeout, secureConnect: timeout, socket: timeout, // read timeout response: timeout, send: timeout, request: operationTimeout, // whole operation timeout }, agent: { http: this.httpRequestService.httpAgent, https: this.httpRequestService.httpsAgent, }, http2: false, // default retry: { limit: 0, }, enableUnixSockets: false, }).on('response', (res: Got.Response) => { if ((process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'test') && !this.config.proxy && res.ip) { if (this.isPrivateIp(res.ip)) { this.logger.warn(`Blocked address: ${res.ip}`); req.destroy(); } } const contentLength = res.headers['content-length']; if (contentLength != null) { const size = Number(contentLength); if (size > maxSize) { this.logger.warn(`maxSize exceeded (${size} > ${maxSize}) on response`); req.destroy(); } } const contentDisposition = res.headers['content-disposition']; if (contentDisposition != null) { try { const parsed = parse(contentDisposition); if (parsed.parameters.filename) { filename = parsed.parameters.filename; } } catch (e) { this.logger.warn(`Failed to parse content-disposition: ${contentDisposition}`, { stack: e }); } } }).on('downloadProgress', (progress: Got.Progress) => { if (progress.transferred > maxSize) { this.logger.warn(`maxSize exceeded (${progress.transferred} > ${maxSize}) on downloadProgress`); req.destroy(); } }); try { await stream.pipeline(req, fs.createWriteStream(path)); } catch (e) { if (e instanceof Got.HTTPError) { throw new StatusError(`${e.response.statusCode} ${e.response.statusMessage}`, e.response.statusCode, e.response.statusMessage); } else { throw e; } } this.logger.succ(`Download finished: ${chalk.cyan(url)}`); return { filename, }; } @bindThis public async downloadTextFile(url: string): Promise { // Create temp file const [path, cleanup] = await createTemp(); this.logger.info(`text file: Temp file is ${path}`); try { // write content at URL to temp file await this.downloadUrl(url, path); const text = await fs.promises.readFile(path, 'utf8'); return text; } finally { cleanup(); } } @bindThis private isPrivateIp(ip: string): boolean { const parsedIp = ipaddr.parse(ip); for (const net of this.config.allowedPrivateNetworks ?? []) { const cidr = ipaddr.parseCIDR(net); if (cidr[0].kind() === parsedIp.kind() && parsedIp.match(ipaddr.parseCIDR(net))) { return false; } } return parsedIp.range() !== 'unicast'; } }