/* * SPDX-FileCopyrightText: syuilo and misskey-project * SPDX-License-Identifier: AGPL-3.0-only */ import { URL } from 'node:url'; import { Inject, Injectable } from '@nestjs/common'; import { JSDOM } from 'jsdom'; import tinycolor from 'tinycolor2'; import * as Redis from 'ioredis'; import type { MiInstance } from '@/models/Instance.js'; import type Logger from '@/logger.js'; import { DI } from '@/di-symbols.js'; import { LoggerService } from '@/core/LoggerService.js'; import { HttpRequestService } from '@/core/HttpRequestService.js'; import { bindThis } from '@/decorators.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; import type { DOMWindow } from 'jsdom'; type NodeInfo = { openRegistrations?: unknown; software?: { name?: unknown; version?: unknown; }; metadata?: { name?: unknown; nodeName?: unknown; nodeDescription?: unknown; description?: unknown; maintainer?: { name?: unknown; email?: unknown; }; themeColor?: unknown; }; }; @Injectable() export class FetchInstanceMetadataService { private logger: Logger; constructor( private httpRequestService: HttpRequestService, private loggerService: LoggerService, private federatedInstanceService: FederatedInstanceService, @Inject(DI.redis) private redisClient: Redis.Redis, ) { this.logger = this.loggerService.getLogger('metadata', 'cyan'); } @bindThis // public for test public async tryLock(host: string): Promise { // TODO: マイグレーションなのであとで消す (2024.3.1) this.redisClient.del(`fetchInstanceMetadata:mutex:${host}`); return await this.redisClient.set( `fetchInstanceMetadata:mutex:v2:${host}`, '1', 'EX', 30, // 30秒したら自動でロック解除 https://github.com/misskey-dev/misskey/issues/13506#issuecomment-1975375395 'GET' // 古い値を返す(なかったらnull) ); } @bindThis // public for test public unlock(host: string): Promise { return this.redisClient.del(`fetchInstanceMetadata:mutex:v2:${host}`); } @bindThis public async fetchInstanceMetadata(instance: MiInstance, force = false): Promise { const host = instance.host; // finallyでunlockされてしまうのでtry内でロックチェックをしない // (returnであってもfinallyは実行される) if (!force && await this.tryLock(host) === '1') { // 1が返ってきていたらロックされているという意味なので、何もしない return; } try { if (!force) { const _instance = await this.federatedInstanceService.fetch(host); const now = Date.now(); if (_instance && _instance.infoUpdatedAt && (now - _instance.infoUpdatedAt.getTime() < 1000 * 60 * 60 * 24)) { // unlock at the finally caluse return; } } this.logger.info(`Fetching metadata of ${instance.host} ...`); const [info, dom, manifest] = await Promise.all([ this.fetchNodeinfo(instance).catch(() => null), this.fetchDom(instance).catch(() => null), this.fetchManifest(instance).catch(() => null), ]); const [favicon, icon, themeColor, name, description] = await Promise.all([ this.fetchFaviconUrl(instance, dom).catch(() => null), this.fetchIconUrl(instance, dom, manifest).catch(() => null), this.getThemeColor(info, dom, manifest).catch(() => null), this.getSiteName(info, dom, manifest).catch(() => null), this.getDescription(info, dom, manifest).catch(() => null), ]); this.logger.succ(`Successfuly fetched metadata of ${instance.host}`); const updates = { infoUpdatedAt: new Date(), } as Record; if (info) { updates.softwareName = typeof info.software?.name === 'string' ? info.software.name.toLowerCase() : '?'; updates.softwareVersion = info.software?.version; updates.openRegistrations = info.openRegistrations; updates.maintainerName = info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.name ?? null) : null : null; updates.maintainerEmail = info.metadata ? info.metadata.maintainer ? (info.metadata.maintainer.email ?? null) : null : null; } if (name) updates.name = name; if (description) updates.description = description; if (icon ?? favicon) updates.iconUrl = (icon && !icon.includes('data:image/png;base64')) ? icon : favicon; if (favicon) updates.faviconUrl = favicon; if (themeColor) updates.themeColor = themeColor; await this.federatedInstanceService.update(instance.id, updates); this.logger.succ(`Successfuly updated metadata of ${instance.host}`); } catch (e) { this.logger.error(`Failed to update metadata of ${instance.host}: ${e}`); } finally { await this.unlock(host); } } @bindThis private async fetchNodeinfo(instance: MiInstance): Promise { this.logger.info(`Fetching nodeinfo of ${instance.host} ...`); try { const wellknown = await this.httpRequestService.getJson('https://' + instance.host + '/.well-known/nodeinfo') .catch(err => { if (err.statusCode === 404) { throw new Error('No nodeinfo provided'); } else { throw err.statusCode ?? err.message; } }) as Record; if (wellknown.links == null || !Array.isArray(wellknown.links)) { throw new Error('No wellknown links'); } const links = wellknown.links as any[]; const link1_0 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/1.0'); const link2_0 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/2.0'); const link2_1 = links.find(link => link.rel === 'http://nodeinfo.diaspora.software/ns/schema/2.1'); const link = link2_1 ?? link2_0 ?? link1_0; if (link == null) { throw new Error('No nodeinfo link provided'); } const info = await this.httpRequestService.getJson(link.href) .catch(err => { throw err.statusCode ?? err.message; }); this.logger.succ(`Successfuly fetched nodeinfo of ${instance.host}`); return info as NodeInfo; } catch (err) { this.logger.error(`Failed to fetch nodeinfo of ${instance.host}: ${err}`); throw err; } } @bindThis private async fetchDom(instance: MiInstance): Promise { this.logger.info(`Fetching HTML of ${instance.host} ...`); const url = 'https://' + instance.host; const html = await this.httpRequestService.getHtml(url); const { window } = new JSDOM(html); const doc = window.document; return doc; } @bindThis private async fetchManifest(instance: MiInstance): Promise | null> { const url = 'https://' + instance.host; const manifestUrl = url + '/manifest.json'; const manifest = await this.httpRequestService.getJson(manifestUrl) as Record; return manifest; } @bindThis private async fetchFaviconUrl(instance: MiInstance, doc: DOMWindow['document'] | null): Promise { const url = 'https://' + instance.host; if (doc) { // https://github.com/misskey-dev/misskey/pull/8220#issuecomment-1025104043 const href = Array.from(doc.getElementsByTagName('link')).reverse().find(link => link.relList.contains('icon'))?.href; if (href) { return (new URL(href, url)).href; } } const faviconUrl = url + '/favicon.ico'; const favicon = await this.httpRequestService.send(faviconUrl, { method: 'HEAD', }, { throwErrorWhenResponseNotOk: false }); if (favicon.ok) { return faviconUrl; } return null; } @bindThis private async fetchIconUrl(instance: MiInstance, doc: DOMWindow['document'] | null, manifest: Record | null): Promise { if (manifest && manifest.icons && manifest.icons.length > 0 && manifest.icons[0].src) { const url = 'https://' + instance.host; return (new URL(manifest.icons[0].src, url)).href; } if (doc) { const url = 'https://' + instance.host; // https://github.com/misskey-dev/misskey/pull/8220#issuecomment-1025104043 const links = Array.from(doc.getElementsByTagName('link')).reverse(); // https://github.com/misskey-dev/misskey/pull/8220/files/0ec4eba22a914e31b86874f12448f88b3e58dd5a#r796487559 const href = [ links.find(link => link.relList.contains('apple-touch-icon-precomposed'))?.href, links.find(link => link.relList.contains('apple-touch-icon'))?.href, links.find(link => link.relList.contains('icon'))?.href, ] .find(href => href); if (href) { return (new URL(href, url)).href; } } return null; } @bindThis private async getThemeColor(info: NodeInfo | null, doc: DOMWindow['document'] | null, manifest: Record | null): Promise { const themeColor = info?.metadata?.themeColor ?? doc?.querySelector('meta[name="theme-color"]')?.getAttribute('content') ?? manifest?.theme_color; if (themeColor) { const color = new tinycolor(themeColor); if (color.isValid()) return color.toHexString(); } return null; } @bindThis private async getSiteName(info: NodeInfo | null, doc: DOMWindow['document'] | null, manifest: Record | null): Promise { if (info && info.metadata) { if (typeof info.metadata.nodeName === 'string') { return info.metadata.nodeName; } else if (typeof info.metadata.name === 'string') { return info.metadata.name; } } if (doc) { const og = doc.querySelector('meta[property="og:title"]')?.getAttribute('content'); if (og) { return og; } } if (manifest) { return manifest.name ?? manifest.short_name; } return null; } @bindThis private async getDescription(info: NodeInfo | null, doc: DOMWindow['document'] | null, manifest: Record | null): Promise { if (info && info.metadata) { if (typeof info.metadata.nodeDescription === 'string') { return info.metadata.nodeDescription; } else if (typeof info.metadata.description === 'string') { return info.metadata.description; } } if (doc) { const meta = doc.querySelector('meta[name="description"]')?.getAttribute('content'); if (meta) { return meta; } const og = doc.querySelector('meta[property="og:description"]')?.getAttribute('content'); if (og) { return og; } } if (manifest) { return manifest.name ?? manifest.short_name; } return null; } }