feat: ✨ verify links with rel=me

Creates background job to re-check every local link once a week

CALCKEY.md

@@ -16,7 +16,6 @@
 
 ## Work in progress
 
-- Link verification
 - Better Messaging UI
 - Better API Documentation
 - Remote follow button
@@ -118,6 +117,7 @@
 - Non-mangled unicode emojis
   - Skin tone selection support
 - [DragonflyDB](https://dragonflydb.io/) support as a Redis alternative
+- Link verification
 
 ## Implemented (remote)
 

locales/en-US.yml

@@ -1124,6 +1124,7 @@ remindMeLater: "Maybe later"
 removeQuote: "Remove quote"
 removeRecipient: "Remove recipient"
 removeMember: "Remove member"
+verifiedLink: "Verified link"
 
 _sensitiveMediaDetection:
   description: "Reduces the effort of server moderation through automatically recognizing

packages/backend/src/models/entities/user-profile.ts

@@ -51,6 +51,8 @@ export class UserProfile {
 	public fields: {
 		name: string;
 		value: string;
+		verified?: boolean;
+		lastVerified?: Date;
 	}[];
 
 	@Column("varchar", {

packages/backend/src/queue/index.ts

@@ -576,6 +576,16 @@ export default function () {
 		{ removeOnComplete: true, removeOnFail: true },
 	);
 
+	systemQueue.add(
+		"verifyLinks",
+		{},
+		{
+			repeat: { cron: "0 0 * * 0" },
+			removeOnComplete: true,
+			removeOnFail: true
+		},
+	);
+
 	processSystemQueue(systemQueue);
 }
 

packages/backend/src/queue/processors/background/index.ts

@@ -3,6 +3,7 @@ import indexAllNotes from "./index-all-notes.js";
 
 const jobs = {
 	indexAllNotes,
+
 } as Record<string, Bull.ProcessCallbackFunction<Record<string, unknown>>>;
 
 export default function (q: Bull.Queue) {

packages/backend/src/queue/processors/system/index.ts

@@ -5,6 +5,7 @@ import { cleanCharts } from "./clean-charts.js";
 import { checkExpiredMutings } from "./check-expired-mutings.js";
 import { clean } from "./clean.js";
 import { setLocalEmojiSizes } from "./local-emoji-size.js";
+import { verifyLinks } from "./verify-links.js";
 
 const jobs = {
 	tickCharts,
@@ -13,6 +14,7 @@ const jobs = {
 	checkExpiredMutings,
 	clean,
 	setLocalEmojiSizes,
+	verifyLinks,
 } as Record<
 	string,
 	| Bull.ProcessCallbackFunction<Record<string, unknown>>

packages/backend/src/queue/processors/system/verify-links.ts

@@ -0,0 +1,62 @@
+import type Bull from "bull";
+
+import { UserProfiles } from "@/models/index.js";
+import { Not } from "typeorm";
+import { queueLogger } from "../../logger.js";
+import { getRelMeLinks } from "@/services/fetch-rel-me.js";
+import config from "@/config/index.js";
+
+const logger = queueLogger.createSubLogger("verify-links");
+
+export async function verifyLinks(
+	job: Bull.Job<Record<string, unknown>>,
+	done: any,
+): Promise<void> {
+	logger.info("Verifying links...");
+
+	const usersToVerify = await UserProfiles.findBy({
+		fields: Not(null),
+		userHost: "",
+	});
+	for (const user of usersToVerify) {
+		const fields = user.fields
+			.filter(
+				(x) =>
+					typeof x.name === "string" &&
+					x.name !== "" &&
+					typeof x.value === "string" &&
+					x.value !== "" &&
+					((x.lastVerified &&
+						x.lastVerified.getTime() < Date.now() - 1000 * 60 * 60 * 24 * 14) ||
+						!x.lastVerified),
+			)
+			.map(async (x) => {
+				const relMeLinks = await getRelMeLinks(x.value);
+				const verified = relMeLinks.some((link) =>
+					link.includes(`${config.host}/@${user.user?.host}`),
+				);
+				return {
+					name: x.name,
+					value: x.value,
+					verified: verified,
+					lastVerified: new Date(),
+				};
+			});
+		if (fields.length > 0) {
+			const fieldsFinal = await Promise.all(fields);
+			try {
+				await UserProfiles.update(user.userId, {
+					fields: fieldsFinal,
+				});
+			}
+			catch (e: any) {
+				logger.error(`Failed to update user ${user.userId} ${e}`);
+				done(e);
+				break;
+			}
+		}
+	}
+
+	logger.succ("All links successfully verified.");
+	done();
+}

packages/backend/src/server/api/endpoints/i/update.ts

@@ -12,7 +12,9 @@ import type { UserProfile } from "@/models/entities/user-profile.js";
 import { notificationTypes } from "@/types.js";
 import { normalizeForSearch } from "@/misc/normalize-for-search.js";
 import { langmap } from "@/misc/langmap.js";
+import { getRelMeLinks } from "@/services/fetch-rel-me.js";
 import { ApiError } from "../../error.js";
+import config from "@/config/index.js";
 import define from "../../define.js";
 
 export const meta = {
@@ -242,8 +244,17 @@ export default define(meta, paramDef, async (ps, _user, token) => {
 					typeof x.value === "string" &&
 					x.value !== "",
 			)
-			.map((x) => {
+			.map(async (x) => {
-				return { name: x.name, value: x.value };
+				const relMeLinks = await getRelMeLinks(x.value);
+				const verified = relMeLinks.some((link) =>
+					link.includes(`${config.host}/@${user.username}`),
+				);
+				return {
+					name: x.name,
+					value: x.value,
+					verified: verified,
+					lastVerified: new Date(),
+				};
 			});
 	}
 

packages/backend/src/services/fetch-rel-me.ts

@@ -0,0 +1,16 @@
+import { getHtml } from "@/misc/fetch.js";
+import { JSDOM } from "jsdom";
+
+export async function getRelMeLinks(url: string): Promise<string[]> {
+	try {
+		const html = await getHtml(url);
+		const dom = new JSDOM(html);
+		const relMeLinks = [
+			...dom.window.document.querySelectorAll("a[rel='me']"),
+		].map((a) => (a as HTMLAnchorElement).href);
+		return relMeLinks;
+	}
+	catch {
+		return [];
+	}
+}

packages/client/src/pages/user/home.vue

@@ -288,10 +288,17 @@
 						<div v-if="user.fields.length > 0" class="fields">
 							<dl
 								v-for="(field, i) in user.fields"
+								:class="field.verified ?? 'verified'"
 								:key="i"
 								class="field"
 							>
 								<dt class="name">
+									<i
+										class="ph-bold ph-seal-check ph-lg ph-fw"
+										style="padding: 5px"
+										:v-tooltip="i18n.ts.verifiedLink"
+										:aria-label="i18n.t('verifiedLink')"
+									></i>
 									<Mfm
 										:text="field.name"
 										:plain="true"
@@ -744,6 +751,12 @@ onUnmounted(() => {
 						margin: 0;
 						align-items: center;
 
+						> .verified {
+							background-color: var(--hover);
+							border-radius: 10px;
+							color: var(--badge) !important;
+						}
+
 						&:not(:last-child) {
 							margin-bottom: 8px;
 						}