feat: ✨ verify links with rel=me (#10506)

Adds Mastodon-style `rel=me` link verification, and creates a background job to verify said links

Closes #9341

![image](/attachments/861e01eb-660f-4c62-8d83-d824cb79da48)

Co-authored-by: ThatOneCalculator <kainoa@t1c.dev>
Co-authored-by: Namekuji <nmkj@waah.day>
Reviewed-on: https://codeberg.org/calckey/calckey/pulls/10506

.gitignore

@@ -46,6 +46,7 @@ files
 ormconfig.json
 packages/backend/assets/instance.css
 packages/backend/assets/sounds/None.mp3
+packages/backend/assets/LICENSE
 
 !packages/backend/src/db
 

CALCKEY.md

@@ -16,7 +16,6 @@
 
 ## Work in progress
 
-- Link verification
 - Better Messaging UI
 - Better API Documentation
 - Remote follow button
@@ -118,6 +117,7 @@
 - Non-mangled unicode emojis
   - Skin tone selection support
 - [DragonflyDB](https://dragonflydb.io/) support as a Redis alternative
+- Link verification
 
 ## Implemented (remote)
 

locales/ar-SA.yml

@@ -1179,7 +1179,6 @@ _profile:
   youCanIncludeHashtags: "يمكنك أيضًا إضافة وسوم إلى سيرتك التعريفية."
   metadata: "معلومات إضافية"
   metadataEdit: "عدّل المعلومات الإضافية"
-  metadataDescription: "يُمكنك عرض 4 حقول معلومات في ملفك الشخصي"
   metadataLabel: "التسمية"
   metadataContent: "المحتوى"
   changeAvatar: "غيّر الصورة الرمزية"

locales/bn-BD.yml

@@ -1268,7 +1268,7 @@ _profile:
   youCanIncludeHashtags: "হ্যাশট্যাগ অন্তর্ভুক্ত করা যেতে পারে।"
   metadata: "অতিরিক্ত তথ্য"
   metadataEdit: "অতিরিক্ত তথ্য সম্পাদনা করুন"
-  metadataDescription: "আপনি আপনার প্রোফাইলে একটি টেবিল হিসাবে চারটি অতিরিক্ত তথ্য দেখাতে পারেন।"
+  metadataDescription: "আপনি আপনার প্রোফাইলে একটি টেবিল হিসাবে চারটি অতিরিক্ত তথ্য দেখাতে পারেন।. আপনি আপনার প্রোফাইলে লিঙ্কটি যাচাই করতে {rel} এর সাথে একটি {a} ট্যাগ বা {l} ট্যাগ যোগ করতে পারেন!"
   metadataLabel: "লেবেল"
   metadataContent: "বিষয়বস্তু"
   changeAvatar: "অ্যাভাটার পরিবর্তন করুন"

locales/ca-ES.yml

@@ -409,8 +409,8 @@ _profile:
   locationDescription: Si primer introduïu la vostra ciutat, es mostrarà l'hora local
     a altres usuaris.
   name: Nom
-  metadataDescription: Fent servir això, podràs mostrar camps d'informació addicionals
+  metadataDescription: "Fent servir això, podràs mostrar camps d'informació addicionals
-    al vostre perfil.
+    al vostre perfil. Podeu afegir una etiqueta {a} o una etiqueta {l} amb {rel} per verificar l'enllaç al vostre perfil."
 _exportOrImport:
   followingList: "Usuaris que segueixes"
   muteList: "Silencia"

locales/de-DE.yml

@@ -1551,7 +1551,7 @@ _profile:
   metadata: "Zusätzliche Informationen"
   metadataEdit: "Zusätzliche Informationen bearbeiten"
   metadataDescription: "Hierdurch kannst du auf deinem Profil zusätzliche Informationsblöcke
-    anzeigen lassen."
+    anzeigen lassen. Sie können ein {a}-Tag oder ein {l}-Tag mit {rel} hinzufügen, um den Link in Ihrem Profil zu überprüfen!"
   metadataLabel: "Beschriftung"
   metadataContent: "Inhalt"
   changeAvatar: "Profilbild ändern"

locales/en-US.yml

@@ -1124,6 +1124,7 @@ remindMeLater: "Maybe later"
 removeQuote: "Remove quote"
 removeRecipient: "Remove recipient"
 removeMember: "Remove member"
+verifiedLink: "Verified link"
 
 _sensitiveMediaDetection:
   description: "Reduces the effort of server moderation through automatically recognizing
@@ -1676,8 +1677,10 @@ _profile:
   youCanIncludeHashtags: "You can also include hashtags in your bio."
   metadata: "Additional Information"
   metadataEdit: "Edit additional Information"
-  metadataDescription: "Using these, you can display additional information fields
+  metadataDescription:
-    in your profile."
+    "Using these, you can display additional information fields
+    in your profile. You can add an {a} tag or {l} tag with {rel}
+    to verify the link on your profile!"
   metadataLabel: "Label"
   metadataContent: "Content"
   changeAvatar: "Change avatar"

locales/es-ES.yml

@@ -1475,7 +1475,7 @@ _profile:
   youCanIncludeHashtags: "Puedes añadir hashtags"
   metadata: "información adicional"
   metadataEdit: "Editar información adicional"
-  metadataDescription: "Muestra la información adicional en el perfil"
+  metadataDescription: "Muestra la información adicional en el perfil. ¡Puede agregar una etiqueta {a} o una etiqueta {l} con {rel} para verificar el enlace en su perfil!"
   metadataLabel: "Etiqueta"
   metadataContent: "Contenido"
   changeAvatar: "Cambiar avatar"

locales/fr-FR.yml

@@ -1413,7 +1413,7 @@ _profile:
   metadata: "Informations supplémentaires"
   metadataEdit: "Éditer les informations supplémentaires"
   metadataDescription: "Vous pouvez afficher jusqu'à quatre informations supplémentaires
-    dans votre profil."
+    dans votre profil. Vous pouvez ajouter une balise {a} ou une balise {l} avec {rel} pour vérifier le lien sur votre profil!"
   metadataLabel: "Étiquette"
   metadataContent: "Contenu"
   changeAvatar: "Changer l'image de profil"

locales/id-ID.yml

@@ -1399,7 +1399,7 @@ _profile:
   metadata: "Informasi tambahan"
   metadataEdit: "Sunting informasi tambahan"
   metadataDescription: "Kamu dapat menampilkan hingga 4 bagian informasi tambahan\
-    \ ke dalam profilmu."
+    \ ke dalam profilmu. Anda dapat menambahkan tag {a} atau tag {l} dengan {rel} untuk memverifikasi tautan di profil Anda!"
   metadataLabel: "Label"
   metadataContent: "Isi"
   changeAvatar: "Ubah avatar"

locales/it-IT.yml

@@ -1266,7 +1266,7 @@ _profile:
   metadata: "Informazioni aggiuntive"
   metadataEdit: "Modifica informazioni aggiuntive"
   metadataDescription: "Puoi pubblicare fino a quattro informazioni aggiuntive sul
-    profilo."
+    profilo. Puoi aggiungere un tag {a} o {l} con {rel} per verificare il link sul tuo profilo!"
   metadataLabel: "Etichetta"
   metadataContent: "Contenuto"
   changeAvatar: "Modifica immagine profilo"

locales/ja-JP.yml

@@ -1491,7 +1491,7 @@ _profile:
   youCanIncludeHashtags: "ハッシュタグを含められます。"
   metadata: "追加情報"
   metadataEdit: "追加情報を編集"
-  metadataDescription: "プロフィールに表として追加情報を表示できます。"
+  metadataDescription: "プロフィールに表として追加情報を表示できます。{a}タグまたは{l}タグを{rel}とともに追加すると、プロフィールのリンクを確認できます。"
   metadataLabel: "ラベル"
   metadataContent: "内容"
   changeAvatar: "アバター画像を変更"

locales/ko-KR.yml

@@ -1319,7 +1319,7 @@ _profile:
   youCanIncludeHashtags: "해시 태그를 포함할 수 있습니다."
   metadata: "추가 정보"
   metadataEdit: "추가 정보 편집"
-  metadataDescription: "프로필에 추가 정보를 표시할 수 있어요"
+  metadataDescription: "프로필에 추가 정보를 표시할 수 있어요. {rel}과 함께 {a} 태그 또는 {l} 태그를 추가하여 프로필의 링크를 확인할 수 있습니다!"
   metadataLabel: "라벨"
   metadataContent: "내용"
   changeAvatar: "아바타 이미지 변경"

locales/pl-PL.yml

@@ -1404,7 +1404,7 @@ _profile:
   metadata: "Dodatkowe informacje"
   metadataEdit: "Edytuj dodatkowe informacje"
   metadataDescription: "Możesz wyświetlać do czterech sekcji dodatkowych informacji
-    na swoim profilu."
+    na swoim profilu. Możesz dodać tag {a} lub tag {l} z {rel}, aby zweryfikować link w swoim profilu!"
   metadataLabel: "Etykieta"
   metadataContent: "Treść"
   changeAvatar: "Zmień awatar"

locales/ru-RU.yml

@@ -1398,7 +1398,7 @@ _profile:
   youCanIncludeHashtags: "Можете использовать здесь хэштеги."
   metadata: "Дополнительные сведения"
   metadataEdit: "Редактировать дополнительные сведения"
-  metadataDescription: "Можно добавить до четырёх дополнительных граф в профиль."
+  metadataDescription: "Можно добавить до четырёх дополнительных граф в профиль. Вы можете добавить тег {a} или тег {l} с {rel}, чтобы подтвердить ссылку в своем профиле!"
   metadataLabel: "Метка"
   metadataContent: "Содержимое"
   changeAvatar: "Поменять аватар"

locales/sk-SK.yml

@@ -1337,7 +1337,7 @@ _profile:
   youCanIncludeHashtags: "Vo svojom bio môžete mať aj hashtagy."
   metadata: "Dodatočné informácie"
   metadataEdit: "Upraviť dodatočné informácie"
-  metadataDescription: "Vo svojom profile môžete uviesť až štyri dodatočné informačné polia."
+  metadataDescription: "Vo svojom profile môžete uviesť až štyri dodatočné informačné polia. Dodate lahko oznako {a} ali oznako {l} z {rel}, da preverite povezavo v svojem profile!"
   metadataLabel: "Popisok"
   metadataContent: "Obsah"
   changeAvatar: "Zmeniť avatara"

locales/tr-TR.yml

@@ -182,7 +182,7 @@ _profile:
     gösterecektir.
   youCanIncludeHashtags: Hakkımdan'da etiket kullanabilirsin.
   description: Hakkımda
-  metadataDescription: Bunları kullanarak profilinizde ek bilgi alanları görüntüleyebilirsiniz.
+  metadataDescription: 'Bunları kullanarak profilinizde ek bilgi alanları görüntüleyebilirsiniz. Profilinizdeki bağlantıyı doğrulamak için {rel} ile bir {a} etiketi veya {l} etiketi ekleyebilirsiniz!'
   metadata: Ek Bilgi
   metadataContent: İçerik
   metadataLabel: Etiket

locales/uk-UA.yml

@@ -1277,7 +1277,7 @@ _profile:
   metadata: "Додаткова інформація"
   metadataEdit: "Редагувати додаткову інформацію"
   metadataDescription: "Ви можете вказати до чотирьох пунктів додаткової інформації
-    у своєму профілі."
+    у своєму профілі. Ви можете додати тег {a} або {l} за допомогою {rel}, щоб підтвердити посилання у своєму профілі!"
   metadataLabel: "Назва"
   metadataContent: "Вміст"
   changeAvatar: "Змінити аватар"

locales/vi-VN.yml

@@ -1342,7 +1342,7 @@ _profile:
   youCanIncludeHashtags: "Bạn có thể dùng hashtag trong tiểu sử."
   metadata: "Thông tin bổ sung"
   metadataEdit: "Sửa thông tin bổ sung"
-  metadataDescription: "Sử dụng phần này, bạn có thể hiển thị các mục thông tin bổ sung trong hồ sơ của mình."
+  metadataDescription: "Sử dụng phần này, bạn có thể hiển thị các mục thông tin bổ sung trong hồ sơ của mình. Bạn có thể thêm thẻ {a} hoặc thẻ {l} với {rel} để xác minh liên kết trên tiểu sử của mình!"
   metadataLabel: "Nhãn"
   metadataContent: "Nội dung"
   changeAvatar: "Đổi ảnh đại diện"

locales/zh-CN.yml

@@ -1402,7 +1402,7 @@ _profile:
   youCanIncludeHashtags: "您可以包含一个话题标签。"
   metadata: "附加信息"
   metadataEdit: "附加信息编辑"
-  metadataDescription: "使用这些，您可以在您的个人资料中显示其它信息字段。"
+  metadataDescription: "使用这些，您可以在您的个人资料中显示其它信息字段。您可以添加带有 {rel} 的 {a} 标签或 {l} 标签来验证您个人资料上的链接！"
   metadataLabel: "标签"
   metadataContent: "内容"
   changeAvatar: "修改头像"

locales/zh-TW.yml

@@ -1361,7 +1361,7 @@ _profile:
   youCanIncludeHashtags: "你也可以在「關於我」中加上 #tag。"
   metadata: "進階資訊"
   metadataEdit: "編輯進階資訊"
-  metadataDescription: "可以在個人資料中以表格形式顯示其他資訊。"
+  metadataDescription: "可以在個人資料中以表格形式顯示其他資訊。您可以添加帶有 {rel} 的 {a} 標籤或 {l} 標籤來驗證您個人資料上的鏈接！"
   metadataLabel: "標籤"
   metadataContent: "内容"
   changeAvatar: "更換大頭貼"

packages/backend/src/models/entities/user-profile.ts

@@ -51,6 +51,7 @@ export class UserProfile {
 	public fields: {
 		name: string;
 		value: string;
+		verified?: boolean;
 	}[];
 
 	@Column("varchar", {

packages/backend/src/queue/index.ts

@@ -576,6 +576,16 @@ export default function () {
 		{ removeOnComplete: true, removeOnFail: true },
 	);
 
+	systemQueue.add(
+		"verifyLinks",
+		{},
+		{
+			repeat: { cron: "0 0 * * 0" },
+			removeOnComplete: true,
+			removeOnFail: true,
+		},
+	);
+
 	processSystemQueue(systemQueue);
 }
 

packages/backend/src/queue/processors/system/index.ts

@@ -5,6 +5,7 @@ import { cleanCharts } from "./clean-charts.js";
 import { checkExpiredMutings } from "./check-expired-mutings.js";
 import { clean } from "./clean.js";
 import { setLocalEmojiSizes } from "./local-emoji-size.js";
+import { verifyLinks } from "./verify-links.js";
 
 const jobs = {
 	tickCharts,
@@ -13,6 +14,7 @@ const jobs = {
 	checkExpiredMutings,
 	clean,
 	setLocalEmojiSizes,
+	verifyLinks,
 } as Record<
 	string,
 	| Bull.ProcessCallbackFunction<Record<string, unknown>>

packages/backend/src/queue/processors/system/verify-links.ts

@@ -0,0 +1,44 @@
+import type Bull from "bull";
+
+import { UserProfiles } from "@/models/index.js";
+import { Not } from "typeorm";
+import { queueLogger } from "../../logger.js";
+import { verifyLink } from "@/services/fetch-rel-me.js";
+import config from "@/config/index.js";
+
+const logger = queueLogger.createSubLogger("verify-links");
+
+export async function verifyLinks(
+	job: Bull.Job<Record<string, unknown>>,
+	done: any,
+): Promise<void> {
+	logger.info("Verifying links...");
+
+	const usersToVerify = await UserProfiles.findBy({
+		fields: Not(null),
+		userHost: "",
+	});
+	for (const user of usersToVerify) {
+		for (const field of user.fields) {
+			if (!field || field.name === "" || field.value === "") {
+				continue;
+			}
+			if (field.value.startsWith("http") && user.user?.username) {
+				field.verified = await verifyLink(field.value, user.user.username);
+			}
+		}
+		if (user.fields.length > 0) {
+			try {
+				await UserProfiles.update(user.userId, {
+					fields: user.fields,
+				});
+			} catch (e) {
+				logger.error(`Failed to update user ${user.userId} ${e}`);
+				done(e);
+			}
+		}
+	}
+
+	logger.succ("All links successfully verified.");
+	done();
+}

packages/backend/src/server/api/endpoints/i/update.ts

@@ -12,7 +12,9 @@ import type { UserProfile } from "@/models/entities/user-profile.js";
 import { notificationTypes } from "@/types.js";
 import { normalizeForSearch } from "@/misc/normalize-for-search.js";
 import { langmap } from "@/misc/langmap.js";
+import { verifyLink } from "@/services/fetch-rel-me.js";
 import { ApiError } from "../../error.js";
+import config from "@/config/index.js";
 import define from "../../define.js";
 
 export const meta = {
@@ -58,6 +60,18 @@ export const meta = {
 			code: "INVALID_REGEXP",
 			id: "0d786918-10df-41cd-8f33-8dec7d9a89a5",
 		},
+
+		invalidFieldName: {
+			message: "Invalid field name.",
+			code: "INVALID_FIELD_NAME",
+			id: "8f81972e-8b53-4d30-b0d2-efb026dda673",
+		},
+
+		invalidFieldValue: {
+			message: "Invalid field value.",
+			code: "INVALID_FIELD_VALUE",
+			id: "aede7444-244b-11ee-be56-0242ac120002",
+		},
 	},
 
 	res: {
@@ -234,16 +248,29 @@ export default define(meta, paramDef, async (ps, _user, token) => {
 	}
 
 	if (ps.fields) {
+		for (const field of ps.fields) {
+			if (!field || field.name === "" || field.value === "") {
+				continue;
+			}
+			if (typeof field.name !== "string" || field.name === "") {
+				throw new ApiError(meta.errors.invalidFieldName);
+			}
+			if (typeof field.value !== "string" || field.value === "") {
+				throw new ApiError(meta.errors.invalidFieldValue);
+			}
+			if (field.value.startsWith("http")) {
+				field.verified = await verifyLink(field.value, user.username);
+			}
+		}
+
 		profileUpdates.fields = ps.fields
-			.filter(
+			.filter((x) => Object.keys(x).length !== 0)
-				(x) =>
-					typeof x.name === "string" &&
-					x.name !== "" &&
-					typeof x.value === "string" &&
-					x.value !== "",
-			)
 			.map((x) => {
-				return { name: x.name, value: x.value };
+				return {
+					name: x.name,
+					value: x.value,
+					verified: x.verified,
+				};
 			});
 	}
 

packages/backend/src/services/fetch-rel-me.ts

@@ -0,0 +1,33 @@
+import { getHtml } from "@/misc/fetch.js";
+import { JSDOM } from "jsdom";
+import config from "@/config/index.js";
+
+async function getRelMeLinks(url: string): Promise<string[]> {
+	try {
+		const html = await getHtml(url);
+		const dom = new JSDOM(html);
+		const relMeLinks = [
+			...dom.window.document.querySelectorAll("a[rel='me']"),
+			...dom.window.document.querySelectorAll("link[rel='me']"),
+		].map((a) => (a as HTMLAnchorElement | HTMLLinkElement).href);
+		return relMeLinks;
+	} catch {
+		return [];
+	}
+}
+
+export async function verifyLink(link: string, username: string): Promise<boolean> {
+	let verified = false;
+	if (link.startsWith("http")) {
+		const relMeLinks = await getRelMeLinks(link);
+		verified = relMeLinks.some((href) =>
+			new RegExp(
+				`^https?:\/\/${config.host.replace(
+					/[.*+\-?^${}()|[\]\\]/g,
+					"\\$&",
+				)}\/@${username.replace(/[.*+\-?^${}()|[\]\\]/g, "\\$&")}$`,
+			).test(href),
+		);
+	}
+	return verified;
+}

packages/calckey-js/src/entities.ts

@@ -38,7 +38,11 @@ export type UserDetailed = UserLite & {
 	createdAt: DateString;
 	description: string | null;
 	ffVisibility: "public" | "followers" | "private";
-	fields: { name: string; value: string }[];
+	fields: {
+		name: string;
+		value: string;
+		verified?: boolean;
+	}[];
 	followersCount: number;
 	followingCount: number;
 	hasPendingFollowRequestFromYou: boolean;

packages/client/src/pages/settings/profile.vue

@@ -126,7 +126,11 @@
 				</div>
 			</FormFolder>
 			<template #caption>{{
-				i18n.ts._profile.metadataDescription
+				i18n.t("_profile.metadataDescription", {
+					a: '<code><a></code>',
+					l: '<code><a></code>',
+					rel: `rel="me" href="https://${host}/@${$i.username}"`
+				})
 			}}</template>
 		</FormSlot>
 
@@ -173,6 +177,7 @@ import { i18n } from "@/i18n";
 import { $i } from "@/account";
 import { langmap } from "@/scripts/langmap";
 import { definePageMetadata } from "@/scripts/page-metadata";
+import { host } from "@/config";
 
 const profile = reactive({
 	name: $i?.name,

packages/client/src/pages/user/home.vue

@@ -288,10 +288,17 @@
 						<div v-if="user.fields.length > 0" class="fields">
 							<dl
 								v-for="(field, i) in user.fields"
+								:class="field.verified ? 'verified' : ''"
 								:key="i"
 								class="field"
 							>
 								<dt class="name">
+									<i
+										v-if="field.verified"
+										class="ph-bold ph-seal-check ph-lg ph-fw"
+										style="padding: 5px"
+										v-tooltip="i18n.ts.verifiedLink"
+									></i>
 									<Mfm
 										:text="field.name"
 										:plain="true"
@@ -748,6 +755,12 @@ onUnmounted(() => {
 							margin-bottom: 8px;
 						}
 
+					 &.verified {
+							background-color: var(--hover);
+							border-radius: 10px;
+							color: var(--badge) !important;
+						}
+
 						> .name {
 							width: 30%;
 							overflow: hidden;

packages/megalodon/src/misskey/entities/field.ts

@@ -2,5 +2,6 @@ namespace MisskeyEntity {
   export type Field = {
     name: string
     value: string
+		verified?: string
   }
 }