save trend tags to redis

packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/down.cql

@@ -1,22 +1,22 @@
-DROP TABLE poll_vote;
+DROP TABLE IF EXISTS poll_vote;
-DROP MATERIALIZED VIEW reaction_by_id;
+DROP MATERIALIZED VIEW IF EXISTS reaction_by_id;
-DROP MATERIALIZED VIEW reaction_by_user_id;
+DROP MATERIALIZED VIEW IF EXISTS reaction_by_user_id;
-DROP INDEX reaction_by_id;
+DROP INDEX IF EXISTS reaction_by_id;
-DROP TABLE reaction;
+DROP TABLE IF EXISTS reaction;
-DROP INDEX home_by_id;
+DROP INDEX IF EXISTS home_by_id;
-DROP TABLE home_timeline;
+DROP TABLE IF EXISTS home_timeline;
-DROP MATERIALIZED VIEW local_timeline;
+DROP MATERIALIZED VIEW IF EXISTS local_timeline;
-DROP MATERIALIZED VIEW global_timeline;
+DROP MATERIALIZED VIEW IF EXISTS global_timeline;
-DROP MATERIALIZED VIEW note_by_channel_id;
+DROP MATERIALIZED VIEW IF EXISTS note_by_channel_id;
-DROP MATERIALIZED VIEW note_by_renote_id_and_user_id;
+DROP MATERIALIZED VIEW IF EXISTS note_by_renote_id_and_user_id;
-DROP MATERIALIZED VIEW note_by_renote_id;
+DROP MATERIALIZED VIEW IF EXISTS note_by_renote_id;
-DROP MATERIALIZED VIEW note_by_user_id;
+DROP MATERIALIZED VIEW IF EXISTS note_by_user_id;
-DROP MATERIALIZED VIEW note_by_id;
+DROP MATERIALIZED VIEW IF EXISTS note_by_id;
-DROP INDEX note_by_reply_id;
+DROP INDEX IF EXISTS note_by_reply_id;
-DROP INDEX note_by_uri;
+DROP INDEX IF EXISTS note_by_uri;
-DROP INDEX note_by_url;
+DROP INDEX IF EXISTS note_by_url;
-DROP TABLE note;
+DROP TABLE IF EXISTS note;
-DROP TYPE poll;
+DROP TYPE IF EXISTS poll;
-DROP TYPE emoji;
+DROP TYPE IF EXISTS emoji;
-DROP TYPE note_edit_history;
+DROP TYPE IF EXISTS note_edit_history;
-DROP TYPE drive_file;
+DROP TYPE IF EXISTS drive_file;

packages/backend/native-utils/scylla-migration/cql/1692402463768_notification/down.cql

@@ -1,2 +1,2 @@
-DROP INDEX notification_by_id;
+DROP INDEX IF EXISTS notification_by_id;
-DROP TABLE notification;
+DROP TABLE IF EXISTS notification;

packages/backend/src/server/api/endpoints/hashtags/trend.ts

@@ -1,10 +1,11 @@
-import { Brackets } from "typeorm";
 import define from "../../define.js";
 import { fetchMeta } from "@/misc/fetch-meta.js";
 import { Notes } from "@/models/index.js";
 import type { Note } from "@/models/entities/note.js";
 import { safeForSql } from "@/misc/safe-for-sql.js";
 import { normalizeForSearch } from "@/misc/normalize-for-search.js";
+import { redisClient } from "@/db/redis.js";
+import { scyllaClient } from "@/db/scylla.js";
 
 /*
 トレンドに載るためには「『直近a分間のユニーク投稿数が今からa分前～今からb分前の間のユニーク投稿数のn倍以上』のハッシュタグの上位5位以内に入る」ことが必要
@@ -73,19 +74,7 @@ export default define(meta, paramDef, async () => {
 	const now = new Date(); // 5分単位で丸めた現在日時
 	now.setMinutes(Math.round(now.getMinutes() / 5) * 5, 0, 0);
 
-	const tagNotes = await Notes.createQueryBuilder("note")
+	const tagNotes = await redisClient.xrange("trendtag", "-", "+");
-		.where("note.createdAt > :date", { date: new Date(now.getTime() - rangeA) })
-		.andWhere(
-			new Brackets((qb) => {
-				qb.where(`note.visibility = 'public'`).orWhere(
-					`note.visibility = 'home'`,
-				);
-			}),
-		)
-		.andWhere(`note.tags != '{}'`)
-		.select(["note.tags", "note.userId"])
-		.cache(60000) // 1 min
-		.getMany();
 
 	if (tagNotes.length === 0) {
 		return [];
@@ -96,30 +85,35 @@ export default define(meta, paramDef, async () => {
 		users: Note["userId"][];
 	}[] = [];
 
-	for (const note of tagNotes) {
+	for (const [_, fields] of tagNotes) {
-		for (const tag of note.tags) {
+		const name = fields[1];
-			if (hiddenTags.includes(tag)) continue;
+		const userId = fields[3];
+		if (hiddenTags.includes(name)) continue;
 
-			const x = tags.find((x) => x.name === tag);
+		const index = tags.findIndex((tag) => tag.name === name);
-			if (x) {
+		if (index >= 0 && !tags[index].users.includes(userId)) {
-				if (!x.users.includes(note.userId)) {
+			tags[index].users.push(userId);
-					x.users.push(note.userId);
+		} else if (index < 0) {
-				}
+			tags.push({ name, users: [userId] });
-			} else {
-				tags.push({
-					name: tag,
-					users: [note.userId],
-				});
-			}
 		}
 	}
 
-	// タグを人気順に並べ替え
+	// Sort tags by their popularity
 	const hots = tags
 		.sort((a, b) => b.users.length - a.users.length)
 		.map((tag) => tag.name)
 		.slice(0, max);
 
+	if (scyllaClient) {
+		const stats = hots.map((tag, i) => ({
+			tag,
+			chart: [], // Really needed?
+			usersCount: tags[i].users.length,
+		}));
+
+		return stats;
+	}
+
 	//#region 2(または3)で話題と判定されたタグそれぞれについて過去の投稿数グラフを取得する
 	const countPromises: Promise<number[]>[] = [];
 

packages/backend/src/services/update-hashtag.ts

@@ -4,14 +4,29 @@ import { hashtagChart } from "@/services/chart/index.js";
 import { genId } from "@/misc/gen-id.js";
 import type { Hashtag } from "@/models/entities/hashtag.js";
 import { normalizeForSearch } from "@/misc/normalize-for-search.js";
+import { redisClient } from "@/db/redis.js";
 
 export async function updateHashtags(
 	user: { id: User["id"]; host: User["host"] },
 	tags: string[],
 ) {
+	const pipe = redisClient.multi();
 	for (const tag of tags) {
+		const normalizedTag = normalizeForSearch(tag);
+		pipe.xadd(
+			"trendtag",
+			"MINID",
+			"~",
+			Date.now() - 60 * 60 * 1000,
+			"*",
+			"tag",
+			normalizedTag,
+			"user",
+			user.id,
+		);
 		await updateHashtag(user, tag);
 	}
+	await pipe.exec();
 }
 
 export async function updateUsertags(user: User, tags: string[]) {
@@ -28,23 +43,23 @@ export async function updateHashtag(
 	user: { id: User["id"]; host: User["host"] },
 	tag: string,
 	isUserAttached = false,
-	inc = true,
+	increment = true,
 ) {
-	tag = normalizeForSearch(tag);
+	const normalizedTag = normalizeForSearch(tag);
 
-	const index = await Hashtags.findOneBy({ name: tag });
+	const index = await Hashtags.findOneBy({ name: normalizedTag });
 
-	if (index == null && !inc) return;
+	if (index == null && !increment) return;
 
 	if (index != null) {
 		const q = Hashtags.createQueryBuilder("tag")
 			.update()
-			.where("name = :name", { name: tag });
+			.where("name = :name", { name: normalizedTag });
 
 		const set = {} as any;
 
 		if (isUserAttached) {
-			if (inc) {
+			if (increment) {
 				// 自分が初めてこのタグを使ったなら
 				if (!index.attachedUserIds.some((id) => id === user.id)) {
 					set.attachedUserIds = () =>
@@ -118,7 +133,7 @@ export async function updateHashtag(
 		if (isUserAttached) {
 			Hashtags.insert({
 				id: genId(),
-				name: tag,
+				name: normalizedTag,
 				mentionedUserIds: [],
 				mentionedUsersCount: 0,
 				mentionedLocalUserIds: [],
@@ -135,7 +150,7 @@ export async function updateHashtag(
 		} else {
 			Hashtags.insert({
 				id: genId(),
-				name: tag,
+				name: normalizedTag,
 				mentionedUserIds: [user.id],
 				mentionedUsersCount: 1,
 				mentionedLocalUserIds: Users.isLocalUser(user) ? [user.id] : [],
@@ -153,6 +168,6 @@ export async function updateHashtag(
 	}
 
 	if (!isUserAttached) {
-		hashtagChart.update(tag, user);
+		hashtagChart.update(normalizedTag, user);
 	}
 }