From de6ffef2a531e330363f2fb647c7235b1078c7a4 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Tue, 22 Aug 2023 02:00:01 -0400 Subject: [PATCH] save trend tags to redis --- .../cql/1689400417034_timeline/down.cql | 44 ++++++++-------- .../cql/1692402463768_notification/down.cql | 4 +- .../server/api/endpoints/hashtags/trend.ts | 52 ++++++++----------- .../backend/src/services/update-hashtag.ts | 33 ++++++++---- 4 files changed, 71 insertions(+), 62 deletions(-) diff --git a/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/down.cql b/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/down.cql index 2116ddb7a1..a7a06bd0f9 100644 --- a/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/down.cql +++ b/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/down.cql @@ -1,22 +1,22 @@ -DROP TABLE poll_vote; -DROP MATERIALIZED VIEW reaction_by_id; -DROP MATERIALIZED VIEW reaction_by_user_id; -DROP INDEX reaction_by_id; -DROP TABLE reaction; -DROP INDEX home_by_id; -DROP TABLE home_timeline; -DROP MATERIALIZED VIEW local_timeline; -DROP MATERIALIZED VIEW global_timeline; -DROP MATERIALIZED VIEW note_by_channel_id; -DROP MATERIALIZED VIEW note_by_renote_id_and_user_id; -DROP MATERIALIZED VIEW note_by_renote_id; -DROP MATERIALIZED VIEW note_by_user_id; -DROP MATERIALIZED VIEW note_by_id; -DROP INDEX note_by_reply_id; -DROP INDEX note_by_uri; -DROP INDEX note_by_url; -DROP TABLE note; -DROP TYPE poll; -DROP TYPE emoji; -DROP TYPE note_edit_history; -DROP TYPE drive_file; +DROP TABLE IF EXISTS poll_vote; +DROP MATERIALIZED VIEW IF EXISTS reaction_by_id; +DROP MATERIALIZED VIEW IF EXISTS reaction_by_user_id; +DROP INDEX IF EXISTS reaction_by_id; +DROP TABLE IF EXISTS reaction; +DROP INDEX IF EXISTS home_by_id; +DROP TABLE IF EXISTS home_timeline; +DROP MATERIALIZED VIEW IF EXISTS local_timeline; +DROP MATERIALIZED VIEW IF EXISTS global_timeline; +DROP MATERIALIZED VIEW IF EXISTS note_by_channel_id; +DROP MATERIALIZED VIEW IF EXISTS note_by_renote_id_and_user_id; +DROP MATERIALIZED VIEW IF EXISTS note_by_renote_id; +DROP MATERIALIZED VIEW IF EXISTS note_by_user_id; +DROP MATERIALIZED VIEW IF EXISTS note_by_id; +DROP INDEX IF EXISTS note_by_reply_id; +DROP INDEX IF EXISTS note_by_uri; +DROP INDEX IF EXISTS note_by_url; +DROP TABLE IF EXISTS note; +DROP TYPE IF EXISTS poll; +DROP TYPE IF EXISTS emoji; +DROP TYPE IF EXISTS note_edit_history; +DROP TYPE IF EXISTS drive_file; diff --git a/packages/backend/native-utils/scylla-migration/cql/1692402463768_notification/down.cql b/packages/backend/native-utils/scylla-migration/cql/1692402463768_notification/down.cql index c2ee64922d..88ca25aed1 100644 --- a/packages/backend/native-utils/scylla-migration/cql/1692402463768_notification/down.cql +++ b/packages/backend/native-utils/scylla-migration/cql/1692402463768_notification/down.cql @@ -1,2 +1,2 @@ -DROP INDEX notification_by_id; -DROP TABLE notification; +DROP INDEX IF EXISTS notification_by_id; +DROP TABLE IF EXISTS notification; diff --git a/packages/backend/src/server/api/endpoints/hashtags/trend.ts b/packages/backend/src/server/api/endpoints/hashtags/trend.ts index e2a8345112..cd7dbe77cf 100644 --- a/packages/backend/src/server/api/endpoints/hashtags/trend.ts +++ b/packages/backend/src/server/api/endpoints/hashtags/trend.ts @@ -1,10 +1,11 @@ -import { Brackets } from "typeorm"; import define from "../../define.js"; import { fetchMeta } from "@/misc/fetch-meta.js"; import { Notes } from "@/models/index.js"; import type { Note } from "@/models/entities/note.js"; import { safeForSql } from "@/misc/safe-for-sql.js"; import { normalizeForSearch } from "@/misc/normalize-for-search.js"; +import { redisClient } from "@/db/redis.js"; +import { scyllaClient } from "@/db/scylla.js"; /* トレンドに載るためには「『直近a分間のユニーク投稿数が今からa分前～今からb分前の間のユニーク投稿数のn倍以上』のハッシュタグの上位5位以内に入る」ことが必要 @@ -73,19 +74,7 @@ export default define(meta, paramDef, async () => { const now = new Date(); // 5分単位で丸めた現在日時 now.setMinutes(Math.round(now.getMinutes() / 5) * 5, 0, 0); - const tagNotes = await Notes.createQueryBuilder("note") - .where("note.createdAt > :date", { date: new Date(now.getTime() - rangeA) }) - .andWhere( - new Brackets((qb) => { - qb.where(`note.visibility = 'public'`).orWhere( - `note.visibility = 'home'`, - ); - }), - ) - .andWhere(`note.tags != '{}'`) - .select(["note.tags", "note.userId"]) - .cache(60000) // 1 min - .getMany(); + const tagNotes = await redisClient.xrange("trendtag", "-", "+"); if (tagNotes.length === 0) { return []; @@ -96,30 +85,35 @@ export default define(meta, paramDef, async () => { users: Note["userId"][]; }[] = []; - for (const note of tagNotes) { - for (const tag of note.tags) { - if (hiddenTags.includes(tag)) continue; + for (const [_, fields] of tagNotes) { + const name = fields[1]; + const userId = fields[3]; + if (hiddenTags.includes(name)) continue; - const x = tags.find((x) => x.name === tag); - if (x) { - if (!x.users.includes(note.userId)) { - x.users.push(note.userId); - } - } else { - tags.push({ - name: tag, - users: [note.userId], - }); - } + const index = tags.findIndex((tag) => tag.name === name); + if (index >= 0 && !tags[index].users.includes(userId)) { + tags[index].users.push(userId); + } else if (index < 0) { + tags.push({ name, users: [userId] }); } } - // タグを人気順に並べ替え + // Sort tags by their popularity const hots = tags .sort((a, b) => b.users.length - a.users.length) .map((tag) => tag.name) .slice(0, max); + if (scyllaClient) { + const stats = hots.map((tag, i) => ({ + tag, + chart: [], // Really needed? + usersCount: tags[i].users.length, + })); + + return stats; + } + //#region 2(または3)で話題と判定されたタグそれぞれについて過去の投稿数グラフを取得する const countPromises: Promise[] = []; diff --git a/packages/backend/src/services/update-hashtag.ts b/packages/backend/src/services/update-hashtag.ts index 0c65b08f0a..89de60e1bb 100644 --- a/packages/backend/src/services/update-hashtag.ts +++ b/packages/backend/src/services/update-hashtag.ts @@ -4,14 +4,29 @@ import { hashtagChart } from "@/services/chart/index.js"; import { genId } from "@/misc/gen-id.js"; import type { Hashtag } from "@/models/entities/hashtag.js"; import { normalizeForSearch } from "@/misc/normalize-for-search.js"; +import { redisClient } from "@/db/redis.js"; export async function updateHashtags( user: { id: User["id"]; host: User["host"] }, tags: string[], ) { + const pipe = redisClient.multi(); for (const tag of tags) { + const normalizedTag = normalizeForSearch(tag); + pipe.xadd( + "trendtag", + "MINID", + "~", + Date.now() - 60 * 60 * 1000, + "*", + "tag", + normalizedTag, + "user", + user.id, + ); await updateHashtag(user, tag); } + await pipe.exec(); } export async function updateUsertags(user: User, tags: string[]) { @@ -28,23 +43,23 @@ export async function updateHashtag( user: { id: User["id"]; host: User["host"] }, tag: string, isUserAttached = false, - inc = true, + increment = true, ) { - tag = normalizeForSearch(tag); + const normalizedTag = normalizeForSearch(tag); - const index = await Hashtags.findOneBy({ name: tag }); + const index = await Hashtags.findOneBy({ name: normalizedTag }); - if (index == null && !inc) return; + if (index == null && !increment) return; if (index != null) { const q = Hashtags.createQueryBuilder("tag") .update() - .where("name = :name", { name: tag }); + .where("name = :name", { name: normalizedTag }); const set = {} as any; if (isUserAttached) { - if (inc) { + if (increment) { // 自分が初めてこのタグを使ったなら if (!index.attachedUserIds.some((id) => id === user.id)) { set.attachedUserIds = () => @@ -118,7 +133,7 @@ export async function updateHashtag( if (isUserAttached) { Hashtags.insert({ id: genId(), - name: tag, + name: normalizedTag, mentionedUserIds: [], mentionedUsersCount: 0, mentionedLocalUserIds: [], @@ -135,7 +150,7 @@ export async function updateHashtag( } else { Hashtags.insert({ id: genId(), - name: tag, + name: normalizedTag, mentionedUserIds: [user.id], mentionedUsersCount: 1, mentionedLocalUserIds: Users.isLocalUser(user) ? [user.id] : [], @@ -153,6 +168,6 @@ export async function updateHashtag( } if (!isUserAttached) { - hashtagChart.update(tag, user); + hashtagChart.update(normalizedTag, user); } }