From ee6795ac9d6600df018ab4f28ebcc417117a6815 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Mon, 24 Jul 2023 07:52:35 -0400 Subject: [PATCH] insert reactions to scylla --- .../cql/1689400417034_timeline/down.cql | 2 + .../cql/1689400417034_timeline/up.cql | 25 ++++++-- packages/backend/src/db/scylla.ts | 17 +++--- packages/backend/src/misc/populate-emojis.ts | 10 ++-- packages/backend/src/misc/reaction-lib.ts | 48 +++++++++++---- .../src/services/note/reaction/create.ts | 59 +++++++++++++------ 6 files changed, 113 insertions(+), 48 deletions(-) diff --git a/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/down.cql b/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/down.cql index fff4c12856..7d90c985c5 100644 --- a/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/down.cql +++ b/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/down.cql @@ -1,3 +1,5 @@ +DROP MATERIALIZED VIEW IF EXISTS reaction_by_user_id; +DROP INDEX IF EXISTS reaction_by_id; DROP TABLE IF EXISTS reaction; DROP MATERIALIZED VIEW IF EXISTS note_by_user_id; DROP INDEX IF EXISTS note_by_id; diff --git a/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/up.cql b/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/up.cql index 94ce482c49..ad364e32fc 100644 --- a/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/up.cql +++ b/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/up.cql @@ -57,7 +57,7 @@ CREATE TABLE IF NOT EXISTS note ( -- Models timeline "reactionEmojis" map>, "noteEdit" set>, -- Edit History "updatedAt" timestamp, - PRIMARY KEY ("createdAtDate", "createdAt") + PRIMARY KEY ("createdAtDate", "createdAt", "id") ) WITH CLUSTERING ORDER BY ("createdAt" DESC); CREATE INDEX IF NOT EXISTS note_by_id ON note (id); @@ -69,13 +69,26 @@ CREATE MATERIALIZED VIEW IF NOT EXISTS note_by_userid AS WHERE "userId" IS NOT NULL AND "createdAt" IS NOT NULL AND "createdAtDate" IS NOT NULL - PRIMARY KEY ("userId", "createdAt", "createdAtDate") + AND "id" IS NOT NULL + PRIMARY KEY ("userId", "createdAt", "createdAtDate", "id") WITH CLUSTERING ORDER BY ("createdAt" DESC); CREATE TABLE IF NOT EXISTS reaction ( + "id" text, "noteId" ascii, - "createdAt" timestamp, "userId" ascii, - "reaction" frozen, - PRIMARY KEY ("noteId", "createdAt", "userId") -) WITH CLUSTERING ORDER BY ("createdAt" DESC); + "reaction" text, + "emoji" frozen, + "createdAt" timestamp, + PRIMARY KEY ("noteId", "userId") +); + +CREATE INDEX IF NOT EXISTS reaction_by_id ON reaction (id); + +CREATE MATERIALIZED VIEW IF NOT EXISTS reaction_by_userid AS + SELECT * FROM reaction + WHERE "userId" IS NOT NULL + AND "createdAt" IS NOT NULL + AND "noteId" IS NOT NULL + PRIMARY KEY ("userId", "createdAt", "noteId") + WITH CLUSTERING ORDER BY ("createdAt" DESC); diff --git a/packages/backend/src/db/scylla.ts b/packages/backend/src/db/scylla.ts index e7b524e6ee..2b6dd800ef 100644 --- a/packages/backend/src/db/scylla.ts +++ b/packages/backend/src/db/scylla.ts @@ -53,18 +53,21 @@ export const prepared = { (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, select: { byDate: `SELECT * FROM note WHERE "createdAtDate" = ? AND "createdAt" < ?`, - byId: "SELECT * FROM note WHERE id IN ?", - byUri: "SELECT * FROM note WHERE uri = ?", - byUrl: "SELECT * FROM note WHERE url = ?", + byId: `SELECT * FROM note WHERE "id" IN ?`, + byUri: `SELECT * FROM note WHERE "uri" = ?`, + byUrl: `SELECT * FROM note WHERE "url" = ?`, byUserId: `SELECT * FROM note_by_userid WHERE "userId" = ? AND "createdAt" < ?`, }, - delete: "DELETE FROM note WHERE id IN ?", + delete: `DELETE FROM note WHERE "createdAtDate" = ? AND "createdAt" = ?`, update: { - renoteCount: `UPDATE note SET "renoteCount" = ?, "score" = ? WHERE "id" = ? IF EXISTS`, - } + renoteCount: `UPDATE note SET "renoteCount" = ?, "score" = ? WHERE "createdAtDate" = ? AND "createdAt" = ? IF EXISTS`, + reactions: `UPDATE note SET "reactions" = ?, "score" = ? WHERE "createdAtDate" = ? AND "createdAt" = ? IF EXISTS`, + }, }, reaction: { - insert: `INSERT INTO reaction ("noteId", "createdAt", "userId", "reaction") VALUES (?, ?, ?, ?)`, + insert: `INSERT INTO reaction + ("id", "noteId", "userId", "reaction", "emoji", "createdAt") + VALUES (?, ?, ?, ?, ?, ?)`, }, }; diff --git a/packages/backend/src/misc/populate-emojis.ts b/packages/backend/src/misc/populate-emojis.ts index 795a267f91..b53e065892 100644 --- a/packages/backend/src/misc/populate-emojis.ts +++ b/packages/backend/src/misc/populate-emojis.ts @@ -9,7 +9,7 @@ import config from "@/config/index.js"; import { query } from "@/prelude/url.js"; import { redisClient } from "@/db/redis.js"; -const cache = new Cache("populateEmojis", 60 * 60 * 12); +export const EmojiCache = new Cache("populateEmojis", 60 * 60 * 12); /** * 添付用絵文字情報 @@ -72,11 +72,11 @@ export async function populateEmoji( })) || null; const cacheKey = `${name} ${host}`; - let emoji = await cache.fetch(cacheKey, queryOrNull); + let emoji = await EmojiCache.fetch(cacheKey, queryOrNull); if (emoji && !(emoji.width && emoji.height)) { emoji = await queryOrNull(); - await cache.set(cacheKey, emoji); + await EmojiCache.set(cacheKey, emoji); } if (emoji == null) return null; @@ -151,7 +151,7 @@ export async function prefetchEmojis( emojis: { name: string; host: string | null }[], ): Promise { const notCachedEmojis = emojis.filter( - async (emoji) => !(await cache.get(`${emoji.name} ${emoji.host}`)), + async (emoji) => !(await EmojiCache.get(`${emoji.name} ${emoji.host}`)), ); const emojisQuery: any[] = []; const hosts = new Set(notCachedEmojis.map((e) => e.host)); @@ -172,7 +172,7 @@ export async function prefetchEmojis( : []; const trans = redisClient.multi(); for (const emoji of _emojis) { - cache.set(`${emoji.name} ${emoji.host}`, emoji, trans); + EmojiCache.set(`${emoji.name} ${emoji.host}`, emoji, trans); } await trans.exec(); } diff --git a/packages/backend/src/misc/reaction-lib.ts b/packages/backend/src/misc/reaction-lib.ts index e25b2d6614..69645f3283 100644 --- a/packages/backend/src/misc/reaction-lib.ts +++ b/packages/backend/src/misc/reaction-lib.ts @@ -3,6 +3,8 @@ import { fetchMeta } from "./fetch-meta.js"; import { Emojis } from "@/models/index.js"; import { toPunyNullable } from "./convert-host.js"; import { IsNull } from "typeorm"; +import { EmojiCache } from "@/misc/populate-emojis.js"; +import type { Emoji } from "@/models/entities/emoji.js"; const legacies = new Map([ ["like", "👍"], @@ -18,9 +20,24 @@ const legacies = new Map([ ["star", "⭐"], ]); -export async function getFallbackReaction() { +async function getFallbackReaction() { const meta = await fetchMeta(); - return meta.defaultReaction; + const name = meta.defaultReaction; + + const match = emojiRegex.exec(name); + if (match) { + const unicode = match[0]; + return { name: unicode, emoji: null }; + } + + const emoji = await EmojiCache.fetch(`${name} ${null}`, () => + Emojis.findOneBy({ + name, + host: IsNull(), + }), + ); + + return { name, emoji }; } export function convertLegacyReactions(reactions: Record) { @@ -38,7 +55,7 @@ export function convertLegacyReactions(reactions: Record) { decodedReactions.set(reaction, decodedReaction); } - let emoji = legacies.get(decodedReaction.reaction); + const emoji = legacies.get(decodedReaction.reaction); if (emoji) { _reactions.set(emoji, (_reactions.get(emoji) || 0) + reactions[reaction]); } else { @@ -61,31 +78,36 @@ export function convertLegacyReactions(reactions: Record) { export async function toDbReaction( reaction?: string | null, reacterHost?: string | null, -): Promise { +): Promise<{ name: string; emoji: Emoji | null }> { if (!reaction) return await getFallbackReaction(); - reacterHost = toPunyNullable(reacterHost); + const _reacterHost = toPunyNullable(reacterHost); // Convert string-type reactions to unicode const emoji = legacies.get(reaction) || (reaction === "♥️" ? "❤️" : null); - if (emoji) return emoji; + if (emoji) return { name: emoji, emoji: null }; // Allow unicode reactions const match = emojiRegex.exec(reaction); if (match) { const unicode = match[0]; - return unicode; + return { name: unicode, emoji: null }; } const custom = reaction.match(/^:([\w+-]+)(?:@\.)?:$/); if (custom) { const name = custom[1]; - const emoji = await Emojis.findOneBy({ - host: reacterHost || IsNull(), - name, - }); + const emoji = await EmojiCache.fetch(`${name} ${_reacterHost}`, () => + Emojis.findOneBy({ + name, + host: _reacterHost || IsNull(), + }), + ); - if (emoji) return reacterHost ? `:${name}@${reacterHost}:` : `:${name}:`; + if (emoji) { + const emojiName = _reacterHost ? `:${name}@${_reacterHost}:` : `:${name}:`; + return { name: emojiName, emoji }; + } } return await getFallbackReaction(); @@ -93,7 +115,7 @@ export async function toDbReaction( type DecodedReaction = { /** - * リアクション名 (Unicode Emoji or ':name@hostname' or ':name@.') + * リアクション名 (Unicode Emoji or ':name@hostname' or ':name@.:') */ reaction: string; diff --git a/packages/backend/src/services/note/reaction/create.ts b/packages/backend/src/services/note/reaction/create.ts index 4cf1fd0941..038e904db5 100644 --- a/packages/backend/src/services/note/reaction/create.ts +++ b/packages/backend/src/services/note/reaction/create.ts @@ -21,6 +21,7 @@ import deleteReaction from "./delete.js"; import { isDuplicateKeyValueError } from "@/misc/is-duplicate-key-value-error.js"; import type { NoteReaction } from "@/models/entities/note-reaction.js"; import { IdentifiableError } from "@/misc/identifiable-error.js"; +import { prepared, scyllaClient } from "@/db/scylla.js"; export default async ( user: { id: User["id"]; host: User["host"] }, @@ -46,20 +47,33 @@ export default async ( ); } - // TODO: cache - reaction = await toDbReaction(reaction, user.host); + // Emoji data will be cached in toDbReaction. + const { name: _reaction, emoji: emojiData } = await toDbReaction( + reaction, + user.host, + ); const record: NoteReaction = { id: genId(), createdAt: new Date(), noteId: note.id, userId: user.id, - reaction, + reaction: _reaction, }; // Create reaction try { - await NoteReactions.insert(record); + if (scyllaClient) { + // INSERT to ScyllaDB is upsert, and the primary key of reaction table is ("noteId", "userId"). + // Thus, a reaction by the same user will be replaced if exists. + await scyllaClient.execute( + prepared.reaction.insert, + [record.id, record.noteId, record.userId, _reaction, emojiData, record.createdAt], + { prepare: true }, + ); + } else { + await NoteReactions.insert(record); + } } catch (e) { if (isDuplicateKeyValueError(e)) { const exists = await NoteReactions.findOneByOrFail({ @@ -67,7 +81,7 @@ export default async ( userId: user.id, }); - if (exists.reaction !== reaction) { + if (exists.reaction !== _reaction) { // 別のリアクションがすでにされていたら置き換える await deleteReaction(user, note); await NoteReactions.insert(record); @@ -81,20 +95,31 @@ export default async ( } // Increment reactions count - const sql = `jsonb_set("reactions", '{${reaction}}', (COALESCE("reactions"->>'${reaction}', '0')::int + 1)::text::jsonb)`; - await Notes.createQueryBuilder() - .update() - .set({ - reactions: () => sql, - score: () => '"score" + 1', - }) - .where("id = :id", { id: note.id }) - .execute(); + if (scyllaClient) { + const current = Math.max(note.reactions[_reaction] ?? 0, 0); + note.reactions[_reaction] = current + 1; + const date = new Date(note.createdAt.getTime()); + await scyllaClient.execute( + prepared.note.update.reactions, + [note.reactions, (note.score ?? 0) + 1, date, date], + { prepare: true }, + ); + } else { + const sql = `jsonb_set("reactions", '{${_reaction}}', (COALESCE("reactions"->>'${_reaction}', '0')::int + 1)::text::jsonb)`; + await Notes.createQueryBuilder() + .update() + .set({ + reactions: () => sql, + score: () => '"score" + 1', + }) + .where("id = :id", { id: note.id }) + .execute(); + } perUserReactionsChart.update(user, note); // カスタム絵文字リアクションだったら絵文字情報も送る - const decodedReaction = decodeReaction(reaction); + const decodedReaction = decodeReaction(_reaction); const emoji = await Emojis.findOne({ where: { @@ -124,7 +149,7 @@ export default async ( notifierId: user.id, note: note, noteId: note.id, - reaction: reaction, + reaction: _reaction, }); } @@ -138,7 +163,7 @@ export default async ( notifierId: user.id, note: note, noteId: note.id, - reaction: reaction, + reaction: _reaction, }); } });