diff --git a/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/down.cql b/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/down.cql index 28ac42bb67..abf85789b1 100644 --- a/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/down.cql +++ b/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/down.cql @@ -1,3 +1,4 @@ +DROP MATERIALIZED VIEW IF EXISTS reaction_by_id; DROP MATERIALIZED VIEW IF EXISTS reaction_by_userid; DROP INDEX IF EXISTS reaction_by_id; DROP TABLE IF EXISTS reaction; diff --git a/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/up.cql b/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/up.cql index bb2b4a68f3..c38657df2b 100644 --- a/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/up.cql +++ b/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/up.cql @@ -99,11 +99,9 @@ CREATE TABLE IF NOT EXISTS reaction ( "reaction" text, "emoji" frozen, "createdAt" timestamp, - PRIMARY KEY ("noteId", "userId") + PRIMARY KEY ("noteId", "userId") -- this key constraints one reaction per user for the same post ); -CREATE INDEX IF NOT EXISTS reaction_by_id ON reaction ("id"); - CREATE MATERIALIZED VIEW IF NOT EXISTS reaction_by_userid AS SELECT * FROM reaction WHERE "userId" IS NOT NULL @@ -111,3 +109,10 @@ CREATE MATERIALIZED VIEW IF NOT EXISTS reaction_by_userid AS AND "noteId" IS NOT NULL PRIMARY KEY ("userId", "createdAt", "noteId") WITH CLUSTERING ORDER BY ("createdAt" DESC); + +CREATE MATERIALIZED VIEW IF NOT EXISTS reaction_by_id AS + SELECT * FROM reaction + WHERE "noteId" IS NOT NULL + AND "reaction" IS NOT NULL + AND "userId" IS NOT NULL + PRIMARY KEY ("noteId", "reaction", "userId"); diff --git a/packages/backend/src/db/scylla.ts b/packages/backend/src/db/scylla.ts index 9a76bda85c..99e6a1b839 100644 --- a/packages/backend/src/db/scylla.ts +++ b/packages/backend/src/db/scylla.ts @@ -135,7 +135,7 @@ export const prepared = { ("id", "noteId", "userId", "reaction", "emoji", "createdAt") VALUES (?, ?, ?, ?, ?, ?)`, select: { - byNoteId: `SELECT * FROM reaction WHERE "noteId" IN ?`, + byNoteId: `SELECT * FROM reaction_by_id WHERE "noteId" IN ?`, byUserId: `SELECT * FROM reaction_by_userid WHERE "userId" IN ?`, byNoteAndUser: `SELECT * FROM reaction WHERE "noteId" IN ? AND "userId" IN ?`, byId: `SELECT * FROM reaction WHERE "id" IN ?`, @@ -274,7 +274,7 @@ export function prepareTimelineQuery(ps: { queryParts.push(`AND "createdAt" > ?`); } - queryParts.push("LIMIT 50"); // Hardcoded to issue a prepared query + queryParts.push("LIMIT ?"); const query = queryParts.join(" "); return { @@ -304,10 +304,11 @@ export async function execTimelineQuery( // Try to get posts of at most in the single request while (foundNotes.length < ps.limit && scannedEmptyPartitions < maxDays) { - const params: (Date | string | string[])[] = [untilDate, untilDate]; + const params: (Date | string | string[] | number)[] = [untilDate, untilDate]; if (sinceDate) { params.push(sinceDate); } + params.push(ps.limit) const result = await scyllaClient.execute(query, params, { prepare: true, diff --git a/packages/backend/src/misc/fetch-meta.ts b/packages/backend/src/misc/fetch-meta.ts index b3a5e30ae4..75c0d1ebdd 100644 --- a/packages/backend/src/misc/fetch-meta.ts +++ b/packages/backend/src/misc/fetch-meta.ts @@ -1,7 +1,8 @@ import { db } from "@/db/postgre.js"; import { Meta } from "@/models/entities/meta.js"; +import { Cache } from "@/misc/cache.js"; -let cache: Meta; +export const metaCache = new Cache("meta", 10); export function metaToPugArgs(meta: Meta): object { let motd = ["Loading..."]; @@ -30,43 +31,38 @@ export function metaToPugArgs(meta: Meta): object { } export async function fetchMeta(noCache = false): Promise { - if (!noCache && cache) return cache; + const fetcher = () => + db.transaction(async (transactionalEntityManager) => { + // New IDs are prioritized because multiple records may have been created due to past bugs. + const metas = await transactionalEntityManager.find(Meta, { + order: { + id: "DESC", + }, + }); - return await db.transaction(async (transactionalEntityManager) => { - // New IDs are prioritized because multiple records may have been created due to past bugs. - const metas = await transactionalEntityManager.find(Meta, { - order: { - id: "DESC", - }, + if (metas.length > 0) { + return metas[0]; + } else { + // If fetchMeta is called at the same time when meta is empty, this part may be called at the same time, so use fail-safe upsert. + const saved = await transactionalEntityManager + .upsert( + Meta, + { + id: "x", + }, + ["id"], + ) + .then((x) => + transactionalEntityManager.findOneByOrFail(Meta, x.identifiers[0]), + ); + + return saved; + } }); - const meta = metas[0]; + if (noCache) { + return await fetcher(); + } - if (meta) { - cache = meta; - return meta; - } else { - // If fetchMeta is called at the same time when meta is empty, this part may be called at the same time, so use fail-safe upsert. - const saved = await transactionalEntityManager - .upsert( - Meta, - { - id: "x", - }, - ["id"], - ) - .then((x) => - transactionalEntityManager.findOneByOrFail(Meta, x.identifiers[0]), - ); - - cache = saved; - return saved; - } - }); + return await metaCache.fetch(null, fetcher); } - -setInterval(() => { - fetchMeta(true).then((meta) => { - cache = meta; - }); -}, 1000 * 10); diff --git a/packages/backend/src/server/api/endpoints/admin/update-meta.ts b/packages/backend/src/server/api/endpoints/admin/update-meta.ts index 2142c7df73..f8bf52d215 100644 --- a/packages/backend/src/server/api/endpoints/admin/update-meta.ts +++ b/packages/backend/src/server/api/endpoints/admin/update-meta.ts @@ -2,6 +2,8 @@ import { Meta } from "@/models/entities/meta.js"; import { insertModerationLog } from "@/services/insert-moderation-log.js"; import { db } from "@/db/postgre.js"; import define from "../../define.js"; +import { Metas } from "@/models/index.js"; +import { metaCache } from "@/misc/fetch-meta.js"; export const meta = { tags: ["admin"], @@ -594,21 +596,17 @@ export default define(meta, paramDef, async (ps, me) => { } } - await db.transaction(async (transactionalEntityManager) => { - const metas = await transactionalEntityManager.find(Meta, { - order: { - id: "DESC", - }, - }); - + const metas = await Metas.find({ order: { id: "DESC" }}); + let newMeta: Meta; + if (metas.length > 0) { const meta = metas[0]; - - if (meta) { - await transactionalEntityManager.update(Meta, meta.id, set); - } else { - await transactionalEntityManager.save(Meta, set); - } - }); + await Metas.update(meta.id, set); + newMeta = {...meta, ...set}; + } else { + await Metas.save(set); + newMeta = set as Meta; + } + await metaCache.set(null, newMeta); insertModerationLog(me, "updateMeta"); }); diff --git a/packages/backend/src/server/api/endpoints/notes/reactions.ts b/packages/backend/src/server/api/endpoints/notes/reactions.ts index 6ff7548355..5936c17446 100644 --- a/packages/backend/src/server/api/endpoints/notes/reactions.ts +++ b/packages/backend/src/server/api/endpoints/notes/reactions.ts @@ -4,6 +4,7 @@ import type { NoteReaction } from "@/models/entities/note-reaction.js"; import define from "../../define.js"; import { ApiError } from "../../error.js"; import { getNote } from "../../common/getters.js"; +import { parseScyllaReaction, prepared, scyllaClient } from "@/db/scylla.js"; export const meta = { tags: ["notes", "reactions"], @@ -50,7 +51,7 @@ export const paramDef = { export default define(meta, paramDef, async (ps, user) => { // check note visibility - const note = await getNote(ps.noteId, user).catch((err) => { + await getNote(ps.noteId, user).catch((err) => { if (err.id === "9725d0ce-ba28-4dde-95a7-2cbb2c15de24") throw new ApiError(meta.errors.noSuchNote); throw err; @@ -61,8 +62,7 @@ export default define(meta, paramDef, async (ps, user) => { } as FindOptionsWhere; if (ps.type) { - // ローカルリアクションはホスト名が . とされているが - // DB 上ではそうではないので、必要に応じて変換 + // Remove "." suffix of local emojis here because they are actually null in DB. const suffix = "@.:"; const type = ps.type.endsWith(suffix) ? `${ps.type.slice(0, ps.type.length - suffix.length)}:` @@ -70,15 +70,39 @@ export default define(meta, paramDef, async (ps, user) => { query.reaction = type; } - const reactions = await NoteReactions.find({ - where: query, - take: ps.limit, - skip: ps.offset, - order: { - id: -1, - }, - relations: ["user", "user.avatar", "user.banner", "note"], - }); + let reactions: NoteReaction[] = []; + if (scyllaClient) { + const scyllaQuery = [prepared.reaction.select.byNoteId] + const params: (string | string[] | number)[] = [[ps.noteId]]; + if (ps.type) { + scyllaQuery.push(`AND "reaction" = ?`); + params.push(query.reaction as string) + } + scyllaQuery.push("LIMIT ?"); + params.push(ps.limit); + + // Note: This query fails if the number of returned rows exceeds 5000 by + // default, i.e., 5000 reactions with the same emoji from different users. + // This limitation can be relaxed via "fetchSize" option. + // Note: Remote emojis and local emojis are different. + // Ref: https://github.com/datastax/nodejs-driver#paging + const result = await scyllaClient.execute( + scyllaQuery.join(" "), + params, + { prepare: true }, + ); + reactions = result.rows.map(parseScyllaReaction); + } else { + reactions = await NoteReactions.find({ + where: query, + take: ps.limit, + skip: ps.offset, + order: { + id: -1, + }, + relations: ["user", "user.avatar", "user.banner", "note"], + }); + } return await NoteReactions.packMany(reactions, user); });