From 98ffb20c6aec09a8090332a3034242eb152a386a Mon Sep 17 00:00:00 2001 From: Namekuji Date: Tue, 5 Sep 2023 07:43:17 -0400 Subject: [PATCH] fix: post count for charts --- .../cql/1689400417034_timeline/up.cql | 20 +++++------ packages/backend/src/db/scylla.ts | 35 +++++++++++++++++-- .../processors/background/index-all-notes.ts | 6 ++-- .../src/server/api/mastodon/endpoints/meta.ts | 6 ++-- packages/backend/src/server/nodeinfo.ts | 6 ++-- .../src/services/chart/charts/instance.ts | 4 ++- .../src/services/chart/charts/notes.ts | 9 +++-- .../services/chart/charts/per-user-notes.ts | 11 +++++- 8 files changed, 69 insertions(+), 28 deletions(-) diff --git a/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/up.cql b/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/up.cql index 78cad49165..db2a664e1d 100644 --- a/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/up.cql +++ b/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/up.cql @@ -104,6 +104,16 @@ CREATE MATERIALIZED VIEW note_by_user_id AS PRIMARY KEY ("userId", "createdAt", "createdAtDate", "userHost", "visibility") WITH CLUSTERING ORDER BY ("createdAt" DESC); +CREATE MATERIALIZED VIEW local_note_by_user_id AS + SELECT "userId", "createdAt", "createdAtDate", "userHost", "visibility" FROM note + WHERE "userId" IS NOT NULL + AND "createdAt" IS NOT NULL + AND "createdAtDate" IS NOT NULL + AND "userHost" = 'local' + AND "visibility" IS NOT NULL + PRIMARY KEY ("userId", "createdAt", "createdAtDate", "userHost", "visibility") + WITH CLUSTERING ORDER BY ("createdAt" DESC); + CREATE MATERIALIZED VIEW note_by_renote_id AS SELECT * FROM note WHERE "renoteId" IS NOT NULL @@ -157,16 +167,6 @@ CREATE MATERIALIZED VIEW local_timeline AS PRIMARY KEY ("createdAtDate", "createdAt", "userId", "userHost", "visibility") WITH CLUSTERING ORDER BY ("createdAt" DESC); -CREATE MATERIALIZED VIEW local_note AS - SELECT "createdAtDate", "createdAt", "userId", "userHost", "visibility" FROM note - WHERE "createdAtDate" IS NOT NULL - AND "createdAt" IS NOT NULL - AND "userId" IS NOT NULL - AND "userHost" = 'local' - AND "visibility" IS NOT NULL - PRIMARY KEY ("createdAtDate", "createdAt", "userId", "userHost", "visibility") - WITH CLUSTERING ORDER BY ("createdAt" DESC); - CREATE MATERIALIZED VIEW score_feed AS SELECT * FROM note WHERE "createdAtDate" IS NOT NULL diff --git a/packages/backend/src/db/scylla.ts b/packages/backend/src/db/scylla.ts index 1ed9c23eb1..7e0e822b9a 100644 --- a/packages/backend/src/db/scylla.ts +++ b/packages/backend/src/db/scylla.ts @@ -6,6 +6,7 @@ import type { NoteReaction } from "@/models/entities/note-reaction.js"; import { Client, types, tracker } from "cassandra-driver"; import type { User } from "@/models/entities/user.js"; import { + Cache, ChannelFollowingsCache, InstanceMutingsCache, LocalFollowingsCache, @@ -67,6 +68,32 @@ export const scyllaClient = newClient(); export const prepared = scyllaQueries; +const localPostCountCache = new Cache("localPostCount", 1000 * 60 * 10); +export const allPostCountCache = new Cache( + "allPostCount", + 1000 * 60 * 10, +); + +export async function fetchPostCount(local = false): Promise { + if (!scyllaClient) { + throw new Error("ScyllaDB is disabled"); + } + + if (local) { + return await localPostCountCache.fetch(null, () => + scyllaClient + .execute("SELECT COUNT(*) FROM local_note_by_user_id") + .then((result) => result.first().get("count") as number), + ); + } + + return await allPostCountCache.fetch(null, () => + scyllaClient + .execute("SELECT COUNT(*) FROM note") + .then((result) => result.first().get("count") as number), + ); +} + export interface ScyllaNotification { targetId: string; createdAtDate: Date; @@ -444,11 +471,15 @@ export async function execPaginationQuery( untilDate = notifications[notifications.length - 1].createdAt; } else if (kind === "reaction") { const reactions = result.rows.map(parseScyllaReaction); - (found as ScyllaNoteReaction[]).push(...(filter?.reaction ? await filter.reaction(reactions) : reactions)); + (found as ScyllaNoteReaction[]).push( + ...(filter?.reaction ? await filter.reaction(reactions) : reactions), + ); untilDate = reactions[reactions.length - 1].createdAt; } else { const notes = result.rows.map(parseScyllaNote); - (found as ScyllaNote[]).push(...(filter?.note ? await filter.note(notes) : notes)); + (found as ScyllaNote[]).push( + ...(filter?.note ? await filter.note(notes) : notes), + ); untilDate = notes[notes.length - 1].createdAt; } } diff --git a/packages/backend/src/queue/processors/background/index-all-notes.ts b/packages/backend/src/queue/processors/background/index-all-notes.ts index 67f1ae14ff..e78dfca829 100644 --- a/packages/backend/src/queue/processors/background/index-all-notes.ts +++ b/packages/backend/src/queue/processors/background/index-all-notes.ts @@ -7,7 +7,7 @@ import { MoreThan } from "typeorm"; import { index } from "@/services/note/create.js"; import { Note } from "@/models/entities/note.js"; import meilisearch from "../../../db/meilisearch.js"; -import { scyllaClient } from "@/db/scylla.js"; +import { fetchPostCount, scyllaClient } from "@/db/scylla.js"; const logger = queueLogger.createSubLogger("index-all-notes"); @@ -57,9 +57,7 @@ export default async function indexAllNotes( try { const count = await (scyllaClient - ? scyllaClient - .execute("SELECT COUNT(1) FROM note") - .then((result) => result.first().get("count") as number) + ? fetchPostCount(false) : Notes.count()); total = count; await job.update({ indexedCount, cursor, total }); diff --git a/packages/backend/src/server/api/mastodon/endpoints/meta.ts b/packages/backend/src/server/api/mastodon/endpoints/meta.ts index 5872ae4a49..5940084556 100644 --- a/packages/backend/src/server/api/mastodon/endpoints/meta.ts +++ b/packages/backend/src/server/api/mastodon/endpoints/meta.ts @@ -4,7 +4,7 @@ import { fetchMeta } from "@/misc/fetch-meta.js"; import { Users, Notes } from "@/models/index.js"; import { IsNull } from "typeorm"; import { MAX_NOTE_TEXT_LENGTH, FILE_TYPE_BROWSERSAFE } from "@/const.js"; -import { scyllaClient } from "@/db/scylla"; +import { fetchPostCount, scyllaClient } from "@/db/scylla"; export async function getInstance( response: Entity.Instance, @@ -14,9 +14,7 @@ export async function getInstance( fetchMeta(true), Users.count({ where: { host: IsNull() } }), scyllaClient - ? scyllaClient - .execute("SELECT COUNT(1) FROM note") - .then((result) => result.first().get("count") as number) + ? fetchPostCount(true) : Notes.count({ where: { userHost: IsNull() } }), ]); diff --git a/packages/backend/src/server/nodeinfo.ts b/packages/backend/src/server/nodeinfo.ts index 695d854607..6bde5cab45 100644 --- a/packages/backend/src/server/nodeinfo.ts +++ b/packages/backend/src/server/nodeinfo.ts @@ -5,7 +5,7 @@ import { Users, Notes } from "@/models/index.js"; import { IsNull, MoreThan } from "typeorm"; import { MAX_NOTE_TEXT_LENGTH, MAX_CAPTION_TEXT_LENGTH } from "@/const.js"; import { Cache } from "@/misc/cache.js"; -import { scyllaClient } from "@/db/scylla"; +import { fetchPostCount, scyllaClient } from "@/db/scylla"; const router = new Router(); @@ -43,9 +43,7 @@ const nodeinfo2 = async () => { }, }), scyllaClient - ? scyllaClient - .execute("SELECT COUNT(1) FROM local_note") - .then((result) => result.first().get("count") as number) + ? fetchPostCount(true) : Notes.count({ where: { userHost: IsNull() } }), ]); diff --git a/packages/backend/src/services/chart/charts/instance.ts b/packages/backend/src/services/chart/charts/instance.ts index d6e3483d8e..82d4279975 100644 --- a/packages/backend/src/services/chart/charts/instance.ts +++ b/packages/backend/src/services/chart/charts/instance.ts @@ -5,6 +5,7 @@ import type { DriveFile } from "@/models/entities/drive-file.js"; import type { Note } from "@/models/entities/note.js"; import { toPuny } from "@/misc/convert-host.js"; import { name, schema } from "./entities/instance.js"; +import { scyllaClient } from "@/db/scylla.js"; /** * インスタンスごとのチャート @@ -18,9 +19,10 @@ export default class InstanceChart extends Chart { protected async tickMajor( group: string, ): Promise>> { + const zero = async () => 0; const [notesCount, usersCount, followingCount, followersCount, driveFiles] = await Promise.all([ - Notes.countBy({ userHost: group }), + scyllaClient ? zero() : Notes.countBy({ userHost: group }), Users.countBy({ host: group }), Followings.countBy({ followerHost: group }), Followings.countBy({ followeeHost: group }), diff --git a/packages/backend/src/services/chart/charts/notes.ts b/packages/backend/src/services/chart/charts/notes.ts index 42db60d0cf..d736a0bd22 100644 --- a/packages/backend/src/services/chart/charts/notes.ts +++ b/packages/backend/src/services/chart/charts/notes.ts @@ -4,6 +4,7 @@ import { Notes } from "@/models/index.js"; import { Not, IsNull } from "typeorm"; import type { Note } from "@/models/entities/note.js"; import { name, schema } from "./entities/notes.js"; +import { fetchPostCount, scyllaClient } from "@/db/scylla.js"; /** * ノートに関するチャート @@ -16,8 +17,12 @@ export default class NotesChart extends Chart { protected async tickMajor(): Promise>> { const [localCount, remoteCount] = await Promise.all([ - Notes.countBy({ userHost: IsNull() }), - Notes.countBy({ userHost: Not(IsNull()) }), + scyllaClient + ? fetchPostCount(true) + : Notes.countBy({ userHost: IsNull() }), + scyllaClient + ? fetchPostCount(false) + : Notes.countBy({ userHost: Not(IsNull()) }), ]); return { diff --git a/packages/backend/src/services/chart/charts/per-user-notes.ts b/packages/backend/src/services/chart/charts/per-user-notes.ts index 22f3fddb77..fc4b21fded 100644 --- a/packages/backend/src/services/chart/charts/per-user-notes.ts +++ b/packages/backend/src/services/chart/charts/per-user-notes.ts @@ -4,6 +4,7 @@ import type { User } from "@/models/entities/user.js"; import { Notes } from "@/models/index.js"; import type { Note } from "@/models/entities/note.js"; import { name, schema } from "./entities/per-user-notes.js"; +import { scyllaClient } from "@/db/scylla.js"; /** * ユーザーごとのノートに関するチャート @@ -17,7 +18,15 @@ export default class PerUserNotesChart extends Chart { protected async tickMajor( group: string, ): Promise>> { - const [count] = await Promise.all([Notes.countBy({ userId: group })]); + const count = await (scyllaClient + ? scyllaClient + .execute( + `SELECT COUNT(*) note_by_user_id WHERE "userId" = ?`, + [group], + { prepare: true }, + ) + .then((result) => result.first().get("count") as number) + : Notes.countBy({ userId: group })); return { total: count,