fix: post count for charts

packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/up.cql

@@ -104,6 +104,16 @@ CREATE MATERIALIZED VIEW note_by_user_id AS
 	PRIMARY KEY ("userId", "createdAt", "createdAtDate", "userHost", "visibility")
 	WITH CLUSTERING ORDER BY ("createdAt" DESC);
 
+CREATE MATERIALIZED VIEW local_note_by_user_id AS
+	SELECT "userId", "createdAt", "createdAtDate", "userHost", "visibility" FROM note
+	WHERE "userId" IS NOT NULL
+	AND "createdAt" IS NOT NULL
+	AND "createdAtDate" IS NOT NULL
+	AND "userHost" = 'local'
+	AND "visibility" IS NOT NULL
+	PRIMARY KEY ("userId", "createdAt", "createdAtDate", "userHost", "visibility")
+	WITH CLUSTERING ORDER BY ("createdAt" DESC);
+
 CREATE MATERIALIZED VIEW note_by_renote_id AS
 	SELECT * FROM note
 	WHERE "renoteId" IS NOT NULL
@@ -157,16 +167,6 @@ CREATE MATERIALIZED VIEW local_timeline AS
 	PRIMARY KEY ("createdAtDate", "createdAt", "userId", "userHost", "visibility")
 	WITH CLUSTERING ORDER BY ("createdAt" DESC);
 
-CREATE MATERIALIZED VIEW local_note AS
-	SELECT "createdAtDate", "createdAt", "userId", "userHost", "visibility" FROM note
-	WHERE "createdAtDate" IS NOT NULL
-	AND "createdAt" IS NOT NULL
-	AND "userId" IS NOT NULL
-	AND "userHost" = 'local'
-	AND "visibility" IS NOT NULL
-	PRIMARY KEY ("createdAtDate", "createdAt", "userId", "userHost", "visibility")
-	WITH CLUSTERING ORDER BY ("createdAt" DESC);
-
 CREATE MATERIALIZED VIEW score_feed AS
 	SELECT * FROM note
 	WHERE "createdAtDate" IS NOT NULL

packages/backend/src/db/scylla.ts

@@ -6,6 +6,7 @@ import type { NoteReaction } from "@/models/entities/note-reaction.js";
 import { Client, types, tracker } from "cassandra-driver";
 import type { User } from "@/models/entities/user.js";
 import {
+	Cache,
 	ChannelFollowingsCache,
 	InstanceMutingsCache,
 	LocalFollowingsCache,
@@ -67,6 +68,32 @@ export const scyllaClient = newClient();
 
 export const prepared = scyllaQueries;
 
+const localPostCountCache = new Cache<number>("localPostCount", 1000 * 60 * 10);
+export const allPostCountCache = new Cache<number>(
+	"allPostCount",
+	1000 * 60 * 10,
+);
+
+export async function fetchPostCount(local = false): Promise<number> {
+	if (!scyllaClient) {
+		throw new Error("ScyllaDB is disabled");
+	}
+
+	if (local) {
+		return await localPostCountCache.fetch(null, () =>
+			scyllaClient
+				.execute("SELECT COUNT(*) FROM local_note_by_user_id")
+				.then((result) => result.first().get("count") as number),
+		);
+	}
+
+	return await allPostCountCache.fetch(null, () =>
+		scyllaClient
+			.execute("SELECT COUNT(*) FROM note")
+			.then((result) => result.first().get("count") as number),
+	);
+}
+
 export interface ScyllaNotification {
 	targetId: string;
 	createdAtDate: Date;
@@ -444,11 +471,15 @@ export async function execPaginationQuery(
 				untilDate = notifications[notifications.length - 1].createdAt;
 			} else if (kind === "reaction") {
 				const reactions = result.rows.map(parseScyllaReaction);
-				(found as ScyllaNoteReaction[]).push(...(filter?.reaction ? await filter.reaction(reactions) : reactions));
+				(found as ScyllaNoteReaction[]).push(
+					...(filter?.reaction ? await filter.reaction(reactions) : reactions),
+				);
 				untilDate = reactions[reactions.length - 1].createdAt;
 			} else {
 				const notes = result.rows.map(parseScyllaNote);
-				(found as ScyllaNote[]).push(...(filter?.note ? await filter.note(notes) : notes));
+				(found as ScyllaNote[]).push(
+					...(filter?.note ? await filter.note(notes) : notes),
+				);
 				untilDate = notes[notes.length - 1].createdAt;
 			}
 		}

packages/backend/src/queue/processors/background/index-all-notes.ts

@@ -7,7 +7,7 @@ import { MoreThan } from "typeorm";
 import { index } from "@/services/note/create.js";
 import { Note } from "@/models/entities/note.js";
 import meilisearch from "../../../db/meilisearch.js";
-import { scyllaClient } from "@/db/scylla.js";
+import { fetchPostCount, scyllaClient } from "@/db/scylla.js";
 
 const logger = queueLogger.createSubLogger("index-all-notes");
 
@@ -57,9 +57,7 @@ export default async function indexAllNotes(
 
 		try {
 			const count = await (scyllaClient
-				? scyllaClient
+				? fetchPostCount(false)
-						.execute("SELECT COUNT(1) FROM note")
-						.then((result) => result.first().get("count") as number)
 				: Notes.count());
 			total = count;
 			await job.update({ indexedCount, cursor, total });

packages/backend/src/server/api/mastodon/endpoints/meta.ts

@@ -4,7 +4,7 @@ import { fetchMeta } from "@/misc/fetch-meta.js";
 import { Users, Notes } from "@/models/index.js";
 import { IsNull } from "typeorm";
 import { MAX_NOTE_TEXT_LENGTH, FILE_TYPE_BROWSERSAFE } from "@/const.js";
-import { scyllaClient } from "@/db/scylla";
+import { fetchPostCount, scyllaClient } from "@/db/scylla";
 
 export async function getInstance(
 	response: Entity.Instance,
@@ -14,9 +14,7 @@ export async function getInstance(
 		fetchMeta(true),
 		Users.count({ where: { host: IsNull() } }),
 		scyllaClient
-			? scyllaClient
+			? fetchPostCount(true)
-					.execute("SELECT COUNT(1) FROM note")
-					.then((result) => result.first().get("count") as number)
 			: Notes.count({ where: { userHost: IsNull() } }),
 	]);
 

packages/backend/src/server/nodeinfo.ts

@@ -5,7 +5,7 @@ import { Users, Notes } from "@/models/index.js";
 import { IsNull, MoreThan } from "typeorm";
 import { MAX_NOTE_TEXT_LENGTH, MAX_CAPTION_TEXT_LENGTH } from "@/const.js";
 import { Cache } from "@/misc/cache.js";
-import { scyllaClient } from "@/db/scylla";
+import { fetchPostCount, scyllaClient } from "@/db/scylla";
 
 const router = new Router();
 
@@ -43,9 +43,7 @@ const nodeinfo2 = async () => {
 				},
 			}),
 			scyllaClient
-				? scyllaClient
+				? fetchPostCount(true)
-						.execute("SELECT COUNT(1) FROM local_note")
-						.then((result) => result.first().get("count") as number)
 				: Notes.count({ where: { userHost: IsNull() } }),
 		]);
 

packages/backend/src/services/chart/charts/instance.ts

@@ -5,6 +5,7 @@ import type { DriveFile } from "@/models/entities/drive-file.js";
 import type { Note } from "@/models/entities/note.js";
 import { toPuny } from "@/misc/convert-host.js";
 import { name, schema } from "./entities/instance.js";
+import { scyllaClient } from "@/db/scylla.js";
 
 /**
  * インスタンスごとのチャート
@@ -18,9 +19,10 @@ export default class InstanceChart extends Chart<typeof schema> {
 	protected async tickMajor(
 		group: string,
 	): Promise<Partial<KVs<typeof schema>>> {
+		const zero = async () => 0;
 		const [notesCount, usersCount, followingCount, followersCount, driveFiles] =
 			await Promise.all([
-				Notes.countBy({ userHost: group }),
+				scyllaClient ? zero() : Notes.countBy({ userHost: group }),
 				Users.countBy({ host: group }),
 				Followings.countBy({ followerHost: group }),
 				Followings.countBy({ followeeHost: group }),

packages/backend/src/services/chart/charts/notes.ts

@@ -4,6 +4,7 @@ import { Notes } from "@/models/index.js";
 import { Not, IsNull } from "typeorm";
 import type { Note } from "@/models/entities/note.js";
 import { name, schema } from "./entities/notes.js";
+import { fetchPostCount, scyllaClient } from "@/db/scylla.js";
 
 /**
  * ノートに関するチャート
@@ -16,8 +17,12 @@ export default class NotesChart extends Chart<typeof schema> {
 
 	protected async tickMajor(): Promise<Partial<KVs<typeof schema>>> {
 		const [localCount, remoteCount] = await Promise.all([
-			Notes.countBy({ userHost: IsNull() }),
+			scyllaClient
-			Notes.countBy({ userHost: Not(IsNull()) }),
+				? fetchPostCount(true)
+				: Notes.countBy({ userHost: IsNull() }),
+			scyllaClient
+				? fetchPostCount(false)
+				: Notes.countBy({ userHost: Not(IsNull()) }),
 		]);
 
 		return {

packages/backend/src/services/chart/charts/per-user-notes.ts

@@ -4,6 +4,7 @@ import type { User } from "@/models/entities/user.js";
 import { Notes } from "@/models/index.js";
 import type { Note } from "@/models/entities/note.js";
 import { name, schema } from "./entities/per-user-notes.js";
+import { scyllaClient } from "@/db/scylla.js";
 
 /**
  * ユーザーごとのノートに関するチャート
@@ -17,7 +18,15 @@ export default class PerUserNotesChart extends Chart<typeof schema> {
 	protected async tickMajor(
 		group: string,
 	): Promise<Partial<KVs<typeof schema>>> {
-		const [count] = await Promise.all([Notes.countBy({ userId: group })]);
+		const count = await (scyllaClient
+			? scyllaClient
+					.execute(
+						`SELECT COUNT(*) note_by_user_id WHERE "userId" = ?`,
+						[group],
+						{ prepare: true },
+					)
+					.then((result) => result.first().get("count") as number)
+			: Notes.countBy({ userId: group }));
 
 		return {
 			total: count,