HoovesSocial/hippofish
2
0
Fork 0
Code Issues 3 Pull requests 1 Projects Releases Packages Wiki Activity Actions

fix: post count for charts

Browse source
This commit is contained in:
Namekuji 2023-09-05 07:43:17 -04:00
parent 4439d5210f
commit 98ffb20c6a
No known key found for this signature in database
GPG key ID: 1D62332C07FBA532
8 changed files with 69 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

20
packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/up.cql
View file

@ -104,6 +104,16 @@ CREATE MATERIALIZED VIEW note_by_user_id AS
PRIMARY KEY ("userId", "createdAt", "createdAtDate", "userHost", "visibility")
WITH CLUSTERING ORDER BY ("createdAt" DESC);
CREATE MATERIALIZED VIEW local_note_by_user_id AS
SELECT "userId", "createdAt", "createdAtDate", "userHost", "visibility" FROM note
WHERE "userId" IS NOT NULL
AND "createdAt" IS NOT NULL
AND "createdAtDate" IS NOT NULL
AND "userHost" = 'local'
AND "visibility" IS NOT NULL
PRIMARY KEY ("userId", "createdAt", "createdAtDate", "userHost", "visibility")
WITH CLUSTERING ORDER BY ("createdAt" DESC);
CREATE MATERIALIZED VIEW note_by_renote_id AS
SELECT * FROM note
WHERE "renoteId" IS NOT NULL
@ -157,16 +167,6 @@ CREATE MATERIALIZED VIEW local_timeline AS
PRIMARY KEY ("createdAtDate", "createdAt", "userId", "userHost", "visibility")
WITH CLUSTERING ORDER BY ("createdAt" DESC);
CREATE MATERIALIZED VIEW local_note AS
SELECT "createdAtDate", "createdAt", "userId", "userHost", "visibility" FROM note
WHERE "createdAtDate" IS NOT NULL
AND "createdAt" IS NOT NULL
AND "userId" IS NOT NULL
AND "userHost" = 'local'
AND "visibility" IS NOT NULL
PRIMARY KEY ("createdAtDate", "createdAt", "userId", "userHost", "visibility")
WITH CLUSTERING ORDER BY ("createdAt" DESC);
CREATE MATERIALIZED VIEW score_feed AS
SELECT * FROM note
WHERE "createdAtDate" IS NOT NULL

35
packages/backend/src/db/scylla.ts
View file

@ -6,6 +6,7 @@ import type { NoteReaction } from "@/models/entities/note-reaction.js";
import { Client, types, tracker } from "cassandra-driver";
import type { User } from "@/models/entities/user.js";
import {
Cache,
ChannelFollowingsCache,
InstanceMutingsCache,
LocalFollowingsCache,
@ -67,6 +68,32 @@ export const scyllaClient = newClient();
export const prepared = scyllaQueries;
const localPostCountCache = new Cache<number>("localPostCount", 1000 * 60 * 10);
export const allPostCountCache = new Cache<number>(
"allPostCount",
1000 * 60 * 10,
);
export async function fetchPostCount(local = false): Promise<number> {
if (!scyllaClient) {
throw new Error("ScyllaDB is disabled");
}
if (local) {
return await localPostCountCache.fetch(null, () =>
scyllaClient
.execute("SELECT COUNT(*) FROM local_note_by_user_id")
.then((result) => result.first().get("count") as number),
);
}
return await allPostCountCache.fetch(null, () =>
scyllaClient
.execute("SELECT COUNT(*) FROM note")
.then((result) => result.first().get("count") as number),
);
}
export interface ScyllaNotification {
targetId: string;
createdAtDate: Date;
@ -444,11 +471,15 @@ export async function execPaginationQuery(
untilDate = notifications[notifications.length - 1].createdAt;
} else if (kind === "reaction") {
const reactions = result.rows.map(parseScyllaReaction);
(found as ScyllaNoteReaction[]).push(...(filter?.reaction ? await filter.reaction(reactions) : reactions));
(found as ScyllaNoteReaction[]).push(
...(filter?.reaction ? await filter.reaction(reactions) : reactions),
);
untilDate = reactions[reactions.length - 1].createdAt;
} else {
const notes = result.rows.map(parseScyllaNote);
(found as ScyllaNote[]).push(...(filter?.note ? await filter.note(notes) : notes));
(found as ScyllaNote[]).push(
...(filter?.note ? await filter.note(notes) : notes),
);
untilDate = notes[notes.length - 1].createdAt;
}
}

6
packages/backend/src/queue/processors/background/index-all-notes.ts
View file

@ -7,7 +7,7 @@ import { MoreThan } from "typeorm";
import { index } from "@/services/note/create.js";
import { Note } from "@/models/entities/note.js";
import meilisearch from "../../../db/meilisearch.js";
import { scyllaClient } from "@/db/scylla.js";
import { fetchPostCount, scyllaClient } from "@/db/scylla.js";
const logger = queueLogger.createSubLogger("index-all-notes");
@ -57,9 +57,7 @@ export default async function indexAllNotes(
try {
const count = await (scyllaClient
? scyllaClient
.execute("SELECT COUNT(1) FROM note")
.then((result) => result.first().get("count") as number)
? fetchPostCount(false)
: Notes.count());
total = count;
await job.update({ indexedCount, cursor, total });

6
packages/backend/src/server/api/mastodon/endpoints/meta.ts
View file

@ -4,7 +4,7 @@ import { fetchMeta } from "@/misc/fetch-meta.js";
import { Users, Notes } from "@/models/index.js";
import { IsNull } from "typeorm";
import { MAX_NOTE_TEXT_LENGTH, FILE_TYPE_BROWSERSAFE } from "@/const.js";
import { scyllaClient } from "@/db/scylla";
import { fetchPostCount, scyllaClient } from "@/db/scylla";
export async function getInstance(
response: Entity.Instance,
@ -14,9 +14,7 @@ export async function getInstance(
fetchMeta(true),
Users.count({ where: { host: IsNull() } }),
scyllaClient
? scyllaClient
.execute("SELECT COUNT(1) FROM note")
.then((result) => result.first().get("count") as number)
? fetchPostCount(true)
: Notes.count({ where: { userHost: IsNull() } }),
]);

6
packages/backend/src/server/nodeinfo.ts
View file

@ -5,7 +5,7 @@ import { Users, Notes } from "@/models/index.js";
import { IsNull, MoreThan } from "typeorm";
import { MAX_NOTE_TEXT_LENGTH, MAX_CAPTION_TEXT_LENGTH } from "@/const.js";
import { Cache } from "@/misc/cache.js";
import { scyllaClient } from "@/db/scylla";
import { fetchPostCount, scyllaClient } from "@/db/scylla";
const router = new Router();
@ -43,9 +43,7 @@ const nodeinfo2 = async () => {
},
}),
scyllaClient
? scyllaClient
.execute("SELECT COUNT(1) FROM local_note")
.then((result) => result.first().get("count") as number)
? fetchPostCount(true)
: Notes.count({ where: { userHost: IsNull() } }),
]);

4
packages/backend/src/services/chart/charts/instance.ts
View file

@ -5,6 +5,7 @@ import type { DriveFile } from "@/models/entities/drive-file.js";
import type { Note } from "@/models/entities/note.js";
import { toPuny } from "@/misc/convert-host.js";
import { name, schema } from "./entities/instance.js";
import { scyllaClient } from "@/db/scylla.js";
/**
* インスタンスごとのチャート
@ -18,9 +19,10 @@ export default class InstanceChart extends Chart<typeof schema> {
protected async tickMajor(
group: string,
): Promise<Partial<KVs<typeof schema>>> {
const zero = async () => 0;
const [notesCount, usersCount, followingCount, followersCount, driveFiles] =
await Promise.all([
Notes.countBy({ userHost: group }),
scyllaClient ? zero() : Notes.countBy({ userHost: group }),
Users.countBy({ host: group }),
Followings.countBy({ followerHost: group }),
Followings.countBy({ followeeHost: group }),

9
packages/backend/src/services/chart/charts/notes.ts
View file

@ -4,6 +4,7 @@ import { Notes } from "@/models/index.js";
import { Not, IsNull } from "typeorm";
import type { Note } from "@/models/entities/note.js";
import { name, schema } from "./entities/notes.js";
import { fetchPostCount, scyllaClient } from "@/db/scylla.js";
/**
* ノートに関するチャート
@ -16,8 +17,12 @@ export default class NotesChart extends Chart<typeof schema> {
protected async tickMajor(): Promise<Partial<KVs<typeof schema>>> {
const [localCount, remoteCount] = await Promise.all([
Notes.countBy({ userHost: IsNull() }),
Notes.countBy({ userHost: Not(IsNull()) }),
scyllaClient
? fetchPostCount(true)
: Notes.countBy({ userHost: IsNull() }),
scyllaClient
? fetchPostCount(false)
: Notes.countBy({ userHost: Not(IsNull()) }),
]);
return {

11
packages/backend/src/services/chart/charts/per-user-notes.ts
View file

@ -4,6 +4,7 @@ import type { User } from "@/models/entities/user.js";
import { Notes } from "@/models/index.js";
import type { Note } from "@/models/entities/note.js";
import { name, schema } from "./entities/per-user-notes.js";
import { scyllaClient } from "@/db/scylla.js";
/**
* ユーザーごとのノートに関するチャート
@ -17,7 +18,15 @@ export default class PerUserNotesChart extends Chart<typeof schema> {
protected async tickMajor(
group: string,
): Promise<Partial<KVs<typeof schema>>> {
const [count] = await Promise.all([Notes.countBy({ userId: group })]);
const count = await (scyllaClient
? scyllaClient
.execute(
`SELECT COUNT(*) note_by_user_id WHERE "userId" = ?`,
[group],
{ prepare: true },
)
.then((result) => result.first().get("count") as number)
: Notes.countBy({ userId: group }));
return {
total: count,