From 292c7651dc2128a4ece9f69d8d401e808d9bc007 Mon Sep 17 00:00:00 2001
From: Namekuji <nmkj@waah.day>
Date: Wed, 9 Aug 2023 21:59:13 -0400
Subject: [PATCH] perf: add materialized view for renotes
---
.../cql/1689400417034_timeline/down.cql | 1 +
.../cql/1689400417034_timeline/up.cql | 13 ++++++++--
packages/backend/src/db/scylla.ts | 24 ++++++++++++++-----
.../api/endpoints/notes/global-timeline.ts | 4 ++--
.../api/endpoints/notes/hybrid-timeline.ts | 4 ++--
.../api/endpoints/notes/local-timeline.ts | 4 ++--
.../endpoints/notes/recommended-timeline.ts | 4 ++--
.../src/server/api/endpoints/notes/renotes.ts | 6 ++---
.../server/api/endpoints/notes/timeline.ts | 4 ++--
9 files changed, 43 insertions(+), 21 deletions(-)
diff --git a/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/down.cql b/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/down.cql
index abf85789b1..c1746c8a49 100644
--- a/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/down.cql
+++ b/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/down.cql
@@ -2,6 +2,7 @@ DROP MATERIALIZED VIEW IF EXISTS reaction_by_id;
DROP MATERIALIZED VIEW IF EXISTS reaction_by_userid;
DROP INDEX IF EXISTS reaction_by_id;
DROP TABLE IF EXISTS reaction;
+DROP MATERIALIZED VIEW IF EXISTS note_by_renote_id;
DROP MATERIALIZED VIEW IF EXISTS note_by_userid;
DROP MATERIALIZED VIEW IF EXISTS note_by_id;
DROP INDEX IF EXISTS note_by_uri;
diff --git a/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/up.cql b/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/up.cql
index c38657df2b..64fba60827 100644
--- a/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/up.cql
+++ b/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/up.cql
@@ -83,7 +83,7 @@ CREATE MATERIALIZED VIEW IF NOT EXISTS note_by_id AS
PRIMARY KEY ("id", "createdAt", "createdAtDate")
WITH CLUSTERING ORDER BY ("createdAt" DESC);
-CREATE MATERIALIZED VIEW IF NOT EXISTS note_by_userid AS
+CREATE MATERIALIZED VIEW IF NOT EXISTS note_by_user_id AS
SELECT * FROM note
WHERE "userId" IS NOT NULL
AND "createdAt" IS NOT NULL
@@ -92,6 +92,15 @@ CREATE MATERIALIZED VIEW IF NOT EXISTS note_by_userid AS
PRIMARY KEY ("userId", "createdAt", "createdAtDate", "id")
WITH CLUSTERING ORDER BY ("createdAt" DESC);
+CREATE MATERIALIZED VIEW IF NOT EXISTS note_by_renote_id AS
+ SELECT * FROM note
+ WHERE "renoteId" IS NOT NULL
+ AND "createdAt" IS NOT NULL
+ AND "createdAtDate" IS NOT NULL
+ AND "id" IS NOT NULL
+ PRIMARY KEY ("renoteId", "createdAt", "createdAtDate", "id")
+ WITH CLUSTERING ORDER BY ("createdAt" DESC);
+
CREATE TABLE IF NOT EXISTS reaction (
"id" text,
"noteId" ascii,
@@ -102,7 +111,7 @@ CREATE TABLE IF NOT EXISTS reaction (
PRIMARY KEY ("noteId", "userId") -- this key constraints one reaction per user for the same post
);
-CREATE MATERIALIZED VIEW IF NOT EXISTS reaction_by_userid AS
+CREATE MATERIALIZED VIEW IF NOT EXISTS reaction_by_user_id AS
SELECT * FROM reaction
WHERE "userId" IS NOT NULL
AND "createdAt" IS NOT NULL
diff --git a/packages/backend/src/db/scylla.ts b/packages/backend/src/db/scylla.ts
index cba13fb89f..d168a4ceb9 100644
--- a/packages/backend/src/db/scylla.ts
+++ b/packages/backend/src/db/scylla.ts
@@ -112,7 +112,8 @@ export const prepared = {
byUri: `SELECT * FROM note WHERE "uri" = ?`,
byUrl: `SELECT * FROM note WHERE "url" = ?`,
byId: `SELECT * FROM note_by_id WHERE "id" IN ?`,
- byUserId: `SELECT * FROM note_by_userid WHERE "userId" IN ?`,
+ byUserId: `SELECT * FROM note_by_user_id WHERE "userId" IN ?`,
+ byRenoteId: `SELECT * FROM note_by_renote_id WHERE "renoteId" IN ?`,
},
delete: `DELETE FROM note WHERE "createdAtDate" = ? AND "createdAt" = ? AND "id" = ?`,
update: {
@@ -136,7 +137,7 @@ export const prepared = {
VALUES (?, ?, ?, ?, ?, ?)`,
select: {
byNoteId: `SELECT * FROM reaction_by_id WHERE "noteId" IN ?`,
- byUserId: `SELECT * FROM reaction_by_userid WHERE "userId" IN ?`,
+ byUserId: `SELECT * FROM reaction_by_user_id WHERE "userId" IN ?`,
byNoteAndUser: `SELECT * FROM reaction WHERE "noteId" IN ? AND "userId" IN ?`,
byId: `SELECT * FROM reaction WHERE "id" IN ?`,
},
@@ -253,8 +254,13 @@ export function prepareTimelineQuery(ps: {
untilDate?: number;
sinceId?: string;
sinceDate?: number;
+ noteId?: string;
}): { query: string; untilDate: Date; sinceDate: Date | null } {
- const queryParts = [`${prepared.note.select.byDate} AND "createdAt" < ?`];
+ const queryParts = [
+ `${
+ ps.noteId ? prepared.note.select.byRenoteId : prepared.note.select.byDate
+ } AND "createdAt" < ?`,
+ ];
let until = new Date();
if (ps.untilId) {
@@ -284,13 +290,14 @@ export function prepareTimelineQuery(ps: {
};
}
-export async function execTimelineQuery(
+export async function execNotePaginationQuery(
ps: {
limit: number;
untilId?: string;
untilDate?: number;
sinceId?: string;
sinceDate?: number;
+ noteId?: string;
},
filter?: (_: ScyllaNote[]) => Promise<ScyllaNote[]>,
maxDays = 30,
@@ -304,11 +311,16 @@ export async function execTimelineQuery(
// Try to get posts of at most <maxDays> in the single request
while (foundNotes.length < ps.limit && scannedEmptyPartitions < maxDays) {
- const params: (Date | string | string[] | number)[] = [untilDate, untilDate];
+ const params: (Date | string | string[] | number)[] = [];
+ if (ps.noteId) {
+ params.push(ps.noteId);
+ } else {
+ params.push(untilDate, untilDate);
+ }
if (sinceDate) {
params.push(sinceDate);
}
- params.push(ps.limit)
+ params.push(ps.limit);
const result = await scyllaClient.execute(query, params, {
prepare: true,
diff --git a/packages/backend/src/server/api/endpoints/notes/global-timeline.ts b/packages/backend/src/server/api/endpoints/notes/global-timeline.ts
index f660dd8f2b..5f504aeed8 100644
--- a/packages/backend/src/server/api/endpoints/notes/global-timeline.ts
+++ b/packages/backend/src/server/api/endpoints/notes/global-timeline.ts
@@ -11,7 +11,7 @@ import { generateBlockedUserQuery } from "../../common/generate-block-query.js";
import { generateMutedUserRenotesQueryForNotes } from "../../common/generated-muted-renote-query.js";
import {
ScyllaNote,
- execTimelineQuery,
+ execNotePaginationQuery,
filterBlockedUser,
filterMutedNote,
filterMutedRenotes,
@@ -138,7 +138,7 @@ export default define(meta, paramDef, async (ps, user) => {
return filtered;
};
- const foundNotes = await execTimelineQuery(ps, filter);
+ const foundNotes = await execNotePaginationQuery(ps, filter);
return await Notes.packMany(foundNotes.slice(0, ps.limit), user, {
scyllaNote: true,
});
diff --git a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts
index 4443946509..9b97b20b28 100644
--- a/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts
+++ b/packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts
@@ -14,7 +14,7 @@ import { generateBlockedUserQuery } from "../../common/generate-block-query.js";
import { generateMutedUserRenotesQueryForNotes } from "../../common/generated-muted-renote-query.js";
import {
ScyllaNote,
- execTimelineQuery,
+ execNotePaginationQuery,
filterBlockedUser,
filterChannel,
filterMutedNote,
@@ -165,7 +165,7 @@ export default define(meta, paramDef, async (ps, user) => {
return filtered;
};
- const foundNotes = await execTimelineQuery(ps, filter);
+ const foundNotes = await execNotePaginationQuery(ps, filter);
return await Notes.packMany(foundNotes.slice(0, ps.limit), user, {
scyllaNote: true,
});
diff --git a/packages/backend/src/server/api/endpoints/notes/local-timeline.ts b/packages/backend/src/server/api/endpoints/notes/local-timeline.ts
index c7ddc49052..246548c983 100644
--- a/packages/backend/src/server/api/endpoints/notes/local-timeline.ts
+++ b/packages/backend/src/server/api/endpoints/notes/local-timeline.ts
@@ -14,7 +14,7 @@ import { generateBlockedUserQuery } from "../../common/generate-block-query.js";
import { generateMutedUserRenotesQueryForNotes } from "../../common/generated-muted-renote-query.js";
import {
ScyllaNote,
- execTimelineQuery,
+ execNotePaginationQuery,
filterBlockedUser,
filterChannel,
filterMutedNote,
@@ -180,7 +180,7 @@ export default define(meta, paramDef, async (ps, user) => {
return filtered;
};
- const foundNotes = await execTimelineQuery(ps, filter);
+ const foundNotes = await execNotePaginationQuery(ps, filter);
return await Notes.packMany(foundNotes.slice(0, ps.limit), user, {
scyllaNote: true,
});
diff --git a/packages/backend/src/server/api/endpoints/notes/recommended-timeline.ts b/packages/backend/src/server/api/endpoints/notes/recommended-timeline.ts
index 6bdc017482..f15a112acc 100644
--- a/packages/backend/src/server/api/endpoints/notes/recommended-timeline.ts
+++ b/packages/backend/src/server/api/endpoints/notes/recommended-timeline.ts
@@ -14,7 +14,7 @@ import { generateBlockedUserQuery } from "../../common/generate-block-query.js";
import { generateMutedUserRenotesQueryForNotes } from "../../common/generated-muted-renote-query.js";
import {
ScyllaNote,
- execTimelineQuery,
+ execNotePaginationQuery,
filterBlockedUser,
filterMutedNote,
filterMutedRenotes,
@@ -177,7 +177,7 @@ export default define(meta, paramDef, async (ps, user) => {
return filtered;
};
- const foundNotes = await execTimelineQuery(ps, filter);
+ const foundNotes = await execNotePaginationQuery(ps, filter);
return await Notes.packMany(foundNotes.slice(0, ps.limit), user, {
scyllaNote: true,
});
diff --git a/packages/backend/src/server/api/endpoints/notes/renotes.ts b/packages/backend/src/server/api/endpoints/notes/renotes.ts
index ae48c165c0..550981aba4 100644
--- a/packages/backend/src/server/api/endpoints/notes/renotes.ts
+++ b/packages/backend/src/server/api/endpoints/notes/renotes.ts
@@ -8,7 +8,7 @@ import { makePaginationQuery } from "../../common/make-pagination-query.js";
import { generateBlockedUserQuery } from "../../common/generate-block-query.js";
import {
ScyllaNote,
- execTimelineQuery,
+ execNotePaginationQuery,
filterBlockedUser,
filterMutedUser,
filterVisibility,
@@ -85,7 +85,7 @@ export default define(meta, paramDef, async (ps, user) => {
}
const filter = async (notes: ScyllaNote[]) => {
- let filtered = notes.filter((n) => n.renoteId === note.id);
+ let filtered = notes;
if (ps.userId) {
filtered = filtered.filter((n) => n.userId === ps.userId);
}
@@ -102,7 +102,7 @@ export default define(meta, paramDef, async (ps, user) => {
return filtered;
};
- const foundNotes = await execTimelineQuery(ps, filter, 1);
+ const foundNotes = await execNotePaginationQuery(ps, filter);
return await Notes.packMany(foundNotes.slice(0, ps.limit), user, {
scyllaNote: true,
});
diff --git a/packages/backend/src/server/api/endpoints/notes/timeline.ts b/packages/backend/src/server/api/endpoints/notes/timeline.ts
index 3852b85a89..50e4adbe89 100644
--- a/packages/backend/src/server/api/endpoints/notes/timeline.ts
+++ b/packages/backend/src/server/api/endpoints/notes/timeline.ts
@@ -17,7 +17,7 @@ import {
filterChannel,
filterReply,
filterVisibility,
- execTimelineQuery,
+ execNotePaginationQuery,
filterMutedUser,
filterMutedNote,
filterBlockedUser,
@@ -152,7 +152,7 @@ export default define(meta, paramDef, async (ps, user) => {
return filtered;
};
- const foundNotes = await execTimelineQuery(ps, filter);
+ const foundNotes = await execNotePaginationQuery(ps, filter);
return await Notes.packMany(foundNotes.slice(0, ps.limit), user, {
scyllaNote: true,
});