HoovesSocial/hippofish
2
0
Fork 0
Code Issues 3 Pull requests 1 Projects Releases Packages Wiki Activity Actions

perf: add materialized view for renotes

Browse source
This commit is contained in:
Namekuji 2023-08-09 21:59:13 -04:00
parent 18a172cd29
commit 292c7651dc
No known key found for this signature in database
GPG key ID: 1D62332C07FBA532
9 changed files with 43 additions and 21 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/down.cql
View file

@ -2,6 +2,7 @@ DROP MATERIALIZED VIEW IF EXISTS reaction_by_id;
DROP MATERIALIZED VIEW IF EXISTS reaction_by_userid;
DROP INDEX IF EXISTS reaction_by_id;
DROP TABLE IF EXISTS reaction;
DROP MATERIALIZED VIEW IF EXISTS note_by_renote_id;
DROP MATERIALIZED VIEW IF EXISTS note_by_userid;
DROP MATERIALIZED VIEW IF EXISTS note_by_id;
DROP INDEX IF EXISTS note_by_uri;

13
packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/up.cql
View file

@ -83,7 +83,7 @@ CREATE MATERIALIZED VIEW IF NOT EXISTS note_by_id AS
PRIMARY KEY ("id", "createdAt", "createdAtDate")
WITH CLUSTERING ORDER BY ("createdAt" DESC);
CREATE MATERIALIZED VIEW IF NOT EXISTS note_by_userid AS
CREATE MATERIALIZED VIEW IF NOT EXISTS note_by_user_id AS
SELECT * FROM note
WHERE "userId" IS NOT NULL
AND "createdAt" IS NOT NULL
@ -92,6 +92,15 @@ CREATE MATERIALIZED VIEW IF NOT EXISTS note_by_userid AS
PRIMARY KEY ("userId", "createdAt", "createdAtDate", "id")
WITH CLUSTERING ORDER BY ("createdAt" DESC);
CREATE MATERIALIZED VIEW IF NOT EXISTS note_by_renote_id AS
SELECT * FROM note
WHERE "renoteId" IS NOT NULL
AND "createdAt" IS NOT NULL
AND "createdAtDate" IS NOT NULL
AND "id" IS NOT NULL
PRIMARY KEY ("renoteId", "createdAt", "createdAtDate", "id")
WITH CLUSTERING ORDER BY ("createdAt" DESC);
CREATE TABLE IF NOT EXISTS reaction (
"id" text,
"noteId" ascii,
@ -102,7 +111,7 @@ CREATE TABLE IF NOT EXISTS reaction (
PRIMARY KEY ("noteId", "userId") -- this key constraints one reaction per user for the same post
);
CREATE MATERIALIZED VIEW IF NOT EXISTS reaction_by_userid AS
CREATE MATERIALIZED VIEW IF NOT EXISTS reaction_by_user_id AS
SELECT * FROM reaction
WHERE "userId" IS NOT NULL
AND "createdAt" IS NOT NULL

24
packages/backend/src/db/scylla.ts
View file

@ -112,7 +112,8 @@ export const prepared = {
byUri: `SELECT * FROM note WHERE "uri" = ?`,
byUrl: `SELECT * FROM note WHERE "url" = ?`,
byId: `SELECT * FROM note_by_id WHERE "id" IN ?`,
byUserId: `SELECT * FROM note_by_userid WHERE "userId" IN ?`,
byUserId: `SELECT * FROM note_by_user_id WHERE "userId" IN ?`,
byRenoteId: `SELECT * FROM note_by_renote_id WHERE "renoteId" IN ?`,
},
delete: `DELETE FROM note WHERE "createdAtDate" = ? AND "createdAt" = ? AND "id" = ?`,
update: {
@ -136,7 +137,7 @@ export const prepared = {
VALUES (?, ?, ?, ?, ?, ?)`,
select: {
byNoteId: `SELECT * FROM reaction_by_id WHERE "noteId" IN ?`,
byUserId: `SELECT * FROM reaction_by_userid WHERE "userId" IN ?`,
byUserId: `SELECT * FROM reaction_by_user_id WHERE "userId" IN ?`,
byNoteAndUser: `SELECT * FROM reaction WHERE "noteId" IN ? AND "userId" IN ?`,
byId: `SELECT * FROM reaction WHERE "id" IN ?`,
},
@ -253,8 +254,13 @@ export function prepareTimelineQuery(ps: {
untilDate?: number;
sinceId?: string;
sinceDate?: number;
noteId?: string;
}): { query: string; untilDate: Date; sinceDate: Date | null } {
const queryParts = [`${prepared.note.select.byDate} AND "createdAt" < ?`];
const queryParts = [
`${
ps.noteId ? prepared.note.select.byRenoteId : prepared.note.select.byDate
} AND "createdAt" < ?`,
];
let until = new Date();
if (ps.untilId) {
@ -284,13 +290,14 @@ export function prepareTimelineQuery(ps: {
};
}
export async function execTimelineQuery(
export async function execNotePaginationQuery(
ps: {
limit: number;
untilId?: string;
untilDate?: number;
sinceId?: string;
sinceDate?: number;
noteId?: string;
},
filter?: (_: ScyllaNote[]) => Promise<ScyllaNote[]>,
maxDays = 30,
@ -304,11 +311,16 @@ export async function execTimelineQuery(
// Try to get posts of at most <maxDays> in the single request
while (foundNotes.length < ps.limit && scannedEmptyPartitions < maxDays) {
const params: (Date | string | string[] | number)[] = [untilDate, untilDate];
const params: (Date | string | string[] | number)[] = [];
if (ps.noteId) {
params.push(ps.noteId);
} else {
params.push(untilDate, untilDate);
}
if (sinceDate) {
params.push(sinceDate);
}
params.push(ps.limit)
params.push(ps.limit);
const result = await scyllaClient.execute(query, params, {
prepare: true,

4
packages/backend/src/server/api/endpoints/notes/global-timeline.ts
View file

@ -11,7 +11,7 @@ import { generateBlockedUserQuery } from "../../common/generate-block-query.js";
import { generateMutedUserRenotesQueryForNotes } from "../../common/generated-muted-renote-query.js";
import {
ScyllaNote,
execTimelineQuery,
execNotePaginationQuery,
filterBlockedUser,
filterMutedNote,
filterMutedRenotes,
@ -138,7 +138,7 @@ export default define(meta, paramDef, async (ps, user) => {
return filtered;
};
const foundNotes = await execTimelineQuery(ps, filter);
const foundNotes = await execNotePaginationQuery(ps, filter);
return await Notes.packMany(foundNotes.slice(0, ps.limit), user, {
scyllaNote: true,
});

4
packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts
View file

@ -14,7 +14,7 @@ import { generateBlockedUserQuery } from "../../common/generate-block-query.js";
import { generateMutedUserRenotesQueryForNotes } from "../../common/generated-muted-renote-query.js";
import {
ScyllaNote,
execTimelineQuery,
execNotePaginationQuery,
filterBlockedUser,
filterChannel,
filterMutedNote,
@ -165,7 +165,7 @@ export default define(meta, paramDef, async (ps, user) => {
return filtered;
};
const foundNotes = await execTimelineQuery(ps, filter);
const foundNotes = await execNotePaginationQuery(ps, filter);
return await Notes.packMany(foundNotes.slice(0, ps.limit), user, {
scyllaNote: true,
});

4
packages/backend/src/server/api/endpoints/notes/local-timeline.ts
View file

@ -14,7 +14,7 @@ import { generateBlockedUserQuery } from "../../common/generate-block-query.js";
import { generateMutedUserRenotesQueryForNotes } from "../../common/generated-muted-renote-query.js";
import {
ScyllaNote,
execTimelineQuery,
execNotePaginationQuery,
filterBlockedUser,
filterChannel,
filterMutedNote,
@ -180,7 +180,7 @@ export default define(meta, paramDef, async (ps, user) => {
return filtered;
};
const foundNotes = await execTimelineQuery(ps, filter);
const foundNotes = await execNotePaginationQuery(ps, filter);
return await Notes.packMany(foundNotes.slice(0, ps.limit), user, {
scyllaNote: true,
});

4
packages/backend/src/server/api/endpoints/notes/recommended-timeline.ts
View file

@ -14,7 +14,7 @@ import { generateBlockedUserQuery } from "../../common/generate-block-query.js";
import { generateMutedUserRenotesQueryForNotes } from "../../common/generated-muted-renote-query.js";
import {
ScyllaNote,
execTimelineQuery,
execNotePaginationQuery,
filterBlockedUser,
filterMutedNote,
filterMutedRenotes,
@ -177,7 +177,7 @@ export default define(meta, paramDef, async (ps, user) => {
return filtered;
};
const foundNotes = await execTimelineQuery(ps, filter);
const foundNotes = await execNotePaginationQuery(ps, filter);
return await Notes.packMany(foundNotes.slice(0, ps.limit), user, {
scyllaNote: true,
});

6
packages/backend/src/server/api/endpoints/notes/renotes.ts
View file

@ -8,7 +8,7 @@ import { makePaginationQuery } from "../../common/make-pagination-query.js";
import { generateBlockedUserQuery } from "../../common/generate-block-query.js";
import {
ScyllaNote,
execTimelineQuery,
execNotePaginationQuery,
filterBlockedUser,
filterMutedUser,
filterVisibility,
@ -85,7 +85,7 @@ export default define(meta, paramDef, async (ps, user) => {
}
const filter = async (notes: ScyllaNote[]) => {
let filtered = notes.filter((n) => n.renoteId === note.id);
let filtered = notes;
if (ps.userId) {
filtered = filtered.filter((n) => n.userId === ps.userId);
}
@ -102,7 +102,7 @@ export default define(meta, paramDef, async (ps, user) => {
return filtered;
};
const foundNotes = await execTimelineQuery(ps, filter, 1);
const foundNotes = await execNotePaginationQuery(ps, filter);
return await Notes.packMany(foundNotes.slice(0, ps.limit), user, {
scyllaNote: true,
});

4
packages/backend/src/server/api/endpoints/notes/timeline.ts
View file

@ -17,7 +17,7 @@ import {
filterChannel,
filterReply,
filterVisibility,
execTimelineQuery,
execNotePaginationQuery,
filterMutedUser,
filterMutedNote,
filterBlockedUser,
@ -152,7 +152,7 @@ export default define(meta, paramDef, async (ps, user) => {
return filtered;
};
const foundNotes = await execTimelineQuery(ps, filter);
const foundNotes = await execNotePaginationQuery(ps, filter);
return await Notes.packMany(foundNotes.slice(0, ps.limit), user, {
scyllaNote: true,
});