HoovesSocial/hippofish
2
0
Fork 0
Code Issues 3 Pull requests 1 Projects Releases Packages Wiki Activity Actions

perf: read timelines from scylla

Browse source
This commit is contained in:
Namekuji 2023-08-09 06:03:09 -04:00
parent 36d7493264
commit dfcac3750c
No known key found for this signature in database
GPG key ID: 1D62332C07FBA532
3 changed files with 277 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

82
packages/backend/src/server/api/endpoints/notes/global-timeline.ts
View file

@ -1,5 +1,5 @@
import { fetchMeta } from "@/misc/fetch-meta.js";
import { Notes } from "@/models/index.js";
import { Notes, UserProfiles } from "@/models/index.js";
import { activeUsersChart } from "@/services/chart/index.js";
import define from "../../define.js";
import { ApiError } from "../../error.js";
@ -9,6 +9,23 @@ import { generateRepliesQuery } from "../../common/generate-replies-query.js";
import { generateMutedNoteQuery } from "../../common/generate-muted-note-query.js";
import { generateBlockedUserQuery } from "../../common/generate-block-query.js";
import { generateMutedUserRenotesQueryForNotes } from "../../common/generated-muted-renote-query.js";
import {
ScyllaNote,
execTimelineQuery,
filterBlockedUser,
filterMutedNote,
filterMutedRenotes,
filterMutedUser,
filterReply,
scyllaClient,
} from "@/db/scylla.js";
import {
InstanceMutingsCache,
RenoteMutingsCache,
UserBlockedCache,
UserMutingsCache,
userWordMuteCache,
} from "@/misc/cache.js";
export const meta = {
tags: ["notes"],
@ -70,6 +87,63 @@ export default define(meta, paramDef, async (ps, user) => {
}
}
process.nextTick(() => {
if (user) {
activeUsersChart.read(user);
}
});
if (scyllaClient) {
let [mutedUserIds, mutedInstances, blockerIds, renoteMutedIds]: string[][] =
[];
let mutedWords: string[][];
if (user) {
[mutedUserIds, mutedInstances, mutedWords, blockerIds, renoteMutedIds] =
await Promise.all([
UserMutingsCache.init(user.id).then((cache) => cache.getAll()),
InstanceMutingsCache.init(user.id).then((cache) => cache.getAll()),
userWordMuteCache
.fetchMaybe(user.id, () =>
UserProfiles.findOne({
select: ["mutedWords"],
where: { userId: user.id },
}).then((profile) => profile?.mutedWords),
)
.then((words) => words ?? []),
UserBlockedCache.init(user.id).then((cache) => cache.getAll()),
RenoteMutingsCache.init(user.id).then((cache) => cache.getAll()),
]);
}
const filter = async (notes: ScyllaNote[]) => {
let filtered = notes.filter(
(n) => n.visibility === "public" && !n.channelId,
);
filtered = await filterReply(filtered, ps.withReplies, user);
if (user) {
filtered = await filterMutedUser(
filtered,
user,
mutedUserIds,
mutedInstances,
);
filtered = await filterMutedNote(filtered, user, mutedWords);
filtered = await filterBlockedUser(filtered, user, blockerIds);
filtered = await filterMutedRenotes(filtered, user, renoteMutedIds);
}
if (ps.withFiles) {
filtered = filtered.filter((n) => n.files.length > 0);
}
filtered = filtered.filter((n) => n.visibility !== "hidden");
return filtered;
};
const foundNotes = await execTimelineQuery(ps, filter);
return await Notes.packMany(foundNotes.slice(0, ps.limit), user, {
scyllaNote: true,
});
}
//#region Construct query
const query = makePaginationQuery(
Notes.createQueryBuilder("note"),
@ -95,12 +169,6 @@ export default define(meta, paramDef, async (ps, user) => {
query.andWhere("note.visibility != 'hidden'");
//#endregion
process.nextTick(() => {
if (user) {
activeUsersChart.read(user);
}
});
// We fetch more than requested because some may be filtered out, and if there's less than
// requested, the pagination stops.
const found = [];

102
packages/backend/src/server/api/endpoints/notes/hybrid-timeline.ts
View file

@ -1,6 +1,6 @@
import { Brackets } from "typeorm";
import { fetchMeta } from "@/misc/fetch-meta.js";
import { Followings, Notes } from "@/models/index.js";
import { Followings, Notes, UserProfiles } from "@/models/index.js";
import { activeUsersChart } from "@/services/chart/index.js";
import define from "../../define.js";
import { ApiError } from "../../error.js";
@ -12,6 +12,27 @@ import { generateMutedNoteQuery } from "../../common/generate-muted-note-query.j
import { generateChannelQuery } from "../../common/generate-channel-query.js";
import { generateBlockedUserQuery } from "../../common/generate-block-query.js";
import { generateMutedUserRenotesQueryForNotes } from "../../common/generated-muted-renote-query.js";
import {
ScyllaNote,
execTimelineQuery,
filterBlockedUser,
filterChannel,
filterMutedNote,
filterMutedRenotes,
filterMutedUser,
filterReply,
filterVisibility,
scyllaClient,
} from "@/db/scylla.js";
import {
ChannelFollowingsCache,
InstanceMutingsCache,
LocalFollowingsCache,
RenoteMutingsCache,
UserBlockedCache,
UserMutingsCache,
userWordMuteCache,
} from "@/misc/cache.js";
export const meta = {
tags: ["notes"],
@ -75,6 +96,81 @@ export default define(meta, paramDef, async (ps, user) => {
throw new ApiError(meta.errors.stlDisabled);
}
process.nextTick(() => {
activeUsersChart.read(user);
});
if (scyllaClient) {
const [
followingChannelIds,
followingUserIds,
mutedUserIds,
mutedInstances,
mutedWords,
blockerIds,
renoteMutedIds,
] = await Promise.all([
ChannelFollowingsCache.init(user.id).then((cache) => cache.getAll()),
LocalFollowingsCache.init(user.id).then((cache) => cache.getAll()),
UserMutingsCache.init(user.id).then((cache) => cache.getAll()),
InstanceMutingsCache.init(user.id).then((cache) => cache.getAll()),
userWordMuteCache
.fetchMaybe(user.id, () =>
UserProfiles.findOne({
select: ["mutedWords"],
where: { userId: user.id },
}).then((profile) => profile?.mutedWords),
)
.then((words) => words ?? []),
UserBlockedCache.init(user.id).then((cache) => cache.getAll()),
RenoteMutingsCache.init(user.id).then((cache) => cache.getAll()),
]);
const validUserIds = [user.id].concat(followingUserIds);
const optFilter = (n: ScyllaNote) =>
!n.renoteId || !!n.text || n.files.length > 0 || n.hasPoll;
const filter = async (notes: ScyllaNote[]) => {
let filtered = notes.filter(
(n) =>
validUserIds.includes(n.userId) ||
(n.visibility === "public" && !n.userHost),
);
filtered = await filterChannel(filtered, user, followingChannelIds);
filtered = await filterReply(filtered, ps.withReplies, user);
filtered = await filterVisibility(filtered, user, followingUserIds);
filtered = await filterMutedUser(
filtered,
user,
mutedUserIds,
mutedInstances,
);
filtered = await filterMutedNote(filtered, user, mutedWords);
filtered = await filterBlockedUser(filtered, user, blockerIds);
filtered = await filterMutedRenotes(filtered, user, renoteMutedIds);
if (!ps.includeMyRenotes) {
filtered = filtered.filter((n) => n.userId !== user.id || optFilter(n));
}
if (!ps.includeRenotedMyNotes) {
filtered = filtered.filter(
(n) => n.renoteUserId !== user.id || optFilter(n),
);
}
if (!ps.includeLocalRenotes) {
filtered = filtered.filter((n) => n.renoteUserHost || optFilter(n));
}
if (ps.withFiles) {
filtered = filtered.filter((n) => n.files.length > 0);
}
filtered = filtered.filter((n) => n.visibility !== "hidden");
return filtered;
};
const foundNotes = await execTimelineQuery(ps, filter);
return await Notes.packMany(foundNotes.slice(0, ps.limit), user, {
scyllaNote: true,
});
}
//#region Construct query
const followingQuery = Followings.createQueryBuilder("following")
.select("following.followeeId")
@ -154,10 +250,6 @@ export default define(meta, paramDef, async (ps, user) => {
query.andWhere("note.visibility != 'hidden'");
//#endregion
process.nextTick(() => {
activeUsersChart.read(user);
});
// We fetch more than requested because some may be filtered out, and if there's less than
// requested, the pagination stops.
const found = [];

118
packages/backend/src/server/api/endpoints/notes/recommended-timeline.ts
View file

@ -1,6 +1,6 @@
import { Brackets } from "typeorm";
import { fetchMeta } from "@/misc/fetch-meta.js";
import { Notes } from "@/models/index.js";
import { Notes, UserProfiles } from "@/models/index.js";
import { activeUsersChart } from "@/services/chart/index.js";
import define from "../../define.js";
import { ApiError } from "../../error.js";
@ -12,6 +12,25 @@ import { generateMutedNoteQuery } from "../../common/generate-muted-note-query.j
import { generateChannelQuery } from "../../common/generate-channel-query.js";
import { generateBlockedUserQuery } from "../../common/generate-block-query.js";
import { generateMutedUserRenotesQueryForNotes } from "../../common/generated-muted-renote-query.js";
import {
ScyllaNote,
execTimelineQuery,
filterBlockedUser,
filterMutedNote,
filterMutedRenotes,
filterMutedUser,
filterReply,
filterVisibility,
scyllaClient,
} from "@/db/scylla.js";
import {
InstanceMutingsCache,
LocalFollowingsCache,
RenoteMutingsCache,
UserBlockedCache,
UserMutingsCache,
userWordMuteCache,
} from "@/misc/cache.js";
export const meta = {
tags: ["notes"],
@ -80,6 +99,90 @@ export default define(meta, paramDef, async (ps, user) => {
}
}
process.nextTick(() => {
if (user) {
activeUsersChart.read(user);
}
});
if (scyllaClient) {
let [
followingUserIds,
mutedUserIds,
mutedInstances,
blockerIds,
renoteMutedIds,
]: string[][] = [];
let mutedWords: string[][];
if (user) {
[
followingUserIds,
mutedUserIds,
mutedInstances,
mutedWords,
blockerIds,
renoteMutedIds,
] = await Promise.all([
LocalFollowingsCache.init(user.id).then((cache) => cache.getAll()),
UserMutingsCache.init(user.id).then((cache) => cache.getAll()),
InstanceMutingsCache.init(user.id).then((cache) => cache.getAll()),
userWordMuteCache
.fetchMaybe(user.id, () =>
UserProfiles.findOne({
select: ["mutedWords"],
where: { userId: user.id },
}).then((profile) => profile?.mutedWords),
)
.then((words) => words ?? []),
UserBlockedCache.init(user.id).then((cache) => cache.getAll()),
RenoteMutingsCache.init(user.id).then((cache) => cache.getAll()),
]);
}
const filter = async (notes: ScyllaNote[]) => {
let filtered = notes.filter(
(n) =>
n.visibility === "public" &&
n.userHost &&
m.recommendedInstances.includes(n.userHost) &&
!n.channelId,
);
filtered = await filterReply(filtered, ps.withReplies, user);
filtered = await filterVisibility(filtered, user, followingUserIds);
if (user) {
filtered = await filterMutedUser(
filtered,
user,
mutedUserIds,
mutedInstances,
);
filtered = await filterMutedNote(filtered, user, mutedWords);
filtered = await filterBlockedUser(filtered, user, blockerIds);
filtered = await filterMutedRenotes(filtered, user, renoteMutedIds);
}
if (ps.withFiles) {
filtered = filtered.filter((n) => n.files.length > 0);
}
if (ps.fileType) {
filtered = filtered.filter((n) =>
n.files.some((f) => ps.fileType?.includes(f.type)),
);
}
if (ps.excludeNsfw) {
filtered = filtered.filter(
(n) => !n.cw && n.files.every((f) => !f.isSensitive),
);
}
filtered = filtered.filter((n) => n.visibility !== "hidden");
return filtered;
};
const foundNotes = await execTimelineQuery(ps, filter);
return await Notes.packMany(foundNotes.slice(0, ps.limit), user, {
scyllaNote: true,
});
}
//#region Construct query
const query = makePaginationQuery(
Notes.createQueryBuilder("note"),
@ -91,12 +194,7 @@ export default define(meta, paramDef, async (ps, user) => {
.andWhere(
`(note.userHost = ANY ('{"${m.recommendedInstances.join('","')}"}'))`,
)
.andWhere("(note.visibility = 'public')")
.innerJoinAndSelect("note.user", "user")
.leftJoinAndSelect("note.reply", "reply")
.leftJoinAndSelect("note.renote", "renote")
.leftJoinAndSelect("reply.user", "replyUser")
.leftJoinAndSelect("renote.user", "renoteUser");
.andWhere("(note.visibility = 'public')");
generateChannelQuery(query, user);
generateRepliesQuery(query, ps.withReplies, user);
@ -133,12 +231,6 @@ export default define(meta, paramDef, async (ps, user) => {
query.andWhere("note.visibility != 'hidden'");
//#endregion
process.nextTick(() => {
if (user) {
activeUsersChart.read(user);
}
});
// We fetch more than requested because some may be filtered out, and if there's less than
// requested, the pagination stops.
const found = [];