HoovesSocial/hippofish
2
0
Fork 0
Code Issues 3 Pull requests 1 Projects Releases Packages Wiki Activity Actions

fix: include quotes

Browse source
This commit is contained in:
Namekuji 2023-08-13 20:38:26 -04:00
parent aecc243033
commit 37215ae0fa
No known key found for this signature in database
GPG key ID: 1D62332C07FBA532
5 changed files with 167 additions and 66 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
packages/backend/src/db/cql.ts
View file

@ -46,7 +46,7 @@ export const scyllaQueries = {
byUri: `SELECT * FROM note WHERE "uri" = ?`,
byUrl: `SELECT * FROM note WHERE "url" = ?`,
byId: `SELECT * FROM note WHERE "id" = ?`,
byUserId: `SELECT * FROM note_by_user_id WHERE "userId" IN ?`,
byUserId: `SELECT * FROM note_by_user_id WHERE "userId" = ?`,
byRenoteId: `SELECT * FROM note_by_renote_id WHERE "renoteId" = ?`,
byReplyId: `SELECT * FROM note WHERE "replyId" = ?`
},

40
packages/backend/src/db/scylla.ts
View file

@ -82,7 +82,10 @@ export interface ScyllaDriveFile {
height: number | null;
}
export function getScyllaDrivePublicUrl(file: ScyllaDriveFile, thumbnail = false): string | null {
export function getScyllaDrivePublicUrl(
file: ScyllaDriveFile,
thumbnail = false,
): string | null {
const isImage =
file.type &&
[
@ -190,12 +193,13 @@ export interface ScyllaNoteReaction extends NoteReaction {
const QUERY_LIMIT = 1000; // TODO: should this be configurable?
export type TimelineKind =
export type FeedType =
| "home"
| "local"
| "recommended"
| "global"
| "renotes";
| "renotes"
| "user";
export function parseScyllaReaction(row: types.Row): ScyllaNoteReaction {
return {
@ -209,7 +213,7 @@ export function parseScyllaReaction(row: types.Row): ScyllaNoteReaction {
}
export function prepareNoteQuery(
kind: TimelineKind,
kind: FeedType,
ps: {
untilId?: string;
untilDate?: number;
@ -233,6 +237,9 @@ export function prepareNoteQuery(
case "renotes":
queryParts.push(prepared.note.select.byRenoteId);
break;
case "user":
queryParts.push(prepared.note.select.byUserId);
break;
default:
queryParts.push(prepared.note.select.byDate);
}
@ -269,7 +276,7 @@ export function prepareNoteQuery(
}
export async function execNotePaginationQuery(
kind: TimelineKind,
kind: FeedType,
ps: {
limit: number;
untilId?: string;
@ -284,11 +291,15 @@ export async function execNotePaginationQuery(
): Promise<ScyllaNote[]> {
if (!scyllaClient) return [];
if (kind === "home" && !userId) {
throw new Error("Query of home timeline needs userId");
}
if (kind === "renotes" && !ps.noteId) {
throw new Error("Query of renotes needs noteId");
switch (kind) {
case "home":
case "user":
if (!userId)
throw new Error("Query of home and user timelines needs userId");
break;
case "renotes":
if (!ps.noteId) throw new Error("Query of renotes needs noteId");
break;
}
let { query, untilDate, sinceDate } = prepareNoteQuery(kind, ps);
@ -300,14 +311,15 @@ export async function execNotePaginationQuery(
while (foundNotes.length < ps.limit && scannedPartitions < maxPartitions) {
const params: (Date | string | string[] | number)[] = [];
if (kind === "home" && userId) {
params.push(userId);
}
if (kind === "renotes" && ps.noteId) {
params.push(userId, untilDate, untilDate);
} else if (kind === "user" && userId) {
params.push(userId, untilDate);
} else if (kind === "renotes" && ps.noteId) {
params.push(ps.noteId, untilDate);
} else {
params.push(untilDate, untilDate);
}
if (sinceDate) {
params.push(sinceDate);
}

13
packages/backend/src/server/api/endpoints/notes.ts
View file

@ -41,18 +41,7 @@ export default define(meta, paramDef, async (ps) => {
ps.untilId,
)
.andWhere("note.visibility = 'public'")
.andWhere("note.localOnly = FALSE")
.innerJoinAndSelect("note.user", "user")
.leftJoinAndSelect("user.avatar", "avatar")
.leftJoinAndSelect("user.banner", "banner")
.leftJoinAndSelect("note.reply", "reply")
.leftJoinAndSelect("note.renote", "renote")
.leftJoinAndSelect("reply.user", "replyUser")
.leftJoinAndSelect("replyUser.avatar", "replyUserAvatar")
.leftJoinAndSelect("replyUser.banner", "replyUserBanner")
.leftJoinAndSelect("renote.user", "renoteUser")
.leftJoinAndSelect("renoteUser.avatar", "renoteUserAvatar")
.leftJoinAndSelect("renoteUser.banner", "renoteUserBanner");
.andWhere("note.localOnly = FALSE");
if (ps.local) {
query.andWhere("note.userHost IS NULL");

61
packages/backend/src/server/api/endpoints/notes/children.ts
View file

@ -5,6 +5,7 @@ import { generateVisibilityQuery } from "../../common/generate-visibility-query.
import { generateMutedUserQuery } from "../../common/generate-muted-user-query.js";
import { generateBlockedUserQuery } from "../../common/generate-block-query.js";
import {
ScyllaNote,
filterBlockedUser,
filterMutedNote,
filterMutedUser,
@ -89,44 +90,54 @@ export default define(meta, paramDef, async (ps, user) => {
return await Notes.packMany([]);
}
// Find replies in BFS manner
const queue = [root];
const foundReplies: Note[] = [];
let depth = 0;
const filter = async (notes: ScyllaNote[]) => {
let filtered = await filterVisibility(notes, user, followingUserIds);
if (user) {
filtered = await filterMutedUser(
filtered,
user,
mutedUserIds,
mutedInstances,
);
filtered = await filterMutedNote(filtered, user, mutedWords);
filtered = await filterBlockedUser(filtered, user, blockerIds);
}
return filtered;
};
while (
queue.length > 0 &&
foundReplies.length < ps.limit &&
depth < ps.depth
) {
// Find quotes first
const renoteResult = await scyllaClient.execute(
prepared.note.select.byRenoteId,
[root.id],
{ prepare: true },
);
const foundNotes = await filter(
renoteResult.rows.map(parseScyllaNote).filter((note) => !!note.text),
);
// Then find replies in BFS manner
const queue = [root];
let depth = 0;
const limit = ps.limit + foundNotes.length;
while (queue.length > 0 && foundNotes.length < limit && depth < ps.depth) {
const note = queue.shift();
if (note) {
const result = await scyllaClient.execute(
const replyResult = await scyllaClient.execute(
prepared.note.select.byReplyId,
[note.id],
{ prepare: true },
);
if (result.rowLength > 0) {
let replies = result.rows.map(parseScyllaNote);
replies = await filterVisibility(replies, user, followingUserIds);
if (user) {
replies = await filterMutedUser(
replies,
user,
mutedUserIds,
mutedInstances,
);
replies = await filterMutedNote(replies, user, mutedWords);
replies = await filterBlockedUser(replies, user, blockerIds);
}
foundReplies.push(...replies);
const replies = await filter(replyResult.rows.map(parseScyllaNote));
if (replies.length > 0) {
foundNotes.push(...replies);
queue.push(...replies);
depth++;
}
}
}
return await Notes.packMany(foundReplies, user, {
return await Notes.packMany(foundNotes, user, {
detail: false,
scyllaNote: true,
});

117
packages/backend/src/server/api/endpoints/users/notes.ts
View file

@ -1,5 +1,5 @@
import { Brackets } from "typeorm";
import { Notes } from "@/models/index.js";
import { Notes, UserProfiles } from "@/models/index.js";
import define from "../../define.js";
import { ApiError } from "../../error.js";
import { getUser } from "../../common/getters.js";
@ -7,6 +7,22 @@ import { makePaginationQuery } from "../../common/make-pagination-query.js";
import { generateVisibilityQuery } from "../../common/generate-visibility-query.js";
import { generateMutedUserQuery } from "../../common/generate-muted-user-query.js";
import { generateBlockedUserQuery } from "../../common/generate-block-query.js";
import {
ScyllaNote,
execNotePaginationQuery,
filterBlockedUser,
filterMutedNote,
filterMutedUser,
filterVisibility,
scyllaClient,
} from "@/db/scylla.js";
import {
InstanceMutingsCache,
LocalFollowingsCache,
UserBlockedCache,
UserMutingsCache,
userWordMuteCache,
} from "@/misc/cache.js";
export const meta = {
tags: ["users", "notes"],
@ -66,6 +82,91 @@ export default define(meta, paramDef, async (ps, me) => {
throw e;
});
if (scyllaClient) {
const [
followingUserIds,
mutedUserIds,
mutedInstances,
mutedWords,
blockerIds,
] = await Promise.all([
LocalFollowingsCache.init(user.id).then((cache) => cache.getAll()),
UserMutingsCache.init(user.id).then((cache) => cache.getAll()),
InstanceMutingsCache.init(user.id).then((cache) => cache.getAll()),
userWordMuteCache
.fetchMaybe(user.id, () =>
UserProfiles.findOne({
select: ["mutedWords"],
where: { userId: user.id },
}).then((profile) => profile?.mutedWords),
)
.then((words) => words ?? []),
UserBlockedCache.init(user.id).then((cache) => cache.getAll()),
]);
if (
mutedUserIds.includes(user.id) ||
blockerIds.includes(user.id) ||
(user.host && mutedInstances.includes(user.host))
) {
return Notes.packMany([]);
}
const filter = async (notes: ScyllaNote[]) => {
let filtered = notes.filter((n) => n.userId === ps.userId);
filtered = await filterVisibility(filtered, user, followingUserIds);
filtered = await filterMutedUser(
filtered,
user,
mutedUserIds,
mutedInstances,
);
filtered = await filterMutedNote(filtered, user, mutedWords);
filtered = await filterBlockedUser(filtered, user, blockerIds);
if (ps.withFiles) {
filtered = filtered.filter((n) => n.files.length > 0);
}
if (ps.fileType) {
filtered = filtered.filter((n) =>
n.files.some((f) => ps.fileType?.includes(f.type)),
);
}
if (ps.excludeNsfw) {
filtered = filtered.filter(
(n) => !n.cw && n.files.every((f) => !f.isSensitive),
);
}
if (!ps.includeMyRenotes) {
filtered = filtered.filter(
(n) =>
n.userId !== user.id ||
!n.renoteId ||
!!n.text ||
n.files.length > 0 ||
n.hasPoll,
);
}
if (!ps.includeReplies) {
filtered = filtered.filter((n) => !n.replyId);
}
return filtered;
};
const foundPacked = [];
while (foundPacked.length < ps.limit) {
const foundNotes = (
await execNotePaginationQuery("user", ps, filter, user.id)
).slice(0, ps.limit * 1.5); // Some may filtered out by Notes.packMany, thus we take more than ps.limit.
foundPacked.push(
...(await Notes.packMany(foundNotes, user, { scyllaNote: true })),
);
if (foundNotes.length < ps.limit) break;
ps.untilDate = foundNotes[foundNotes.length - 1].createdAt.getTime();
}
return foundPacked.slice(0, ps.limit);
}
//#region Construct query
const query = makePaginationQuery(
Notes.createQueryBuilder("note"),
@ -73,19 +174,7 @@ export default define(meta, paramDef, async (ps, me) => {
ps.untilId,
ps.sinceDate,
ps.untilDate,
)
.andWhere("note.userId = :userId", { userId: user.id })
.innerJoinAndSelect("note.user", "user")
.leftJoinAndSelect("user.avatar", "avatar")
.leftJoinAndSelect("user.banner", "banner")
.leftJoinAndSelect("note.reply", "reply")
.leftJoinAndSelect("note.renote", "renote")
.leftJoinAndSelect("reply.user", "replyUser")
.leftJoinAndSelect("replyUser.avatar", "replyUserAvatar")
.leftJoinAndSelect("replyUser.banner", "replyUserBanner")
.leftJoinAndSelect("renote.user", "renoteUser")
.leftJoinAndSelect("renoteUser.avatar", "renoteUserAvatar")
.leftJoinAndSelect("renoteUser.banner", "renoteUserBanner");
).andWhere("note.userId = :userId", { userId: user.id });
generateVisibilityQuery(query, me);
if (me) {