From c9dd562145d5d01edfef8286a0a6d2800d16448a Mon Sep 17 00:00:00 2001 From: Namekuji Date: Mon, 14 Aug 2023 02:49:59 -0400 Subject: [PATCH] fix: pins and cascading delete --- .../cql/1689400417034_timeline/down.cql | 2 +- .../cql/1689400417034_timeline/up.cql | 12 ++- packages/backend/src/db/cql.ts | 7 +- packages/backend/src/db/postgre.ts | 5 +- .../src/models/entities/user-note-pining.ts | 9 ++- packages/backend/src/models/index.ts | 5 +- .../backend/src/models/repositories/user.ts | 46 ++++++++---- .../src/server/api/endpoints/notes/delete.ts | 9 ++- packages/backend/src/services/note/delete.ts | 74 +++++++++++-------- 9 files changed, 109 insertions(+), 60 deletions(-) diff --git a/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/down.cql b/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/down.cql index 5d48bbcc0d..64783e2aa5 100644 --- a/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/down.cql +++ b/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/down.cql @@ -9,8 +9,8 @@ DROP MATERIALIZED VIEW IF EXISTS global_timeline; DROP MATERIALIZED VIEW IF EXISTS note_by_renote_id_and_user_id; DROP MATERIALIZED VIEW IF EXISTS note_by_renote_id; DROP MATERIALIZED VIEW IF EXISTS note_by_user_id; +DROP MATERIALIZED VIEW IF EXISTS note_by_id; DROP INDEX IF EXISTS note_by_reply_id; -DROP INDEX IF EXISTS note_by_id; DROP INDEX IF EXISTS note_by_uri; DROP INDEX IF EXISTS note_by_url; DROP TABLE IF EXISTS note; diff --git a/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/up.cql b/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/up.cql index 50c952e6ec..de3693dd02 100644 --- a/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/up.cql +++ b/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/up.cql @@ -74,9 +74,19 @@ CREATE TABLE IF NOT EXISTS note ( -- Store all posts CREATE INDEX IF NOT EXISTS note_by_uri ON note ("uri"); CREATE INDEX IF NOT EXISTS note_by_url ON note ("url"); -CREATE INDEX IF NOT EXISTS note_by_id ON note ("id"); CREATE INDEX IF NOT EXISTS note_by_reply_id ON note ("replyId"); +CREATE MATERIALIZED VIEW IF NOT EXISTS note_by_id AS + SELECT * FROM note + WHERE "id" IS NOT NULL + AND "createdAt" IS NOT NULL + AND "createdAtDate" IS NOT NULL + AND "userId" IS NOT NULL + AND "userHost" IS NOT NULL + AND "visibility" IS NOT NULL + PRIMARY KEY ("id", "createdAt", "createdAtDate", "userId", "userHost", "visibility") + WITH CLUSTERING ORDER BY ("createdAt" DESC); + CREATE MATERIALIZED VIEW IF NOT EXISTS note_by_user_id AS SELECT * FROM note WHERE "userId" IS NOT NULL diff --git a/packages/backend/src/db/cql.ts b/packages/backend/src/db/cql.ts index c2c4f52213..c2c7207f63 100644 --- a/packages/backend/src/db/cql.ts +++ b/packages/backend/src/db/cql.ts @@ -45,12 +45,13 @@ export const scyllaQueries = { byDate: `SELECT * FROM note WHERE "createdAtDate" = ?`, byUri: `SELECT * FROM note WHERE "uri" = ?`, byUrl: `SELECT * FROM note WHERE "url" = ?`, - byId: `SELECT * FROM note WHERE "id" = ?`, + byId: `SELECT * FROM note_by_id WHERE "id" = ?`, + byIds: `SELECT * FROM note_by_id WHERE "id" IN ?`, byUserId: `SELECT * FROM note_by_user_id WHERE "userId" = ?`, byRenoteId: `SELECT * FROM note_by_renote_id WHERE "renoteId" = ?`, byReplyId: `SELECT * FROM note WHERE "replyId" = ?` }, - delete: `DELETE FROM note WHERE "createdAtDate" = ? AND "createdAt" = ? AND "userId" = ? AND "userHost" = ? AND "visibility" = ?`, + delete: `DELETE FROM note WHERE ("createdAtDate", "createdAt", "userId", "userHost", "visibility") IN ?`, update: { renoteCount: `UPDATE note SET "renoteCount" = ?, @@ -113,7 +114,7 @@ export const scyllaQueries = { byUserAndDate: `SELECT * FROM home_timeline WHERE "feedUserId" = ? AND "createdAtDate" = ?`, byId: `SELECT * FROM home_timeline WHERE "id" = ?`, }, - delete: `DELETE FROM home_timeline WHERE "feedUserId" = ? AND "createdAtDate" = ? AND "createdAt" = ? AND "userId" = ?`, + delete: `DELETE FROM home_timeline WHERE ("feedUserId", "createdAtDate", "createdAt", "userId") IN ?`, update: { renoteCount: `UPDATE home_timeline SET "renoteCount" = ?, diff --git a/packages/backend/src/db/postgre.ts b/packages/backend/src/db/postgre.ts index 10ea5b15f6..560be89c90 100644 --- a/packages/backend/src/db/postgre.ts +++ b/packages/backend/src/db/postgre.ts @@ -40,7 +40,7 @@ import { Signin } from "@/models/entities/signin.js"; import { AuthSession } from "@/models/entities/auth-session.js"; import { FollowRequest } from "@/models/entities/follow-request.js"; import { Emoji } from "@/models/entities/emoji.js"; -import { UserNotePining } from "@/models/entities/user-note-pining.js"; +import { UserNotePining, UserNotePiningScylla } from "@/models/entities/user-note-pining.js"; import { Poll } from "@/models/entities/poll.js"; import { UserKeypair } from "@/models/entities/user-keypair.js"; import { UserPublickey } from "@/models/entities/user-publickey.js"; @@ -74,7 +74,6 @@ import { UserIp } from "@/models/entities/user-ip.js"; import { NoteEdit } from "@/models/entities/note-edit.js"; import { entities as charts } from "@/services/chart/entities.js"; -import { envOption } from "../env.js"; import { dbLogger } from "./logger.js"; import { redisClient } from "./redis.js"; import { nativeInitDatabase } from "native-utils/built/index.js"; @@ -131,7 +130,7 @@ export const entities = [ UserGroup, UserGroupJoining, UserGroupInvitation, - UserNotePining, + config.scylla ? UserNotePiningScylla : UserNotePining, UserSecurityKey, UsedUsername, AttestationChallenge, diff --git a/packages/backend/src/models/entities/user-note-pining.ts b/packages/backend/src/models/entities/user-note-pining.ts index c30fe1e028..e662edb1fb 100644 --- a/packages/backend/src/models/entities/user-note-pining.ts +++ b/packages/backend/src/models/entities/user-note-pining.ts @@ -10,9 +10,8 @@ import { Note } from "./note.js"; import { User } from "./user.js"; import { id } from "../id.js"; -@Entity() @Index(["userId", "noteId"], { unique: true }) -export class UserNotePining { +class UserNotePiningBase { @PrimaryColumn(id()) public id: string; @@ -33,10 +32,16 @@ export class UserNotePining { @Column(id()) public noteId: Note["id"]; +} +@Entity() +export class UserNotePining extends UserNotePiningBase { @ManyToOne((type) => Note, { onDelete: "CASCADE", }) @JoinColumn() public note: Note | null; } + +@Entity({ name: "user_note_pining" }) +export class UserNotePiningScylla extends UserNotePiningBase {} diff --git a/packages/backend/src/models/index.ts b/packages/backend/src/models/index.ts index 8782d57408..f757e12638 100644 --- a/packages/backend/src/models/index.ts +++ b/packages/backend/src/models/index.ts @@ -17,7 +17,7 @@ import { NoteRepository } from "./repositories/note.js"; import { DriveFileRepository } from "./repositories/drive-file.js"; import { DriveFolderRepository } from "./repositories/drive-folder.js"; import { AccessToken } from "./entities/access-token.js"; -import { UserNotePining } from "./entities/user-note-pining.js"; +import { UserNotePining, UserNotePiningScylla } from "./entities/user-note-pining.js"; import { SigninRepository } from "./repositories/signin.js"; import { MessagingMessageRepository } from "./repositories/messaging-message.js"; import { UserListRepository } from "./repositories/user-list.js"; @@ -67,6 +67,7 @@ import { InstanceRepository } from "./repositories/instance.js"; import { Webhook } from "./entities/webhook.js"; import { UserIp } from "./entities/user-ip.js"; import { NoteEdit } from "./entities/note-edit.js"; +import config from "@/config/index.js"; export const Announcements = db.getRepository(Announcement); export const AnnouncementReads = db.getRepository(AnnouncementRead); @@ -92,7 +93,7 @@ export const UserListJoinings = db.getRepository(UserListJoining); export const UserGroups = UserGroupRepository; export const UserGroupJoinings = db.getRepository(UserGroupJoining); export const UserGroupInvitations = UserGroupInvitationRepository; -export const UserNotePinings = db.getRepository(UserNotePining); +export const UserNotePinings = db.getRepository(config.scylla ? UserNotePiningScylla : UserNotePining); export const UserIps = db.getRepository(UserIp); export const UsedUsernames = db.getRepository(UsedUsername); export const Followings = FollowingRepository; diff --git a/packages/backend/src/models/repositories/user.ts b/packages/backend/src/models/repositories/user.ts index 0fa8fdf189..f40b84e8ee 100644 --- a/packages/backend/src/models/repositories/user.ts +++ b/packages/backend/src/models/repositories/user.ts @@ -38,6 +38,9 @@ import { } from "../index.js"; import type { Instance } from "../entities/instance.js"; import { userDenormalizedCache } from "@/services/user-cache.js"; +import { parseScyllaNote, prepared, scyllaClient } from "@/db/scylla.js"; +import type { UserNotePining } from "@/models/entities/user-note-pining.js"; +import type { Note } from "@/models/entities/note.js"; const userInstanceCache = new Cache( "userInstance", @@ -404,13 +407,29 @@ export const UserRepository = db.getRepository(User).extend({ meId && !isMe && opts.detail ? await this.getRelation(meId, user.id) : null; - const pins = opts.detail - ? await UserNotePinings.createQueryBuilder("pin") - .where("pin.userId = :userId", { userId: user.id }) - .innerJoinAndSelect("pin.note", "note") - .orderBy("pin.id", "DESC") - .getMany() - : []; + + let pinnedNoteIds: UserNotePining["noteId"][] = []; + let pinnedNotes: Note[] = []; + if (opts.detail) { + pinnedNoteIds = await UserNotePinings.find({ + select: ["noteId"], + where: { + userId: user.id, + }, + }).then((notes) => notes.map(({ noteId }) => noteId)); + + if (scyllaClient) { + const result = await scyllaClient.execute( + prepared.note.select.byIds, + [pinnedNoteIds], + { prepare: true }, + ); + pinnedNotes = result.rows.map(parseScyllaNote); + } else { + pinnedNotes = await Notes.findBy({ id: In(pinnedNoteIds) }); + } + } + const profile = opts.detail ? await UserProfiles.findOneByOrFail({ userId: user.id }) : null; @@ -506,14 +525,11 @@ export const UserRepository = db.getRepository(User).extend({ followersCount: followersCount || 0, followingCount: followingCount || 0, notesCount: user.notesCount, - pinnedNoteIds: pins.map((pin) => pin.noteId), - pinnedNotes: Notes.packMany( - pins.map((pin) => pin.note!), - me, - { - detail: true, - }, - ), + pinnedNoteIds, + pinnedNotes: Notes.packMany(pinnedNotes, me, { + detail: true, + scyllaNote: !!scyllaClient, + }), pinnedPageId: profile!.pinnedPageId, pinnedPage: profile!.pinnedPageId ? Pages.pack(profile!.pinnedPageId, me) diff --git a/packages/backend/src/server/api/endpoints/notes/delete.ts b/packages/backend/src/server/api/endpoints/notes/delete.ts index 5fc79db7d1..f5856ab5fd 100644 --- a/packages/backend/src/server/api/endpoints/notes/delete.ts +++ b/packages/backend/src/server/api/endpoints/notes/delete.ts @@ -4,6 +4,7 @@ import define from "../../define.js"; import { getNote } from "../../common/getters.js"; import { ApiError } from "../../error.js"; import { SECOND, HOUR } from "@/const.js"; +import { userByIdCache } from "@/services/user-cache.js"; export const meta = { tags: ["notes"], @@ -52,6 +53,10 @@ export default define(meta, paramDef, async (ps, user) => { throw new ApiError(meta.errors.accessDenied); } - // この操作を行うのが投稿者とは限らない(例えばモデレーター)ため - await deleteNote(await Users.findOneByOrFail({ id: note.userId }), note); + await deleteNote( + await userByIdCache.fetch(note.userId, () => + Users.findOneByOrFail({ id: note.userId }), + ), + note, + ); }); diff --git a/packages/backend/src/services/note/delete.ts b/packages/backend/src/services/note/delete.ts index 9813ef9c15..4099c06904 100644 --- a/packages/backend/src/services/note/delete.ts +++ b/packages/backend/src/services/note/delete.ts @@ -28,6 +28,7 @@ import { prepared, scyllaClient, } from "@/db/scylla.js"; +import { LocalFollowersCache } from "@/misc/cache.js"; /** * Delete a post @@ -143,6 +144,9 @@ export default async function ( } } + const cascadingNotes = + scyllaClient || !quiet ? await findCascadingNotes(note) : []; + if (!quiet) { publishNoteStream(note.id, "deleted", { deletedAt: deletedAt, @@ -192,10 +196,10 @@ export default async function ( } // also deliever delete activity to cascaded notes - const cascadingNotes = (await findCascadingNotes(note)).filter( - (note) => !note.localOnly, + const cascadingLocalNotes = cascadingNotes.filter( + (note) => note.userHost === null && !note.localOnly, ); // filter out local-only notes - for (const cascadingNote of cascadingNotes) { + for (const cascadingNote of cascadingLocalNotes) { if (!cascadingNote.user) continue; if (!Users.isLocalUser(cascadingNote.user)) continue; const content = renderActivity( @@ -221,34 +225,42 @@ export default async function ( } if (scyllaClient) { - const date = new Date(note.createdAt.getTime()); - await scyllaClient.execute( - prepared.note.delete, - [date, date, note.userId, note.userHost ?? "local", note.visibility], - { - prepare: true, - }, - ); + const notesToDelete = [note, ...cascadingNotes]; - const homeTimelines = await scyllaClient - .execute(prepared.homeTimeline.select.byId, [note.id], { prepare: true }) - .then((result) => result.rows.map(parseHomeTimeline)); - for (const timeline of homeTimelines) { - // No need to wait - scyllaClient.execute(prepared.homeTimeline.delete, [ - timeline.feedUserId, - timeline.createdAtDate, - timeline.createdAt, - timeline.userId, - ]); + const noteDeleteParams = notesToDelete.map((n) => { + const date = new Date(n.createdAt.getTime()); + return [date, date, n.userId, n.userHost ?? "local", n.visibility]; + }); + await scyllaClient.execute(prepared.note.delete, noteDeleteParams, { + prepare: true, + }); + + const noteUserIds = new Set(notesToDelete.map((n) => n.userId)); + const followers: string[] = []; + for (const id of noteUserIds) { + const list = await LocalFollowersCache.init(id).then((cache) => + cache.getAll(), + ); + followers.push(...list); } + const localFollowers = new Set(followers); + const homeDeleteParams = notesToDelete.map((n) => { + const tuples: [string, Date, Date, string][] = []; + for (const feedUserId of localFollowers) { + tuples.push([feedUserId, n.createdAt, n.createdAt, n.userId]); + } + return tuples; + }); + await scyllaClient.execute(prepared.homeTimeline.delete, homeDeleteParams, { + prepare: true, + }); + } else { + await Notes.delete({ + id: note.id, + userId: user.id, + }); } - await Notes.delete({ - id: note.id, - userId: user.id, - }); - if (meilisearch) { await meilisearch.deleteNotes(note.id); } @@ -293,14 +305,14 @@ async function findCascadingNotes(note: Note) { notes = await query.getMany(); } - for (const reply of notes) { - cascadingNotes.push(reply); - await recursive(reply.id); + for (const note of notes) { + cascadingNotes.push(note); + await recursive(note.id); } }; await recursive(note.id); - return cascadingNotes.filter((note) => note.userHost === null); // filter out non-local users + return cascadingNotes; } async function getMentionedRemoteUsers(note: Note) {