From 93cf2f9045e4af485a29690313b7d74b0f54cab6 Mon Sep 17 00:00:00 2001 From: Hazelnoot Date: Sun, 13 Oct 2024 18:11:16 -0400 Subject: [PATCH] factor out latest_note logic into LatestNoteService --- packages/backend/src/core/CoreModule.ts | 6 + .../backend/src/core/LatestNoteService.ts | 139 ++++++++++++++++++ .../backend/src/core/NoteCreateService.ts | 41 +----- .../backend/src/core/NoteDeleteService.ts | 84 +---------- packages/backend/src/models/LatestNote.ts | 12 ++ 5 files changed, 167 insertions(+), 115 deletions(-) create mode 100644 packages/backend/src/core/LatestNoteService.ts diff --git a/packages/backend/src/core/CoreModule.ts b/packages/backend/src/core/CoreModule.ts index 049d858189..1d807d2aa0 100644 --- a/packages/backend/src/core/CoreModule.ts +++ b/packages/backend/src/core/CoreModule.ts @@ -42,6 +42,7 @@ import { ModerationLogService } from './ModerationLogService.js'; import { NoteCreateService } from './NoteCreateService.js'; import { NoteEditService } from './NoteEditService.js'; import { NoteDeleteService } from './NoteDeleteService.js'; +import { LatestNoteService } from './LatestNoteService.js'; import { NotePiningService } from './NotePiningService.js'; import { NoteReadService } from './NoteReadService.js'; import { NotificationService } from './NotificationService.js'; @@ -185,6 +186,7 @@ const $ModerationLogService: Provider = { provide: 'ModerationLogService', useEx const $NoteCreateService: Provider = { provide: 'NoteCreateService', useExisting: NoteCreateService }; const $NoteEditService: Provider = { provide: 'NoteEditService', useExisting: NoteEditService }; const $NoteDeleteService: Provider = { provide: 'NoteDeleteService', useExisting: NoteDeleteService }; +const $LatestNoteService: Provider = { provide: 'LatestNoteService', useExisting: LatestNoteService }; const $NotePiningService: Provider = { provide: 'NotePiningService', useExisting: NotePiningService }; const $NoteReadService: Provider = { provide: 'NoteReadService', useExisting: NoteReadService }; const $NotificationService: Provider = { provide: 'NotificationService', useExisting: NotificationService }; @@ -335,6 +337,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp NoteCreateService, NoteEditService, NoteDeleteService, + LatestNoteService, NotePiningService, NoteReadService, NotificationService, @@ -481,6 +484,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp $NoteCreateService, $NoteEditService, $NoteDeleteService, + $LatestNoteService, $NotePiningService, $NoteReadService, $NotificationService, @@ -628,6 +632,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp NoteCreateService, NoteEditService, NoteDeleteService, + LatestNoteService, NotePiningService, NoteReadService, NotificationService, @@ -773,6 +778,7 @@ const $SponsorsService: Provider = { provide: 'SponsorsService', useExisting: Sp $NoteCreateService, $NoteEditService, $NoteDeleteService, + $LatestNoteService, $NotePiningService, $NoteReadService, $NotificationService, diff --git a/packages/backend/src/core/LatestNoteService.ts b/packages/backend/src/core/LatestNoteService.ts new file mode 100644 index 0000000000..c379805506 --- /dev/null +++ b/packages/backend/src/core/LatestNoteService.ts @@ -0,0 +1,139 @@ +import { Inject, Injectable } from '@nestjs/common'; +import { Not } from 'typeorm'; +import { MiNote } from '@/models/Note.js'; +import { isPureRenote } from '@/misc/is-renote.js'; +import { SkLatestNote } from '@/models/LatestNote.js'; +import { DI } from '@/di-symbols.js'; +import type { LatestNotesRepository, NotesRepository } from '@/models/_.js'; +import { LoggerService } from '@/core/LoggerService.js'; +import Logger from '@/logger.js'; + +@Injectable() +export class LatestNoteService { + private readonly logger: Logger; + + constructor( + @Inject(DI.notesRepository) + private notesRepository: NotesRepository, + + @Inject(DI.latestNotesRepository) + private latestNotesRepository: LatestNotesRepository, + + loggerService: LoggerService, + ) { + this.logger = loggerService.getLogger('LatestNoteService'); + } + + handleUpdatedNoteBG(before: MiNote, after: MiNote): void { + this + .handleUpdatedNote(before, after) + .catch(err => this.logger.error('Unhandled exception while updating latest_note (after update):', err)); + } + + async handleUpdatedNote(before: MiNote, after: MiNote): Promise { + // If the key didn't change, then there's nothing to update + if (SkLatestNote.areEquivalent(before, after)) return; + + // Simulate update as delete + create + await this.handleDeletedNote(before); + await this.handleCreatedNote(after); + } + + handleCreatedNoteBG(note: MiNote): void { + this + .handleCreatedNote(note) + .catch(err => this.logger.error('Unhandled exception while updating latest_note (after create):', err)); + } + + async handleCreatedNote(note: MiNote): Promise { + // Ignore DMs. + // Followers-only posts are *included*, as this table is used to back the "following" feed. + if (note.visibility === 'specified') return; + + // Ignore pure renotes + if (isPureRenote(note)) return; + + // Compute the compound key of the entry to check + const key = SkLatestNote.keyFor(note); + + // Make sure that this isn't an *older* post. + // We can get older posts through replies, lookups, updates, etc. + const currentLatest = await this.latestNotesRepository.findOneBy(key); + if (currentLatest != null && currentLatest.noteId >= note.id) return; + + // Record this as the latest note for the given user + const latestNote = new SkLatestNote({ + ...key, + noteId: note.id, + }); + await this.latestNotesRepository.upsert(latestNote, ['userId', 'isPublic', 'isReply', 'isQuote']); + } + + handleDeletedNoteBG(note: MiNote): void { + this + .handleDeletedNote(note) + .catch(err => this.logger.error('Unhandled exception while updating latest_note (after delete):', err)); + } + + async handleDeletedNote(note: MiNote): Promise { + // If it's a DM, then it can't possibly be the latest note so we can safely skip this. + if (note.visibility === 'specified') return; + + // If it's a pure renote, then it can't possibly be the latest note so we can safely skip this. + if (isPureRenote(note)) return; + + // Compute the compound key of the entry to check + const key = SkLatestNote.keyFor(note); + + // Check if the deleted note was possibly the latest for the user + const existingLatest = await this.latestNotesRepository.findOneBy(key); + if (existingLatest == null || existingLatest.noteId !== note.id) return; + + // Find the newest remaining note for the user. + // We exclude DMs and pure renotes. + const nextLatest = await this.notesRepository + .createQueryBuilder('note') + .select() + .where({ + userId: key.userId, + visibility: key.isPublic + ? 'public' + : Not('specified'), + replyId: key.isReply + ? Not(null) + : null, + renoteId: key.isQuote + ? Not(null) + : null, + }) + .andWhere(` + ( + note."renoteId" IS NULL + OR note.text IS NOT NULL + OR note.cw IS NOT NULL + OR note."replyId" IS NOT NULL + OR note."hasPoll" + OR note."fileIds" != '{}' + ) + `) + .orderBy({ id: 'DESC' }) + .getOne(); + if (!nextLatest) return; + + // Record it as the latest + const latestNote = new SkLatestNote({ + ...key, + noteId: nextLatest.id, + }); + + // When inserting the latest note, it's possible that another worker has "raced" the insert and already added a newer note. + // We must use orIgnore() to ensure that the query ignores conflicts, otherwise an exception may be thrown. + await this.latestNotesRepository + .createQueryBuilder('latest') + .insert() + .into(SkLatestNote) + .values(latestNote) + .orIgnore() + .execute(); + } +} diff --git a/packages/backend/src/core/NoteCreateService.ts b/packages/backend/src/core/NoteCreateService.ts index 55c55589ad..cd497a7d87 100644 --- a/packages/backend/src/core/NoteCreateService.ts +++ b/packages/backend/src/core/NoteCreateService.ts @@ -14,8 +14,7 @@ import { extractCustomEmojisFromMfm } from '@/misc/extract-custom-emojis-from-mf import { extractHashtags } from '@/misc/extract-hashtags.js'; import type { IMentionedRemoteUsers } from '@/models/Note.js'; import { MiNote } from '@/models/Note.js'; -import { SkLatestNote } from '@/models/LatestNote.js'; -import type { ChannelFollowingsRepository, ChannelsRepository, FollowingsRepository, InstancesRepository, LatestNotesRepository, MiFollowing, MutingsRepository, NotesRepository, NoteThreadMutingsRepository, UserListMembershipsRepository, UserProfilesRepository, UsersRepository } from '@/models/_.js'; +import type { ChannelFollowingsRepository, ChannelsRepository, FollowingsRepository, InstancesRepository, MiFollowing, MutingsRepository, NotesRepository, NoteThreadMutingsRepository, UserListMembershipsRepository, UserProfilesRepository, UsersRepository } from '@/models/_.js'; import type { MiDriveFile } from '@/models/DriveFile.js'; import type { MiApp } from '@/models/App.js'; import { concat } from '@/misc/prelude/array.js'; @@ -63,7 +62,7 @@ import { isReply } from '@/misc/is-reply.js'; import { trackPromise } from '@/misc/promise-tracker.js'; import { isUserRelated } from '@/misc/is-user-related.js'; import { IdentifiableError } from '@/misc/identifiable-error.js'; -import { isPureRenote } from '@/misc/is-renote.js'; +import { LatestNoteService } from '@/core/LatestNoteService.js'; type NotificationType = 'reply' | 'renote' | 'quote' | 'mention'; @@ -172,9 +171,6 @@ export class NoteCreateService implements OnApplicationShutdown { @Inject(DI.notesRepository) private notesRepository: NotesRepository, - @Inject(DI.latestNotesRepository) - private latestNotesRepository: LatestNotesRepository, - @Inject(DI.mutingsRepository) private mutingsRepository: MutingsRepository, @@ -226,6 +222,7 @@ export class NoteCreateService implements OnApplicationShutdown { private utilityService: UtilityService, private userBlockingService: UserBlockingService, private cacheService: CacheService, + private latestNoteService: LatestNoteService, ) { } @bindThis @@ -814,7 +811,7 @@ export class NoteCreateService implements OnApplicationShutdown { } // Update the Latest Note index / following feed - this.updateLatestNoteBG(note); + this.latestNoteService.handleCreatedNoteBG(note); // Register to search database if (!user.noindex) this.index(note); @@ -1145,34 +1142,4 @@ export class NoteCreateService implements OnApplicationShutdown { public onApplicationShutdown(signal?: string | undefined): void { this.dispose(); } - - private updateLatestNoteBG(note: MiNote): void { - this - .updateLatestNote(note) - .catch(err => console.error('Unhandled exception while updating latest_note (after create):', err)); - } - - private async updateLatestNote(note: MiNote): Promise { - // Ignore DMs. - // Followers-only posts are *included*, as this table is used to back the "following" feed. - if (note.visibility === 'specified') return; - - // Ignore pure renotes - if (isPureRenote(note)) return; - - // Compute the compound key of the entry to check - const key = SkLatestNote.keyFor(note); - - // Make sure that this isn't an *older* post. - // We can get older posts through replies, lookups, etc. - const currentLatest = await this.latestNotesRepository.findOneBy(key); - if (currentLatest != null && currentLatest.noteId >= note.id) return; - - // Record this as the latest note for the given user - const latestNote = new SkLatestNote({ - ...key, - noteId: note.id, - }); - await this.latestNotesRepository.upsert(latestNote, ['userId', 'isPublic', 'isReply', 'isQuote']); - } } diff --git a/packages/backend/src/core/NoteDeleteService.ts b/packages/backend/src/core/NoteDeleteService.ts index 3f1eef39c9..bdafad3f02 100644 --- a/packages/backend/src/core/NoteDeleteService.ts +++ b/packages/backend/src/core/NoteDeleteService.ts @@ -3,12 +3,11 @@ * SPDX-License-Identifier: AGPL-3.0-only */ -import { Brackets, In, Not } from 'typeorm'; +import { Brackets, In } from 'typeorm'; import { Injectable, Inject } from '@nestjs/common'; import type { MiUser, MiLocalUser, MiRemoteUser } from '@/models/User.js'; import { MiNote, IMentionedRemoteUsers } from '@/models/Note.js'; -import { SkLatestNote } from '@/models/LatestNote.js'; -import type { InstancesRepository, LatestNotesRepository, NotesRepository, UsersRepository } from '@/models/_.js'; +import type { InstancesRepository, NotesRepository, UsersRepository } from '@/models/_.js'; import { RelayService } from '@/core/RelayService.js'; import { FederatedInstanceService } from '@/core/FederatedInstanceService.js'; import { DI } from '@/di-symbols.js'; @@ -20,12 +19,12 @@ import { GlobalEventService } from '@/core/GlobalEventService.js'; import { ApRendererService } from '@/core/activitypub/ApRendererService.js'; import { ApDeliverManagerService } from '@/core/activitypub/ApDeliverManagerService.js'; import { UserEntityService } from '@/core/entities/UserEntityService.js'; -import { NoteEntityService } from '@/core/entities/NoteEntityService.js'; import { bindThis } from '@/decorators.js'; import { MetaService } from '@/core/MetaService.js'; import { SearchService } from '@/core/SearchService.js'; import { ModerationLogService } from '@/core/ModerationLogService.js'; -import { isPureRenote, isQuote, isRenote } from '@/misc/is-renote.js'; +import { isQuote, isRenote } from '@/misc/is-renote.js'; +import { LatestNoteService } from '@/core/LatestNoteService.js'; @Injectable() export class NoteDeleteService { @@ -39,14 +38,10 @@ export class NoteDeleteService { @Inject(DI.notesRepository) private notesRepository: NotesRepository, - @Inject(DI.latestNotesRepository) - private latestNotesRepository: LatestNotesRepository, - @Inject(DI.instancesRepository) private instancesRepository: InstancesRepository, private userEntityService: UserEntityService, - private noteEntityService: NoteEntityService, private globalEventService: GlobalEventService, private relayService: RelayService, private federatedInstanceService: FederatedInstanceService, @@ -58,6 +53,7 @@ export class NoteDeleteService { private notesChart: NotesChart, private perUserNotesChart: PerUserNotesChart, private instanceChart: InstanceChart, + private latestNoteService: LatestNoteService, ) {} /** @@ -152,7 +148,7 @@ export class NoteDeleteService { userId: user.id, }); - this.updateLatestNoteBG(note); + this.latestNoteService.handleDeletedNoteBG(note); if (deleter && (note.userId !== deleter.id)) { const user = await this.usersRepository.findOneByOrFail({ id: note.userId }); @@ -235,72 +231,4 @@ export class NoteDeleteService { this.apDeliverManagerService.deliverToUser(user, content, remoteUser); } } - - private updateLatestNoteBG(note: MiNote): void { - this - .updateLatestNote(note) - .catch(err => console.error('Unhandled exception while updating latest_note (after delete):', err)); - } - - private async updateLatestNote(note: MiNote) { - // If it's a DM, then it can't possibly be the latest note so we can safely skip this. - if (note.visibility === 'specified') return; - - // If it's a pure renote, then it can't possibly be the latest note so we can safely skip this. - if (isPureRenote(note)) return; - - // Compute the compound key of the entry to check - const key = SkLatestNote.keyFor(note); - - // Check if the deleted note was possibly the latest for the user - const hasLatestNote = await this.latestNotesRepository.existsBy(key); - if (hasLatestNote) return; - - // Find the newest remaining note for the user. - // We exclude DMs and pure renotes. - const nextLatest = await this.notesRepository - .createQueryBuilder('note') - .select() - .where({ - userId: key.userId, - visibility: key.isPublic - ? 'public' - : Not('specified'), - replyId: key.isReply - ? Not(null) - : null, - renoteId: key.isQuote - ? Not(null) - : null, - }) - .andWhere(` - ( - note."renoteId" IS NULL - OR note.text IS NOT NULL - OR note.cw IS NOT NULL - OR note."replyId" IS NOT NULL - OR note."hasPoll" - OR note."fileIds" != '{}' - ) - `) - .orderBy({ id: 'DESC' }) - .getOne(); - if (!nextLatest) return; - - // Record it as the latest - const latestNote = new SkLatestNote({ - ...key, - noteId: nextLatest.id, - }); - - // When inserting the latest note, it's possible that another worker has "raced" the insert and already added a newer note. - // We must use orIgnore() to ensure that the query ignores conflicts, otherwise an exception may be thrown. - await this.latestNotesRepository - .createQueryBuilder('latest') - .insert() - .into(SkLatestNote) - .values(latestNote) - .orIgnore() - .execute(); - } } diff --git a/packages/backend/src/models/LatestNote.ts b/packages/backend/src/models/LatestNote.ts index d36a4d568a..ff8815bd42 100644 --- a/packages/backend/src/models/LatestNote.ts +++ b/packages/backend/src/models/LatestNote.ts @@ -82,4 +82,16 @@ export class SkLatestNote { isQuote: isRenote(note) && isQuote(note), }; } + + /** + * Checks if two notes would produce equivalent compound keys. + */ + static areEquivalent(first: MiNote, second: MiNote): boolean { + return ( + first.userId === second.userId && + first.visibility === second.visibility && + (first.replyId != null) === (second.replyId != null) && + (isRenote(first) && isQuote(first)) === (isRenote(second) && isQuote(second)) + ); + } }