From 6f76a3a1f82131210b120139a5322f054a866c7f Mon Sep 17 00:00:00 2001
From: Mar0xy <marie@kaifa.ch>
Date: Mon, 13 Nov 2023 13:07:49 +0100
Subject: [PATCH] upd: import replies to own posts on *key
---
packages/backend/src/core/QueueService.ts | 5 ++-
.../processors/ImportNotesProcessorService.ts | 42 ++++++++++++++++---
packages/backend/src/queue/types.ts | 9 +++-
3 files changed, 47 insertions(+), 9 deletions(-)
diff --git a/packages/backend/src/core/QueueService.ts b/packages/backend/src/core/QueueService.ts
index ed24cfa56a..887048349f 100644
--- a/packages/backend/src/core/QueueService.ts
+++ b/packages/backend/src/core/QueueService.ts
@@ -16,6 +16,7 @@ import type { DbQueue, DeliverQueue, EndedPollNotificationQueue, InboxQueue, Obj
import type { DbJobData, DeliverJobData, RelationshipJobData, ThinUser } from '../queue/types.js';
import type httpSignature from '@peertube/http-signature';
import type * as Bull from 'bullmq';
+import { MiNote } from '@/models/Note.js';
@Injectable()
export class QueueService {
@@ -289,8 +290,8 @@ export class QueueService {
}
@bindThis
- public createImportKeyNotesToDbJob(user: ThinUser, targets: string[]) {
- const jobs = targets.map(rel => this.generateToDbJobData('importKeyNotesToDb', { user, target: rel }));
+ public createImportKeyNotesToDbJob(user: ThinUser, targets: string[], note: MiNote['id'] | null) {
+ const jobs = targets.map(rel => this.generateToDbJobData('importKeyNotesToDb', { user, target: rel, note }));
return this.dbQueue.addBulk(jobs);
}
diff --git a/packages/backend/src/queue/processors/ImportNotesProcessorService.ts b/packages/backend/src/queue/processors/ImportNotesProcessorService.ts
index 3f9d472716..58f10998bf 100644
--- a/packages/backend/src/queue/processors/ImportNotesProcessorService.ts
+++ b/packages/backend/src/queue/processors/ImportNotesProcessorService.ts
@@ -4,7 +4,7 @@ import { Inject, Injectable } from '@nestjs/common';
import { IsNull } from 'typeorm';
import { ZipReader } from 'slacc';
import { DI } from '@/di-symbols.js';
-import type { UsersRepository, DriveFilesRepository, MiDriveFile, MiNote } from '@/models/_.js';
+import type { UsersRepository, DriveFilesRepository, MiDriveFile, MiNote, NotesRepository } from '@/models/_.js';
import type Logger from '@/logger.js';
import { DownloadService } from '@/core/DownloadService.js';
import { UtilityService } from '@/core/UtilityService.js';
@@ -18,7 +18,7 @@ import { ApNoteService } from '@/core/activitypub/models/ApNoteService.js';
import { extractApHashtagObjects } from '@/core/activitypub/models/tag.js';
import { QueueLoggerService } from '../QueueLoggerService.js';
import type * as Bull from 'bullmq';
-import type { DbNoteImportToDbJobData, DbNoteImportJobData } from '../types.js';
+import type { DbNoteImportToDbJobData, DbNoteImportJobData, DbKeyNoteImportToDbJobData } from '../types.js';
@Injectable()
export class ImportNotesProcessorService {
@@ -31,6 +31,9 @@ export class ImportNotesProcessorService {
@Inject(DI.driveFilesRepository)
private driveFilesRepository: DriveFilesRepository,
+ @Inject(DI.notesRepository)
+ private notesRepository: NotesRepository,
+
private queueService: QueueService,
private utilityService: UtilityService,
private noteCreateService: NoteCreateService,
@@ -66,6 +69,30 @@ export class ImportNotesProcessorService {
}
}
+ // Function was taken from Firefish and edited to remove renoteId and make it run in only one for loop instead of two
+ @bindThis
+ private async recreateChain(arr: any[]) {
+ type NotesMap = {
+ [id: string]: any;
+ };
+ const notesTree: any[] = [];
+ const lookup: NotesMap = {};
+ for await (const note of arr) {
+ lookup[`${note.id}`] = note;
+ note.childNotes = [];
+ let parent = null;
+
+ if (note.replyId == null) {
+ notesTree.push(note);
+ } else {
+ parent = lookup[`${note.replyId}`];
+ }
+
+ if (parent) parent.childNotes.push(note);
+ }
+ return notesTree;
+ }
+
@bindThis
private isIterable(obj: any) {
if (obj == null) {
@@ -195,7 +222,8 @@ export class ImportNotesProcessorService {
const notesJson = fs.readFileSync(path, 'utf-8');
const notes = JSON.parse(notesJson);
- this.queueService.createImportKeyNotesToDbJob(job.data.user, notes);
+ const processedNotes = await this.recreateChain(notes);
+ this.queueService.createImportKeyNotesToDbJob(job.data.user, processedNotes, null);
cleanup();
}
@@ -203,7 +231,7 @@ export class ImportNotesProcessorService {
}
@bindThis
- public async processKeyNotesToDb(job: Bull.Job<DbNoteImportToDbJobData>): Promise<void> {
+ public async processKeyNotesToDb(job: Bull.Job<DbKeyNoteImportToDbJobData>): Promise<void> {
const note = job.data.target;
const user = await this.usersRepository.findOneBy({ id: job.data.user.id });
if (user == null) {
@@ -212,6 +240,8 @@ export class ImportNotesProcessorService {
if (note.renoteId) return;
+ const parentNote = job.data.note ? await this.notesRepository.findOneBy({ id: job.data.note }) : null;
+
const files: MiDriveFile[] = [];
const date = new Date(note.createdAt);
@@ -243,8 +273,8 @@ export class ImportNotesProcessorService {
}
}
- await this.noteCreateService.import(user, { createdAt: date, text: note.text, apMentions: new Array(0), visibility: note.visibility, localOnly: note.localOnly, files: files, cw: note.cw });
- if (note.childNotes) this.queueService.createImportKeyNotesToDbJob(user, note.childNotes);
+ const createdNote = await this.noteCreateService.import(user, { createdAt: date, reply: parentNote, text: note.text, apMentions: new Array(0), visibility: note.visibility, localOnly: note.localOnly, files: files, cw: note.cw });
+ if (note.childNotes) this.queueService.createImportKeyNotesToDbJob(user, note.childNotes, createdNote.id);
}
@bindThis
diff --git a/packages/backend/src/queue/types.ts b/packages/backend/src/queue/types.ts
index 9de3a41f03..493d3911b4 100644
--- a/packages/backend/src/queue/types.ts
+++ b/packages/backend/src/queue/types.ts
@@ -54,7 +54,7 @@ export type DbJobMap = {
importIGToDb: DbNoteImportToDbJobData;
importMastoToDb: DbNoteImportToDbJobData;
importPleroToDb: DbNoteImportToDbJobData;
- importKeyNotesToDb: DbNoteImportToDbJobData;
+ importKeyNotesToDb: DbKeyNoteImportToDbJobData;
importFollowing: DbUserImportJobData;
importFollowingToDb: DbUserImportToDbJobData;
importMuting: DbUserImportJobData;
@@ -110,6 +110,13 @@ export type DbUserImportToDbJobData = {
export type DbNoteImportToDbJobData = {
user: ThinUser;
target: any;
+ note?: MiNote['id'] | null;
+};
+
+export type DbKeyNoteImportToDbJobData = {
+ user: ThinUser;
+ target: any;
+ note: MiNote['id'] | null;
};
export type ObjectStorageJobData = ObjectStorageFileJobData | Record<string, unknown>;