From 109ed0aa80055298fc019aeac90e2003f01eec7b Mon Sep 17 00:00:00 2001 From: CGsama Date: Mon, 31 Jul 2023 15:42:31 -0400 Subject: [PATCH] import all misskey self note thread and allow import multiple times to restore missing parts --- packages/backend/src/misc/post.ts | 6 +- packages/backend/src/queue/index.ts | 3 + .../processors/db/import-firefish-post.ts | 88 +++++++++++++++---- .../queue/processors/db/import-masto-post.ts | 57 ++++++++---- .../src/queue/processors/db/import-posts.ts | 32 ++++++- packages/backend/src/queue/types.ts | 1 + packages/backend/src/services/note/create.ts | 5 +- 7 files changed, 157 insertions(+), 35 deletions(-) diff --git a/packages/backend/src/misc/post.ts b/packages/backend/src/misc/post.ts index 90f4f75283..dbe703d1a0 100644 --- a/packages/backend/src/misc/post.ts +++ b/packages/backend/src/misc/post.ts @@ -1,16 +1,18 @@ export type Post = { - text: string | null; + text: string | undefined; cw: string | null; localOnly: boolean; createdAt: Date; + visibility: string; }; export function parse(acct: any): Post { return { - text: acct.text, + text: acct.text || undefined, cw: acct.cw, localOnly: acct.localOnly, createdAt: new Date(acct.createdAt), + visibility: "hidden" + (acct.visibility || ""), }; } diff --git a/packages/backend/src/queue/index.ts b/packages/backend/src/queue/index.ts index 93aed7cb8b..0500737a48 100644 --- a/packages/backend/src/queue/index.ts +++ b/packages/backend/src/queue/index.ts @@ -28,6 +28,7 @@ import { backgroundQueue, } from "./queues.js"; import type { ThinUser } from "./types.js"; +import { Note } from "@/models/entities/note.js"; function renderError(e: Error): any { return { @@ -358,6 +359,7 @@ export function createImportCkPostJob( user: ThinUser, post: any, signatureCheck: boolean, + parent: Note | null = null, ) { return dbQueue.add( "importCkPost", @@ -365,6 +367,7 @@ export function createImportCkPostJob( user: user, post: post, signatureCheck: signatureCheck, + parent: parent, }, { removeOnComplete: true, diff --git a/packages/backend/src/queue/processors/db/import-firefish-post.ts b/packages/backend/src/queue/processors/db/import-firefish-post.ts index 504cf9e507..4ec88d32cc 100644 --- a/packages/backend/src/queue/processors/db/import-firefish-post.ts +++ b/packages/backend/src/queue/processors/db/import-firefish-post.ts @@ -3,7 +3,13 @@ import create from "@/services/note/create.js"; import { Users } from "@/models/index.js"; import type { DbUserImportMastoPostJobData } from "@/queue/types.js"; import { queueLogger } from "../../logger.js"; +import { uploadFromUrl } from "@/services/drive/upload-from-url.js"; +import type { DriveFile } from "@/models/entities/drive-file.js"; import type Bull from "bull"; +import { createImportCkPostJob } from "@/queue/index.js"; +import { Notes, NoteEdits } from "@/models/index.js"; +import type { Note } from "@/models/entities/note.js"; +import { genId } from "@/misc/gen-id.js"; const logger = queueLogger.createSubLogger("import-firefish-post"); @@ -17,6 +23,7 @@ export async function importCkPost( return; } const post = job.data.post; + /* if (post.replyId != null) { done(); return; @@ -29,23 +36,74 @@ export async function importCkPost( done(); return; } - const { text, cw, localOnly, createdAt } = Post.parse(post); - const note = await create(user, { + */ + const urls = (post.files || []) + .map((x: any) => x.url) + .filter((x: String) => x.startsWith("http")); + const files: DriveFile[] = []; + for (const url of urls) { + try { + const file = await uploadFromUrl({ + url: url, + user: user, + }); + files.push(file); + } catch (e) { + logger.error(`Skipped adding file to drive: ${url}`); + } + } + const { text, cw, localOnly, createdAt, visibility } = Post.parse(post); + let note = await Notes.findOneBy({ createdAt: createdAt, - files: undefined, - poll: undefined, - text: text || undefined, - reply: null, - renote: null, - cw: cw, - localOnly, - visibility: "hidden", - visibleUsers: [], - channel: null, - apMentions: new Array(0), - apHashtags: undefined, - apEmojis: undefined, + text: text, + userId: user.id, }); + + if (note && (note?.fileIds?.length || 0) < files.length) { + const update: Partial = {}; + update.fileIds = files.map((x) => x.id); + await Notes.update(note.id, update); + await NoteEdits.insert({ + id: genId(), + noteId: note.id, + text: note.text || undefined, + cw: note.cw, + fileIds: note.fileIds, + updatedAt: new Date(), + }); + logger.info(`Note file updated`); + } + if (!note) { + note = await create(user, { + createdAt: createdAt, + files: files.length == 0 ? undefined : files, + poll: undefined, + text: text || undefined, + reply: post.replyId ? job.data.parent : null, + renote: post.renoteId ? job.data.parent : null, + cw: cw, + localOnly, + visibility: visibility, + visibleUsers: [], + channel: null, + apMentions: new Array(0), + apHashtags: undefined, + apEmojis: undefined, + }); + logger.info(`Create new note`); + } else { + logger.info(`Note exist`); + } logger.succ("Imported"); + if (post.childNotes) { + for (const child of post.childNotes) { + createImportCkPostJob( + job.data.user, + child, + job.data.signatureCheck, + note, + ); + } + } done(); } diff --git a/packages/backend/src/queue/processors/db/import-masto-post.ts b/packages/backend/src/queue/processors/db/import-masto-post.ts index 1d18008a0f..9b86a5903f 100644 --- a/packages/backend/src/queue/processors/db/import-masto-post.ts +++ b/packages/backend/src/queue/processors/db/import-masto-post.ts @@ -8,6 +8,9 @@ import { resolveNote } from "@/remote/activitypub/models/note.js"; import { Note } from "@/models/entities/note.js"; import { uploadFromUrl } from "@/services/drive/upload-from-url.js"; import type { DriveFile } from "@/models/entities/drive-file.js"; +import { Notes, NoteEdits } from "@/models/index.js"; +import type { Note } from "@/models/entities/note.js"; +import { genId } from "@/misc/gen-id.js"; const logger = queueLogger.createSubLogger("import-masto-post"); @@ -67,23 +70,47 @@ export async function importMastoPost( } } } - - const note = await create(user, { + let note = await Notes.findOneBy({ createdAt: new Date(post.object.published), - files: files.length == 0 ? undefined : files, - poll: undefined, - text: text || undefined, - reply, - renote: null, - cw: post.object.sensitive ? post.object.summary : undefined, - localOnly: false, - visibility: "hidden", - visibleUsers: [], - channel: null, - apMentions: new Array(0), - apHashtags: undefined, - apEmojis: undefined, + text: text, + userId: user.id, }); + + if (note && (note?.fileIds?.length || 0) < files.length) { + const update: Partial = {}; + update.fileIds = files.map((x) => x.id); + await Notes.update(note.id, update); + await NoteEdits.insert({ + id: genId(), + noteId: note.id, + text: note.text || undefined, + cw: note.cw, + fileIds: note.fileIds, + updatedAt: new Date(), + }); + logger.info(`Note file updated`); + } + if (!note) { + note = await create(user, { + createdAt: new Date(post.object.published), + files: files.length == 0 ? undefined : files, + poll: undefined, + text: text || undefined, + reply, + renote: null, + cw: post.object.sensitive ? post.object.summary : undefined, + localOnly: false, + visibility: "hiddenpublic", + visibleUsers: [], + channel: null, + apMentions: new Array(0), + apHashtags: undefined, + apEmojis: undefined, + }); + logger.info(`Create new note`); + } else { + logger.info(`Note exist`); + } job.progress(100); done(); diff --git a/packages/backend/src/queue/processors/db/import-posts.ts b/packages/backend/src/queue/processors/db/import-posts.ts index 9bde7479ed..526489c634 100644 --- a/packages/backend/src/queue/processors/db/import-posts.ts +++ b/packages/backend/src/queue/processors/db/import-posts.ts @@ -57,7 +57,8 @@ export async function importPosts( const parsed = JSON.parse(json); if (parsed instanceof Array) { logger.info("Parsing key style posts"); - for (const post of JSON.parse(json)) { + const arr = recreateChain(parsed); + for (const post of arr) { createImportCkPostJob(job.data.user, post, job.data.signatureCheck); } } else if (parsed instanceof Object) { @@ -74,3 +75,32 @@ export async function importPosts( logger.succ("Imported"); done(); } + +function recreateChain(arr: any[]): any { + type NotesMap = { + [id: string]: any; + }; + const notesTree: any[] = []; + const lookup: NotesMap = {}; + for (const note of arr) { + lookup[`${note.id}`] = note; + note.childNotes = []; + if (note.replyId == null && note.renoteId == null) { + notesTree.push(note); + } + } + for (const note of arr) { + let parent = null; + if (note.replyId != null) { + parent = lookup[`${note.replyId}`]; + } + if (note.renoteId != null) { + parent = lookup[`${note.renoteId}`]; + } + + if (parent) { + parent.childNotes.push(note); + } + } + return notesTree; +} diff --git a/packages/backend/src/queue/types.ts b/packages/backend/src/queue/types.ts index b72b127894..6383f3fdd5 100644 --- a/packages/backend/src/queue/types.ts +++ b/packages/backend/src/queue/types.ts @@ -52,6 +52,7 @@ export type DbUserImportMastoPostJobData = { user: ThinUser; post: any; signatureCheck: boolean; + parent: Note | null; }; export type ObjectStorageJobData = diff --git a/packages/backend/src/services/note/create.ts b/packages/backend/src/services/note/create.ts index 095c75f427..adebc22a40 100644 --- a/packages/backend/src/services/note/create.ts +++ b/packages/backend/src/services/note/create.ts @@ -172,7 +172,7 @@ export default async ( // rome-ignore lint/suspicious/noAsyncPromiseExecutor: FIXME new Promise(async (res, rej) => { const dontFederateInitially = - data.localOnly || data.visibility === "hidden"; + data.localOnly || data.visibility?.startsWith("hidden"); // If you reply outside the channel, match the scope of the target. // TODO (I think it's a process that could be done on the client side, but it's server side for now.) @@ -206,7 +206,8 @@ export default async ( if (data.channel != null) data.visibility = "public"; if (data.channel != null) data.visibleUsers = []; if (data.channel != null) data.localOnly = true; - if (data.visibility === "hidden") data.visibility = "public"; + if (data.visibility.startsWith("hidden")) + data.visibility = data.visibility.slice(6); // enforce silent clients on server if (