From b211788d3c47e0a4d513b014ae5f5e77dd4bdb3f Mon Sep 17 00:00:00 2001 From: Namekuji Date: Tue, 25 Jul 2023 07:28:08 -0400 Subject: [PATCH] adjusting schema --- .../cql/1689400417034_timeline/up.cql | 5 +- packages/backend/src/db/scylla.ts | 77 ++++++++++++++++--- packages/backend/src/misc/populate-emojis.ts | 18 +++-- .../backend/src/server/api/common/getters.ts | 17 +++- .../src/server/api/endpoints/i/update.ts | 11 ++- .../src/server/api/endpoints/notes/create.ts | 25 ++++-- packages/backend/src/services/note/create.ts | 7 +- packages/backend/src/services/note/delete.ts | 2 +- .../src/services/note/reaction/create.ts | 11 ++- .../src/services/note/reaction/delete.ts | 12 ++- 10 files changed, 147 insertions(+), 38 deletions(-) diff --git a/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/up.cql b/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/up.cql index ad364e32fc..55591ae75f 100644 --- a/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/up.cql +++ b/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/up.cql @@ -42,9 +42,9 @@ CREATE TABLE IF NOT EXISTS note ( -- Models timeline "url" text, "score" int, "files" set>, - "visibleUsersId" set, + "visibleUserIds" set, "mentions" set, - "emojis" set>, + "emojis" set, "tags" set, "hasPoll" boolean, "threadId" ascii, @@ -54,7 +54,6 @@ CREATE TABLE IF NOT EXISTS note ( -- Models timeline "replyId" ascii, -- Reply "renoteId" ascii, -- Boost "reactions" map, - "reactionEmojis" map>, "noteEdit" set>, -- Edit History "updatedAt" timestamp, PRIMARY KEY ("createdAtDate", "createdAt", "id") diff --git a/packages/backend/src/db/scylla.ts b/packages/backend/src/db/scylla.ts index eacb7fc95f..96c90d206d 100644 --- a/packages/backend/src/db/scylla.ts +++ b/packages/backend/src/db/scylla.ts @@ -1,5 +1,6 @@ import config from "@/config/index.js"; import type { PopulatedEmoji } from "@/misc/populate-emojis.js"; +import type { Note } from "@/models/entities/note.js"; import type { NoteReaction } from "@/models/entities/note-reaction.js"; import { Client, types } from "cassandra-driver"; @@ -34,7 +35,7 @@ export const prepared = { "url", "score", "files", - "visibleUsersId", + "visibleUserIds", "mentions", "emojis", "tags", @@ -46,12 +47,11 @@ export const prepared = { "replyId", "renoteId", "reactions", - "reactionEmojis", "noteEdit", "updatedAt" ) VALUES - (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, select: { byDate: `SELECT * FROM note WHERE "createdAtDate" IN ?`, byId: `SELECT * FROM note WHERE "id" IN ?`, @@ -59,10 +59,17 @@ export const prepared = { byUrl: `SELECT * FROM note WHERE "url" IN ?`, byUserId: `SELECT * FROM note_by_userid WHERE "userId" IN ?`, }, - delete: `DELETE FROM note WHERE "createdAtDate" = ? AND "createdAt" = ?`, + delete: `DELETE FROM note WHERE "createdAtDate" = ? AND "createdAt" = ? AND "id" = ?`, update: { - renoteCount: `UPDATE note SET "renoteCount" = ?, "score" = ? WHERE "createdAtDate" = ? AND "createdAt" = ? IF EXISTS`, - reactions: `UPDATE note SET "reactions" = ?, "score" = ? WHERE "createdAtDate" = ? AND "createdAt" = ? IF EXISTS`, + renoteCount: `UPDATE note SET + "renoteCount" = ?, + "score" = ? + WHERE "createdAtDate" = ? AND "createdAt" = ? AND "id" = ? IF EXISTS`, + reactions: `UPDATE note SET + "emojis" = ?, + "reactions" = ?, + "score" = ? + WHERE "createdAtDate" = ? AND "createdAt" = ? AND "id" = ? IF EXISTS`, }, }, reaction: { @@ -92,12 +99,60 @@ export interface ScyllaDriveFile { isLink: boolean; md5: string; size: number; - width: number; - height: number; } -export type ScyllaNoteReaction = NoteReaction & { - emoji: PopulatedEmoji +export interface ScyllaNoteEditHistory { + content: string; + cw: string; + files: ScyllaDriveFile[]; + updatedAt: Date; +} + +export type ScyllaNote = Partial & { + createdAtDate: Date; + files: ScyllaDriveFile[]; + channelName: string; + noteEdit: ScyllaNoteEditHistory[]; +}; + +export function parseScyllaNote(row: types.Row): ScyllaNote { + const files: ScyllaDriveFile[] = row.get("files"); + return { + createdAtDate: row.get("createdAtDate"), + createdAt: row.get("createdAt"), + id: row.get("id"), + visibility: row.get("visibility"), + text: row.get("content"), + name: row.get("name"), + cw: row.get("cw"), + localOnly: row.get("localOnly"), + renoteCount: row.get("renoteCount"), + repliesCount: row.get("repliesCount"), + uri: row.get("uri"), + url: row.get("url"), + score: row.get("score"), + files, + fileIds: files.map((file) => file.id), + attachedFileTypes: files.map((file) => file.type), + visibleUserIds: row.get("visibleUserIds"), + mentions: row.get("mentions"), + emojis: row.get("emojis"), + tags: row.get("tags"), + hasPoll: row.get("hasPoll"), + threadId: row.get("threadId"), + channelId: row.get("channelId"), + channelName: row.get("channelName"), + userId: row.get("userId"), + replyId: row.get("replyId"), + renoteId: row.get("replyId"), + reactions: row.get("reactions"), + noteEdit: row.get("noteEdit"), + updatedAt: row.get("updatedAt"), + } +} + +export interface ScyllaNoteReaction extends NoteReaction { + emoji: PopulatedEmoji; } export function parseScyllaReaction(row: types.Row): ScyllaNoteReaction { @@ -108,5 +163,5 @@ export function parseScyllaReaction(row: types.Row): ScyllaNoteReaction { reaction: row.get("reaction"), createdAt: row.get("createdAt"), emoji: row.get("emoji"), - } + }; } diff --git a/packages/backend/src/misc/populate-emojis.ts b/packages/backend/src/misc/populate-emojis.ts index c9ba9e323d..8b9ab35534 100644 --- a/packages/backend/src/misc/populate-emojis.ts +++ b/packages/backend/src/misc/populate-emojis.ts @@ -9,7 +9,10 @@ import config from "@/config/index.js"; import { query } from "@/prelude/url.js"; import { redisClient } from "@/db/redis.js"; -export const EmojiCache = new Cache("populateEmojis", 60 * 60 * 12); +export const EmojiCache = new Cache( + "populateEmojis", + 60 * 60 * 12, +); /** * 添付用絵文字情報 @@ -71,13 +74,12 @@ export async function populateEmoji( host: host ?? IsNull(), })) || null; - const cacheKey = `${name} ${host}`; - let emoji = await EmojiCache.fetch(cacheKey, queryOrNull); - - if (emoji && !(emoji.width && emoji.height)) { - emoji = await queryOrNull(); - await EmojiCache.set(cacheKey, emoji); - } + const emoji = await EmojiCache.fetch( + `${name} ${host}`, + queryOrNull, + false, + (cache) => !!cache?.width && !!cache?.height, + ); if (emoji == null) return null; diff --git a/packages/backend/src/server/api/common/getters.ts b/packages/backend/src/server/api/common/getters.ts index fd7580775a..df6161d393 100644 --- a/packages/backend/src/server/api/common/getters.ts +++ b/packages/backend/src/server/api/common/getters.ts @@ -3,6 +3,7 @@ import type { User } from "@/models/entities/user.js"; import type { Note } from "@/models/entities/note.js"; import { Notes, Users } from "@/models/index.js"; import { generateVisibilityQuery } from "./generate-visibility-query.js"; +import { prepared, scyllaClient } from "@/db/scylla.js"; /** * Get note for API processing, taking into account visibility. @@ -11,13 +12,27 @@ export async function getNote( noteId: Note["id"], me: { id: User["id"] } | null, ) { + let note: Note | null = null; + if (scyllaClient) { + const result = await scyllaClient.execute( + prepared.note.select.byId, + [noteId], + { prepare: true }, + ); + if (result.rowLength > 0) { + const visibility: string = result.rows[0].get("visibility"); + if (!me) { + } + } + } + const query = Notes.createQueryBuilder("note").where("note.id = :id", { id: noteId, }); generateVisibilityQuery(query, me); - const note = await query.getOne(); + note = await query.getOne(); if (note == null) { throw new IdentifiableError( diff --git a/packages/backend/src/server/api/endpoints/i/update.ts b/packages/backend/src/server/api/endpoints/i/update.ts index 6d3bde2b8d..56c15de048 100644 --- a/packages/backend/src/server/api/endpoints/i/update.ts +++ b/packages/backend/src/server/api/endpoints/i/update.ts @@ -16,6 +16,7 @@ import { verifyLink } from "@/services/fetch-rel-me.js"; import { ApiError } from "../../error.js"; import config from "@/config/index.js"; import define from "../../define.js"; +import { userByIdCache } from "@/services/user-cache.js"; export const meta = { tags: ["account"], @@ -305,7 +306,13 @@ export default define(meta, paramDef, async (ps, _user, token) => { updateUsertags(user, tags); //#endregion - if (Object.keys(updates).length > 0) await Users.update(user.id, updates); + if (Object.keys(updates).length > 0) { + await Users.update(user.id, updates); + await userByIdCache.set( + user.id, + await Users.findOneByOrFail({ id: user.id }), + ); + } if (Object.keys(profileUpdates).length > 0) await UserProfiles.update(user.id, profileUpdates); @@ -319,7 +326,7 @@ export default define(meta, paramDef, async (ps, _user, token) => { publishUserEvent( user.id, "updateUserProfile", - await UserProfiles.findOneBy({ userId: user.id }), + await UserProfiles.findOneByOrFail({ userId: user.id }), ); // 鍵垢を解除したとき、溜まっていたフォローリクエストがあるならすべて承認 diff --git a/packages/backend/src/server/api/endpoints/notes/create.ts b/packages/backend/src/server/api/endpoints/notes/create.ts index e15221046f..f16f64b671 100644 --- a/packages/backend/src/server/api/endpoints/notes/create.ts +++ b/packages/backend/src/server/api/endpoints/notes/create.ts @@ -1,6 +1,6 @@ import { In } from "typeorm"; import create from "@/services/note/create.js"; -import type { User } from "@/models/entities/user.js"; +import type { CacheableUser, User } from "@/models/entities/user.js"; import { Users, DriveFiles, @@ -17,6 +17,7 @@ import { ApiError } from "../../error.js"; import define from "../../define.js"; import { HOUR } from "@/const.js"; import { getNote } from "../../common/getters.js"; +import { userByIdCache } from "@/services/user-cache.js"; export const meta = { tags: ["notes"], @@ -190,15 +191,25 @@ export default define(meta, paramDef, async (ps, user) => { if (user.movedToUri != null) throw new ApiError(meta.errors.accountLocked); let visibleUsers: User[] = []; if (ps.visibleUserIds) { - visibleUsers = await Users.findBy({ - id: In(ps.visibleUserIds), - }); + visibleUsers = ( + await Promise.all( + ps.visibleUserIds.map((id) => + userByIdCache.fetchMaybe(id, () => + Users.findOneBy({ id }).then((user) => user ?? undefined), + ), + ), + ) + ).filter((user) => user !== undefined) as CacheableUser[]; } let files: DriveFile[] = []; const fileIds = - ps.fileIds != null ? ps.fileIds : ps.mediaIds != null ? ps.mediaIds : null; - if (fileIds != null) { + ps.fileIds && ps.fileIds.length > 0 + ? ps.fileIds + : ps.mediaIds && ps.mediaIds.length > 0 + ? ps.mediaIds + : null; + if (fileIds && fileIds.length > 0) { files = await DriveFiles.createQueryBuilder("file") .where("file.userId = :userId AND file.id IN (:...fileIds)", { userId: user.id, @@ -210,7 +221,7 @@ export default define(meta, paramDef, async (ps, user) => { } let renote: Note | null = null; - if (ps.renoteId != null) { + if (ps.renoteId) { // Fetch renote to note renote = await getNote(ps.renoteId, user).catch((e) => { if (e.id === "9725d0ce-ba28-4dde-95a7-2cbb2c15de24") diff --git a/packages/backend/src/services/note/create.ts b/packages/backend/src/services/note/create.ts index e396b0b232..46386e4ed1 100644 --- a/packages/backend/src/services/note/create.ts +++ b/packages/backend/src/services/note/create.ts @@ -677,9 +677,12 @@ async function renderNoteOrRenoteActivity(data: Option, note: Note) { function incRenoteCount(renote: Note) { if (scyllaClient) { + const date = new Date(renote.createdAt.getTime()); scyllaClient.execute(prepared.note.update.renoteCount, [ renote.renoteCount + 1, renote.score + 1, + date, + date, renote.id, ]); } else { @@ -768,7 +771,6 @@ async function insertNote( // 投稿を作成 try { if (scyllaClient) { - const noteEmojis = await populateEmojis(insert.emojis, user.host); await scyllaClient.execute( prepared.note.insert, [ @@ -788,7 +790,7 @@ async function insertNote( data.files, insert.visibleUserIds, insert.mentions, - noteEmojis, + insert.emojis, insert.tags, insert.hasPoll, insert.threadId, @@ -800,7 +802,6 @@ async function insertNote( null, null, null, - null, ], { prepare: true }, ); diff --git a/packages/backend/src/services/note/delete.ts b/packages/backend/src/services/note/delete.ts index b810744bee..9cc800bbef 100644 --- a/packages/backend/src/services/note/delete.ts +++ b/packages/backend/src/services/note/delete.ts @@ -119,7 +119,7 @@ export default async function ( if (scyllaClient) { const date = new Date(note.createdAt.getTime()); - await scyllaClient.execute(prepared.note.delete, [date, date], { + await scyllaClient.execute(prepared.note.delete, [date, date, note.id], { prepare: true, }); } diff --git a/packages/backend/src/services/note/reaction/create.ts b/packages/backend/src/services/note/reaction/create.ts index 893f239632..a5ef8670b4 100644 --- a/packages/backend/src/services/note/reaction/create.ts +++ b/packages/backend/src/services/note/reaction/create.ts @@ -22,6 +22,7 @@ import { isDuplicateKeyValueError } from "@/misc/is-duplicate-key-value-error.js import type { NoteReaction } from "@/models/entities/note-reaction.js"; import { IdentifiableError } from "@/misc/identifiable-error.js"; import { prepared, scyllaClient } from "@/db/scylla.js"; +import { populateEmojis } from "@/misc/populate-emojis.js"; export default async ( user: { id: User["id"]; host: User["host"] }, @@ -105,10 +106,18 @@ export default async ( if (scyllaClient) { const current = Math.max(note.reactions[_reaction] ?? 0, 0); note.reactions[_reaction] = current + 1; + const emojiName = decodeReaction(_reaction).reaction.replaceAll(":", ""); const date = new Date(note.createdAt.getTime()); await scyllaClient.execute( prepared.note.update.reactions, - [note.reactions, (note.score ?? 0) + 1, date, date], + [ + note.emojis.concat(emojiName), + note.reactions, + (note.score ?? 0) + 1, + date, + date, + note.id, + ], { prepare: true }, ); } else { diff --git a/packages/backend/src/services/note/reaction/delete.ts b/packages/backend/src/services/note/reaction/delete.ts index c044596274..c61821cebf 100644 --- a/packages/backend/src/services/note/reaction/delete.ts +++ b/packages/backend/src/services/note/reaction/delete.ts @@ -56,9 +56,19 @@ export default async ( note.reactions[reaction.reaction] = count; } const date = new Date(note.createdAt.getTime()); + const emojiName = reaction.reaction.replaceAll(":", ""); + const emojiIndex = note.emojis.indexOf(emojiName); + if (emojiIndex >= 0 && count === 0) note.emojis.splice(emojiIndex, 1); await scyllaClient.execute( prepared.note.update.reactions, - [note.reactions, Math.max((note.score ?? 0) - 1, 0), date, date], + [ + note.emojis, + note.reactions, + Math.max((note.score ?? 0) - 1, 0), + date, + date, + note.id, + ], { prepare: true }, ); } else {