diff --git a/.config/example.yml b/.config/example.yml index f73f4f1d79..12665bcb38 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -67,7 +67,7 @@ redis: #db: 1 #user: default -# ┌─────────────────────────────┐ +# ┌────────────────────────────┐ #───┘ Cache server configuration └───────────────────────────────────── # A Redis-compatible server (DragonflyDB, Keydb, Redis) for caching @@ -81,6 +81,14 @@ redis: #prefix: example-prefix #db: 1 +# ┌────────────────────────┐ +#───┘ ScyllaDB configuration └───────────────────────────────────── + +# scylla: +# nodes: ['localhost:9042'] +# keyspace: calckey +# replicationFactor: 1 + # Please configure either MeiliSearch *or* Sonic. # If both MeiliSearch and Sonic configurations are present, MeiliSearch will take precedence. diff --git a/packages/backend/package.json b/packages/backend/package.json index cedd9bee71..cc94f8ee24 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -53,6 +53,7 @@ "bull": "4.10.4", "cacheable-lookup": "7.0.0", "calckey-js": "workspace:*", + "cassandra-driver": "^4.6.4", "cbor": "8.1.0", "chalk": "5.3.0", "chalk-template": "0.4.0", diff --git a/packages/backend/src/config/types.ts b/packages/backend/src/config/types.ts index 7789c26e07..49a96642f2 100644 --- a/packages/backend/src/config/types.ts +++ b/packages/backend/src/config/types.ts @@ -16,6 +16,11 @@ export type Source = { disableCache?: boolean; extra?: { [x: string]: string }; }; + scylla?: { + nodes: string[]; + keyspace: string; + replicationFactor: number; + }, redis: { host: string; port: number; diff --git a/packages/backend/src/db/scylla.ts b/packages/backend/src/db/scylla.ts new file mode 100644 index 0000000000..443f23793b --- /dev/null +++ b/packages/backend/src/db/scylla.ts @@ -0,0 +1,28 @@ +import config from "@/config/index.js"; +import { Client } from "cassandra-driver"; + +function newClient(): Client | null { + if (!config.scylla) { + return null; + } + + return new Client({ + contactPoints: config.scylla.nodes, + keyspace: config.scylla.keyspace, + }); +} + +export const scyllaClient = newClient(); + +export const prepared = { + timeline: { + insert: "", + select: "", + delete: "", + }, + notification: { + insert: "", + select: "", + delete: "", + } +} diff --git a/packages/backend/src/models/repositories/user.ts b/packages/backend/src/models/repositories/user.ts index e4aab896d0..7c3f418e50 100644 --- a/packages/backend/src/models/repositories/user.ts +++ b/packages/backend/src/models/repositories/user.ts @@ -38,6 +38,7 @@ import { UserSecurityKeys, } from "../index.js"; import type { Instance } from "../entities/instance.js"; +import { userByIdCache } from "@/services/user-cache.js"; const userInstanceCache = new Cache( "userInstance", @@ -382,23 +383,24 @@ export const UserRepository = db.getRepository(User).extend({ options, ); - let user: User; + let id: string; if (typeof src === "object") { - user = src; - if (src.avatar === undefined && src.avatarId) - src.avatar = (await DriveFiles.findOneBy({ id: src.avatarId })) ?? null; - if (src.banner === undefined && src.bannerId) - src.banner = (await DriveFiles.findOneBy({ id: src.bannerId })) ?? null; + id = src.id; } else { - user = await this.findOneOrFail({ - where: { id: src }, + id = src; + } + + const user = await userByIdCache.fetch(id, () => + this.findOneOrFail({ + where: { id }, relations: { avatar: true, banner: true, }, - }); - } + }), + true, + ); const meId = me ? me.id : null; const isMe = meId === user.id; diff --git a/packages/backend/src/remote/activitypub/db-resolver.ts b/packages/backend/src/remote/activitypub/db-resolver.ts index a710b9f115..cbd8b4312c 100644 --- a/packages/backend/src/remote/activitypub/db-resolver.ts +++ b/packages/backend/src/remote/activitypub/db-resolver.ts @@ -128,8 +128,12 @@ export default class DbResolver { (await userByIdCache.fetchMaybe( parsed.id, () => - Users.findOneBy({ - id: parsed.id, + Users.findOne({ + where: { id: parsed.id }, + relations: { + avatar: true, + banner: true, + }, }).then((x) => x ?? undefined), true, )) ?? null @@ -173,7 +177,11 @@ export default class DbResolver { return { user: (await userByIdCache.fetch( key.userId, - () => Users.findOneByOrFail({ id: key.userId }), + () => + Users.findOneOrFail({ + where: { id: key.userId }, + relations: { avatar: true, banner: true }, + }), true, )) as CacheableRemoteUser, key, diff --git a/packages/backend/src/server/api/endpoints/i/update.ts b/packages/backend/src/server/api/endpoints/i/update.ts index 0637251a6b..273ac422a7 100644 --- a/packages/backend/src/server/api/endpoints/i/update.ts +++ b/packages/backend/src/server/api/endpoints/i/update.ts @@ -14,6 +14,8 @@ import { normalizeForSearch } from "@/misc/normalize-for-search.js"; import { langmap } from "@/misc/langmap.js"; import { ApiError } from "../../error.js"; import define from "../../define.js"; +import { userByIdCache } from "@/services/user-cache.js"; +import type { DriveFile } from "@/models/entities/drive-file.js"; export const meta = { tags: ["account"], @@ -204,8 +206,9 @@ export default define(meta, paramDef, async (ps, _user, token) => { if (ps.emailNotificationTypes !== undefined) profileUpdates.emailNotificationTypes = ps.emailNotificationTypes; + let avatar: DriveFile | null = null; if (ps.avatarId) { - const avatar = await DriveFiles.findOneBy({ id: ps.avatarId }); + avatar = await DriveFiles.findOneBy({ id: ps.avatarId }); if (avatar == null || avatar.userId !== user.id) throw new ApiError(meta.errors.noSuchAvatar); @@ -213,8 +216,9 @@ export default define(meta, paramDef, async (ps, _user, token) => { throw new ApiError(meta.errors.avatarNotAnImage); } + let banner: DriveFile | null = null; if (ps.bannerId) { - const banner = await DriveFiles.findOneBy({ id: ps.bannerId }); + banner = await DriveFiles.findOneBy({ id: ps.bannerId }); if (banner == null || banner.userId !== user.id) throw new ApiError(meta.errors.noSuchBanner); @@ -278,7 +282,10 @@ export default define(meta, paramDef, async (ps, _user, token) => { updateUsertags(user, tags); //#endregion - if (Object.keys(updates).length > 0) await Users.update(user.id, updates); + if (Object.keys(updates).length > 0) { + await Users.update(user.id, updates); + await userByIdCache.set(user.id, { ...user, ...updates, avatar, banner }); + } if (Object.keys(profileUpdates).length > 0) await UserProfiles.update(user.id, profileUpdates); diff --git a/packages/backend/src/server/api/endpoints/notes/local-timeline.ts b/packages/backend/src/server/api/endpoints/notes/local-timeline.ts index 82e93e371f..7967dbaec1 100644 --- a/packages/backend/src/server/api/endpoints/notes/local-timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/local-timeline.ts @@ -89,17 +89,10 @@ export default define(meta, paramDef, async (ps, user) => { ps.untilDate, ) .andWhere("(note.visibility = 'public') AND (note.userHost IS NULL)") - .innerJoinAndSelect("note.user", "user") - .leftJoinAndSelect("user.avatar", "avatar") - .leftJoinAndSelect("user.banner", "banner") .leftJoinAndSelect("note.reply", "reply") .leftJoinAndSelect("note.renote", "renote") - .leftJoinAndSelect("reply.user", "replyUser") - .leftJoinAndSelect("replyUser.avatar", "replyUserAvatar") - .leftJoinAndSelect("replyUser.banner", "replyUserBanner") - .leftJoinAndSelect("renote.user", "renoteUser") - .leftJoinAndSelect("renoteUser.avatar", "renoteUserAvatar") - .leftJoinAndSelect("renoteUser.banner", "renoteUserBanner"); + + // TODO: Use materialized view of postgres as much for non-denormalizable columns as possible. generateChannelQuery(query, user); generateRepliesQuery(query, ps.withReplies, user); diff --git a/packages/backend/src/server/api/endpoints/notes/timeline.ts b/packages/backend/src/server/api/endpoints/notes/timeline.ts index d629deebb6..606e83ad1f 100644 --- a/packages/backend/src/server/api/endpoints/notes/timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/timeline.ts @@ -92,16 +92,10 @@ export default define(meta, paramDef, async (ps, user) => { }), ) .innerJoinAndSelect("note.user", "user") - .leftJoinAndSelect("user.avatar", "avatar") - .leftJoinAndSelect("user.banner", "banner") .leftJoinAndSelect("note.reply", "reply") .leftJoinAndSelect("note.renote", "renote") .leftJoinAndSelect("reply.user", "replyUser") - .leftJoinAndSelect("replyUser.avatar", "replyUserAvatar") - .leftJoinAndSelect("replyUser.banner", "replyUserBanner") .leftJoinAndSelect("renote.user", "renoteUser") - .leftJoinAndSelect("renoteUser.avatar", "renoteUserAvatar") - .leftJoinAndSelect("renoteUser.banner", "renoteUserBanner") .setParameters(followingQuery.getParameters()); generateChannelQuery(query, user); diff --git a/packages/backend/src/services/user-cache.ts b/packages/backend/src/services/user-cache.ts index ed700185df..790fd648f2 100644 --- a/packages/backend/src/services/user-cache.ts +++ b/packages/backend/src/services/user-cache.ts @@ -40,7 +40,10 @@ subscriber.on("message", async (_, data) => { case "userChangeSilencedState": case "userChangeModeratorState": case "remoteUserUpdated": { - const user = await Users.findOneByOrFail({ id: body.id }); + const user = await Users.findOneOrFail({ + where: { id: body.id }, + relations: { avatar: true, banner: true }, + }); await userByIdCache.set(user.id, user); const trans = redisClient.multi(); for (const [k, v] of (await uriPersonCache.getAll()).entries()) { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3ad273c464..b0dabbab5e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -159,6 +159,9 @@ importers: calckey-js: specifier: workspace:* version: link:../calckey-js + cassandra-driver: + specifier: ^4.6.4 + version: 4.6.4 cbor: specifier: 8.1.0 version: 8.1.0 @@ -6220,6 +6223,16 @@ packages: /caseless@0.12.0: resolution: {integrity: sha512-4tYFyifaFfGacoiObjJegolkwSU4xQNGbVgUiNYVUxbQ2x2lUsFvY4hVgVzGiIe6WLOPqycWXA40l+PWsxthUw==} + /cassandra-driver@4.6.4: + resolution: {integrity: sha512-SksbIK0cZ2QZRx8ti7w+PnLqldyY+6kU2gRWFChwXFTtrD/ce8cQICDEHxyPwx+DeILwRnMrPf9cjUGizYw9Vg==} + engines: {node: '>=8'} + dependencies: + '@types/long': 4.0.2 + '@types/node': 20.4.1 + adm-zip: 0.5.10 + long: 2.4.0 + dev: false + /cbor@8.1.0: resolution: {integrity: sha512-DwGjNW9omn6EwP70aXsn7FQJx5kO12tX0bZkaTjzdVFM6/7nhA4t0EENocKGx6D2Bch9PE2KzCUf5SceBdeijg==} engines: {node: '>=12.19'} @@ -12887,6 +12900,11 @@ packages: wrap-ansi: 6.2.0 dev: true + /long@2.4.0: + resolution: {integrity: sha512-ijUtjmO/n2A5PaosNG9ZGDsQ3vxJg7ZW8vsY8Kp0f2yIZWhSJvjmegV7t+9RPQKxKrvj8yKGehhS+po14hPLGQ==} + engines: {node: '>=0.6'} + dev: false + /long@4.0.0: resolution: {integrity: sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==} dev: false