From 41930bda52c97f3e400713d998cde087954d3c92 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Sun, 30 Jul 2023 17:35:34 -0400 Subject: [PATCH] wip: timeline query --- .../cql/1689400417034_timeline/down.cql | 4 +-- .../cql/1689400417034_timeline/up.cql | 1 - packages/backend/src/db/scylla.ts | 7 ++-- packages/backend/src/misc/cache.ts | 14 ++++++-- .../backend/src/models/repositories/note.ts | 4 +-- .../backend/src/models/repositories/user.ts | 17 ++++++---- .../src/remote/activitypub/models/person.ts | 17 +++++++++- .../src/server/api/endpoints/i/update.ts | 14 ++++++-- .../server/api/endpoints/notes/timeline.ts | 34 +++++++++++++++---- packages/backend/src/services/note/create.ts | 3 +- .../src/services/note/reaction/delete.ts | 2 +- 11 files changed, 83 insertions(+), 34 deletions(-) diff --git a/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/down.cql b/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/down.cql index 7d90c985c5..17fd261d9c 100644 --- a/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/down.cql +++ b/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/down.cql @@ -1,7 +1,7 @@ -DROP MATERIALIZED VIEW IF EXISTS reaction_by_user_id; +DROP MATERIALIZED VIEW IF EXISTS reaction_by_userid; DROP INDEX IF EXISTS reaction_by_id; DROP TABLE IF EXISTS reaction; -DROP MATERIALIZED VIEW IF EXISTS note_by_user_id; +DROP MATERIALIZED VIEW IF EXISTS note_by_userid; DROP INDEX IF EXISTS note_by_id; DROP INDEX IF EXISTS note_by_uri; DROP INDEX IF EXISTS note_by_url; diff --git a/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/up.cql b/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/up.cql index 10137086d3..517fff1227 100644 --- a/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/up.cql +++ b/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/up.cql @@ -49,7 +49,6 @@ CREATE TABLE IF NOT EXISTS note ( -- Models timeline "hasPoll" boolean, "threadId" ascii, "channelId" ascii, -- Channel - "channelName" text, "userId" ascii, -- User "userHost" text, "replyId" ascii, -- Reply diff --git a/packages/backend/src/db/scylla.ts b/packages/backend/src/db/scylla.ts index bc9d688397..42d62a244d 100644 --- a/packages/backend/src/db/scylla.ts +++ b/packages/backend/src/db/scylla.ts @@ -44,7 +44,6 @@ export const prepared = { "hasPoll", "threadId", "channelId", - "channelName", "userId", "userHost", "replyId", @@ -58,9 +57,9 @@ export const prepared = { "updatedAt" ) VALUES - (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, select: { - byDate: `SELECT * FROM note WHERE "createdAtDate" IN ?`, + byDate: `SELECT * FROM note WHERE "createdAtDate" = ?`, byId: `SELECT * FROM note WHERE "id" IN ?`, byUri: `SELECT * FROM note WHERE "uri" IN ?`, byUrl: `SELECT * FROM note WHERE "url" IN ?`, @@ -118,7 +117,6 @@ export interface ScyllaNoteEditHistory { export type ScyllaNote = Note & { createdAtDate: Date; files: ScyllaDriveFile[]; - channelName: string; noteEdit: ScyllaNoteEditHistory[]; }; @@ -148,7 +146,6 @@ export function parseScyllaNote(row: types.Row): ScyllaNote { hasPoll: row.get("hasPoll"), threadId: row.get("threadId"), channelId: row.get("channelId"), - channelName: row.get("channelName"), userId: row.get("userId"), userHost: row.get("userHost"), replyId: row.get("replyId"), diff --git a/packages/backend/src/misc/cache.ts b/packages/backend/src/misc/cache.ts index a6dab9c89f..7f0803b47c 100644 --- a/packages/backend/src/misc/cache.ts +++ b/packages/backend/src/misc/cache.ts @@ -140,11 +140,11 @@ export class LocalFollowingsCache { this.key = `follow:${userId}`; } - public static async init(userId: string) { + public static async init(userId: string): Promise { const cache = new LocalFollowingsCache(userId); - // Sync from DB if no relationships is cached - if ((await redisClient.scard(cache.key)) === 0) { + // Sync from DB if no followings are cached + if (!(await cache.hasFollowing())) { const rel = await Followings.find({ select: { followeeId: true }, where: { followerId: cache.myId }, @@ -172,4 +172,12 @@ export class LocalFollowingsCache { public async isFollowing(targetId: string): Promise { return (await redisClient.sismember(this.key, targetId)) === 1; } + + public async hasFollowing(): Promise { + return (await redisClient.scard(this.key)) !== 0; + } + + public async getAll(): Promise { + return (await redisClient.smembers(this.key)) + } } diff --git a/packages/backend/src/models/repositories/note.ts b/packages/backend/src/models/repositories/note.ts index 453179bd6f..8293b5d663 100644 --- a/packages/backend/src/models/repositories/note.ts +++ b/packages/backend/src/models/repositories/note.ts @@ -296,8 +296,8 @@ export const NoteRepository = db.getRepository(Note).extend({ const myReactionsMap = new Map(); if (meId) { const renoteIds = notes - .filter((n) => n.renoteId != null) - .map((n) => n.renoteId!); + .filter((n) => !!n.renoteId) + .map((n) => n.renoteId) as string[]; const targets = [...notes.map((n) => n.id), ...renoteIds]; const myReactions = await NoteReactions.findBy({ userId: meId, diff --git a/packages/backend/src/models/repositories/user.ts b/packages/backend/src/models/repositories/user.ts index e7c4b6f008..2e09ed58a3 100644 --- a/packages/backend/src/models/repositories/user.ts +++ b/packages/backend/src/models/repositories/user.ts @@ -37,6 +37,7 @@ import { UserSecurityKeys, } from "../index.js"; import type { Instance } from "../entities/instance.js"; +import { userByIdCache, userDenormalizedCache } from "@/services/user-cache.js"; const userInstanceCache = new Cache( "userInstance", @@ -391,13 +392,15 @@ export const UserRepository = db.getRepository(User).extend({ if (src.banner === undefined && src.bannerId) src.banner = (await DriveFiles.findOneBy({ id: src.bannerId })) ?? null; } else { - user = await this.findOneOrFail({ - where: { id: src }, - relations: { - avatar: true, - banner: true, - }, - }); + user = await userDenormalizedCache.fetch(src, () => + this.findOneOrFail({ + where: { id: src }, + relations: { + avatar: true, + banner: true, + }, + }), + ); } const meId = me ? me.id : null; diff --git a/packages/backend/src/remote/activitypub/models/person.ts b/packages/backend/src/remote/activitypub/models/person.ts index 5528e34eef..dcdca1ee41 100644 --- a/packages/backend/src/remote/activitypub/models/person.ts +++ b/packages/backend/src/remote/activitypub/models/person.ts @@ -28,7 +28,11 @@ import { fetchInstanceMetadata } from "@/services/fetch-instance-metadata.js"; import { normalizeForSearch } from "@/misc/normalize-for-search.js"; import { truncate } from "@/misc/truncate.js"; import { StatusError } from "@/misc/fetch.js"; -import { uriPersonCache, userByIdCache } from "@/services/user-cache.js"; +import { + uriPersonCache, + userByIdCache, + userDenormalizedCache, +} from "@/services/user-cache.js"; import { publishInternalEvent } from "@/services/stream.js"; import { db } from "@/db/postgre.js"; import { apLogger } from "../logger.js"; @@ -373,6 +377,10 @@ export async function createPerson( await updateFeatured(user!.id, resolver).catch((err) => logger.error(err)); + user!.avatar = avatar; + user!.banner = banner; + await userDenormalizedCache.set(user!.id, user!); + return user!; } @@ -518,6 +526,13 @@ export async function updatePerson( // Update user await Users.update(user.id, updates); + const updatedUser = await Users.findOneByOrFail({ id: user.id }); + updatedUser.avatarId = avatar?.id ?? null; + updatedUser.avatar = avatar; + updatedUser.bannerId = banner?.id ?? null; + updatedUser.banner = banner; + await userDenormalizedCache.set(updatedUser.id, updatedUser); + if (person.publicKey) { await UserPublickeys.update( { userId: user.id }, diff --git a/packages/backend/src/server/api/endpoints/i/update.ts b/packages/backend/src/server/api/endpoints/i/update.ts index 56c15de048..f12a6693f3 100644 --- a/packages/backend/src/server/api/endpoints/i/update.ts +++ b/packages/backend/src/server/api/endpoints/i/update.ts @@ -16,7 +16,7 @@ import { verifyLink } from "@/services/fetch-rel-me.js"; import { ApiError } from "../../error.js"; import config from "@/config/index.js"; import define from "../../define.js"; -import { userByIdCache } from "@/services/user-cache.js"; +import { userByIdCache, userDenormalizedCache } from "@/services/user-cache.js"; export const meta = { tags: ["account"], @@ -308,10 +308,18 @@ export default define(meta, paramDef, async (ps, _user, token) => { if (Object.keys(updates).length > 0) { await Users.update(user.id, updates); + const data = await Users.findOneByOrFail({ id: user.id }); await userByIdCache.set( - user.id, - await Users.findOneByOrFail({ id: user.id }), + data.id, + data, ); + if (data.avatarId) { + data.avatar = await DriveFiles.findOneBy({ id: data.avatarId }); + } + if (data.bannerId) { + data.banner = await DriveFiles.findOneBy({ id: data.bannerId }); + } + await userDenormalizedCache.set(data.id, data); } if (Object.keys(profileUpdates).length > 0) await UserProfiles.update(user.id, profileUpdates); diff --git a/packages/backend/src/server/api/endpoints/notes/timeline.ts b/packages/backend/src/server/api/endpoints/notes/timeline.ts index d629deebb6..a910342223 100644 --- a/packages/backend/src/server/api/endpoints/notes/timeline.ts +++ b/packages/backend/src/server/api/endpoints/notes/timeline.ts @@ -11,6 +11,8 @@ import { generateChannelQuery } from "../../common/generate-channel-query.js"; import { generateBlockedUserQuery } from "../../common/generate-block-query.js"; import { generateMutedUserRenotesQueryForNotes } from "../../common/generated-muted-renote-query.js"; import { ApiError } from "../../error.js"; +import { parseScyllaNote, prepared, scyllaClient } from "@/db/scylla.js"; +import { LocalFollowingsCache } from "@/misc/cache.js"; export const meta = { tags: ["notes"], @@ -64,13 +66,31 @@ export const paramDef = { } as const; export default define(meta, paramDef, async (ps, user) => { - const hasFollowing = - (await Followings.count({ - where: { - followerId: user.id, - }, - take: 1, - })) !== 0; + const followingsCache = await LocalFollowingsCache.init(user.id); + + if (scyllaClient) { + const untilDate = ps.untilDate ? new Date(ps.untilDate) : new Date(); + const query = [`${prepared.note.select.byDate} AND "createdAt" <= ?`]; + const params: (Date | string | string[])[] = [untilDate, untilDate]; + if (ps.sinceDate) { + query.push(`AND "createdAt" >= ?`); + params.push(new Date(ps.sinceDate)); + } + if (ps.untilId) { + query.push(`AND "id" <= ?`); + params.push(ps.untilId); + } + if (ps.sinceId) { + query.push(`AND "id" >= ?`); + params.push(ps.sinceId); + } + + const result = await scyllaClient.execute(query.join(" "), params, { prepare: true }); + const notes = result.rows.map(parseScyllaNote); + return Notes.packMany(notes, user); + } + + const hasFollowing = await followingsCache.hasFollowing(); //#region Construct query const followingQuery = Followings.createQueryBuilder("following") diff --git a/packages/backend/src/services/note/create.ts b/packages/backend/src/services/note/create.ts index 78d98ac7c7..9b9a795e13 100644 --- a/packages/backend/src/services/note/create.ts +++ b/packages/backend/src/services/note/create.ts @@ -794,8 +794,7 @@ async function insertNote( insert.tags, insert.hasPoll, insert.threadId, - data.channel?.id, - data.channel?.name, + insert.channelId, insert.userId, insert.userHost, insert.replyId, diff --git a/packages/backend/src/services/note/reaction/delete.ts b/packages/backend/src/services/note/reaction/delete.ts index c61821cebf..8a04bd8ce2 100644 --- a/packages/backend/src/services/note/reaction/delete.ts +++ b/packages/backend/src/services/note/reaction/delete.ts @@ -8,7 +8,7 @@ import type { User, IRemoteUser } from "@/models/entities/user.js"; import type { Note } from "@/models/entities/note.js"; import { NoteReactions, Users, Notes } from "@/models/index.js"; import { decodeReaction } from "@/misc/reaction-lib.js"; -import { parseScyllaReaction, prepared, scyllaClient } from "@/db/scylla"; +import { parseScyllaReaction, prepared, scyllaClient } from "@/db/scylla.js"; import type { NoteReaction } from "@/models/entities/note-reaction.js"; export default async (