wip: timeline query

packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/down.cql

@@ -1,7 +1,7 @@
-DROP MATERIALIZED VIEW IF EXISTS reaction_by_user_id;
+DROP MATERIALIZED VIEW IF EXISTS reaction_by_userid;
 DROP INDEX IF EXISTS reaction_by_id;
 DROP TABLE IF EXISTS reaction;
-DROP MATERIALIZED VIEW IF EXISTS note_by_user_id;
+DROP MATERIALIZED VIEW IF EXISTS note_by_userid;
 DROP INDEX IF EXISTS note_by_id;
 DROP INDEX IF EXISTS note_by_uri;
 DROP INDEX IF EXISTS note_by_url;

packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/up.cql

@@ -49,7 +49,6 @@ CREATE TABLE IF NOT EXISTS note ( -- Models timeline
 	"hasPoll" boolean,
 	"threadId" ascii,
 	"channelId" ascii, -- Channel
-	"channelName" text,
 	"userId" ascii, -- User
 	"userHost" text,
 	"replyId" ascii, -- Reply

packages/backend/src/db/scylla.ts

@@ -44,7 +44,6 @@ export const prepared = {
 				"hasPoll",
 				"threadId",
 				"channelId",
-				"channelName",
 				"userId",
 				"userHost",
 				"replyId",
@@ -58,9 +57,9 @@ export const prepared = {
 				"updatedAt"
 			)
 			VALUES
-			(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+			(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
 		select: {
-			byDate: `SELECT * FROM note WHERE "createdAtDate" IN ?`,
+			byDate: `SELECT * FROM note WHERE "createdAtDate" = ?`,
 			byId: `SELECT * FROM note WHERE "id" IN ?`,
 			byUri: `SELECT * FROM note WHERE "uri" IN ?`,
 			byUrl: `SELECT * FROM note WHERE "url" IN ?`,
@@ -118,7 +117,6 @@ export interface ScyllaNoteEditHistory {
 export type ScyllaNote = Note & {
 	createdAtDate: Date;
 	files: ScyllaDriveFile[];
-	channelName: string;
 	noteEdit: ScyllaNoteEditHistory[];
 };
 
@@ -148,7 +146,6 @@ export function parseScyllaNote(row: types.Row): ScyllaNote {
 		hasPoll: row.get("hasPoll"),
 		threadId: row.get("threadId"),
 		channelId: row.get("channelId"),
-		channelName: row.get("channelName"),
 		userId: row.get("userId"),
 		userHost: row.get("userHost"),
 		replyId: row.get("replyId"),

packages/backend/src/misc/cache.ts

@@ -140,11 +140,11 @@ export class LocalFollowingsCache {
 		this.key = `follow:${userId}`;
 	}
 
-	public static async init(userId: string) {
+	public static async init(userId: string): Promise<LocalFollowingsCache> {
 		const cache = new LocalFollowingsCache(userId);
 
-		// Sync from DB if no relationships is cached
+		// Sync from DB if no followings are cached
-		if ((await redisClient.scard(cache.key)) === 0) {
+		if (!(await cache.hasFollowing())) {
 			const rel = await Followings.find({
 				select: { followeeId: true },
 				where: { followerId: cache.myId },
@@ -172,4 +172,12 @@ export class LocalFollowingsCache {
 	public async isFollowing(targetId: string): Promise<boolean> {
 		return (await redisClient.sismember(this.key, targetId)) === 1;
 	}
+
+	public async hasFollowing(): Promise<boolean> {
+		return (await redisClient.scard(this.key)) !== 0;
+	}
+
+	public async getAll(): Promise<string[]> {
+		return (await redisClient.smembers(this.key))
+	}
 }

packages/backend/src/models/repositories/note.ts

@@ -296,8 +296,8 @@ export const NoteRepository = db.getRepository(Note).extend({
 		const myReactionsMap = new Map<Note["id"], NoteReaction | null>();
 		if (meId) {
 			const renoteIds = notes
-				.filter((n) => n.renoteId != null)
+				.filter((n) => !!n.renoteId)
-				.map((n) => n.renoteId!);
+				.map((n) => n.renoteId) as string[];
 			const targets = [...notes.map((n) => n.id), ...renoteIds];
 			const myReactions = await NoteReactions.findBy({
 				userId: meId,

packages/backend/src/models/repositories/user.ts

@@ -37,6 +37,7 @@ import {
 	UserSecurityKeys,
 } from "../index.js";
 import type { Instance } from "../entities/instance.js";
+import { userByIdCache, userDenormalizedCache } from "@/services/user-cache.js";
 
 const userInstanceCache = new Cache<Instance | null>(
 	"userInstance",
@@ -391,13 +392,15 @@ export const UserRepository = db.getRepository(User).extend({
 			if (src.banner === undefined && src.bannerId)
 				src.banner = (await DriveFiles.findOneBy({ id: src.bannerId })) ?? null;
 		} else {
-			user = await this.findOneOrFail({
+			user = await userDenormalizedCache.fetch(src, () =>
+				this.findOneOrFail({
 					where: { id: src },
 					relations: {
 						avatar: true,
 						banner: true,
 					},
-			});
+				}),
+			);
 		}
 
 		const meId = me ? me.id : null;

packages/backend/src/remote/activitypub/models/person.ts

@@ -28,7 +28,11 @@ import { fetchInstanceMetadata } from "@/services/fetch-instance-metadata.js";
 import { normalizeForSearch } from "@/misc/normalize-for-search.js";
 import { truncate } from "@/misc/truncate.js";
 import { StatusError } from "@/misc/fetch.js";
-import { uriPersonCache, userByIdCache } from "@/services/user-cache.js";
+import {
+	uriPersonCache,
+	userByIdCache,
+	userDenormalizedCache,
+} from "@/services/user-cache.js";
 import { publishInternalEvent } from "@/services/stream.js";
 import { db } from "@/db/postgre.js";
 import { apLogger } from "../logger.js";
@@ -373,6 +377,10 @@ export async function createPerson(
 
 	await updateFeatured(user!.id, resolver).catch((err) => logger.error(err));
 
+	user!.avatar = avatar;
+	user!.banner = banner;
+	await userDenormalizedCache.set(user!.id, user!);
+
 	return user!;
 }
 
@@ -518,6 +526,13 @@ export async function updatePerson(
 	// Update user
 	await Users.update(user.id, updates);
 
+	const updatedUser = await Users.findOneByOrFail({ id: user.id });
+	updatedUser.avatarId = avatar?.id ?? null;
+	updatedUser.avatar = avatar;
+	updatedUser.bannerId = banner?.id ?? null;
+	updatedUser.banner = banner;
+	await userDenormalizedCache.set(updatedUser.id, updatedUser);
+
 	if (person.publicKey) {
 		await UserPublickeys.update(
 			{ userId: user.id },

packages/backend/src/server/api/endpoints/i/update.ts

@@ -16,7 +16,7 @@ import { verifyLink } from "@/services/fetch-rel-me.js";
 import { ApiError } from "../../error.js";
 import config from "@/config/index.js";
 import define from "../../define.js";
-import { userByIdCache } from "@/services/user-cache.js";
+import { userByIdCache, userDenormalizedCache } from "@/services/user-cache.js";
 
 export const meta = {
 	tags: ["account"],
@@ -308,10 +308,18 @@ export default define(meta, paramDef, async (ps, _user, token) => {
 
 	if (Object.keys(updates).length > 0) {
 		await Users.update(user.id, updates);
+		const data = await Users.findOneByOrFail({ id: user.id });
 		await userByIdCache.set(
-			user.id,
+			data.id,
-			await Users.findOneByOrFail({ id: user.id }),
+			data,
 		);
+		if (data.avatarId) {
+			data.avatar = await DriveFiles.findOneBy({ id: data.avatarId });
+		}
+		if (data.bannerId) {
+			data.banner = await DriveFiles.findOneBy({ id: data.bannerId });
+		}
+		await userDenormalizedCache.set(data.id, data);
 	}
 	if (Object.keys(profileUpdates).length > 0)
 		await UserProfiles.update(user.id, profileUpdates);

packages/backend/src/server/api/endpoints/notes/timeline.ts

@@ -11,6 +11,8 @@ import { generateChannelQuery } from "../../common/generate-channel-query.js";
 import { generateBlockedUserQuery } from "../../common/generate-block-query.js";
 import { generateMutedUserRenotesQueryForNotes } from "../../common/generated-muted-renote-query.js";
 import { ApiError } from "../../error.js";
+import { parseScyllaNote, prepared, scyllaClient } from "@/db/scylla.js";
+import { LocalFollowingsCache } from "@/misc/cache.js";
 
 export const meta = {
 	tags: ["notes"],
@@ -64,13 +66,31 @@ export const paramDef = {
 } as const;
 
 export default define(meta, paramDef, async (ps, user) => {
-	const hasFollowing =
+	const followingsCache = await LocalFollowingsCache.init(user.id);
-		(await Followings.count({
+
-			where: {
+	if (scyllaClient) {
-				followerId: user.id,
+		const untilDate = ps.untilDate ? new Date(ps.untilDate) : new Date();
-			},
+		const query = [`${prepared.note.select.byDate} AND "createdAt" <= ?`];
-			take: 1,
+		const params: (Date | string | string[])[] = [untilDate, untilDate];
-		})) !== 0;
+		if (ps.sinceDate) {
+			query.push(`AND "createdAt" >= ?`);
+			params.push(new Date(ps.sinceDate));
+		}
+		if (ps.untilId) {
+			query.push(`AND "id" <= ?`);
+			params.push(ps.untilId);
+		}
+		if (ps.sinceId) {
+			query.push(`AND "id" >= ?`);
+			params.push(ps.sinceId);
+		}
+
+		const result = await scyllaClient.execute(query.join(" "), params, { prepare: true });
+		const notes = result.rows.map(parseScyllaNote);
+		return Notes.packMany(notes, user);
+	}
+
+	const hasFollowing = await followingsCache.hasFollowing();
 
 	//#region Construct query
 	const followingQuery = Followings.createQueryBuilder("following")

packages/backend/src/services/note/create.ts

@@ -794,8 +794,7 @@ async function insertNote(
 					insert.tags,
 					insert.hasPoll,
 					insert.threadId,
-					data.channel?.id,
+					insert.channelId,
-					data.channel?.name,
 					insert.userId,
 					insert.userHost,
 					insert.replyId,

packages/backend/src/services/note/reaction/delete.ts

@@ -8,7 +8,7 @@ import type { User, IRemoteUser } from "@/models/entities/user.js";
 import type { Note } from "@/models/entities/note.js";
 import { NoteReactions, Users, Notes } from "@/models/index.js";
 import { decodeReaction } from "@/misc/reaction-lib.js";
-import { parseScyllaReaction, prepared, scyllaClient } from "@/db/scylla";
+import { parseScyllaReaction, prepared, scyllaClient } from "@/db/scylla.js";
 import type { NoteReaction } from "@/models/entities/note-reaction.js";
 
 export default async (