insert post to scylladb

2023-07-24 05:35:44 -04:00 · 2023-07-24 05:35:44 -04:00 · 03da4eb57a
commit 03da4eb57a
parent 271d75c35c
5 changed files with 135 additions and 126 deletions
--- a/.config/example.yml
+++ b/.config/example.yml
@ -57,8 +57,9 @@ db:

 # scylla:
 #   nodes: ['localhost:9042']
-#   keyspace: calckey
+#   keyspace: firefish
 #   replicationFactor: 1
+#   localDataCentre: "datacenter1"

 #   ┌─────────────────────┐
 #───┘ Redis configuration └─────────────────────────────────────
--- a/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/up.cql
+++ b/packages/backend/native-utils/scylla-migration/cql/1689400417034_timeline/up.cql
@ -1,83 +1,81 @@
 CREATE TYPE IF NOT EXISTS drive_file (
-	id ascii,
-	type ascii,
-	createdAt timestamp,
-	name text,
-	comment text,
-	blurhash text,
-	url text,
-	thumbnailUrl text,
-	isSensitive boolean,
-	isLink boolean,
-	md5 ascii,
-	size int,
-	width int,
-	height int,
+	"id" ascii,
+	"type" ascii,
+	"createdAt" timestamp,
+	"name" text,
+	"comment" text,
+	"blurhash" text,
+	"url" text,
+	"thumbnailUrl" text,
+	"isSensitive" boolean,
+	"isLink" boolean,
+	"md5" ascii,
+	"size" int
 );

 CREATE TYPE IF NOT EXISTS note_edit_history (
-	content text,
-	cw text,
-	files set<frozen<drive_file>>,
-	updatedAt timestamp,
+	"content" text,
+	"cw" text,
+	"files" set<frozen<drive_file>>,
+	"updatedAt" timestamp,
 );

 CREATE TYPE IF NOT EXISTS emoji (
-	name text,
-	url text,
-	width int,
-	height int,
+	"name" text,
+	"url" text,
+	"width" int,
+	"height" int,
 );

 CREATE TABLE IF NOT EXISTS note ( -- Models timeline
-	createdAtDate date, -- For partitioning
-	createdAt timestamp,
-	id ascii, -- Post
-	visibility ascii,
-	content text,
-	name text,
-	cw text,
-	localOnly boolean,
-	renoteCount int,
-	repliesCount int,
-	uri text,
-	url text,
-	score int,
-	files set<frozen<drive_file>>,
-	visibleUsersId set<ascii>,
-	mentions set<ascii>,
-	emojis set<frozen<emoji>>,
-	tags set<text>,
-	hasPoll boolean,
-	threadId ascii,
-	channelId ascii, -- Channel
-	channelName text,
-	userId ascii, -- User
-	replyId ascii, -- Reply
-	renoteId ascii, -- Boost
-	reactions map<text, int>,
-	reactionEmojis map<text, frozen<emoji>>,
-	noteEdit set<frozen<note_edit_history>>, -- Edit History
-	updatedAt timestamp,
-	PRIMARY KEY (createdAtDate, createdAt)
-) WITH CLUSTERING ORDER BY (createdAt DESC);
+	"createdAtDate" date, -- For partitioning
+	"createdAt" timestamp,
+	"id" ascii, -- Post
+	"visibility" ascii,
+	"content" text,
+	"name" text,
+	"cw" text,
+	"localOnly" boolean,
+	"renoteCount" int,
+	"repliesCount" int,
+	"uri" text,
+	"url" text,
+	"score" int,
+	"files" set<frozen<drive_file>>,
+	"visibleUsersId" set<ascii>,
+	"mentions" set<ascii>,
+	"emojis" set<frozen<emoji>>,
+	"tags" set<text>,
+	"hasPoll" boolean,
+	"threadId" ascii,
+	"channelId" ascii, -- Channel
+	"channelName" text,
+	"userId" ascii, -- User
+	"replyId" ascii, -- Reply
+	"renoteId" ascii, -- Boost
+	"reactions" map<text, int>,
+	"reactionEmojis" map<text, frozen<emoji>>,
+	"noteEdit" set<frozen<note_edit_history>>, -- Edit History
+	"updatedAt" timestamp,
+	PRIMARY KEY ("createdAtDate", "createdAt")
+) WITH CLUSTERING ORDER BY ("createdAt" DESC);

 CREATE INDEX IF NOT EXISTS note_by_id ON note (id);
 CREATE INDEX IF NOT EXISTS note_by_uri ON note (uri);
 CREATE INDEX IF NOT EXISTS note_by_url ON note (url);

-CREATE MATERIALIZED VIEW IF NOT EXISTS note_by_user_id AS
+CREATE MATERIALIZED VIEW IF NOT EXISTS note_by_userid AS
 	SELECT * FROM note
-	WHERE userId IS NOT NULL
-	AND createdAt IS NOT NULL
-	AND createdAtDate IS NOT NULL
-	PRIMARY KEY (userId, createdAt, createdAtDate)
-	WITH CLUSTERING ORDER BY (createdAt DESC);
+	WHERE "userId" IS NOT NULL
+	AND "createdAt" IS NOT NULL
+	AND "createdAtDate" IS NOT NULL
+	PRIMARY KEY ("userId", "createdAt", "createdAtDate")
+	WITH CLUSTERING ORDER BY ("createdAt" DESC);

 CREATE TABLE IF NOT EXISTS reaction (
-	noteId ascii,
-	createdAt timestamp,
-	userId ascii,
-	reaction frozen<emoji>,
-	PRIMARY KEY (noteId, createdAt, userId)
-);
+	"noteId" ascii,
+	"createdAt" timestamp,
+	"userId" ascii,
+	"reaction" frozen<emoji>,
+	PRIMARY KEY ("noteId", "createdAt", "userId")
+) WITH CLUSTERING ORDER BY ("createdAt" DESC);
--- a/packages/backend/src/config/types.ts
+++ b/packages/backend/src/config/types.ts
@ -20,6 +20,7 @@ export type Source = {
 		nodes: string[];
 		keyspace: string;
 		replicationFactor: number;
+		localDataCentre: string;
 	},
 	redis: {
 		host: string;
--- a/packages/backend/src/db/scylla.ts
+++ b/packages/backend/src/db/scylla.ts
@ -9,6 +9,7 @@ function newClient(): Client | null {

 	return new Client({
 		contactPoints: config.scylla.nodes,
+		localDataCenter: config.scylla.localDataCentre,
 		keyspace: config.scylla.keyspace,
 	});
 }
@ -16,51 +17,56 @@ function newClient(): Client | null {
 export const scyllaClient = newClient();

 export const prepared = {
-	timeline: {
+	note: {
 		insert: `INSERT INTO note (
-				createdAtDate,
-				createdAt,
-				id,
-				visibility,
-				content,
-				name,
-				cw,
-				localOnly,
-				renoteCount,
-				repliesCount,
-				uri,
-				url,
-				score,
-				files,
-				visibleUsersId,
-				mentions,
-				emojis,
-				tags,
-				hasPoll,
-				threadId,
-				channelId,
-				channelName,
-				userId,
-				userId,
-				replyId,
-				renoteId,
-				reactions,
-				reactionEmojis
-				noteEdit,
-				updatedAt,
+				"createdAtDate",
+				"createdAt",
+				"id",
+				"visibility",
+				"content",
+				"name",
+				"cw",
+				"localOnly",
+				"renoteCount",
+				"repliesCount",
+				"uri",
+				"url",
+				"score",
+				"files",
+				"visibleUsersId",
+				"mentions",
+				"emojis",
+				"tags",
+				"hasPoll",
+				"threadId",
+				"channelId",
+				"channelName",
+				"userId",
+				"replyId",
+				"renoteId",
+				"reactions",
+				"reactionEmojis",
+				"noteEdit",
+				"updatedAt"
 			)
 			VALUES
-			(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+			(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
 		select: {
-			byDate: "SELECT * FROM note WHERE createdAtDate = ? AND createdAt < ?",
+			byDate: `SELECT * FROM note WHERE "createdAtDate" = ? AND "createdAt" < ?`,
 			byId: "SELECT * FROM note WHERE id IN ?",
 			byUri: "SELECT * FROM note WHERE uri = ?",
 			byUrl: "SELECT * FROM note WHERE url = ?",
-			byUserId: "SELECT * FROM note WHERE userId = ? AND createdAt < ?",
+			byUserId: `SELECT * FROM note_by_userid WHERE "userId" = ? AND "createdAt" < ?`,
 		},
 		delete: "DELETE FROM note WHERE id IN ?",
+		update: {
+			renoteCount: `UPDATE note SET "renoteCount" = ?, "score" = ? WHERE "id" = ? IF EXISTS`,
+		}
 	},
-}
+	reaction: {
+		insert: `INSERT INTO reaction ("noteId", "createdAt", "userId", "reaction") VALUES (?, ?, ?, ?)`,
+	},
+};

 export interface ScyllaDriveFile {
 	id: string;
--- a/packages/backend/src/services/note/create.ts
+++ b/packages/backend/src/services/note/create.ts
@ -70,6 +70,7 @@ import { Mutex } from "redis-semaphore";
 import { prepared, scyllaClient, ScyllaDriveFile } from "@/db/scylla.js";
 import { populateEmojis } from "@/misc/populate-emojis.js";
 import { decodeReaction } from "@/misc/reaction-lib.js";
+import { types } from "cassandra-driver";

 const mutedWordsCache = new Cache<
 	{ userId: UserProfile["userId"]; mutedWords: UserProfile["mutedWords"] }[]
@ -411,7 +412,6 @@ export default async (
 			saveReply(data.reply, note);
 		}

-		// この投稿を除く指定したユーザーによる指定したノートのリノートが存在しないとき
 		if (
 			data.renote &&
 			!user.isBot &&
@ -678,14 +678,22 @@ async function renderNoteOrRenoteActivity(data: Option, note: Note) {
 }

 function incRenoteCount(renote: Note) {
-	Notes.createQueryBuilder()
-		.update()
-		.set({
-			renoteCount: () => '"renoteCount" + 1',
-			score: () => '"score" + 1',
-		})
-		.where("id = :id", { id: renote.id })
-		.execute();
+	if (scyllaClient) {
+		scyllaClient.execute(prepared.note.update.renoteCount, [
+			renote.renoteCount + 1,
+			renote.score + 1,
+			renote.id,
+		]);
+	} else {
+		Notes.createQueryBuilder()
+			.update()
+			.set({
+				renoteCount: () => '"renoteCount" + 1',
+				score: () => '"score" + 1',
+			})
+			.where("id = :id", { id: renote.id })
+			.execute();
+	}
 }

 async function insertNote(
@ -762,16 +770,9 @@ async function insertNote(
 	// 投稿を作成
 	try {
 		if (scyllaClient) {
-			const reactionEmojiNames = Object.keys(insert.reactions)
-				.filter((x) => x?.startsWith(":"))
-				.map((x) => decodeReaction(x).reaction)
-				.map((x) => x.replace(/:/g, ""));
-			const noteEmojis = await populateEmojis(
-				insert.emojis.concat(reactionEmojiNames),
-				user.host,
-			);
+			const noteEmojis = await populateEmojis(insert.emojis, user.host);
 			await scyllaClient.execute(
-				prepared.timeline.insert,
+				prepared.note.insert,
 				[
 					insert.createdAt,
 					insert.createdAt,
@ -781,11 +782,11 @@ async function insertNote(
 					insert.name,
 					insert.cw,
 					insert.localOnly,
-					insert.renoteCount,
-					insert.repliesCount,
+					insert.renoteCount ?? 0,
+					insert.repliesCount ?? 0,
 					insert.uri,
 					insert.url,
-					insert.score,
+					insert.score ?? 0,
 					data.files,
 					insert.visibleUserIds,
 					insert.mentions,
@ -800,6 +801,8 @@ async function insertNote(
 					insert.renoteId,
 					null,
 					null,
+					null,
+					null,
 				],
 				{ prepare: true },
 			);
@ -809,7 +812,9 @@ async function insertNote(
 			await db.transaction(async (transactionalEntityManager) => {
 				if (!data.poll) throw new Error("Empty poll data");

-				await transactionalEntityManager.insert(Note, insert);
+				if (!scyllaClient) {
+					await transactionalEntityManager.insert(Note, insert);
+				}

 				let expiresAt: Date | null;
 				if (!data.poll.expiresAt || isNaN(data.poll.expiresAt.getTime())) {
@ -831,7 +836,7 @@ async function insertNote(

 				await transactionalEntityManager.insert(Poll, poll);
 			});
-		} else {
+		} else if (!scyllaClient) {
 			await Notes.insert(insert);
 		}

@ -844,8 +849,6 @@ async function insertNote(
 			throw err;
 		}

-		console.error(e);
-
 		throw e;
 	}
 }