From bd1b5a9249d703778b3e00cc7968ab0549e3c923 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Fri, 8 Sep 2023 22:10:36 -0400 Subject: [PATCH] wip: drop emoji column --- packages/backend/native-utils/Cargo.lock | 1 + .../native-utils/scylla-migration/Cargo.toml | 1 + .../1694224890248_drop_emoji_type/down.cql | 7 + .../cql/1694224890248_drop_emoji_type/up.cql | 2 + .../scylla-migration/src/entity/following.rs | 50 +++ .../scylla-migration/src/entity/mod.rs | 6 +- .../scylla-migration/src/entity/note.rs | 30 +- .../scylla-migration/src/entity/note_edit.rs | 39 ++ .../src/entity/note_reaction.rs | 37 ++ .../scylla-migration/src/entity/user.rs | 79 ++++ .../native-utils/scylla-migration/src/lib.rs | 2 +- .../scylla-migration/src/migrator.rs | 2 +- .../scylla-migration/src/setup.rs | 372 ++++++++++++++++-- packages/backend/src/db/cql.ts | 4 +- packages/backend/src/db/scylla.ts | 5 +- .../backend/src/models/repositories/note.ts | 1 + .../src/services/note/reaction/create.ts | 3 +- 17 files changed, 590 insertions(+), 51 deletions(-) create mode 100644 packages/backend/native-utils/scylla-migration/cql/1694224890248_drop_emoji_type/down.cql create mode 100644 packages/backend/native-utils/scylla-migration/cql/1694224890248_drop_emoji_type/up.cql create mode 100644 packages/backend/native-utils/scylla-migration/src/entity/following.rs create mode 100644 packages/backend/native-utils/scylla-migration/src/entity/note_edit.rs create mode 100644 packages/backend/native-utils/scylla-migration/src/entity/note_reaction.rs create mode 100644 packages/backend/native-utils/scylla-migration/src/entity/user.rs diff --git a/packages/backend/native-utils/Cargo.lock b/packages/backend/native-utils/Cargo.lock index 6770f1bfc3..27e1cfef50 100644 --- a/packages/backend/native-utils/Cargo.lock +++ b/packages/backend/native-utils/Cargo.lock @@ -2446,6 +2446,7 @@ dependencies = [ "chrono", "clap", "futures", + "indicatif", "scylla", "sea-orm", "serde", diff --git a/packages/backend/native-utils/scylla-migration/Cargo.toml b/packages/backend/native-utils/scylla-migration/Cargo.toml index e9b6082fab..e23e73b45e 100644 --- a/packages/backend/native-utils/scylla-migration/Cargo.toml +++ b/packages/backend/native-utils/scylla-migration/Cargo.toml @@ -9,6 +9,7 @@ edition = "2021" chrono = "0.4.26" clap = { version = "4.3.11", features = ["derive"] } futures = "0.3.28" +indicatif = { version = "0.17.6", features = ["tokio"] } scylla = "0.8.2" sea-orm = { version = "0.12.2", features = ["sqlx-postgres", "runtime-tokio-rustls"] } serde = { version = "1.0.171", features = ["derive"] } diff --git a/packages/backend/native-utils/scylla-migration/cql/1694224890248_drop_emoji_type/down.cql b/packages/backend/native-utils/scylla-migration/cql/1694224890248_drop_emoji_type/down.cql new file mode 100644 index 0000000000..758fccde6c --- /dev/null +++ b/packages/backend/native-utils/scylla-migration/cql/1694224890248_drop_emoji_type/down.cql @@ -0,0 +1,7 @@ +CREATE TYPE emoji ( + "name" text, + "url" text, + "width" int, + "height" int, +); +ALTER TABLE reaction ADD "emoji" frozen; diff --git a/packages/backend/native-utils/scylla-migration/cql/1694224890248_drop_emoji_type/up.cql b/packages/backend/native-utils/scylla-migration/cql/1694224890248_drop_emoji_type/up.cql new file mode 100644 index 0000000000..8a8700d6ea --- /dev/null +++ b/packages/backend/native-utils/scylla-migration/cql/1694224890248_drop_emoji_type/up.cql @@ -0,0 +1,2 @@ +ALTER TABLE reaction DROP "emoji"; +DROP TYPE emoji; diff --git a/packages/backend/native-utils/scylla-migration/src/entity/following.rs b/packages/backend/native-utils/scylla-migration/src/entity/following.rs new file mode 100644 index 0000000000..fe7672ccaa --- /dev/null +++ b/packages/backend/native-utils/scylla-migration/src/entity/following.rs @@ -0,0 +1,50 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "following")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub id: String, + #[sea_orm(column_name = "createdAt")] + pub created_at: DateTimeWithTimeZone, + #[sea_orm(column_name = "followeeId")] + pub followee_id: String, + #[sea_orm(column_name = "followerId")] + pub follower_id: String, + #[sea_orm(column_name = "followerHost")] + pub follower_host: Option, + #[sea_orm(column_name = "followerInbox")] + pub follower_inbox: Option, + #[sea_orm(column_name = "followerSharedInbox")] + pub follower_shared_inbox: Option, + #[sea_orm(column_name = "followeeHost")] + pub followee_host: Option, + #[sea_orm(column_name = "followeeInbox")] + pub followee_inbox: Option, + #[sea_orm(column_name = "followeeSharedInbox")] + pub followee_shared_inbox: Option, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::user::Entity", + from = "Column::FolloweeId", + to = "super::user::Column::Id", + on_update = "NoAction", + on_delete = "Cascade" + )] + User2, + #[sea_orm( + belongs_to = "super::user::Entity", + from = "Column::FollowerId", + to = "super::user::Column::Id", + on_update = "NoAction", + on_delete = "Cascade" + )] + User1, +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/packages/backend/native-utils/scylla-migration/src/entity/mod.rs b/packages/backend/native-utils/scylla-migration/src/entity/mod.rs index e1453d8df0..55aae352d1 100644 --- a/packages/backend/native-utils/scylla-migration/src/entity/mod.rs +++ b/packages/backend/native-utils/scylla-migration/src/entity/mod.rs @@ -1,6 +1,10 @@ pub mod drive_file; pub mod emoji; +pub mod following; pub mod note; -pub mod poll_vote; +pub mod note_edit; +pub mod note_reaction; pub mod poll; +pub mod poll_vote; +pub mod user; pub mod sea_orm_active_enums; diff --git a/packages/backend/native-utils/scylla-migration/src/entity/note.rs b/packages/backend/native-utils/scylla-migration/src/entity/note.rs index 056b7dd4c9..441b350ff9 100644 --- a/packages/backend/native-utils/scylla-migration/src/entity/note.rs +++ b/packages/backend/native-utils/scylla-migration/src/entity/note.rs @@ -64,6 +64,34 @@ pub struct Model { } #[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] -pub enum Relation {} +pub enum Relation { + #[sea_orm(has_many = "super::note_edit::Entity")] + NoteEdit, + #[sea_orm(has_many = "super::note_reaction::Entity")] + NoteReaction, + #[sea_orm(has_one = "super::poll::Entity")] + Poll, + #[sea_orm(has_many = "super::poll_vote::Entity")] + PollVote, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::NoteEdit.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Poll.def() + } +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::PollVote.def() + } +} + impl ActiveModelBehavior for ActiveModel {} diff --git a/packages/backend/native-utils/scylla-migration/src/entity/note_edit.rs b/packages/backend/native-utils/scylla-migration/src/entity/note_edit.rs new file mode 100644 index 0000000000..8236a218eb --- /dev/null +++ b/packages/backend/native-utils/scylla-migration/src/entity/note_edit.rs @@ -0,0 +1,39 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "note_edit")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub id: String, + #[sea_orm(column_name = "noteId")] + pub note_id: String, + #[sea_orm(column_type = "Text", nullable)] + pub text: Option, + pub cw: Option, + #[sea_orm(column_name = "fileIds")] + pub file_ids: Vec, + #[sea_orm(column_name = "updatedAt")] + pub updated_at: DateTimeWithTimeZone, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::note::Entity", + from = "Column::NoteId", + to = "super::note::Column::Id", + on_update = "NoAction", + on_delete = "Cascade" + )] + Note, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Note.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/packages/backend/native-utils/scylla-migration/src/entity/note_reaction.rs b/packages/backend/native-utils/scylla-migration/src/entity/note_reaction.rs new file mode 100644 index 0000000000..88d4f5ce11 --- /dev/null +++ b/packages/backend/native-utils/scylla-migration/src/entity/note_reaction.rs @@ -0,0 +1,37 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "note_reaction")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub id: String, + #[sea_orm(column_name = "createdAt")] + pub created_at: DateTimeWithTimeZone, + #[sea_orm(column_name = "userId")] + pub user_id: String, + #[sea_orm(column_name = "noteId")] + pub note_id: String, + pub reaction: String, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::note::Entity", + from = "Column::NoteId", + to = "super::note::Column::Id", + on_update = "NoAction", + on_delete = "Cascade" + )] + Note, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Note.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/packages/backend/native-utils/scylla-migration/src/entity/user.rs b/packages/backend/native-utils/scylla-migration/src/entity/user.rs new file mode 100644 index 0000000000..5e2b688c30 --- /dev/null +++ b/packages/backend/native-utils/scylla-migration/src/entity/user.rs @@ -0,0 +1,79 @@ +//! `SeaORM` Entity. Generated by sea-orm-codegen 0.12.2 + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] +#[sea_orm(table_name = "user")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub id: String, + #[sea_orm(column_name = "createdAt")] + pub created_at: DateTimeWithTimeZone, + #[sea_orm(column_name = "updatedAt")] + pub updated_at: Option, + #[sea_orm(column_name = "lastFetchedAt")] + pub last_fetched_at: Option, + pub username: String, + #[sea_orm(column_name = "usernameLower")] + pub username_lower: String, + pub name: Option, + #[sea_orm(column_name = "followersCount")] + pub followers_count: i32, + #[sea_orm(column_name = "followingCount")] + pub following_count: i32, + #[sea_orm(column_name = "notesCount")] + pub notes_count: i32, + #[sea_orm(column_name = "avatarId", unique)] + pub avatar_id: Option, + #[sea_orm(column_name = "bannerId", unique)] + pub banner_id: Option, + pub tags: Vec, + #[sea_orm(column_name = "isSuspended")] + pub is_suspended: bool, + #[sea_orm(column_name = "isSilenced")] + pub is_silenced: bool, + #[sea_orm(column_name = "isLocked")] + pub is_locked: bool, + #[sea_orm(column_name = "isBot")] + pub is_bot: bool, + #[sea_orm(column_name = "isCat")] + pub is_cat: bool, + #[sea_orm(column_name = "isAdmin")] + pub is_admin: bool, + #[sea_orm(column_name = "isModerator")] + pub is_moderator: bool, + pub emojis: Vec, + pub host: Option, + pub inbox: Option, + #[sea_orm(column_name = "sharedInbox")] + pub shared_inbox: Option, + pub featured: Option, + pub uri: Option, + #[sea_orm(unique)] + pub token: Option, + #[sea_orm(column_name = "isExplorable")] + pub is_explorable: bool, + #[sea_orm(column_name = "followersUri")] + pub followers_uri: Option, + #[sea_orm(column_name = "lastActiveDate")] + pub last_active_date: Option, + #[sea_orm(column_name = "hideOnlineStatus")] + pub hide_online_status: bool, + #[sea_orm(column_name = "isDeleted")] + pub is_deleted: bool, + #[sea_orm(column_name = "driveCapacityOverrideMb")] + pub drive_capacity_override_mb: Option, + #[sea_orm(column_name = "movedToUri")] + pub moved_to_uri: Option, + #[sea_orm(column_name = "alsoKnownAs", column_type = "Text", nullable)] + pub also_known_as: Option, + #[sea_orm(column_name = "speakAsCat")] + pub speak_as_cat: bool, + #[sea_orm(column_name = "isIndexable")] + pub is_indexable: bool, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/packages/backend/native-utils/scylla-migration/src/lib.rs b/packages/backend/native-utils/scylla-migration/src/lib.rs index 135b2452cb..7a380a4c9c 100644 --- a/packages/backend/native-utils/scylla-migration/src/lib.rs +++ b/packages/backend/native-utils/scylla-migration/src/lib.rs @@ -2,8 +2,8 @@ pub use clap::Parser; pub mod cli; pub mod config; -pub mod error; pub mod entity; +pub mod error; pub(crate) mod migrator; pub(crate) mod setup; diff --git a/packages/backend/native-utils/scylla-migration/src/migrator.rs b/packages/backend/native-utils/scylla-migration/src/migrator.rs index 62e024cd8f..14ad19d6fd 100644 --- a/packages/backend/native-utils/scylla-migration/src/migrator.rs +++ b/packages/backend/native-utils/scylla-migration/src/migrator.rs @@ -183,7 +183,7 @@ impl Migrator { self.down(1).await?; return Err(e.into()); - }, + } Ok(_) => {} }; } diff --git a/packages/backend/native-utils/scylla-migration/src/setup.rs b/packages/backend/native-utils/scylla-migration/src/setup.rs index 3c66e21e1d..6c3d590ab2 100644 --- a/packages/backend/native-utils/scylla-migration/src/setup.rs +++ b/packages/backend/native-utils/scylla-migration/src/setup.rs @@ -1,15 +1,18 @@ -use std::collections::HashMap; +use std::{collections::HashMap, fmt::Write, sync::OnceLock}; -use futures::TryStreamExt; -use scylla::{Session, SessionBuilder, IntoUserType, FromUserType, ValueList, FromRow}; -use sea_orm::{query::*, entity::*, Database}; +use chrono::{DateTime, NaiveDate, Utc}; +use futures::{future, TryStreamExt}; +use indicatif::{ProgressBar, ProgressState, ProgressStyle}; +use scylla::{ + prepared_statement::PreparedStatement, FromUserType, IntoUserType, Session, SessionBuilder, + ValueList, +}; +use sea_orm::{entity::*, query::*, Database, DatabaseConnection}; use urlencoding::encode; -use chrono::{DateTime, Utc, NaiveDate}; - use crate::{ config::{DbConfig, ScyllaConfig}, - entity::note, + entity::{drive_file, emoji, following, note, note_edit, note_reaction, poll, poll_vote, user}, error::Error, }; @@ -47,13 +50,7 @@ impl Initializer { let pool = Database::connect(&self.postgres_url).await?; let db_backend = pool.get_database_backend(); - let num_notes: i64 = note::Entity::find().select_only().column_as(note::Column::Id.count(), "count").into_tuple().one(&pool).await?.unwrap_or_default(); - - println!("{num_notes} posts are being copied.") - - let mut notes = note::Entity::find().stream(&pool).await?; - while let Some(note) = notes.try_next().await? { - } + self.copy(&pool).await?; let fk_pairs = vec![ ("channel_note_pining", "FK_10b19ef67d297ea9de325cd4502"), @@ -92,13 +89,308 @@ impl Initializer { Ok(()) } + + async fn copy(&self, db: &DatabaseConnection) -> Result<(), Error> { + let note_prepared = self.scylla.prepare(INSERT_NOTE).await?; + let home_prepared = self.scylla.prepare(INSERT_HOME_TIMELINE).await?; + let reaction_prepared = self.scylla.prepare(INSERT_REACTION).await?; + + let num_notes: i64 = note::Entity::find() + .select_only() + .column_as(note::Column::Id.count(), "count") + .into_tuple() + .one(db) + .await? + .unwrap_or_default(); + + let num_reactions: i64 = note_reaction::Entity::find() + .select_only() + .column_as(note_reaction::Column::Id.count(), "count") + .into_tuple() + .one(db) + .await? + .unwrap_or_default(); + + println!("Copying notes from PostgreSQL to ScyllaDB."); + + const PB_TMPL: &str = + "{spinner:.green} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {pos}/{len} ({eta})"; + let pb_style = ProgressStyle::with_template(PB_TMPL) + .unwrap() + .progress_chars("#>-"); + + let note_pb = ProgressBar::new(num_notes as u64).with_style(pb_style.to_owned()); + let reaction_pb = ProgressBar::new(num_reactions as u64).with_style(pb_style.to_owned()); + + let mut notes = note::Entity::find() + .order_by_asc(note::Column::Id) + .stream(db) + .await?; + let mut note_tasks = Vec::new(); + while let Some(note) = notes.try_next().await? { + note_tasks.push(self.copy_note(note, db, ¬e_prepared, &home_prepared, ¬e_pb)); + } + let mut reactions = note_reaction::Entity::find() + .order_by_asc(note_reaction::Column::Id) + .stream(db) + .await?; + let mut reaction_tasks = Vec::new(); + while let Some(reaction) = reactions.try_next().await? { + reaction_tasks.push(self.copy_reaction(reaction, &reaction_prepared, &reaction_pb)); + } + + future::try_join_all(note_tasks).await?; + future::try_join_all(reaction_tasks).await?; + + Ok(()) + } + + async fn copy_note( + &self, + note: note::Model, + db: &DatabaseConnection, + prepared_note: &PreparedStatement, + prepared_home: &PreparedStatement, + pb: &ProgressBar, + ) -> Result<(), Error> { + let reply = match ¬e.reply_id { + None => None, + Some(id) => note::Entity::find_by_id(id).one(db).await?, + }; + let renote = match ¬e.renote_id { + None => None, + Some(id) => note::Entity::find_by_id(id).one(db).await?, + }; + + let files = get_attached_files(Some(note.to_owned()), db).await?; + let reply_files = get_attached_files(reply.to_owned(), db).await?; + let renote_files = get_attached_files(renote.to_owned(), db).await?; + + let note_poll = note.find_related(poll::Entity).one(db).await?; + let poll_type = match note_poll { + None => None, + Some(v) => Some(PollType { + multiple: v.multiple, + expires_at: v.expires_at.map(Into::into), + choices: HashMap::from_iter( + v.choices + .iter() + .enumerate() + .map(|(i, v)| ((i + 1) as i32, v.to_string())), + ), + }), + }; + + let reactions: HashMap = match note.reactions.as_object() { + None => HashMap::new(), + Some(obj) => HashMap::from_iter( + obj.into_iter() + .map(|(k, v)| (k.to_string(), v.as_i64().unwrap_or_default() as i32)), + ), + }; + let edits = note.find_related(note_edit::Entity).all(db).await?; + let edits = edits.iter().map(|v| async { + let v = v.to_owned(); + Ok(NoteEditHistoryType { + content: v.text, + cw: v.cw, + files: get_files(v.file_ids, db).await?, + updated_at: v.updated_at.into(), + }) + }); + let edits: Vec> = futures::future::join_all(edits).await; + let edits: Vec = edits + .iter() + .filter_map(|v| v.as_ref().ok()) + .cloned() + .collect(); + + let scylla_note = NoteTable { + created_at_date: note.created_at.date_naive(), + created_at: note.created_at.into(), + id: note.id.to_owned(), + visibility: note.visibility.to_value(), + content: note.text, + name: note.name, + cw: note.cw, + local_only: note.local_only, + renote_count: note.renote_count as i32, + replies_count: note.replies_count as i32, + uri: note.uri, + url: note.url, + score: note.score, + files, + visible_user_ids: note.visible_user_ids, + mentions: note.mentions, + mentioned_remote_users: note.mentioned_remote_users, + emojis: note.emojis, + tags: note.tags, + has_poll: poll_type.is_some(), + poll: poll_type, + thread_id: note.thread_id, + channel_id: note.channel_id, + user_id: note.user_id.to_owned(), + user_host: note.user_host, + reply_id: note.reply_id, + reply_user_id: note.reply_user_id, + reply_user_host: note.reply_user_host, + reply_content: reply.as_ref().map(|v| v.text.to_owned()).flatten(), + reply_cw: reply.as_ref().map(|v| v.cw.to_owned()).flatten(), + reply_files, + renote_id: note.renote_id, + renote_user_id: note.renote_user_id, + renote_user_host: note.renote_user_host, + renote_content: renote.as_ref().map(|v| v.text.to_owned()).flatten(), + renote_cw: renote.as_ref().map(|v| v.cw.to_owned()).flatten(), + renote_files, + reactions, + note_edit: edits, + updated_at: note.updated_at.map(Into::into), + }; + + self.scylla + .execute(prepared_note, scylla_note.to_owned()) + .await?; + + let mut home_tasks = Vec::new(); + let mut local_followers = following::Entity::find() + .select_only() + .column(following::Column::FollowerId) + .filter(following::Column::FolloweeId.eq(note.user_id)) + .filter(following::Column::FollowerHost.is_null()) + .into_tuple::() + .stream(db) + .await?; + + while let Some(follower_id) = local_followers.try_next().await? { + let s_note = scylla_note.to_owned(); + let home = HomeTimelineTable { + feed_user_id: follower_id, + created_at_date: s_note.created_at_date, + created_at: s_note.created_at, + id: s_note.id, + visibility: s_note.visibility, + content: s_note.content, + name: s_note.name, + cw: s_note.cw, + local_only: s_note.local_only, + renote_count: s_note.renote_count, + replies_count: s_note.replies_count, + uri: s_note.uri, + url: s_note.url, + score: s_note.score, + files: s_note.files, + visible_user_ids: s_note.visible_user_ids, + mentions: s_note.mentions, + mentioned_remote_users: s_note.mentioned_remote_users, + emojis: s_note.emojis, + tags: s_note.tags, + has_poll: s_note.has_poll, + poll: s_note.poll, + thread_id: s_note.thread_id, + channel_id: s_note.channel_id, + user_id: s_note.user_id, + user_host: s_note.user_host, + reply_id: s_note.reply_id, + reply_user_id: s_note.reply_user_id, + reply_user_host: s_note.reply_user_host, + reply_content: s_note.reply_content, + reply_cw: s_note.reply_cw, + reply_files: s_note.reply_files, + renote_id: s_note.renote_id, + renote_user_id: s_note.renote_user_id, + renote_user_host: s_note.renote_user_host, + renote_content: s_note.renote_content, + renote_cw: s_note.renote_cw, + renote_files: s_note.renote_files, + reactions: s_note.reactions, + note_edit: s_note.note_edit, + updated_at: s_note.updated_at, + }; + home_tasks.push(self.scylla.execute(prepared_home, home)); + } + future::try_join_all(home_tasks).await?; + + pb.inc(1); + Ok(()) + } + + async fn copy_reaction( + &self, + reaction: note_reaction::Model, + prepared: &PreparedStatement, + pb: &ProgressBar, + ) -> Result<(), Error> { + let scylla_reaction = ReactionTable { + id: reaction.id, + note_id: reaction.note_id, + user_id: reaction.user_id, + reaction: reaction.reaction, + created_at: reaction.created_at.into(), + }; + self.scylla.execute(prepared, scylla_reaction).await?; + + pb.inc(1); + Ok(()) + } } -#[derive(Debug, IntoUserType, FromUserType)] +fn map_drive_file(file: drive_file::Model) -> DriveFileType { + DriveFileType { + id: file.id, + r#type: file.r#type, + created_at: file.created_at.into(), + name: file.name, + comment: file.comment, + blurhash: file.blurhash, + url: file.url, + thumbnail_url: file.thumbnail_url, + is_sensitive: file.is_sensitive, + is_link: file.is_link, + md5: file.md5, + size: file.size, + width: file + .properties + .get("width") + .filter(|v| v.is_number()) + .map(|v| v.as_i64().unwrap() as i32), + height: file + .properties + .get("height") + .filter(|v| v.is_number()) + .map(|v| v.as_i64().unwrap() as i32), + } +} + +async fn get_attached_files( + note: Option, + db: &DatabaseConnection, +) -> Result, Error> { + match note { + None => Ok(vec![]), + Some(v) => Ok(get_files(v.file_ids, db).await?), + } +} + +async fn get_files( + file_ids: Vec, + db: &DatabaseConnection, +) -> Result, Error> { + if file_ids.is_empty() { + Ok(vec![]) + } else { + let files = drive_file::Entity::find() + .filter(drive_file::Column::Id.is_in(file_ids)) + .all(db) + .await?; + Ok(files.iter().map(|v| map_drive_file(v.to_owned())).collect()) + } +} + +#[derive(Debug, Clone, IntoUserType, FromUserType)] struct DriveFileType { id: String, - #[scylla_crate(rename = "type")] - kind: String, + r#type: String, #[scylla_crate(rename = "createdAt")] created_at: DateTime, name: String, @@ -117,24 +409,16 @@ struct DriveFileType { height: Option, } -#[derive(Debug, IntoUserType, FromUserType)] +#[derive(Debug, Clone, IntoUserType, FromUserType)] struct NoteEditHistoryType { content: Option, cw: Option, files: Vec, #[scylla_crate(rename = "updatedAt")] - updated_at: DateTime + updated_at: DateTime, } -#[derive(Debug, IntoUserType, FromUserType)] -struct EmojiType { - name: String, - url: String, - width: Option, - height: Option, -} - -#[derive(Debug, IntoUserType, FromUserType)] +#[derive(Debug, Clone, IntoUserType, FromUserType)] struct PollType { #[scylla_crate(rename = "expiresAt")] expires_at: Option>, @@ -142,7 +426,7 @@ struct PollType { choices: HashMap, } -#[derive(ValueList)] +#[derive(Clone, ValueList)] struct NoteTable { created_at_date: NaiveDate, created_at: DateTime, @@ -164,11 +448,11 @@ struct NoteTable { emojis: Vec, tags: Vec, has_poll: bool, - poll: PollType, + poll: Option, thread_id: Option, channel_id: Option, user_id: String, - user_host: String, + user_host: Option, reply_id: Option, reply_user_id: Option, reply_user_host: Option, @@ -183,7 +467,7 @@ struct NoteTable { renote_files: Vec, reactions: HashMap, note_edit: Vec, - updated_at: DateTime, + updated_at: Option>, } const INSERT_NOTE: &str = r#" @@ -254,11 +538,11 @@ struct HomeTimelineTable { emojis: Vec, tags: Vec, has_poll: bool, - poll: PollType, + poll: Option, thread_id: Option, channel_id: Option, user_id: String, - user_host: String, + user_host: Option, reply_id: Option, reply_user_id: Option, reply_user_host: Option, @@ -273,7 +557,7 @@ struct HomeTimelineTable { renote_files: Vec, reactions: HashMap, note_edit: Vec, - updated_at: DateTime, + updated_at: Option>, } const INSERT_HOME_TIMELINE: &str = r#" @@ -328,17 +612,16 @@ struct ReactionTable { note_id: String, user_id: String, reaction: String, - emoji: EmojiType, created_at: DateTime, } -const INSERT_REACTION: &str = r#"INSERT INTO reaction ("id", "noteId", "userId", "reaction", "emoji", "createdAt") VALUES (?, ?, ?, ?, ?, ?)"#; +const INSERT_REACTION: &str = r#"INSERT INTO reaction ("id", "noteId", "userId", "reaction", "createdAt") VALUES (?, ?, ?, ?, ?)"#; #[derive(ValueList)] struct PollVoteTable { note_id: String, user_id: String, - user_host: String, + user_host: Option, choice: Vec, created_at: DateTime, } @@ -349,5 +632,16 @@ const INSERT_POLL_VOTE: &str = r#"INSERT INTO poll_vote ("noteId", "userId", "us struct NotificationTable { target_id: String, created_at_date: NaiveDate, - + created_at: DateTime, + id: String, + notifier_id: Option, + notifier_host: Option, + r#type: String, + entity_id: Option, + reatcion: Option, + choice: Option, + custom_body: Option, + custom_icon: Option, } + +const INSERT_NOTIFICATION: &str = r#"INSERT INTO notification ("targetId", "createdAtDate", "createdAt", "id", "notifierId", "notifierHost", "type", "entityId", "reaction", "choice", "customBody", "customHeader", "customIcon") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"#; diff --git a/packages/backend/src/db/cql.ts b/packages/backend/src/db/cql.ts index 942fb4e3a2..90de62409c 100644 --- a/packages/backend/src/db/cql.ts +++ b/packages/backend/src/db/cql.ts @@ -152,8 +152,8 @@ export const scyllaQueries = { }, reaction: { insert: `INSERT INTO reaction - ("id", "noteId", "userId", "reaction", "emoji", "createdAt") - VALUES (?, ?, ?, ?, ?, ?)`, + ("id", "noteId", "userId", "reaction", "createdAt") + VALUES (?, ?, ?, ?, ?)`, select: { byNoteId: `SELECT * FROM reaction_by_id WHERE "noteId" = ?`, byUserId: `SELECT * FROM reaction_by_user_id WHERE "userId" = ?`, diff --git a/packages/backend/src/db/scylla.ts b/packages/backend/src/db/scylla.ts index eb6f0d6927..e6c3c472b7 100644 --- a/packages/backend/src/db/scylla.ts +++ b/packages/backend/src/db/scylla.ts @@ -294,9 +294,7 @@ export function parseHomeTimeline( }; } -export interface ScyllaNoteReaction extends NoteReaction { - emoji: PopulatedEmoji; -} +export type ScyllaNoteReaction = NoteReaction; export type FeedType = | "home" @@ -318,7 +316,6 @@ export function parseScyllaReaction(row: types.Row): ScyllaNoteReaction { userId: row.get("userId"), reaction: row.get("reaction"), createdAt: row.get("createdAt"), - emoji: row.get("emoji"), }; } diff --git a/packages/backend/src/models/repositories/note.ts b/packages/backend/src/models/repositories/note.ts index a65df2b582..01115cbada 100644 --- a/packages/backend/src/models/repositories/note.ts +++ b/packages/backend/src/models/repositories/note.ts @@ -21,6 +21,7 @@ import { } from "@/misc/reaction-lib.js"; import type { NoteReaction } from "@/models/entities/note-reaction.js"; import { + type PopulatedEmoji, aggregateNoteEmojis, populateEmojis, prefetchEmojis, diff --git a/packages/backend/src/services/note/reaction/create.ts b/packages/backend/src/services/note/reaction/create.ts index 657bdee48d..25e67ce64a 100644 --- a/packages/backend/src/services/note/reaction/create.ts +++ b/packages/backend/src/services/note/reaction/create.ts @@ -49,7 +49,7 @@ export default async ( } // Emoji data will be cached in toDbReaction. - const { name: _reaction, emoji: emojiData } = await toDbReaction( + const { name: _reaction } = await toDbReaction( reaction, user.host, ); @@ -74,7 +74,6 @@ export default async ( record.noteId, record.userId, _reaction, - emojiData, record.createdAt, ], { prepare: true },