From 98f2397fbbe34737471a8cf3ed69484773b6d9a2 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Thu, 1 Jun 2023 04:01:25 -0400 Subject: [PATCH] add migration to convert array to jsonb --- .../src/m20230531_180824_stringvec.rs | 395 ++++++++++++++++-- .../native-utils/crates/model/Cargo.toml | 6 +- .../crates/model/src/entity/mod.rs | 2 - .../crates/model/src/entity/prelude.rs | 2 - .../crates/model/src/entity/reversi_game.rs | 69 --- .../model/src/entity/reversi_matching.rs | 38 -- .../native-utils/crates/model/tests/common.rs | 6 +- 7 files changed, 365 insertions(+), 153 deletions(-) delete mode 100644 packages/backend/native-utils/crates/model/src/entity/reversi_game.rs delete mode 100644 packages/backend/native-utils/crates/model/src/entity/reversi_matching.rs diff --git a/packages/backend/native-utils/crates/migration/src/m20230531_180824_stringvec.rs b/packages/backend/native-utils/crates/migration/src/m20230531_180824_stringvec.rs index ba1fc257d0..b7c0a70b3d 100644 --- a/packages/backend/native-utils/crates/migration/src/m20230531_180824_stringvec.rs +++ b/packages/backend/native-utils/crates/migration/src/m20230531_180824_stringvec.rs @@ -1,7 +1,11 @@ -use model::entity::{antenna, newtype::StringVec}; +use model::entity::{ + access_token, antenna, app, emoji, gallery_post, hashtag, messaging_message, meta, + newtype::{I32Vec, StringVec}, + note, note_edit, poll, registry_item, user, user_profile, webhook, +}; use sea_orm_migration::{ prelude::*, - sea_orm::{DbBackend, EntityTrait, Statement}, + sea_orm::{DbBackend, EntityTrait, Statement, TryGetable}, }; use serde_json::json; @@ -11,50 +15,220 @@ pub struct Migration; #[async_trait::async_trait] impl MigrationTrait for Migration { async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .drop_table(Table::drop().table(Alias::new("reversi_game")).to_owned()) + .await?; + + manager + .drop_table( + Table::drop() + .table(Alias::new("reversi_matching")) + .to_owned(), + ) + .await?; + if manager.get_database_backend() == DbBackend::Sqlite { return Ok(()); } let db = manager.get_connection(); - let query = Query::select() - .column(Antenna::Id) - .column(Antenna::Users) - .from(Antenna::Table) - .to_string(PostgresQueryBuilder); - let res: Vec<(String, Vec)> = db - .query_all(Statement::from_string(DbBackend::Postgres, query)) - .await? - .iter() - .filter_map(|r| r.try_get_many_by_index().ok()) - .collect(); - manager - .alter_table( - Table::alter() - .table(Antenna::Table) - .drop_column(Antenna::Users) - .add_column( - ColumnDef::new(Antenna::Users) - .json_binary() - .not_null() - .default(json!([])), - ) - .to_owned(), - ) - .await?; + macro_rules! copy_data { + ($data:ident, $ent:ident, $col:tt) => { + let models: Vec<$ent::ActiveModel> = $data + .iter() + .map(|(id, r)| $ent::ActiveModel { + id: sea_orm::Set(id.to_owned()), + $col: sea_orm::Set(StringVec::from(r.to_owned())), + ..Default::default() + }) + .collect(); + for model in models { + $ent::Entity::update(model).exec(db).await?; + } + }; + } - let models: Vec = res + macro_rules! convert_to_stringvec_json { + ($table:expr, $id:expr, $col:expr, $ent:ident, $col_name:tt) => { + let query = select_query($table, $id, $col); + let res = get_vec::>(db, query).await?; + convert_col(manager, $table, $col).await?; + copy_data!(res, $ent, $col_name); + }; + } + + convert_to_stringvec_json!( + AccessToken::Table, + AccessToken::Id, + AccessToken::Permission, + access_token, + permission + ); + convert_to_stringvec_json!(Antenna::Table, Antenna::Id, Antenna::Users, antenna, users); + convert_to_stringvec_json!(App::Table, App::Id, App::Permission, app, permission); + convert_to_stringvec_json!(Emoji::Table, Emoji::Id, Emoji::Aliases, emoji, aliases); + convert_to_stringvec_json!( + GalleryPost::Table, + GalleryPost::Id, + GalleryPost::FileIds, + gallery_post, + file_ids + ); + convert_to_stringvec_json!( + GalleryPost::Table, + GalleryPost::Id, + GalleryPost::Tags, + gallery_post, + tags + ); + convert_to_stringvec_json!( + Hashtag::Table, + Hashtag::Id, + Hashtag::MentionedUserIds, + hashtag, + mentioned_user_ids + ); + convert_to_stringvec_json!( + Hashtag::Table, + Hashtag::Id, + Hashtag::MentionedLocalUserIds, + hashtag, + mentioned_local_user_ids + ); + convert_to_stringvec_json!( + Hashtag::Table, + Hashtag::Id, + Hashtag::MentionedRemoteUserIds, + hashtag, + mentioned_remote_user_ids + ); + convert_to_stringvec_json!( + Hashtag::Table, + Hashtag::Id, + Hashtag::AttachedUserIds, + hashtag, + attached_user_ids + ); + convert_to_stringvec_json!( + Hashtag::Table, + Hashtag::Id, + Hashtag::AttachedLocalUserIds, + hashtag, + attached_local_user_ids + ); + convert_to_stringvec_json!( + Hashtag::Table, + Hashtag::Id, + Hashtag::AttachedRemoteUserIds, + hashtag, + attached_remote_user_ids + ); + convert_to_stringvec_json!( + MessagingMessage::Table, + MessagingMessage::Id, + MessagingMessage::Reads, + messaging_message, + reads + ); + convert_to_stringvec_json!(Meta::Table, Meta::Id, Meta::Langs, meta, langs); + convert_to_stringvec_json!( + Meta::Table, + Meta::Id, + Meta::BlockedHosts, + meta, + blocked_hosts + ); + convert_to_stringvec_json!(Meta::Table, Meta::Id, Meta::HiddenTags, meta, hidden_tags); + convert_to_stringvec_json!(Meta::Table, Meta::Id, Meta::PinnedUsers, meta, pinned_users); + convert_to_stringvec_json!(Meta::Table, Meta::Id, Meta::PinnedPages, meta, pinned_pages); + convert_to_stringvec_json!( + Meta::Table, + Meta::Id, + Meta::RecommendedInstances, + meta, + recommended_instances + ); + convert_to_stringvec_json!( + Meta::Table, + Meta::Id, + Meta::SilencedHosts, + meta, + silenced_hosts + ); + convert_to_stringvec_json!(Note::Table, Note::Id, Note::FileIds, note, file_ids); + convert_to_stringvec_json!( + Note::Table, + Note::Id, + Note::AttachedFileTypes, + note, + attached_file_types + ); + convert_to_stringvec_json!( + Note::Table, + Note::Id, + Note::VisibleUserIds, + note, + visible_user_ids + ); + convert_to_stringvec_json!(Note::Table, Note::Id, Note::Mentions, note, mentions); + convert_to_stringvec_json!(Note::Table, Note::Id, Note::Emojis, note, emojis); + convert_to_stringvec_json!(Note::Table, Note::Id, Note::Tags, note, tags); + convert_to_stringvec_json!( + NoteEdit::Table, + NoteEdit::Id, + NoteEdit::FileIds, + note_edit, + file_ids + ); + + // Convert poll here because its primary key is not id, but note_id. + let query = select_query(Poll::Table, Poll::NoteId, Poll::Choices); + let res = get_vec::>(db, query).await?; + convert_col(manager, Poll::Table, Poll::Choices).await?; + let poll_models: Vec = res .iter() - .map(|(id, users)| antenna::ActiveModel { - id: sea_orm::Set(id.to_owned()), - users: sea_orm::Set(StringVec::from(users.to_owned())), + .map(|(id, r)| poll::ActiveModel { + note_id: sea_orm::Set(id.to_owned()), + choices: sea_orm::Set(StringVec::from(r.to_owned())), ..Default::default() }) .collect(); - - for model in models { - antenna::Entity::update(model).exec(db).await?; + for model in poll_models { + poll::Entity::update(model).exec(db).await?; } + let query = select_query(Poll::Table, Poll::NoteId, Poll::Votes); + let res = get_vec::>(db, query).await?; + convert_col(manager, Poll::Table, Poll::Votes).await?; + let poll_models: Vec = res + .iter() + .map(|(id, r)| poll::ActiveModel { + note_id: sea_orm::Set(id.to_owned()), + votes: sea_orm::Set(I32Vec::from(r.to_owned())), + ..Default::default() + }) + .collect(); + for model in poll_models { + poll::Entity::update(model).exec(db).await?; + } + + convert_to_stringvec_json!( + RegistryItem::Table, + RegistryItem::Id, + RegistryItem::Scope, + registry_item, + scope + ); + convert_to_stringvec_json!(User::Table, User::Id, User::Tags, user, tags); + convert_to_stringvec_json!(User::Table, User::Id, User::Emojis, user, emojis); + convert_to_stringvec_json!( + UserProfile::Table, + UserProfile::Id, + UserProfile::MutingNotificationTypes, + user_profile, + muting_notification_types + ); + convert_to_stringvec_json!(Webhook::Table, Webhook::Id, Webhook::On, webhook, on); Ok(()) } @@ -66,9 +240,160 @@ impl MigrationTrait for Migration { } /// Learn more at https://docs.rs/sea-query#iden -#[derive(Iden)] +#[derive(Iden, Clone)] enum Antenna { Table, Id, Users, } +#[derive(Iden, Clone)] +enum AccessToken { + Table, + Id, + Permission, +} +#[derive(Iden, Clone)] +enum App { + Table, + Id, + Permission, +} +#[derive(Iden, Clone)] +enum Emoji { + Table, + Id, + Aliases, +} +#[derive(Iden, Clone)] +enum GalleryPost { + Table, + Id, + FileIds, + Tags, +} +#[derive(Iden, Clone)] +enum Hashtag { + Table, + Id, + MentionedUserIds, + MentionedLocalUserIds, + MentionedRemoteUserIds, + AttachedUserIds, + AttachedLocalUserIds, + AttachedRemoteUserIds, +} +#[derive(Iden, Clone)] +enum MessagingMessage { + Table, + Id, + Reads, +} +#[derive(Iden, Clone)] +enum Meta { + Table, + Id, + Langs, + HiddenTags, + BlockedHosts, + PinnedUsers, + PinnedPages, + RecommendedInstances, + SilencedHosts, +} +#[derive(Iden, Clone)] +enum Note { + Table, + Id, + FileIds, + AttachedFileTypes, + VisibleUserIds, + Mentions, + Emojis, + Tags, +} +#[derive(Iden, Clone)] +enum NoteEdit { + Table, + Id, + FileIds, +} +#[derive(Iden, Clone)] +enum Page { + Table, + Id, + VisibleUserIds, +} +#[derive(Iden, Clone)] +enum Poll { + Table, + NoteId, + Choices, + Votes, // I32Vec +} +#[derive(Iden, Clone)] +enum RegistryItem { + Table, + Id, + Scope, +} +#[derive(Iden, Clone)] +enum User { + Table, + Id, + Tags, + Emojis, +} +#[derive(Iden, Clone)] +enum UserProfile { + Table, + Id, + MutingNotificationTypes, +} +#[derive(Iden, Clone)] +enum Webhook { + Table, + Id, + On, +} + +fn select_query(table: T, id: T, col: T) -> String { + Query::select() + .column(id) + .column(col) + .from(table) + .to_string(PostgresQueryBuilder) +} + +async fn get_vec<'a, T: TryGetable>( + db: &SchemaManagerConnection<'a>, + query: String, +) -> Result, DbErr> { + let res: Vec<(String, T)> = db + .query_all(Statement::from_string(DbBackend::Postgres, query)) + .await? + .iter() + .filter_map(|r| r.try_get_many_by_index().ok()) + .collect(); + Ok(res) +} + +async fn convert_col<'a, T: Iden + Clone + 'static>( + manager: &SchemaManager<'a>, + table: T, + col: T, +) -> Result<(), DbErr> { + manager + .alter_table( + Table::alter() + .table(table) + .drop_column(col.to_owned()) + .add_column( + ColumnDef::new(col.to_owned()) + .json_binary() + .not_null() + .default(json!([])), + ) + .to_owned(), + ) + .await +} diff --git a/packages/backend/native-utils/crates/model/Cargo.toml b/packages/backend/native-utils/crates/model/Cargo.toml index e99f8b561d..a511e6e997 100644 --- a/packages/backend/native-utils/crates/model/Cargo.toml +++ b/packages/backend/native-utils/crates/model/Cargo.toml @@ -5,8 +5,8 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [features] -default = ["legacy"] -legacy = ["sea-orm/postgres-array"] +default = [] +legacy = [] [dependencies] async-trait = "0.1.68" @@ -18,7 +18,7 @@ jsonschema = "0.17.0" once_cell = "1.17.1" parse-display = "0.8.0" schemars = { version = "0.8.12", features = ["chrono"] } -sea-orm = { version = "0.11.3", features = ["sqlx-postgres", "sqlx-sqlite", "runtime-tokio-rustls"] } +sea-orm = { version = "0.11.3", features = ["sqlx-postgres", "postgres-array", "sqlx-sqlite", "runtime-tokio-rustls"] } serde = { version = "1.0.163", features = ["derive"] } serde_json = "1.0.96" thiserror = "1.0.40" diff --git a/packages/backend/native-utils/crates/model/src/entity/mod.rs b/packages/backend/native-utils/crates/model/src/entity/mod.rs index 6105b05550..d71057fdee 100644 --- a/packages/backend/native-utils/crates/model/src/entity/mod.rs +++ b/packages/backend/native-utils/crates/model/src/entity/mod.rs @@ -53,8 +53,6 @@ pub mod registration_ticket; pub mod registry_item; pub mod relay; pub mod renote_muting; -pub mod reversi_game; -pub mod reversi_matching; pub mod sea_orm_active_enums; pub mod signin; pub mod sw_subscription; diff --git a/packages/backend/native-utils/crates/model/src/entity/prelude.rs b/packages/backend/native-utils/crates/model/src/entity/prelude.rs index 7cab688bb8..8be696cb40 100644 --- a/packages/backend/native-utils/crates/model/src/entity/prelude.rs +++ b/packages/backend/native-utils/crates/model/src/entity/prelude.rs @@ -50,8 +50,6 @@ pub use super::registration_ticket::Entity as RegistrationTicket; pub use super::registry_item::Entity as RegistryItem; pub use super::relay::Entity as Relay; pub use super::renote_muting::Entity as RenoteMuting; -pub use super::reversi_game::Entity as ReversiGame; -pub use super::reversi_matching::Entity as ReversiMatching; pub use super::signin::Entity as Signin; pub use super::sw_subscription::Entity as SwSubscription; pub use super::used_username::Entity as UsedUsername; diff --git a/packages/backend/native-utils/crates/model/src/entity/reversi_game.rs b/packages/backend/native-utils/crates/model/src/entity/reversi_game.rs deleted file mode 100644 index 0c82fccf74..0000000000 --- a/packages/backend/native-utils/crates/model/src/entity/reversi_game.rs +++ /dev/null @@ -1,69 +0,0 @@ -//! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3 - -use sea_orm::entity::prelude::*; - -use super::newtype::StringVec; - -#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] -#[sea_orm(table_name = "reversi_game")] -pub struct Model { - #[sea_orm(primary_key, auto_increment = false)] - pub id: String, - #[sea_orm(column_name = "createdAt")] - pub created_at: DateTimeWithTimeZone, - #[sea_orm(column_name = "startedAt")] - pub started_at: Option, - #[sea_orm(column_name = "user1Id")] - pub user1_id: String, - #[sea_orm(column_name = "user2Id")] - pub user2_id: String, - #[sea_orm(column_name = "user1Accepted")] - pub user1_accepted: bool, - #[sea_orm(column_name = "user2Accepted")] - pub user2_accepted: bool, - pub black: Option, - #[sea_orm(column_name = "isStarted")] - pub is_started: bool, - #[sea_orm(column_name = "isEnded")] - pub is_ended: bool, - #[sea_orm(column_name = "winnerId")] - pub winner_id: Option, - pub surrendered: Option, - #[sea_orm(column_type = "JsonBinary")] - pub logs: Json, - pub map: StringVec, - pub bw: String, - #[sea_orm(column_name = "isLlotheo")] - pub is_llotheo: bool, - #[sea_orm(column_name = "canPutEverywhere")] - pub can_put_everywhere: bool, - #[sea_orm(column_name = "loopedBoard")] - pub looped_board: bool, - #[sea_orm(column_type = "JsonBinary", nullable)] - pub form1: Option, - #[sea_orm(column_type = "JsonBinary", nullable)] - pub form2: Option, - pub crc32: Option, -} - -#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] -pub enum Relation { - #[sea_orm( - belongs_to = "super::user::Entity", - from = "Column::User2Id", - to = "super::user::Column::Id", - on_update = "NoAction", - on_delete = "Cascade" - )] - User2, - #[sea_orm( - belongs_to = "super::user::Entity", - from = "Column::User1Id", - to = "super::user::Column::Id", - on_update = "NoAction", - on_delete = "Cascade" - )] - User1, -} - -impl ActiveModelBehavior for ActiveModel {} diff --git a/packages/backend/native-utils/crates/model/src/entity/reversi_matching.rs b/packages/backend/native-utils/crates/model/src/entity/reversi_matching.rs deleted file mode 100644 index aafdf13f69..0000000000 --- a/packages/backend/native-utils/crates/model/src/entity/reversi_matching.rs +++ /dev/null @@ -1,38 +0,0 @@ -//! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3 - -use sea_orm::entity::prelude::*; - -#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)] -#[sea_orm(table_name = "reversi_matching")] -pub struct Model { - #[sea_orm(primary_key, auto_increment = false)] - pub id: String, - #[sea_orm(column_name = "createdAt")] - pub created_at: DateTimeWithTimeZone, - #[sea_orm(column_name = "parentId")] - pub parent_id: String, - #[sea_orm(column_name = "childId")] - pub child_id: String, -} - -#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] -pub enum Relation { - #[sea_orm( - belongs_to = "super::user::Entity", - from = "Column::ParentId", - to = "super::user::Column::Id", - on_update = "NoAction", - on_delete = "Cascade" - )] - User2, - #[sea_orm( - belongs_to = "super::user::Entity", - from = "Column::ChildId", - to = "super::user::Column::Id", - on_update = "NoAction", - on_delete = "Cascade" - )] - User1, -} - -impl ActiveModelBehavior for ActiveModel {} diff --git a/packages/backend/native-utils/crates/model/tests/common.rs b/packages/backend/native-utils/crates/model/tests/common.rs index f05c4432dc..0fd32d8f96 100644 --- a/packages/backend/native-utils/crates/model/tests/common.rs +++ b/packages/backend/native-utils/crates/model/tests/common.rs @@ -31,7 +31,7 @@ async fn setup_schema(db: DbConn) { let stmt = schema.create_table_from_entity(antenna::Entity); db.execute(db.get_database_backend().build(&stmt)) .await - .expect("Unable to initialize in-memoty sqlite"); + .expect("Unable to setup schemas for in-memoty sqlite"); } /// Delete all entries in the database. @@ -101,9 +101,7 @@ async fn setup_model(db: &DbConn) { mod int_test { use sea_orm::Database; - use crate::setup_schema; - - use super::{cleanup, prepare}; + use super::{cleanup, prepare, setup_schema}; #[tokio::test] async fn can_prepare_and_cleanup() {