diff --git a/packages/backend/native-utils/crates/migration/src/vec_to_json.rs b/packages/backend/native-utils/crates/migration/src/vec_to_json.rs index eccd801149..c762dcf1cc 100644 --- a/packages/backend/native-utils/crates/migration/src/vec_to_json.rs +++ b/packages/backend/native-utils/crates/migration/src/vec_to_json.rs @@ -1,11 +1,14 @@ #![allow(dead_code)] -use sea_orm_migration::{prelude::*, sea_orm::{DbConn, DbBackend, Statement, TryGetable, Database}}; -use serde_json::json; -use std::time::Duration; -use indicatif::{ProgressBar, ProgressStyle, MultiProgress}; +use indicatif::{MultiProgress, ProgressBar, ProgressStyle}; use model::entity::newtype::{I32Vec, StringVec}; +use sea_orm_migration::{ + prelude::*, + sea_orm::{Database, DbBackend, DbConn, Statement, TryGetable}, +}; +use serde_json::json; use std::env; +use std::time::Duration; pub async fn convert() { let uri = env::var("DATABASE_URL").expect("Environment variable 'DATABASE_URL' not set"); @@ -14,41 +17,251 @@ pub async fn convert() { let mp = MultiProgress::new(); let handlers = vec![ - tokio::spawn(to_json::, StringVec>(db.clone(), mp.clone(), AccessToken::Table, AccessToken::Id, AccessToken::Permission)), - tokio::spawn(to_json::, StringVec>(db.clone(), mp.clone(), Antenna::Table, Antenna::Id, Antenna::Users)), - tokio::spawn(to_json::, StringVec>(db.clone(), mp.clone(), App::Table, App::Id, App::Permission)), - tokio::spawn(to_json::, StringVec>(db.clone(), mp.clone(), Emoji::Table, Emoji::Id, Emoji::Aliases)), - tokio::spawn(to_json::, StringVec>(db.clone(), mp.clone(), GalleryPost::Table, GalleryPost::Id, GalleryPost::FileIds)), - tokio::spawn(to_json::, StringVec>(db.clone(), mp.clone(), GalleryPost::Table, GalleryPost::Id, GalleryPost::Tags)), - tokio::spawn(to_json::, StringVec>(db.clone(), mp.clone(), Hashtag::Table, Hashtag::Id, Hashtag::MentionedUserIds)), - tokio::spawn(to_json::, StringVec>(db.clone(), mp.clone(), Hashtag::Table, Hashtag::Id, Hashtag::MentionedLocalUserIds)), - tokio::spawn(to_json::, StringVec>(db.clone(), mp.clone(), Hashtag::Table, Hashtag::Id, Hashtag::MentionedRemoteUserIds)), - tokio::spawn(to_json::, StringVec>(db.clone(), mp.clone(), Hashtag::Table, Hashtag::Id, Hashtag::AttachedUserIds)), - tokio::spawn(to_json::, StringVec>(db.clone(), mp.clone(), Hashtag::Table, Hashtag::Id, Hashtag::AttachedLocalUserIds)), - tokio::spawn(to_json::, StringVec>(db.clone(), mp.clone(), Hashtag::Table, Hashtag::Id, Hashtag::AttachedRemoteUserIds)), - tokio::spawn(to_json::, StringVec>(db.clone(), mp.clone(), MessagingMessage::Table, MessagingMessage::Id, MessagingMessage::Reads)), - tokio::spawn(to_json::, StringVec>(db.clone(), mp.clone(), Meta::Table, Meta::Id, Meta::Langs)), - tokio::spawn(to_json::, StringVec>(db.clone(), mp.clone(), Meta::Table, Meta::Id, Meta::BlockedHosts)), - tokio::spawn(to_json::, StringVec>(db.clone(), mp.clone(), Meta::Table, Meta::Id, Meta::HiddenTags)), - tokio::spawn(to_json::, StringVec>(db.clone(), mp.clone(), Meta::Table, Meta::Id, Meta::PinnedUsers)), - tokio::spawn(to_json::, StringVec>(db.clone(), mp.clone(), Meta::Table, Meta::Id, Meta::PinnedPages)), - tokio::spawn(to_json::, StringVec>(db.clone(), mp.clone(), Meta::Table, Meta::Id, Meta::RecommendedInstances)), - tokio::spawn(to_json::, StringVec>(db.clone(), mp.clone(), Meta::Table, Meta::Id, Meta::SilencedHosts)), - tokio::spawn(to_json::, StringVec>(db.clone(), mp.clone(), Note::Table, Note::Id, Note::FileIds)), - tokio::spawn(to_json::, StringVec>(db.clone(), mp.clone(), Note::Table, Note::Id, Note::AttachedFileTypes)), - tokio::spawn(to_json::, StringVec>(db.clone(), mp.clone(), Note::Table, Note::Id, Note::VisibleUserIds)), - tokio::spawn(to_json::, StringVec>(db.clone(), mp.clone(), Note::Table, Note::Id, Note::Mentions)), - tokio::spawn(to_json::, StringVec>(db.clone(), mp.clone(), Note::Table, Note::Id, Note::Emojis)), - tokio::spawn(to_json::, StringVec>(db.clone(), mp.clone(), Note::Table, Note::Id, Note::Tags)), - tokio::spawn(to_json::, StringVec>(db.clone(), mp.clone(), NoteEdit::Table, NoteEdit::Id, NoteEdit::FileIds)), - tokio::spawn(to_json::, StringVec>(db.clone(), mp.clone(), Page::Table, Page::Id, Page::VisibleUserIds)), - tokio::spawn(to_json::, StringVec>(db.clone(), mp.clone(), RegistryItem::Table, RegistryItem::Id, RegistryItem::Scope)), - tokio::spawn(to_json::, StringVec>(db.clone(), mp.clone(), User::Table, User::Id, User::Tags)), - tokio::spawn(to_json::, StringVec>(db.clone(), mp.clone(), User::Table, User::Id, User::Emojis)), - tokio::spawn(to_json::, StringVec>(db.clone(), mp.clone(), Webhook::Table, Webhook::Id, Webhook::On)), - tokio::spawn(to_json::, StringVec>(db.clone(), mp.clone(), Poll::Table, Poll::NoteId, Poll::Choices)), - tokio::spawn(to_json::, I32Vec>(db.clone(), mp.clone(), Poll::Table, Poll::NoteId, Poll::Votes)), - tokio::spawn(to_json::, StringVec>(db.clone(), mp.clone(), UserProfile::Table, UserProfile::UserId, UserProfile::MutingNotificationTypes)), + tokio::spawn(to_json::, StringVec>( + db.clone(), + mp.clone(), + AccessToken::Table, + AccessToken::Id, + AccessToken::Permission, + )), + tokio::spawn(to_json::, StringVec>( + db.clone(), + mp.clone(), + Antenna::Table, + Antenna::Id, + Antenna::Users, + )), + tokio::spawn(to_json::, StringVec>( + db.clone(), + mp.clone(), + App::Table, + App::Id, + App::Permission, + )), + tokio::spawn(to_json::, StringVec>( + db.clone(), + mp.clone(), + Emoji::Table, + Emoji::Id, + Emoji::Aliases, + )), + tokio::spawn(to_json::, StringVec>( + db.clone(), + mp.clone(), + GalleryPost::Table, + GalleryPost::Id, + GalleryPost::FileIds, + )), + tokio::spawn(to_json::, StringVec>( + db.clone(), + mp.clone(), + GalleryPost::Table, + GalleryPost::Id, + GalleryPost::Tags, + )), + tokio::spawn(to_json::, StringVec>( + db.clone(), + mp.clone(), + Hashtag::Table, + Hashtag::Id, + Hashtag::MentionedUserIds, + )), + tokio::spawn(to_json::, StringVec>( + db.clone(), + mp.clone(), + Hashtag::Table, + Hashtag::Id, + Hashtag::MentionedLocalUserIds, + )), + tokio::spawn(to_json::, StringVec>( + db.clone(), + mp.clone(), + Hashtag::Table, + Hashtag::Id, + Hashtag::MentionedRemoteUserIds, + )), + tokio::spawn(to_json::, StringVec>( + db.clone(), + mp.clone(), + Hashtag::Table, + Hashtag::Id, + Hashtag::AttachedUserIds, + )), + tokio::spawn(to_json::, StringVec>( + db.clone(), + mp.clone(), + Hashtag::Table, + Hashtag::Id, + Hashtag::AttachedLocalUserIds, + )), + tokio::spawn(to_json::, StringVec>( + db.clone(), + mp.clone(), + Hashtag::Table, + Hashtag::Id, + Hashtag::AttachedRemoteUserIds, + )), + tokio::spawn(to_json::, StringVec>( + db.clone(), + mp.clone(), + MessagingMessage::Table, + MessagingMessage::Id, + MessagingMessage::Reads, + )), + tokio::spawn(to_json::, StringVec>( + db.clone(), + mp.clone(), + Meta::Table, + Meta::Id, + Meta::Langs, + )), + tokio::spawn(to_json::, StringVec>( + db.clone(), + mp.clone(), + Meta::Table, + Meta::Id, + Meta::BlockedHosts, + )), + tokio::spawn(to_json::, StringVec>( + db.clone(), + mp.clone(), + Meta::Table, + Meta::Id, + Meta::HiddenTags, + )), + tokio::spawn(to_json::, StringVec>( + db.clone(), + mp.clone(), + Meta::Table, + Meta::Id, + Meta::PinnedUsers, + )), + tokio::spawn(to_json::, StringVec>( + db.clone(), + mp.clone(), + Meta::Table, + Meta::Id, + Meta::PinnedPages, + )), + tokio::spawn(to_json::, StringVec>( + db.clone(), + mp.clone(), + Meta::Table, + Meta::Id, + Meta::RecommendedInstances, + )), + tokio::spawn(to_json::, StringVec>( + db.clone(), + mp.clone(), + Meta::Table, + Meta::Id, + Meta::SilencedHosts, + )), + tokio::spawn(to_json::, StringVec>( + db.clone(), + mp.clone(), + Note::Table, + Note::Id, + Note::FileIds, + )), + tokio::spawn(to_json::, StringVec>( + db.clone(), + mp.clone(), + Note::Table, + Note::Id, + Note::AttachedFileTypes, + )), + tokio::spawn(to_json::, StringVec>( + db.clone(), + mp.clone(), + Note::Table, + Note::Id, + Note::VisibleUserIds, + )), + tokio::spawn(to_json::, StringVec>( + db.clone(), + mp.clone(), + Note::Table, + Note::Id, + Note::Mentions, + )), + tokio::spawn(to_json::, StringVec>( + db.clone(), + mp.clone(), + Note::Table, + Note::Id, + Note::Emojis, + )), + tokio::spawn(to_json::, StringVec>( + db.clone(), + mp.clone(), + Note::Table, + Note::Id, + Note::Tags, + )), + tokio::spawn(to_json::, StringVec>( + db.clone(), + mp.clone(), + NoteEdit::Table, + NoteEdit::Id, + NoteEdit::FileIds, + )), + tokio::spawn(to_json::, StringVec>( + db.clone(), + mp.clone(), + Page::Table, + Page::Id, + Page::VisibleUserIds, + )), + tokio::spawn(to_json::, StringVec>( + db.clone(), + mp.clone(), + RegistryItem::Table, + RegistryItem::Id, + RegistryItem::Scope, + )), + tokio::spawn(to_json::, StringVec>( + db.clone(), + mp.clone(), + User::Table, + User::Id, + User::Tags, + )), + tokio::spawn(to_json::, StringVec>( + db.clone(), + mp.clone(), + User::Table, + User::Id, + User::Emojis, + )), + tokio::spawn(to_json::, StringVec>( + db.clone(), + mp.clone(), + Webhook::Table, + Webhook::Id, + Webhook::On, + )), + tokio::spawn(to_json::, StringVec>( + db.clone(), + mp.clone(), + Poll::Table, + Poll::NoteId, + Poll::Choices, + )), + tokio::spawn(to_json::, I32Vec>( + db.clone(), + mp.clone(), + Poll::Table, + Poll::NoteId, + Poll::Votes, + )), + tokio::spawn(to_json::, StringVec>( + db.clone(), + mp.clone(), + UserProfile::Table, + UserProfile::UserId, + UserProfile::MutingNotificationTypes, + )), ]; futures::future::join_all(handlers).await; @@ -62,10 +275,7 @@ fn select_query(table: T, id: T, col: T) -> String { .to_string(PostgresQueryBuilder) } -async fn get_vec( - db: &DbConn, - query: String, -) -> Result, DbErr> { +async fn get_vec(db: &DbConn, query: String) -> Result, DbErr> { let res: Vec<(String, T)> = db .query_all(Statement::from_string(DbBackend::Postgres, query)) .await? @@ -81,15 +291,17 @@ async fn convert_col( col: T, ) -> Result<(), DbErr> { let stmt = Table::alter() - .table(table) - .drop_column(col.to_owned()) - .add_column( - ColumnDef::new(col.to_owned()) - .json_binary() - .not_null() - .default(json!([])), - ).to_string(PostgresQueryBuilder); - db.query_one(Statement::from_string(DbBackend::Postgres, stmt)).await?; + .table(table) + .drop_column(col.to_owned()) + .add_column( + ColumnDef::new(col.to_owned()) + .json_binary() + .not_null() + .default(json!([])), + ) + .to_string(PostgresQueryBuilder); + db.query_one(Statement::from_string(DbBackend::Postgres, stmt)) + .await?; Ok(()) } @@ -102,26 +314,35 @@ async fn to_json( ) -> Result<(), DbErr> where T: Iden + Clone + 'static, - U: TryGetable + Clone, + U: TryGetable + IntoIterator + Clone, V: From + Into, { let query = select_query(table.clone(), id.clone(), col.clone()); let loading = ProgressBar::new_spinner() .with_style(ProgressStyle::with_template("{prefix} {msg} {spinner}").unwrap()) .with_prefix("[-]") - .with_message(format!("Loading data from {}.{}", table.to_string(), col.to_string())); - loading.enable_steady_tick(Duration::from_millis(100)); + .with_message(format!( + "Loading data from {}.{}", + table.to_string(), + col.to_string() + )); let loading = mp.add(loading); + loading.enable_steady_tick(Duration::from_millis(100)); let res = get_vec::(&db, query).await?; let models: Vec<(String, V)> = res .iter() + .filter(|(_, r)| r.clone().into_iter().count() > 0) .map(|(id, r)| (id.clone(), ::from(r.clone()))) .collect(); loading.finish_and_clear(); convert_col(&db, table.clone(), col.clone()).await?; let progress = ProgressBar::new(models.len() as u64) - .with_style(ProgressStyle::with_template("{prefix} {msg} {wide_bar} {pos}/{len}").unwrap().progress_chars("##-")) + .with_style( + ProgressStyle::with_template("{prefix} {msg} {wide_bar} {pos}/{len}") + .unwrap() + .progress_chars("##-"), + ) .with_prefix("[*]") .with_message(format!("Copying {}.{}", table.to_string(), col.to_string())); let progress = mp.add(progress); @@ -136,7 +357,7 @@ where db.query_one(Statement::from_string(DbBackend::Postgres, q)) .await?; } - progress.finish_with_message(format!("Complete {}.{}", table.to_string(), col.to_string())); + progress.finish_with_message(format!("Done {}.{}", table.to_string(), col.to_string())); Ok(()) }