From ef00b4034b8963e1fe8836f90993dd54519908a8 Mon Sep 17 00:00:00 2001 From: Namekuji Date: Mon, 10 Jul 2023 01:39:33 -0400 Subject: [PATCH] copy existing posts in antenna to redis at migration --- .../migration/1680491187535-cleanup.js | 4 +- packages/backend/native-utils/Cargo.lock | 145 ++++++++++- packages/backend/native-utils/Cargo.toml | 4 +- .../native-utils/__test__/index.spec.mjs | 8 +- .../backend/native-utils/migration/Cargo.toml | 9 +- .../backend/native-utils/migration/src/lib.rs | 2 + .../m20230709_000510_move_antenna_to_cache.rs | 240 ++++++++++++++++++ .../native-utils/migration/src/main.rs | 81 +++++- .../native-utils/src/model/schema/app.rs | 8 +- packages/backend/native-utils/src/util/id.rs | 82 +++--- packages/backend/native-utils/tests/common.rs | 8 +- .../tests/model/repository/antenna.rs | 2 +- packages/backend/src/misc/gen-id.ts | 2 +- .../src/services/add-note-to-antenna.ts | 7 +- 14 files changed, 516 insertions(+), 86 deletions(-) create mode 100644 packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs diff --git a/packages/backend/migration/1680491187535-cleanup.js b/packages/backend/migration/1680491187535-cleanup.js index 021a368c06..671f7521e4 100644 --- a/packages/backend/migration/1680491187535-cleanup.js +++ b/packages/backend/migration/1680491187535-cleanup.js @@ -1,9 +1,7 @@ export class cleanup1680491187535 { name = "cleanup1680491187535"; - async up(queryRunner) { - await queryRunner.query(`DROP TABLE "antenna_note" `); - } + async up(queryRunner) {} async down(queryRunner) {} } diff --git a/packages/backend/native-utils/Cargo.lock b/packages/backend/native-utils/Cargo.lock index 27ba602448..822778d237 100644 --- a/packages/backend/native-utils/Cargo.lock +++ b/packages/backend/native-utils/Cargo.lock @@ -504,6 +504,20 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "combine" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4" +dependencies = [ + "bytes", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", + "tokio-util", +] + [[package]] name = "console" version = "0.15.7" @@ -532,6 +546,16 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "core-foundation" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.4" @@ -1370,11 +1394,14 @@ dependencies = [ "futures", "indicatif", "native-utils", + "redis", + "sea-orm", "sea-orm-migration", "serde", "serde_json", "serde_yaml", "tokio", + "url", "urlencoding", ] @@ -1651,6 +1678,12 @@ version = "1.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + [[package]] name = "os_str_bytes" version = "6.5.0" @@ -1993,6 +2026,29 @@ dependencies = [ "rand_core", ] +[[package]] +name = "redis" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ea8c51b5dc1d8e5fd3350ec8167f464ec0995e79f2e90a075b63371500d557f" +dependencies = [ + "async-trait", + "bytes", + "combine", + "futures-util", + "itoa", + "percent-encoding", + "pin-project-lite", + "rustls 0.21.3", + "rustls-native-certs", + "ryu", + "sha1_smol", + "tokio", + "tokio-rustls 0.24.1", + "tokio-util", + "url", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -2199,6 +2255,30 @@ dependencies = [ "webpki", ] +[[package]] +name = "rustls" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b19faa85ecb5197342b54f987b142fb3e30d0c90da40f80ef4fa9a726e6676ed" +dependencies = [ + "log", + "ring", + "rustls-webpki", + "sct", +] + +[[package]] +name = "rustls-native-certs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +dependencies = [ + "openssl-probe", + "rustls-pemfile", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pemfile" version = "1.0.2" @@ -2208,6 +2288,16 @@ dependencies = [ "base64 0.21.2", ] +[[package]] +name = "rustls-webpki" +version = "0.101.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f36a6828982f422756984e47912a7a51dcbc2a197aa791158f8ca61cd8204e" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.12" @@ -2232,6 +2322,15 @@ version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +[[package]] +name = "schannel" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" +dependencies = [ + "windows-sys 0.48.0", +] + [[package]] name = "schemars" version = "0.8.12" @@ -2521,6 +2620,29 @@ version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" +[[package]] +name = "security-framework" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fc758eb7bffce5b308734e9b0c1468893cae9ff70ebf13e7090be8dcbcc83a8" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f51d0c0d83bec45f16480d0ce0058397a69e48fcdc52d1dc8855fb68acbd31a7" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.17" @@ -2605,6 +2727,12 @@ dependencies = [ "digest", ] +[[package]] +name = "sha1_smol" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" + [[package]] name = "sha2" version = "0.10.6" @@ -2769,7 +2897,7 @@ dependencies = [ "percent-encoding", "rand", "rust_decimal", - "rustls", + "rustls 0.20.8", "rustls-pemfile", "serde", "serde_json", @@ -2815,7 +2943,7 @@ checksum = "804d3f245f894e61b1e6263c84b23ca675d96753b5abfd5cc8597d86806e8024" dependencies = [ "once_cell", "tokio", - "tokio-rustls", + "tokio-rustls 0.23.4", ] [[package]] @@ -3055,11 +3183,21 @@ version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" dependencies = [ - "rustls", + "rustls 0.20.8", "tokio", "webpki", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.3", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.14" @@ -3253,6 +3391,7 @@ dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] diff --git a/packages/backend/native-utils/Cargo.toml b/packages/backend/native-utils/Cargo.toml index 933186c1a8..a1a6228bef 100644 --- a/packages/backend/native-utils/Cargo.toml +++ b/packages/backend/native-utils/Cargo.toml @@ -9,7 +9,7 @@ members = ["migration", "scylla-migration"] [features] default = [] noarray = [] -napi = ["dep:napi", "dep:napi-derive", "dep:radix_fmt"] +napi = ["dep:napi", "dep:napi-derive"] [lib] crate-type = ["cdylib", "lib"] @@ -31,11 +31,11 @@ serde_json = "1.0.96" thiserror = "1.0.40" tokio = { version = "1.28.1", features = ["full"] } utoipa = "3.3.0" +radix_fmt = "1.0.0" # Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix napi = { version = "2.13.1", default-features = false, features = ["napi6", "tokio_rt"], optional = true } napi-derive = { version = "2.12.0", optional = true } -radix_fmt = { version = "1.0.0", optional = true } [dev-dependencies] pretty_assertions = "1.3.0" diff --git a/packages/backend/native-utils/__test__/index.spec.mjs b/packages/backend/native-utils/__test__/index.spec.mjs index 6e6a91858c..9ff8ead6c9 100644 --- a/packages/backend/native-utils/__test__/index.spec.mjs +++ b/packages/backend/native-utils/__test__/index.spec.mjs @@ -12,18 +12,18 @@ test("convert to mastodon id", (t) => { t.is(convertId("9gf61ehcxv", IdConvertType.MastodonId), "960365976481219"); t.is( convertId("9fbr9z0wbrjqyd3u", IdConvertType.MastodonId), - "3954607381600562394", + "2083785058661759970208986", ); t.is( convertId("9fbs680oyviiqrol9md73p8g", IdConvertType.MastodonId), - "3494513243013053824", + "5878598648988104013828532260828151168", ); }); test("create cuid2 with timestamp prefix", (t) => { nativeInitIdGenerator(16, ""); - t.not(nativeCreateId(BigInt(Date.now())), nativeCreateId(BigInt(Date.now()))); - t.is(nativeCreateId(BigInt(Date.now())).length, 16); + t.not(nativeCreateId(Date.now()), nativeCreateId(Date.now())); + t.is(nativeCreateId(Date.now()).length, 16); }); test("create random string", (t) => { diff --git a/packages/backend/native-utils/migration/Cargo.toml b/packages/backend/native-utils/migration/Cargo.toml index 7ed9fd5f03..1e5fa57c38 100644 --- a/packages/backend/native-utils/migration/Cargo.toml +++ b/packages/backend/native-utils/migration/Cargo.toml @@ -10,17 +10,20 @@ path = "src/lib.rs" [features] default = [] -convert = ["dep:native-utils", "dep:indicatif", "dep:futures"] +convert = ["dep:indicatif"] [dependencies] serde_json = "1.0.96" -native-utils = { path = "../", optional = true } +native-utils = { path = "../" } indicatif = { version = "0.17.4", features = ["tokio"], optional = true } tokio = { version = "1.28.2", features = ["full"] } -futures = { version = "0.3.28", optional = true } +futures = "0.3.28" serde_yaml = "0.9.21" serde = { version = "1.0.163", features = ["derive"] } urlencoding = "2.1.2" +redis = { version = "0.23.0", features = ["tokio-rustls-comp"] } +sea-orm = "0.11.3" +url = { version = "2.4.0", features = ["serde"] } [dependencies.sea-orm-migration] version = "0.11.0" diff --git a/packages/backend/native-utils/migration/src/lib.rs b/packages/backend/native-utils/migration/src/lib.rs index 94e2b08cc4..5ad23f1625 100644 --- a/packages/backend/native-utils/migration/src/lib.rs +++ b/packages/backend/native-utils/migration/src/lib.rs @@ -2,6 +2,7 @@ pub use sea_orm_migration::prelude::*; mod m20230531_180824_drop_reversi; mod m20230627_185451_index_note_url; +mod m20230709_000510_move_antenna_to_cache; pub struct Migrator; @@ -11,6 +12,7 @@ impl MigratorTrait for Migrator { vec![ Box::new(m20230531_180824_drop_reversi::Migration), Box::new(m20230627_185451_index_note_url::Migration), + Box::new(m20230709_000510_move_antenna_to_cache::Migration), ] } } diff --git a/packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs b/packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs new file mode 100644 index 0000000000..7c38fd1bd6 --- /dev/null +++ b/packages/backend/native-utils/migration/src/m20230709_000510_move_antenna_to_cache.rs @@ -0,0 +1,240 @@ +use redis::streams::StreamMaxlen; +use sea_orm::Statement; +use sea_orm_migration::prelude::*; +use std::env; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + let cache_url = env::var("CACHE_URL").unwrap(); + let copy_limit = env::var("ANTENNA_MIGRATION_LIMIT").unwrap_or_default(); + let copy_limit: i64 = match copy_limit.parse() { + Ok(limit) => limit, + Err(_) => 0, + }; + + if cache_url != "no" { + let prefix = env::var("CACHE_PREFIX").unwrap(); + + let db = manager.get_connection(); + let bk = manager.get_database_backend(); + + let count_stmt = + Statement::from_string(bk, "SELECT COUNT(1) FROM antenna_note".to_owned()); + let total_num = db + .query_one(count_stmt) + .await? + .unwrap() + .try_get_by_index::(0)?; + let copy_limit = if copy_limit == 0 { + total_num + } else { + copy_limit + }; + println!( + "Copying {} out of {} entries in antenna_note.", + copy_limit, total_num + ); + + let stmt_base = Query::select() + .column((AntennaNote::Table, AntennaNote::Id)) + .column(AntennaNote::AntennaId) + .column(AntennaNote::NoteId) + .from(AntennaNote::Table) + .order_by((AntennaNote::Table, AntennaNote::Id), Order::Asc) + .limit(1000) + .to_owned(); + + let mut stmt = stmt_base.clone(); + + let client = redis::Client::open(cache_url).unwrap(); + let mut redis_conn = client.get_connection().unwrap(); + + let mut remaining = total_num; + let mut pagination: i64 = 0; + + loop { + let res = db.query_all(bk.build(&stmt)).await?; + if res.len() == 0 { + break; + } + let val: Vec<(String, String, String)> = res + .iter() + .filter_map(|q| q.try_get_many_by_index().ok()) + .collect(); + + remaining -= val.len() as i64; + if remaining <= copy_limit { + let mut pipe = redis::pipe(); + for v in &val { + pipe.xadd_maxlen( + format!("{}:antennaTimeline:{}", prefix, v.1), + StreamMaxlen::Approx(200), + "*", + &[("note", v.2.to_owned())], + ) + .ignore(); + } + pipe.query::<()>(&mut redis_conn).unwrap(); + } + + let copied = total_num - remaining; + let copied = std::cmp::min(copied, total_num); + pagination += 1; + if pagination % 100 == 0 { + println!( + "Migrating antenna [{:.2}%]", + (copied as f64 / total_num as f64) * 100_f64, + ); + } + + if let Some((last_id, _, _)) = val.last() { + stmt = stmt_base + .clone() + .and_where( + Expr::col((AntennaNote::Table, AntennaNote::Id)).gt(last_id.to_owned()), + ) + .to_owned(); + } else { + break; + } + } + } + println!("Migrating antenna [100.00%]"); + + manager + .drop_table( + Table::drop() + .table(AntennaNote::Table) + .if_exists() + .to_owned(), + ) + .await?; + + Ok(()) + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .create_table( + Table::create() + .table(AntennaNote::Table) + .if_not_exists() + .col( + ColumnDef::new(AntennaNote::Id) + .string_len(32) + .not_null() + .primary_key(), + ) + .col( + ColumnDef::new(AntennaNote::NoteId) + .string_len(32) + .not_null(), + ) + .col( + ColumnDef::new(AntennaNote::AntennaId) + .string_len(32) + .not_null(), + ) + .col( + ColumnDef::new(AntennaNote::Read) + .boolean() + .default(false) + .not_null(), + ) + .to_owned(), + ) + .await?; + manager + .create_index( + Index::create() + .name("IDX_0d775946662d2575dfd2068a5f") + .table(AntennaNote::Table) + .col(AntennaNote::AntennaId) + .if_not_exists() + .to_owned(), + ) + .await?; + manager + .create_index( + Index::create() + .name("IDX_bd0397be22147e17210940e125") + .table(AntennaNote::Table) + .col(AntennaNote::NoteId) + .if_not_exists() + .to_owned(), + ) + .await?; + manager + .create_index( + Index::create() + .name("IDX_335a0bf3f904406f9ef3dd51c2") + .table(AntennaNote::Table) + .col(AntennaNote::NoteId) + .col(AntennaNote::AntennaId) + .unique() + .if_not_exists() + .to_owned(), + ) + .await?; + manager + .create_index( + Index::create() + .name("IDX_9937ea48d7ae97ffb4f3f063a4") + .table(AntennaNote::Table) + .col(AntennaNote::Read) + .if_not_exists() + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_0d775946662d2575dfd2068a5f5") + .from(AntennaNote::Table, AntennaNote::AntennaId) + .to(Antenna::Table, Antenna::Id) + .on_delete(ForeignKeyAction::Cascade) + .to_owned(), + ) + .await?; + manager + .create_foreign_key( + ForeignKey::create() + .name("FK_bd0397be22147e17210940e125b") + .from(AntennaNote::Table, AntennaNote::NoteId) + .to(Note::Table, Note::Id) + .on_delete(ForeignKeyAction::Cascade) + .to_owned(), + ) + .await?; + + Ok(()) + } +} + +/// Learn more at https://docs.rs/sea-query#iden +#[derive(Iden)] +enum AntennaNote { + Table, + Id, + #[iden = "noteId"] + NoteId, + #[iden = "antennaId"] + AntennaId, + Read, +} + +#[derive(Iden)] +enum Antenna { + Table, + Id, +} + +#[derive(Iden)] +enum Note { + Table, + Id, +} diff --git a/packages/backend/native-utils/migration/src/main.rs b/packages/backend/native-utils/migration/src/main.rs index f0f761f657..fb9920b67a 100644 --- a/packages/backend/native-utils/migration/src/main.rs +++ b/packages/backend/native-utils/migration/src/main.rs @@ -5,6 +5,10 @@ use urlencoding::encode; use sea_orm_migration::prelude::*; +const DB_URL_ENV: &str = "DATABASE_URL"; +const CACHE_URL_ENV: &str = "CACHE_URL"; +const CACHE_PREFIX_ENV: &str = "CACHE_PREFIX"; + #[cfg(feature = "convert")] mod vec_to_json; @@ -15,17 +19,48 @@ async fn main() { .expect("Failed to open '.config/default.yml'"); let config: Config = serde_yaml::from_reader(yml).expect("Failed to parse yaml"); - env::set_var( - "DATABASE_URL", - format!( - "postgres://{}:{}@{}:{}/{}", - config.db.user, - encode(&config.db.pass), - config.db.host, - config.db.port, - config.db.db, - ), - ); + if env::var_os(DB_URL_ENV).is_none() { + env::set_var( + DB_URL_ENV, + format!( + "postgres://{}:{}@{}:{}/{}", + config.db.user, + encode(&config.db.pass), + config.db.host, + config.db.port, + config.db.db, + ), + ); + }; + + if env::var_os(CACHE_URL_ENV).is_none() { + let redis_conf = match config.cache_server { + None => config.redis, + Some(conf) => conf, + }; + let redis_proto = match redis_conf.tls { + None => "redis", + Some(_) => "rediss", + }; + let redis_uri_userpass = match redis_conf.user { + None => "".to_string(), + Some(user) => format!("{}:{}@", user, redis_conf.pass.unwrap_or_default()), + }; + let redis_uri_hostport = format!("{}:{}", redis_conf.host, redis_conf.port); + let redis_uri = format!( + "{}://{}{}", + redis_proto, redis_uri_userpass, redis_uri_hostport + ); + env::set_var(CACHE_URL_ENV, redis_uri); + env::set_var( + CACHE_PREFIX_ENV, + if redis_conf.prefix.is_empty() { + config.url.host_str().unwrap() + } else { + &redis_conf.prefix + }, + ); + } cli::run_cli(migration::Migrator).await; @@ -36,7 +71,10 @@ async fn main() { #[derive(Debug, PartialEq, Deserialize)] #[serde(rename = "camelCase")] pub struct Config { + pub url: url::Url, pub db: DbConfig, + pub redis: RedisConfig, + pub cache_server: Option, } #[derive(Debug, PartialEq, Deserialize)] @@ -48,3 +86,24 @@ pub struct DbConfig { pub user: String, pub pass: String, } + +#[derive(Debug, PartialEq, Deserialize)] +#[serde(rename = "camelCase")] +pub struct RedisConfig { + pub host: String, + pub port: u32, + pub user: Option, + pub pass: Option, + pub tls: Option, + #[serde(default)] + pub db: u32, + #[serde(default)] + pub prefix: String, +} + +#[derive(Debug, PartialEq, Deserialize)] +#[serde(rename = "camelCase")] +pub struct TlsConfig { + pub host: String, + pub reject_unauthorized: bool, +} diff --git a/packages/backend/native-utils/src/model/schema/app.rs b/packages/backend/native-utils/src/model/schema/app.rs index 9b5691154d..5f859ecb3a 100644 --- a/packages/backend/native-utils/src/model/schema/app.rs +++ b/packages/backend/native-utils/src/model/schema/app.rs @@ -105,9 +105,9 @@ mod unit_test { #[test] fn app_valid() { - init_id(12, ""); + init_id(16, ""); let instance = json!({ - "id": create_id().unwrap(), + "id": create_id(0).unwrap(), "name": "Test App", "secret": gen_string(24), "callbackUrl": "urn:ietf:wg:oauth:2.0:oob", @@ -119,9 +119,9 @@ mod unit_test { #[test] fn app_invalid() { - init_id(12, ""); + init_id(16, ""); let instance = json!({ - "id": create_id().unwrap(), + "id": create_id(0).unwrap(), // "name" is required "name": null, // "permission" must be one of the app permissions diff --git a/packages/backend/native-utils/src/util/id.rs b/packages/backend/native-utils/src/util/id.rs index d922518f9d..b18637fdbb 100644 --- a/packages/backend/native-utils/src/util/id.rs +++ b/packages/backend/native-utils/src/util/id.rs @@ -1,7 +1,10 @@ //! ID generation utility based on [cuid2] use cfg_if::cfg_if; +use chrono::Utc; use once_cell::sync::OnceCell; +use radix_fmt::radix_36; +use std::cmp; use crate::impl_into_napi_error; @@ -14,47 +17,56 @@ impl_into_napi_error!(ErrorUninitialized); static FINGERPRINT: OnceCell = OnceCell::new(); static GENERATOR: OnceCell = OnceCell::new(); +const TIME_2000: i64 = 946_684_800_000; +const TIMESTAMP_LENGTH: u16 = 8; + /// Initializes Cuid2 generator. Must be called before any [create_id]. -pub fn init_id(length: u16, fingerprint: impl Into) { - FINGERPRINT.get_or_init(move || format!("{}{}", fingerprint.into(), cuid2::create_id())); +pub fn init_id<'a>(length: u16, fingerprint: &'a str) { + FINGERPRINT.get_or_init(move || format!("{}{}", fingerprint, cuid2::create_id())); GENERATOR.get_or_init(move || { cuid2::CuidConstructor::new() - .with_length(length) + // length to pass shoule be greater than or equal to 8. + .with_length(cmp::max(length - TIMESTAMP_LENGTH, 8)) .with_fingerprinter(|| FINGERPRINT.get().unwrap().clone()) }); } /// Returns Cuid2 with the length specified by [init_id]. Must be called after /// [init_id], otherwise returns [ErrorUninitialized]. -pub fn create_id() -> Result { +/// The current timestamp via [chrono::Utc] is used if `date_num` is `0`. +pub fn create_id(date_num: i64) -> Result { match GENERATOR.get() { None => Err(ErrorUninitialized), - Some(gen) => Ok(gen.create_id()), + Some(gen) => { + let date_num = if date_num > 0 { + date_num + } else { + Utc::now().timestamp_millis() + }; + let time = cmp::max(date_num - TIME_2000, 0); + Ok(format!( + "{:0>8}{}", + radix_36(time).to_string(), + gen.create_id() + )) + } } } cfg_if! { if #[cfg(feature = "napi")] { - use radix_fmt::radix_36; - use std::cmp; - use napi::bindgen_prelude::BigInt; use napi_derive::napi; - const TIME_2000: u64 = 946_684_800_000; - const TIMESTAMP_LENGTH: u16 = 8; - /// Calls [init_id] inside. Must be called before [native_create_id]. #[napi] pub fn native_init_id_generator(length: u16, fingerprint: String) { - // length to pass init_id shoule be greater than or equal to 8. - init_id(cmp::max(length - TIMESTAMP_LENGTH, 8), fingerprint); + init_id(length, &fingerprint); } /// Generates #[napi] - pub fn native_create_id(date_num: BigInt) -> String { - let time = cmp::max(date_num.get_u64().1 - TIME_2000, 0); - format!("{:0>8}{}", radix_36(time).to_string(), create_id().unwrap()) + pub fn native_create_id(date_num: i64) -> String { + create_id(date_num).unwrap() } } } @@ -62,37 +74,17 @@ cfg_if! { #[cfg(test)] mod unit_test { use crate::util::id; - use cfg_if::cfg_if; use pretty_assertions::{assert_eq, assert_ne}; use std::thread; - cfg_if! { - if #[cfg(feature = "napi")] { - use chrono::Utc; - - #[test] - fn can_generate_aid_compat_ids() { - id::native_init_id_generator(20, "".to_string()); - let id1 = id::native_create_id(Utc::now().timestamp_millis().into()); - assert_eq!(id1.len(), 20); - let id1 = id::native_create_id(Utc::now().timestamp_millis().into()); - let id2 = id::native_create_id(Utc::now().timestamp_millis().into()); - assert_ne!(id1, id2); - let id1 = thread::spawn(|| id::native_create_id(Utc::now().timestamp_millis().into())); - let id2 = thread::spawn(|| id::native_create_id(Utc::now().timestamp_millis().into())); - assert_ne!(id1.join().unwrap(), id2.join().unwrap()); - } - } else { - #[test] - fn can_generate_unique_ids() { - assert_eq!(id::create_id(), Err(id::ErrorUninitialized)); - id::init_id(12, ""); - assert_eq!(id::create_id().unwrap().len(), 12); - assert_ne!(id::create_id().unwrap(), id::create_id().unwrap()); - let id1 = thread::spawn(|| id::create_id().unwrap()); - let id2 = thread::spawn(|| id::create_id().unwrap()); - assert_ne!(id1.join().unwrap(), id2.join().unwrap()); - } - } + #[test] + fn can_generate_unique_ids() { + assert_eq!(id::create_id(0), Err(id::ErrorUninitialized)); + id::init_id(16, ""); + assert_eq!(id::create_id(0).unwrap().len(), 16); + assert_ne!(id::create_id(0).unwrap(), id::create_id(0).unwrap()); + let id1 = thread::spawn(|| id::create_id(0).unwrap()); + let id2 = thread::spawn(|| id::create_id(0).unwrap()); + assert_ne!(id1.join().unwrap(), id2.join().unwrap()); } } diff --git a/packages/backend/native-utils/tests/common.rs b/packages/backend/native-utils/tests/common.rs index b134319ca0..674297606f 100644 --- a/packages/backend/native-utils/tests/common.rs +++ b/packages/backend/native-utils/tests/common.rs @@ -139,11 +139,11 @@ async fn cleanup() { } async fn setup_model(db: &DbConn) { - init_id(12, ""); + init_id(16, ""); db.transaction::<_, (), DbErr>(|txn| { Box::pin(async move { - let user_id = create_id().unwrap(); + let user_id = create_id(0).unwrap(); let name = "Alice"; let user_model = entity::user::Model { id: user_id.to_owned(), @@ -161,7 +161,7 @@ async fn setup_model(db: &DbConn) { .insert(txn) .await?; let antenna_model = entity::antenna::Model { - id: create_id().unwrap(), + id: create_id(0).unwrap(), created_at: Utc::now().into(), user_id: user_id.to_owned(), name: "Alice Antenna".to_string(), @@ -186,7 +186,7 @@ async fn setup_model(db: &DbConn) { .insert(txn) .await?; let note_model = entity::note::Model { - id: create_id().unwrap(), + id: create_id(0).unwrap(), created_at: Utc::now().into(), text: Some("Testing 123".to_string()), user_id: user_id.to_owned(), diff --git a/packages/backend/native-utils/tests/model/repository/antenna.rs b/packages/backend/native-utils/tests/model/repository/antenna.rs index 80eea67718..9b03d3652d 100644 --- a/packages/backend/native-utils/tests/model/repository/antenna.rs +++ b/packages/backend/native-utils/tests/model/repository/antenna.rs @@ -93,7 +93,7 @@ mod int_test { .unwrap() .expect("note not found"); let antenna_note = antenna_note::Model { - id: util::id::create_id().unwrap(), + id: util::id::create_id(0).unwrap(), antenna_id: alice_antenna.id.to_owned(), note_id: note_model.id.to_owned(), read: false, diff --git a/packages/backend/src/misc/gen-id.ts b/packages/backend/src/misc/gen-id.ts index ea0d414e77..580c39c3c1 100644 --- a/packages/backend/src/misc/gen-id.ts +++ b/packages/backend/src/misc/gen-id.ts @@ -17,5 +17,5 @@ nativeInitIdGenerator(length, fingerprint); * Ref: https://github.com/paralleldrive/cuid2#parameterized-length */ export function genId(date?: Date): string { - return nativeCreateId(BigInt((date ?? new Date()).getTime())); + return nativeCreateId((date ?? new Date()).getTime()); } diff --git a/packages/backend/src/services/add-note-to-antenna.ts b/packages/backend/src/services/add-note-to-antenna.ts index 131c0348c7..8d6d3e84d0 100644 --- a/packages/backend/src/services/add-note-to-antenna.ts +++ b/packages/backend/src/services/add-note-to-antenna.ts @@ -8,17 +8,14 @@ import type { User } from "@/models/entities/user.js"; export async function addNoteToAntenna( antenna: Antenna, note: Note, - noteUser: { id: User["id"] }, + _noteUser: { id: User["id"] }, ) { - // 通知しない設定になっているか、自分自身の投稿なら既読にする - const read = !antenna.notify || antenna.userId === noteUser.id; - redisClient.xadd( `antennaTimeline:${antenna.id}`, "MAXLEN", "~", "200", - `${genId(note.createdAt)}-*`, + "*", "note", note.id, );