copy existing posts in antenna to redis at migration
This commit is contained in:
Namekuji
parent
6d7acfb7d3
commit
61f0f52d42
14 changed files with 516 additions and 86 deletions
|
|
@ -1,9 +1,7 @@
|
|||
export class cleanup1680491187535 {
|
||||
name = "cleanup1680491187535";
|
||||
|
||||
async up(queryRunner) {
|
||||
await queryRunner.query(`DROP TABLE "antenna_note" `);
|
||||
}
|
||||
async up(queryRunner) {}
|
||||
|
||||
async down(queryRunner) {}
|
||||
}
|
||||
|
|
|
|||
145
packages/backend/native-utils/Cargo.lock
generated
145
packages/backend/native-utils/Cargo.lock
generated
|
|
@ -458,6 +458,20 @@ version = "1.0.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"
|
||||
|
||||
[[package]]
|
||||
name = "combine"
|
||||
version = "4.6.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"memchr",
|
||||
"pin-project-lite",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "console"
|
||||
version = "0.15.7"
|
||||
|
|
@ -486,6 +500,16 @@ dependencies = [
|
|||
"unicode-segmentation",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "core-foundation"
|
||||
version = "0.9.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146"
|
||||
dependencies = [
|
||||
"core-foundation-sys",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "core-foundation-sys"
|
||||
version = "0.8.4"
|
||||
|
|
@ -1278,11 +1302,14 @@ dependencies = [
|
|||
"futures",
|
||||
"indicatif",
|
||||
"native-utils",
|
||||
"redis",
|
||||
"sea-orm",
|
||||
"sea-orm-migration",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_yaml",
|
||||
"tokio",
|
||||
"url",
|
||||
"urlencoding",
|
||||
]
|
||||
|
||||
|
|
@ -1509,6 +1536,12 @@ version = "1.18.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
|
||||
|
||||
[[package]]
|
||||
name = "openssl-probe"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
|
||||
|
||||
[[package]]
|
||||
name = "os_str_bytes"
|
||||
version = "6.5.0"
|
||||
|
|
@ -1843,6 +1876,29 @@ dependencies = [
|
|||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redis"
|
||||
version = "0.23.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3ea8c51b5dc1d8e5fd3350ec8167f464ec0995e79f2e90a075b63371500d557f"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"combine",
|
||||
"futures-util",
|
||||
"itoa",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"rustls 0.21.3",
|
||||
"rustls-native-certs",
|
||||
"ryu",
|
||||
"sha1_smol",
|
||||
"tokio",
|
||||
"tokio-rustls 0.24.1",
|
||||
"tokio-util",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.2.16"
|
||||
|
|
@ -2043,6 +2099,30 @@ dependencies = [
|
|||
"webpki",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls"
|
||||
version = "0.21.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b19faa85ecb5197342b54f987b142fb3e30d0c90da40f80ef4fa9a726e6676ed"
|
||||
dependencies = [
|
||||
"log",
|
||||
"ring",
|
||||
"rustls-webpki",
|
||||
"sct",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls-native-certs"
|
||||
version = "0.6.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00"
|
||||
dependencies = [
|
||||
"openssl-probe",
|
||||
"rustls-pemfile",
|
||||
"schannel",
|
||||
"security-framework",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls-pemfile"
|
||||
version = "1.0.2"
|
||||
|
|
@ -2052,6 +2132,16 @@ dependencies = [
|
|||
"base64 0.21.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls-webpki"
|
||||
version = "0.101.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "15f36a6828982f422756984e47912a7a51dcbc2a197aa791158f8ca61cd8204e"
|
||||
dependencies = [
|
||||
"ring",
|
||||
"untrusted",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustversion"
|
||||
version = "1.0.12"
|
||||
|
|
@ -2076,6 +2166,15 @@ version = "1.0.13"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041"
|
||||
|
||||
[[package]]
|
||||
name = "schannel"
|
||||
version = "0.1.22"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88"
|
||||
dependencies = [
|
||||
"windows-sys 0.48.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "schemars"
|
||||
version = "0.8.12"
|
||||
|
|
@ -2286,6 +2385,29 @@ version = "4.1.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b"
|
||||
|
||||
[[package]]
|
||||
name = "security-framework"
|
||||
version = "2.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1fc758eb7bffce5b308734e9b0c1468893cae9ff70ebf13e7090be8dcbcc83a8"
|
||||
dependencies = [
|
||||
"bitflags 1.3.2",
|
||||
"core-foundation",
|
||||
"core-foundation-sys",
|
||||
"libc",
|
||||
"security-framework-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "security-framework-sys"
|
||||
version = "2.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f51d0c0d83bec45f16480d0ce0058397a69e48fcdc52d1dc8855fb68acbd31a7"
|
||||
dependencies = [
|
||||
"core-foundation-sys",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "semver"
|
||||
version = "1.0.17"
|
||||
|
|
@ -2370,6 +2492,12 @@ dependencies = [
|
|||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha1_smol"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012"
|
||||
|
||||
[[package]]
|
||||
name = "sha2"
|
||||
version = "0.10.6"
|
||||
|
|
@ -2518,7 +2646,7 @@ dependencies = [
|
|||
"percent-encoding",
|
||||
"rand",
|
||||
"rust_decimal",
|
||||
"rustls",
|
||||
"rustls 0.20.8",
|
||||
"rustls-pemfile",
|
||||
"serde",
|
||||
"serde_json",
|
||||
|
|
@ -2564,7 +2692,7 @@ checksum = "804d3f245f894e61b1e6263c84b23ca675d96753b5abfd5cc8597d86806e8024"
|
|||
dependencies = [
|
||||
"once_cell",
|
||||
"tokio",
|
||||
"tokio-rustls",
|
||||
"tokio-rustls 0.23.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -2778,11 +2906,21 @@ version = "0.23.4"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59"
|
||||
dependencies = [
|
||||
"rustls",
|
||||
"rustls 0.20.8",
|
||||
"tokio",
|
||||
"webpki",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-rustls"
|
||||
version = "0.24.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081"
|
||||
dependencies = [
|
||||
"rustls 0.21.3",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-stream"
|
||||
version = "0.1.14"
|
||||
|
|
@ -2949,6 +3087,7 @@ dependencies = [
|
|||
"form_urlencoded",
|
||||
"idna",
|
||||
"percent-encoding",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ members = ["migration"]
|
|||
[features]
|
||||
default = []
|
||||
noarray = []
|
||||
napi = ["dep:napi", "dep:napi-derive", "dep:radix_fmt"]
|
||||
napi = ["dep:napi", "dep:napi-derive"]
|
||||
|
||||
[lib]
|
||||
crate-type = ["cdylib", "lib"]
|
||||
|
|
@ -31,11 +31,11 @@ serde_json = "1.0.96"
|
|||
thiserror = "1.0.40"
|
||||
tokio = { version = "1.28.1", features = ["full"] }
|
||||
utoipa = "3.3.0"
|
||||
radix_fmt = "1.0.0"
|
||||
|
||||
# Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix
|
||||
napi = { version = "2.13.1", default-features = false, features = ["napi6", "tokio_rt"], optional = true }
|
||||
napi-derive = { version = "2.12.0", optional = true }
|
||||
radix_fmt = { version = "1.0.0", optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
pretty_assertions = "1.3.0"
|
||||
|
|
|
|||
|
|
@ -12,18 +12,18 @@ test("convert to mastodon id", (t) => {
|
|||
t.is(convertId("9gf61ehcxv", IdConvertType.MastodonId), "960365976481219");
|
||||
t.is(
|
||||
convertId("9fbr9z0wbrjqyd3u", IdConvertType.MastodonId),
|
||||
"3954607381600562394",
|
||||
"2083785058661759970208986",
|
||||
);
|
||||
t.is(
|
||||
convertId("9fbs680oyviiqrol9md73p8g", IdConvertType.MastodonId),
|
||||
"3494513243013053824",
|
||||
"5878598648988104013828532260828151168",
|
||||
);
|
||||
});
|
||||
|
||||
test("create cuid2 with timestamp prefix", (t) => {
|
||||
nativeInitIdGenerator(16, "");
|
||||
t.not(nativeCreateId(BigInt(Date.now())), nativeCreateId(BigInt(Date.now())));
|
||||
t.is(nativeCreateId(BigInt(Date.now())).length, 16);
|
||||
t.not(nativeCreateId(Date.now()), nativeCreateId(Date.now()));
|
||||
t.is(nativeCreateId(Date.now()).length, 16);
|
||||
});
|
||||
|
||||
test("create random string", (t) => {
|
||||
|
|
|
|||
|
|
@ -10,17 +10,20 @@ path = "src/lib.rs"
|
|||
|
||||
[features]
|
||||
default = []
|
||||
convert = ["dep:native-utils", "dep:indicatif", "dep:futures"]
|
||||
convert = ["dep:indicatif"]
|
||||
|
||||
[dependencies]
|
||||
serde_json = "1.0.96"
|
||||
native-utils = { path = "../", optional = true }
|
||||
native-utils = { path = "../" }
|
||||
indicatif = { version = "0.17.4", features = ["tokio"], optional = true }
|
||||
tokio = { version = "1.28.2", features = ["full"] }
|
||||
futures = { version = "0.3.28", optional = true }
|
||||
futures = "0.3.28"
|
||||
serde_yaml = "0.9.21"
|
||||
serde = { version = "1.0.163", features = ["derive"] }
|
||||
urlencoding = "2.1.2"
|
||||
redis = { version = "0.23.0", features = ["tokio-rustls-comp"] }
|
||||
sea-orm = "0.11.3"
|
||||
url = { version = "2.4.0", features = ["serde"] }
|
||||
|
||||
[dependencies.sea-orm-migration]
|
||||
version = "0.11.0"
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ pub use sea_orm_migration::prelude::*;
|
|||
|
||||
mod m20230531_180824_drop_reversi;
|
||||
mod m20230627_185451_index_note_url;
|
||||
mod m20230709_000510_move_antenna_to_cache;
|
||||
|
||||
pub struct Migrator;
|
||||
|
||||
|
|
@ -11,6 +12,7 @@ impl MigratorTrait for Migrator {
|
|||
vec![
|
||||
Box::new(m20230531_180824_drop_reversi::Migration),
|
||||
Box::new(m20230627_185451_index_note_url::Migration),
|
||||
Box::new(m20230709_000510_move_antenna_to_cache::Migration),
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,240 @@
|
|||
use redis::streams::StreamMaxlen;
|
||||
use sea_orm::Statement;
|
||||
use sea_orm_migration::prelude::*;
|
||||
use std::env;
|
||||
|
||||
#[derive(DeriveMigrationName)]
|
||||
pub struct Migration;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl MigrationTrait for Migration {
|
||||
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
|
||||
let cache_url = env::var("CACHE_URL").unwrap();
|
||||
let copy_limit = env::var("ANTENNA_MIGRATION_LIMIT").unwrap_or_default();
|
||||
let copy_limit: i64 = match copy_limit.parse() {
|
||||
Ok(limit) => limit,
|
||||
Err(_) => 0,
|
||||
};
|
||||
|
||||
if cache_url != "no" {
|
||||
let prefix = env::var("CACHE_PREFIX").unwrap();
|
||||
|
||||
let db = manager.get_connection();
|
||||
let bk = manager.get_database_backend();
|
||||
|
||||
let count_stmt =
|
||||
Statement::from_string(bk, "SELECT COUNT(1) FROM antenna_note".to_owned());
|
||||
let total_num = db
|
||||
.query_one(count_stmt)
|
||||
.await?
|
||||
.unwrap()
|
||||
.try_get_by_index::<i64>(0)?;
|
||||
let copy_limit = if copy_limit == 0 {
|
||||
total_num
|
||||
} else {
|
||||
copy_limit
|
||||
};
|
||||
println!(
|
||||
"Copying {} out of {} entries in antenna_note.",
|
||||
copy_limit, total_num
|
||||
);
|
||||
|
||||
let stmt_base = Query::select()
|
||||
.column((AntennaNote::Table, AntennaNote::Id))
|
||||
.column(AntennaNote::AntennaId)
|
||||
.column(AntennaNote::NoteId)
|
||||
.from(AntennaNote::Table)
|
||||
.order_by((AntennaNote::Table, AntennaNote::Id), Order::Asc)
|
||||
.limit(1000)
|
||||
.to_owned();
|
||||
|
||||
let mut stmt = stmt_base.clone();
|
||||
|
||||
let client = redis::Client::open(cache_url).unwrap();
|
||||
let mut redis_conn = client.get_connection().unwrap();
|
||||
|
||||
let mut remaining = total_num;
|
||||
let mut pagination: i64 = 0;
|
||||
|
||||
loop {
|
||||
let res = db.query_all(bk.build(&stmt)).await?;
|
||||
if res.len() == 0 {
|
||||
break;
|
||||
}
|
||||
let val: Vec<(String, String, String)> = res
|
||||
.iter()
|
||||
.filter_map(|q| q.try_get_many_by_index().ok())
|
||||
.collect();
|
||||
|
||||
remaining -= val.len() as i64;
|
||||
if remaining <= copy_limit {
|
||||
let mut pipe = redis::pipe();
|
||||
for v in &val {
|
||||
pipe.xadd_maxlen(
|
||||
format!("{}:antennaTimeline:{}", prefix, v.1),
|
||||
StreamMaxlen::Approx(200),
|
||||
"*",
|
||||
&[("note", v.2.to_owned())],
|
||||
)
|
||||
.ignore();
|
||||
}
|
||||
pipe.query::<()>(&mut redis_conn).unwrap();
|
||||
}
|
||||
|
||||
let copied = total_num - remaining;
|
||||
let copied = std::cmp::min(copied, total_num);
|
||||
pagination += 1;
|
||||
if pagination % 100 == 0 {
|
||||
println!(
|
||||
"Migrating antenna [{:.2}%]",
|
||||
(copied as f64 / total_num as f64) * 100_f64,
|
||||
);
|
||||
}
|
||||
|
||||
if let Some((last_id, _, _)) = val.last() {
|
||||
stmt = stmt_base
|
||||
.clone()
|
||||
.and_where(
|
||||
Expr::col((AntennaNote::Table, AntennaNote::Id)).gt(last_id.to_owned()),
|
||||
)
|
||||
.to_owned();
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
println!("Migrating antenna [100.00%]");
|
||||
|
||||
manager
|
||||
.drop_table(
|
||||
Table::drop()
|
||||
.table(AntennaNote::Table)
|
||||
.if_exists()
|
||||
.to_owned(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
|
||||
manager
|
||||
.create_table(
|
||||
Table::create()
|
||||
.table(AntennaNote::Table)
|
||||
.if_not_exists()
|
||||
.col(
|
||||
ColumnDef::new(AntennaNote::Id)
|
||||
.string_len(32)
|
||||
.not_null()
|
||||
.primary_key(),
|
||||
)
|
||||
.col(
|
||||
ColumnDef::new(AntennaNote::NoteId)
|
||||
.string_len(32)
|
||||
.not_null(),
|
||||
)
|
||||
.col(
|
||||
ColumnDef::new(AntennaNote::AntennaId)
|
||||
.string_len(32)
|
||||
.not_null(),
|
||||
)
|
||||
.col(
|
||||
ColumnDef::new(AntennaNote::Read)
|
||||
.boolean()
|
||||
.default(false)
|
||||
.not_null(),
|
||||
)
|
||||
.to_owned(),
|
||||
)
|
||||
.await?;
|
||||
manager
|
||||
.create_index(
|
||||
Index::create()
|
||||
.name("IDX_0d775946662d2575dfd2068a5f")
|
||||
.table(AntennaNote::Table)
|
||||
.col(AntennaNote::AntennaId)
|
||||
.if_not_exists()
|
||||
.to_owned(),
|
||||
)
|
||||
.await?;
|
||||
manager
|
||||
.create_index(
|
||||
Index::create()
|
||||
.name("IDX_bd0397be22147e17210940e125")
|
||||
.table(AntennaNote::Table)
|
||||
.col(AntennaNote::NoteId)
|
||||
.if_not_exists()
|
||||
.to_owned(),
|
||||
)
|
||||
.await?;
|
||||
manager
|
||||
.create_index(
|
||||
Index::create()
|
||||
.name("IDX_335a0bf3f904406f9ef3dd51c2")
|
||||
.table(AntennaNote::Table)
|
||||
.col(AntennaNote::NoteId)
|
||||
.col(AntennaNote::AntennaId)
|
||||
.unique()
|
||||
.if_not_exists()
|
||||
.to_owned(),
|
||||
)
|
||||
.await?;
|
||||
manager
|
||||
.create_index(
|
||||
Index::create()
|
||||
.name("IDX_9937ea48d7ae97ffb4f3f063a4")
|
||||
.table(AntennaNote::Table)
|
||||
.col(AntennaNote::Read)
|
||||
.if_not_exists()
|
||||
.to_owned(),
|
||||
)
|
||||
.await?;
|
||||
manager
|
||||
.create_foreign_key(
|
||||
ForeignKey::create()
|
||||
.name("FK_0d775946662d2575dfd2068a5f5")
|
||||
.from(AntennaNote::Table, AntennaNote::AntennaId)
|
||||
.to(Antenna::Table, Antenna::Id)
|
||||
.on_delete(ForeignKeyAction::Cascade)
|
||||
.to_owned(),
|
||||
)
|
||||
.await?;
|
||||
manager
|
||||
.create_foreign_key(
|
||||
ForeignKey::create()
|
||||
.name("FK_bd0397be22147e17210940e125b")
|
||||
.from(AntennaNote::Table, AntennaNote::NoteId)
|
||||
.to(Note::Table, Note::Id)
|
||||
.on_delete(ForeignKeyAction::Cascade)
|
||||
.to_owned(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Learn more at https://docs.rs/sea-query#iden
|
||||
#[derive(Iden)]
|
||||
enum AntennaNote {
|
||||
Table,
|
||||
Id,
|
||||
#[iden = "noteId"]
|
||||
NoteId,
|
||||
#[iden = "antennaId"]
|
||||
AntennaId,
|
||||
Read,
|
||||
}
|
||||
|
||||
#[derive(Iden)]
|
||||
enum Antenna {
|
||||
Table,
|
||||
Id,
|
||||
}
|
||||
|
||||
#[derive(Iden)]
|
||||
enum Note {
|
||||
Table,
|
||||
Id,
|
||||
}
|
||||
|
|
@ -5,6 +5,10 @@ use urlencoding::encode;
|
|||
|
||||
use sea_orm_migration::prelude::*;
|
||||
|
||||
const DB_URL_ENV: &str = "DATABASE_URL";
|
||||
const CACHE_URL_ENV: &str = "CACHE_URL";
|
||||
const CACHE_PREFIX_ENV: &str = "CACHE_PREFIX";
|
||||
|
||||
#[cfg(feature = "convert")]
|
||||
mod vec_to_json;
|
||||
|
||||
|
|
@ -15,17 +19,48 @@ async fn main() {
|
|||
.expect("Failed to open '.config/default.yml'");
|
||||
let config: Config = serde_yaml::from_reader(yml).expect("Failed to parse yaml");
|
||||
|
||||
env::set_var(
|
||||
"DATABASE_URL",
|
||||
format!(
|
||||
"postgres://{}:{}@{}:{}/{}",
|
||||
config.db.user,
|
||||
encode(&config.db.pass),
|
||||
config.db.host,
|
||||
config.db.port,
|
||||
config.db.db,
|
||||
),
|
||||
);
|
||||
if env::var_os(DB_URL_ENV).is_none() {
|
||||
env::set_var(
|
||||
DB_URL_ENV,
|
||||
format!(
|
||||
"postgres://{}:{}@{}:{}/{}",
|
||||
config.db.user,
|
||||
encode(&config.db.pass),
|
||||
config.db.host,
|
||||
config.db.port,
|
||||
config.db.db,
|
||||
),
|
||||
);
|
||||
};
|
||||
|
||||
if env::var_os(CACHE_URL_ENV).is_none() {
|
||||
let redis_conf = match config.cache_server {
|
||||
None => config.redis,
|
||||
Some(conf) => conf,
|
||||
};
|
||||
let redis_proto = match redis_conf.tls {
|
||||
None => "redis",
|
||||
Some(_) => "rediss",
|
||||
};
|
||||
let redis_uri_userpass = match redis_conf.user {
|
||||
None => "".to_string(),
|
||||
Some(user) => format!("{}:{}@", user, redis_conf.pass.unwrap_or_default()),
|
||||
};
|
||||
let redis_uri_hostport = format!("{}:{}", redis_conf.host, redis_conf.port);
|
||||
let redis_uri = format!(
|
||||
"{}://{}{}",
|
||||
redis_proto, redis_uri_userpass, redis_uri_hostport
|
||||
);
|
||||
env::set_var(CACHE_URL_ENV, redis_uri);
|
||||
env::set_var(
|
||||
CACHE_PREFIX_ENV,
|
||||
if redis_conf.prefix.is_empty() {
|
||||
config.url.host_str().unwrap()
|
||||
} else {
|
||||
&redis_conf.prefix
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
cli::run_cli(migration::Migrator).await;
|
||||
|
||||
|
|
@ -36,7 +71,10 @@ async fn main() {
|
|||
#[derive(Debug, PartialEq, Deserialize)]
|
||||
#[serde(rename = "camelCase")]
|
||||
pub struct Config {
|
||||
pub url: url::Url,
|
||||
pub db: DbConfig,
|
||||
pub redis: RedisConfig,
|
||||
pub cache_server: Option<RedisConfig>,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Deserialize)]
|
||||
|
|
@ -48,3 +86,24 @@ pub struct DbConfig {
|
|||
pub user: String,
|
||||
pub pass: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Deserialize)]
|
||||
#[serde(rename = "camelCase")]
|
||||
pub struct RedisConfig {
|
||||
pub host: String,
|
||||
pub port: u32,
|
||||
pub user: Option<String>,
|
||||
pub pass: Option<String>,
|
||||
pub tls: Option<TlsConfig>,
|
||||
#[serde(default)]
|
||||
pub db: u32,
|
||||
#[serde(default)]
|
||||
pub prefix: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Deserialize)]
|
||||
#[serde(rename = "camelCase")]
|
||||
pub struct TlsConfig {
|
||||
pub host: String,
|
||||
pub reject_unauthorized: bool,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -105,9 +105,9 @@ mod unit_test {
|
|||
|
||||
#[test]
|
||||
fn app_valid() {
|
||||
init_id(12, "");
|
||||
init_id(16, "");
|
||||
let instance = json!({
|
||||
"id": create_id().unwrap(),
|
||||
"id": create_id(0).unwrap(),
|
||||
"name": "Test App",
|
||||
"secret": gen_string(24),
|
||||
"callbackUrl": "urn:ietf:wg:oauth:2.0:oob",
|
||||
|
|
@ -119,9 +119,9 @@ mod unit_test {
|
|||
|
||||
#[test]
|
||||
fn app_invalid() {
|
||||
init_id(12, "");
|
||||
init_id(16, "");
|
||||
let instance = json!({
|
||||
"id": create_id().unwrap(),
|
||||
"id": create_id(0).unwrap(),
|
||||
// "name" is required
|
||||
"name": null,
|
||||
// "permission" must be one of the app permissions
|
||||
|
|
|
|||
|
|
@ -1,7 +1,10 @@
|
|||
//! ID generation utility based on [cuid2]
|
||||
|
||||
use cfg_if::cfg_if;
|
||||
use chrono::Utc;
|
||||
use once_cell::sync::OnceCell;
|
||||
use radix_fmt::radix_36;
|
||||
use std::cmp;
|
||||
|
||||
use crate::impl_into_napi_error;
|
||||
|
||||
|
|
@ -14,47 +17,56 @@ impl_into_napi_error!(ErrorUninitialized);
|
|||
static FINGERPRINT: OnceCell<String> = OnceCell::new();
|
||||
static GENERATOR: OnceCell<cuid2::CuidConstructor> = OnceCell::new();
|
||||
|
||||
const TIME_2000: i64 = 946_684_800_000;
|
||||
const TIMESTAMP_LENGTH: u16 = 8;
|
||||
|
||||
/// Initializes Cuid2 generator. Must be called before any [create_id].
|
||||
pub fn init_id(length: u16, fingerprint: impl Into<String>) {
|
||||
FINGERPRINT.get_or_init(move || format!("{}{}", fingerprint.into(), cuid2::create_id()));
|
||||
pub fn init_id<'a>(length: u16, fingerprint: &'a str) {
|
||||
FINGERPRINT.get_or_init(move || format!("{}{}", fingerprint, cuid2::create_id()));
|
||||
GENERATOR.get_or_init(move || {
|
||||
cuid2::CuidConstructor::new()
|
||||
.with_length(length)
|
||||
// length to pass shoule be greater than or equal to 8.
|
||||
.with_length(cmp::max(length - TIMESTAMP_LENGTH, 8))
|
||||
.with_fingerprinter(|| FINGERPRINT.get().unwrap().clone())
|
||||
});
|
||||
}
|
||||
|
||||
/// Returns Cuid2 with the length specified by [init_id]. Must be called after
|
||||
/// [init_id], otherwise returns [ErrorUninitialized].
|
||||
pub fn create_id() -> Result<String, ErrorUninitialized> {
|
||||
/// The current timestamp via [chrono::Utc] is used if `date_num` is `0`.
|
||||
pub fn create_id(date_num: i64) -> Result<String, ErrorUninitialized> {
|
||||
match GENERATOR.get() {
|
||||
None => Err(ErrorUninitialized),
|
||||
Some(gen) => Ok(gen.create_id()),
|
||||
Some(gen) => {
|
||||
let date_num = if date_num > 0 {
|
||||
date_num
|
||||
} else {
|
||||
Utc::now().timestamp_millis()
|
||||
};
|
||||
let time = cmp::max(date_num - TIME_2000, 0);
|
||||
Ok(format!(
|
||||
"{:0>8}{}",
|
||||
radix_36(time).to_string(),
|
||||
gen.create_id()
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
cfg_if! {
|
||||
if #[cfg(feature = "napi")] {
|
||||
use radix_fmt::radix_36;
|
||||
use std::cmp;
|
||||
use napi::bindgen_prelude::BigInt;
|
||||
use napi_derive::napi;
|
||||
|
||||
const TIME_2000: u64 = 946_684_800_000;
|
||||
const TIMESTAMP_LENGTH: u16 = 8;
|
||||
|
||||
/// Calls [init_id] inside. Must be called before [native_create_id].
|
||||
#[napi]
|
||||
pub fn native_init_id_generator(length: u16, fingerprint: String) {
|
||||
// length to pass init_id shoule be greater than or equal to 8.
|
||||
init_id(cmp::max(length - TIMESTAMP_LENGTH, 8), fingerprint);
|
||||
init_id(length, &fingerprint);
|
||||
}
|
||||
|
||||
/// Generates
|
||||
#[napi]
|
||||
pub fn native_create_id(date_num: BigInt) -> String {
|
||||
let time = cmp::max(date_num.get_u64().1 - TIME_2000, 0);
|
||||
format!("{:0>8}{}", radix_36(time).to_string(), create_id().unwrap())
|
||||
pub fn native_create_id(date_num: i64) -> String {
|
||||
create_id(date_num).unwrap()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -62,37 +74,17 @@ cfg_if! {
|
|||
#[cfg(test)]
|
||||
mod unit_test {
|
||||
use crate::util::id;
|
||||
use cfg_if::cfg_if;
|
||||
use pretty_assertions::{assert_eq, assert_ne};
|
||||
use std::thread;
|
||||
|
||||
cfg_if! {
|
||||
if #[cfg(feature = "napi")] {
|
||||
use chrono::Utc;
|
||||
|
||||
#[test]
|
||||
fn can_generate_aid_compat_ids() {
|
||||
id::native_init_id_generator(20, "".to_string());
|
||||
let id1 = id::native_create_id(Utc::now().timestamp_millis().into());
|
||||
assert_eq!(id1.len(), 20);
|
||||
let id1 = id::native_create_id(Utc::now().timestamp_millis().into());
|
||||
let id2 = id::native_create_id(Utc::now().timestamp_millis().into());
|
||||
assert_ne!(id1, id2);
|
||||
let id1 = thread::spawn(|| id::native_create_id(Utc::now().timestamp_millis().into()));
|
||||
let id2 = thread::spawn(|| id::native_create_id(Utc::now().timestamp_millis().into()));
|
||||
assert_ne!(id1.join().unwrap(), id2.join().unwrap());
|
||||
}
|
||||
} else {
|
||||
#[test]
|
||||
fn can_generate_unique_ids() {
|
||||
assert_eq!(id::create_id(), Err(id::ErrorUninitialized));
|
||||
id::init_id(12, "");
|
||||
assert_eq!(id::create_id().unwrap().len(), 12);
|
||||
assert_ne!(id::create_id().unwrap(), id::create_id().unwrap());
|
||||
let id1 = thread::spawn(|| id::create_id().unwrap());
|
||||
let id2 = thread::spawn(|| id::create_id().unwrap());
|
||||
assert_ne!(id1.join().unwrap(), id2.join().unwrap());
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn can_generate_unique_ids() {
|
||||
assert_eq!(id::create_id(0), Err(id::ErrorUninitialized));
|
||||
id::init_id(16, "");
|
||||
assert_eq!(id::create_id(0).unwrap().len(), 16);
|
||||
assert_ne!(id::create_id(0).unwrap(), id::create_id(0).unwrap());
|
||||
let id1 = thread::spawn(|| id::create_id(0).unwrap());
|
||||
let id2 = thread::spawn(|| id::create_id(0).unwrap());
|
||||
assert_ne!(id1.join().unwrap(), id2.join().unwrap());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -139,11 +139,11 @@ async fn cleanup() {
|
|||
}
|
||||
|
||||
async fn setup_model(db: &DbConn) {
|
||||
init_id(12, "");
|
||||
init_id(16, "");
|
||||
|
||||
db.transaction::<_, (), DbErr>(|txn| {
|
||||
Box::pin(async move {
|
||||
let user_id = create_id().unwrap();
|
||||
let user_id = create_id(0).unwrap();
|
||||
let name = "Alice";
|
||||
let user_model = entity::user::Model {
|
||||
id: user_id.to_owned(),
|
||||
|
|
@ -161,7 +161,7 @@ async fn setup_model(db: &DbConn) {
|
|||
.insert(txn)
|
||||
.await?;
|
||||
let antenna_model = entity::antenna::Model {
|
||||
id: create_id().unwrap(),
|
||||
id: create_id(0).unwrap(),
|
||||
created_at: Utc::now().into(),
|
||||
user_id: user_id.to_owned(),
|
||||
name: "Alice Antenna".to_string(),
|
||||
|
|
@ -186,7 +186,7 @@ async fn setup_model(db: &DbConn) {
|
|||
.insert(txn)
|
||||
.await?;
|
||||
let note_model = entity::note::Model {
|
||||
id: create_id().unwrap(),
|
||||
id: create_id(0).unwrap(),
|
||||
created_at: Utc::now().into(),
|
||||
text: Some("Testing 123".to_string()),
|
||||
user_id: user_id.to_owned(),
|
||||
|
|
|
|||
|
|
@ -95,7 +95,7 @@ mod int_test {
|
|||
.unwrap()
|
||||
.expect("note not found");
|
||||
let antenna_note = antenna_note::Model {
|
||||
id: util::id::create_id().unwrap(),
|
||||
id: util::id::create_id(0).unwrap(),
|
||||
antenna_id: alice_antenna.id.to_owned(),
|
||||
note_id: note_model.id.to_owned(),
|
||||
read: false,
|
||||
|
|
|
|||
|
|
@ -17,5 +17,5 @@ nativeInitIdGenerator(length, fingerprint);
|
|||
* Ref: https://github.com/paralleldrive/cuid2#parameterized-length
|
||||
*/
|
||||
export function genId(date?: Date): string {
|
||||
return nativeCreateId(BigInt((date ?? new Date()).getTime()));
|
||||
return nativeCreateId((date ?? new Date()).getTime());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,17 +8,14 @@ import type { User } from "@/models/entities/user.js";
|
|||
export async function addNoteToAntenna(
|
||||
antenna: Antenna,
|
||||
note: Note,
|
||||
noteUser: { id: User["id"] },
|
||||
_noteUser: { id: User["id"] },
|
||||
) {
|
||||
// 通知しない設定になっているか、自分自身の投稿なら既読にする
|
||||
const read = !antenna.notify || antenna.userId === noteUser.id;
|
||||
|
||||
redisClient.xadd(
|
||||
`antennaTimeline:${antenna.id}`,
|
||||
"MAXLEN",
|
||||
"~",
|
||||
"200",
|
||||
`${genId(note.createdAt)}-*`,
|
||||
"*",
|
||||
"note",
|
||||
note.id,
|
||||
);
|
||||
|
|
|
|||
Loading…
Reference in a new issue