fix: generate stream id with timestamp
This commit is contained in:
parent
33b4b1d18c
commit
738b4933ae
8 changed files with 57 additions and 20 deletions
14
packages/backend/native-utils/Cargo.lock
generated
14
packages/backend/native-utils/Cargo.lock
generated
|
@ -196,6 +196,12 @@ version = "0.21.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d"
|
checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "basen"
|
||||||
|
version = "0.1.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "1dbe4bb73fd931c4d1aaf53b35d1286c8a948ad00ec92c8e3c856f15fd027f43"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "bigdecimal"
|
name = "bigdecimal"
|
||||||
version = "0.3.1"
|
version = "0.3.1"
|
||||||
|
@ -1399,6 +1405,7 @@ name = "native-utils"
|
||||||
version = "0.0.0"
|
version = "0.0.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-trait",
|
"async-trait",
|
||||||
|
"basen",
|
||||||
"cfg-if",
|
"cfg-if",
|
||||||
"chrono",
|
"chrono",
|
||||||
"cuid2",
|
"cuid2",
|
||||||
|
@ -1410,7 +1417,6 @@ dependencies = [
|
||||||
"once_cell",
|
"once_cell",
|
||||||
"parse-display",
|
"parse-display",
|
||||||
"pretty_assertions",
|
"pretty_assertions",
|
||||||
"radix_fmt",
|
|
||||||
"rand",
|
"rand",
|
||||||
"schemars",
|
"schemars",
|
||||||
"sea-orm",
|
"sea-orm",
|
||||||
|
@ -1831,12 +1837,6 @@ version = "0.7.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09"
|
checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09"
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "radix_fmt"
|
|
||||||
version = "1.0.0"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "ce082a9940a7ace2ad4a8b7d0b1eac6aa378895f18be598230c5f2284ac05426"
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rand"
|
name = "rand"
|
||||||
version = "0.8.5"
|
version = "0.8.5"
|
||||||
|
|
|
@ -31,11 +31,11 @@ serde_json = "1.0.96"
|
||||||
thiserror = "1.0.40"
|
thiserror = "1.0.40"
|
||||||
tokio = { version = "1.28.1", features = ["full"] }
|
tokio = { version = "1.28.1", features = ["full"] }
|
||||||
utoipa = "3.3.0"
|
utoipa = "3.3.0"
|
||||||
radix_fmt = "1.0.0"
|
|
||||||
|
|
||||||
# Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix
|
# Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix
|
||||||
napi = { version = "2.13.1", default-features = false, features = ["napi6", "tokio_rt"], optional = true }
|
napi = { version = "2.13.1", default-features = false, features = ["napi6", "tokio_rt"], optional = true }
|
||||||
napi-derive = { version = "2.12.0", optional = true }
|
napi-derive = { version = "2.12.0", optional = true }
|
||||||
|
basen = "0.1.0"
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
pretty_assertions = "1.3.0"
|
pretty_assertions = "1.3.0"
|
||||||
|
|
|
@ -10,11 +10,11 @@ path = "src/lib.rs"
|
||||||
|
|
||||||
[features]
|
[features]
|
||||||
default = []
|
default = []
|
||||||
convert = ["dep:native-utils", "dep:indicatif", "dep:futures"]
|
convert = ["dep:indicatif", "dep:futures"]
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
serde_json = "1.0.96"
|
serde_json = "1.0.96"
|
||||||
native-utils = { path = "../", optional = true }
|
native-utils = { path = "../" }
|
||||||
indicatif = { version = "0.17.4", features = ["tokio"], optional = true }
|
indicatif = { version = "0.17.4", features = ["tokio"], optional = true }
|
||||||
tokio = { version = "1.28.2", features = ["full"] }
|
tokio = { version = "1.28.2", features = ["full"] }
|
||||||
futures = { version = "0.3.28", optional = true }
|
futures = { version = "0.3.28", optional = true }
|
||||||
|
|
|
@ -1,3 +1,4 @@
|
||||||
|
use native_utils::util::id;
|
||||||
use redis::streams::StreamMaxlen;
|
use redis::streams::StreamMaxlen;
|
||||||
use sea_orm::Statement;
|
use sea_orm::Statement;
|
||||||
use sea_orm_migration::prelude::*;
|
use sea_orm_migration::prelude::*;
|
||||||
|
@ -80,12 +81,12 @@ impl MigrationTrait for Migration {
|
||||||
pipe.xadd_maxlen(
|
pipe.xadd_maxlen(
|
||||||
format!("{}:antennaTimeline:{}", prefix, v.1),
|
format!("{}:antennaTimeline:{}", prefix, v.1),
|
||||||
StreamMaxlen::Approx(200),
|
StreamMaxlen::Approx(200),
|
||||||
"*",
|
format!("{}-*", id::get_timestamp(&v.2)),
|
||||||
&[("note", v.2.to_owned())],
|
&[("note", v.2.to_owned())],
|
||||||
)
|
)
|
||||||
.ignore();
|
.ignore();
|
||||||
}
|
}
|
||||||
pipe.query::<()>(&mut redis_conn).unwrap();
|
pipe.query::<()>(&mut redis_conn).unwrap_or(());
|
||||||
}
|
}
|
||||||
|
|
||||||
let copied = total_num - remaining;
|
let copied = total_num - remaining;
|
||||||
|
|
|
@ -1,9 +1,9 @@
|
||||||
//! ID generation utility based on [cuid2]
|
//! ID generation utility based on [cuid2]
|
||||||
|
|
||||||
|
use basen::BASE36;
|
||||||
use cfg_if::cfg_if;
|
use cfg_if::cfg_if;
|
||||||
use chrono::Utc;
|
use chrono::Utc;
|
||||||
use once_cell::sync::OnceCell;
|
use once_cell::sync::OnceCell;
|
||||||
use radix_fmt::radix_36;
|
|
||||||
use std::cmp;
|
use std::cmp;
|
||||||
|
|
||||||
use crate::impl_into_napi_error;
|
use crate::impl_into_napi_error;
|
||||||
|
@ -46,13 +46,21 @@ pub fn create_id(date_num: i64) -> Result<String, ErrorUninitialized> {
|
||||||
let time = cmp::max(date_num - TIME_2000, 0);
|
let time = cmp::max(date_num - TIME_2000, 0);
|
||||||
Ok(format!(
|
Ok(format!(
|
||||||
"{:0>8}{}",
|
"{:0>8}{}",
|
||||||
radix_36(time).to_string(),
|
BASE36.encode_var_len(&(time as u64)),
|
||||||
gen.create_id()
|
gen.create_id()
|
||||||
))
|
))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn get_timestamp(id: &str) -> i64 {
|
||||||
|
let n: Option<u64> = BASE36.decode_var_len(&id[0..8]);
|
||||||
|
match n {
|
||||||
|
None => -1,
|
||||||
|
Some(n) => n as i64 + TIME_2000,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
cfg_if! {
|
cfg_if! {
|
||||||
if #[cfg(feature = "napi")] {
|
if #[cfg(feature = "napi")] {
|
||||||
use napi_derive::napi;
|
use napi_derive::napi;
|
||||||
|
@ -68,17 +76,23 @@ cfg_if! {
|
||||||
pub fn native_create_id(date_num: i64) -> String {
|
pub fn native_create_id(date_num: i64) -> String {
|
||||||
create_id(date_num).unwrap()
|
create_id(date_num).unwrap()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[napi]
|
||||||
|
pub fn native_get_timestamp(id: String) -> i64 {
|
||||||
|
get_timestamp(&id)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod unit_test {
|
mod unit_test {
|
||||||
use crate::util::id;
|
use crate::util::id;
|
||||||
|
use chrono::Utc;
|
||||||
use pretty_assertions::{assert_eq, assert_ne};
|
use pretty_assertions::{assert_eq, assert_ne};
|
||||||
use std::thread;
|
use std::thread;
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn can_generate_unique_ids() {
|
fn can_create_and_decode() {
|
||||||
assert_eq!(id::create_id(0), Err(id::ErrorUninitialized));
|
assert_eq!(id::create_id(0), Err(id::ErrorUninitialized));
|
||||||
id::init_id(16, "");
|
id::init_id(16, "");
|
||||||
assert_eq!(id::create_id(0).unwrap().len(), 16);
|
assert_eq!(id::create_id(0).unwrap().len(), 16);
|
||||||
|
@ -86,5 +100,10 @@ mod unit_test {
|
||||||
let id1 = thread::spawn(|| id::create_id(0).unwrap());
|
let id1 = thread::spawn(|| id::create_id(0).unwrap());
|
||||||
let id2 = thread::spawn(|| id::create_id(0).unwrap());
|
let id2 = thread::spawn(|| id::create_id(0).unwrap());
|
||||||
assert_ne!(id1.join().unwrap(), id2.join().unwrap());
|
assert_ne!(id1.join().unwrap(), id2.join().unwrap());
|
||||||
|
|
||||||
|
let now = Utc::now().timestamp_millis();
|
||||||
|
let test_id = id::create_id(now).unwrap();
|
||||||
|
let timestamp = id::get_timestamp(&test_id);
|
||||||
|
assert_eq!(now, timestamp);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,6 +2,7 @@ import config from "@/config/index.js";
|
||||||
import {
|
import {
|
||||||
nativeCreateId,
|
nativeCreateId,
|
||||||
nativeInitIdGenerator,
|
nativeInitIdGenerator,
|
||||||
|
nativeGetTimestamp,
|
||||||
} from "native-utils/built/index.js";
|
} from "native-utils/built/index.js";
|
||||||
|
|
||||||
const length = Math.min(Math.max(config.cuid?.length ?? 16, 16), 24);
|
const length = Math.min(Math.max(config.cuid?.length ?? 16, 16), 24);
|
||||||
|
@ -19,3 +20,7 @@ nativeInitIdGenerator(length, fingerprint);
|
||||||
export function genId(date?: Date): string {
|
export function genId(date?: Date): string {
|
||||||
return nativeCreateId((date ?? new Date()).getTime());
|
return nativeCreateId((date ?? new Date()).getTime());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function getTimestamp(id: string): number {
|
||||||
|
return nativeGetTimestamp(id);
|
||||||
|
}
|
||||||
|
|
|
@ -2,7 +2,7 @@ import define from "../../define.js";
|
||||||
import readNote from "@/services/note/read.js";
|
import readNote from "@/services/note/read.js";
|
||||||
import { Antennas, Notes } from "@/models/index.js";
|
import { Antennas, Notes } from "@/models/index.js";
|
||||||
import { redisClient } from "@/db/redis.js";
|
import { redisClient } from "@/db/redis.js";
|
||||||
import { genId } from "@/misc/gen-id.js";
|
import { genId, getTimestamp } from "@/misc/gen-id.js";
|
||||||
import { makePaginationQuery } from "../../common/make-pagination-query.js";
|
import { makePaginationQuery } from "../../common/make-pagination-query.js";
|
||||||
import { generateVisibilityQuery } from "../../common/generate-visibility-query.js";
|
import { generateVisibilityQuery } from "../../common/generate-visibility-query.js";
|
||||||
import { generateMutedUserQuery } from "../../common/generate-muted-user-query.js";
|
import { generateMutedUserQuery } from "../../common/generate-muted-user-query.js";
|
||||||
|
@ -61,10 +61,22 @@ export default define(meta, paramDef, async (ps, user) => {
|
||||||
}
|
}
|
||||||
|
|
||||||
const limit = ps.limit + (ps.untilId ? 1 : 0) + (ps.sinceId ? 1 : 0); // untilIdに指定したものも含まれるため+1
|
const limit = ps.limit + (ps.untilId ? 1 : 0) + (ps.sinceId ? 1 : 0); // untilIdに指定したものも含まれるため+1
|
||||||
|
let end = "+";
|
||||||
|
if (ps.untilDate) {
|
||||||
|
end = ps.untilDate.toString();
|
||||||
|
} else if (ps.untilId) {
|
||||||
|
end = getTimestamp(ps.untilId).toString();
|
||||||
|
}
|
||||||
|
let start = "-";
|
||||||
|
if (ps.sinceDate) {
|
||||||
|
start = ps.sinceDate.toString();
|
||||||
|
} else if (ps.sinceId) {
|
||||||
|
start = getTimestamp(ps.sinceId).toString();
|
||||||
|
}
|
||||||
const noteIdsRes = await redisClient.xrevrange(
|
const noteIdsRes = await redisClient.xrevrange(
|
||||||
`antennaTimeline:${antenna.id}`,
|
`antennaTimeline:${antenna.id}`,
|
||||||
ps.untilDate ?? "+",
|
end,
|
||||||
ps.sinceDate ?? "-",
|
start,
|
||||||
"COUNT",
|
"COUNT",
|
||||||
limit,
|
limit,
|
||||||
);
|
);
|
||||||
|
|
|
@ -1,6 +1,6 @@
|
||||||
import type { Antenna } from "@/models/entities/antenna.js";
|
import type { Antenna } from "@/models/entities/antenna.js";
|
||||||
import type { Note } from "@/models/entities/note.js";
|
import type { Note } from "@/models/entities/note.js";
|
||||||
import { genId } from "@/misc/gen-id.js";
|
import { getTimestamp } from "@/misc/gen-id.js";
|
||||||
import { redisClient } from "@/db/redis.js";
|
import { redisClient } from "@/db/redis.js";
|
||||||
import { publishAntennaStream } from "@/services/stream.js";
|
import { publishAntennaStream } from "@/services/stream.js";
|
||||||
import type { User } from "@/models/entities/user.js";
|
import type { User } from "@/models/entities/user.js";
|
||||||
|
@ -15,7 +15,7 @@ export async function addNoteToAntenna(
|
||||||
"MAXLEN",
|
"MAXLEN",
|
||||||
"~",
|
"~",
|
||||||
"200",
|
"200",
|
||||||
"*",
|
`${getTimestamp(note.id)}-*`,
|
||||||
"note",
|
"note",
|
||||||
note.id,
|
note.id,
|
||||||
);
|
);
|
||||||
|
|
Loading…
Reference in a new issue