From 6033a63446ee86c1d9dcaf9c24ac2906fe66c830 Mon Sep 17 00:00:00 2001 From: s1idewhist1e Date: Mon, 8 May 2023 22:42:34 -0700 Subject: [PATCH] Move config to its own crate + fix config parsing --- .config/example.yml | 2 +- packages/backend/Cargo.lock | 18 +- packages/backend/Cargo.toml | 12 +- packages/backend/crates/config/Cargo.toml | 12 + packages/backend/crates/config/src/data.rs | 263 ++++++++++++++++++ .../config.rs => crates/config/src/lib.rs} | 67 +++-- packages/backend/crates/logging/Cargo.toml | 9 + packages/backend/crates/logging/src/lib.rs | 7 + packages/backend/package.json | 4 +- packages/backend/src/config/cfg.rs | 133 --------- packages/backend/src/main.rs | 11 +- 11 files changed, 373 insertions(+), 165 deletions(-) create mode 100644 packages/backend/crates/config/Cargo.toml create mode 100644 packages/backend/crates/config/src/data.rs rename packages/backend/{src/config.rs => crates/config/src/lib.rs} (66%) create mode 100644 packages/backend/crates/logging/Cargo.toml create mode 100644 packages/backend/crates/logging/src/lib.rs delete mode 100644 packages/backend/src/config/cfg.rs diff --git a/.config/example.yml b/.config/example.yml index 3f0ce40311..3c2491542f 100644 --- a/.config/example.yml +++ b/.config/example.yml @@ -106,7 +106,7 @@ id: 'aid' # Max note length, should be < 8000. #maxNoteLength: 3000 -# Maximum lenght of an image caption or file comment (default 1500, max 8192) +# Maximum length of an image caption or file comment (default 1500, max 8192) #maxCaptionLength: 1500 # Whether disable HSTS diff --git a/packages/backend/Cargo.lock b/packages/backend/Cargo.lock index 3a554c205f..75bbeb4637 100644 --- a/packages/backend/Cargo.lock +++ b/packages/backend/Cargo.lock @@ -11,11 +11,20 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "backend" version = "0.0.0" +dependencies = [ + "config", + "logging", + "queue", + "server", +] + +[[package]] +name = "config" +version = "0.1.0" dependencies = [ "once_cell", "serde", "serde_yaml", - "server", ] [[package]] @@ -40,6 +49,13 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" +[[package]] +name = "logging" +version = "0.1.0" +dependencies = [ + "config", +] + [[package]] name = "once_cell" version = "1.17.1" diff --git a/packages/backend/Cargo.toml b/packages/backend/Cargo.toml index 136e092e0c..cd2a0dc480 100644 --- a/packages/backend/Cargo.toml +++ b/packages/backend/Cargo.toml @@ -8,12 +8,18 @@ default-run = "backend" members = ["crates/*"] +[profile.development] +inherits = "dev" + +[profile.production] +inherits = "release" + # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -once_cell = "1.17.1" -serde = { version = "1.0.160", features = [ "derive" ] } -serde_yaml = "0.9.21" server = { path = "crates/server" } +logging = { path = "crates/logging" } +queue = { path = "crates/queue" } +config = { path = "crates/config" } [dev-dependencies] diff --git a/packages/backend/crates/config/Cargo.toml b/packages/backend/crates/config/Cargo.toml new file mode 100644 index 0000000000..0058bdbf2e --- /dev/null +++ b/packages/backend/crates/config/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "config" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +once_cell = "1.17.1" +serde = { version = "1.0.160", features = [ "derive" ] } +serde_yaml = "0.9.21" + diff --git a/packages/backend/crates/config/src/data.rs b/packages/backend/crates/config/src/data.rs new file mode 100644 index 0000000000..9cca4d3f15 --- /dev/null +++ b/packages/backend/crates/config/src/data.rs @@ -0,0 +1,263 @@ +use serde::{de::Visitor, Deserialize}; + +type Port = u16; + +#[derive(Debug, PartialEq, Deserialize)] +pub struct MaxNoteLength(pub u16); + +#[derive(Debug, PartialEq, Deserialize)] +pub struct MaxCommentLength(pub u16); + +#[derive(Debug, PartialEq)] +pub enum IpFamily { + Both, + IPv4, + IPv6, +} + +impl<'de> Deserialize<'de> for IpFamily { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + struct IpFamilyVisitor; + + impl<'de> Visitor<'de> for IpFamilyVisitor { + type Value = IpFamily; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("One of `4` `6` `0`") + } + + fn visit_u8(self, v: u8) -> Result + where + E: serde::de::Error, + { + { + match v { + 0 => Ok(IpFamily::Both), + 4 => Ok(IpFamily::IPv4), + 6 => Ok(IpFamily::IPv6), + _ => Err(E::unknown_variant(&v.to_string(), &["0", "4", "6"])), + } + } + } + } + + deserializer.deserialize_u8(IpFamilyVisitor) + } +} + +impl Default for IpFamily { + fn default() -> Self { + Self::Both + } +} + +impl<'de> Deserialize<'de> for Host { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + struct HostVisitor; + + impl<'de> Visitor<'de> for HostVisitor { + type Value = Host; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("(proto://)host(.tld)(/)") + } + + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + let components: Vec<&str> = v.split("://").collect(); + + match components.len() { + 1 => Ok(Host(None, components[0].into())), + 2 => Ok(Host( + Some(components[0].into()), + components[1].trim_end_matches('/').into(), + )), + _ => Err(E::custom(format!("Invalid url: {}", v))), // FIXME: more descriptive + // error message + } + } + } + deserializer.deserialize_str(HostVisitor) + } +} + +#[derive(Debug, PartialEq, Default)] +// TODO: Convert to uri later and maybe some more enums +pub struct Host(pub Option, pub String); + +#[derive(Debug, PartialEq, Deserialize)] +#[serde(rename = "camelCase")] +pub struct Config { + pub repository_url: Option, + pub feedback_url: Option, + pub url: Host, + pub port: Port, + pub db: db::DbConfig, + pub redis: redis::RedisConfig, + // pub sonic: sonic::SonicConfig, + // pub elasticsearch: elasticsearch::ElasticsearchConfig, + // pub id: IdGenerator, + #[serde(default)] + pub max_note_length: MaxNoteLength, + #[serde(default)] + pub max_caption_length: MaxCommentLength, + // pub disable_hsts: bool, + pub cluster_limit: Option, + // pub deliver_job_concurrency: u16, + // pub inbox_job_concurrency: u16, + // pub deliver_job_per_sec: u16, + // pub inbox_job_per_sec: u16, + // pub deliver_job_max_attempts: u16, + // pub inbox_job_max_attempts: u16, + // pub outgoing_address_family: IpFamily, + // pub syslog: syslog::SyslogConfig, + // pub proxy: Option, + // pub proxy_smtp: Option, + // pub proxy_bypass_hosts: Vec, + // pub allowed_private_networks: Vec, + // pub max_file_size: Option, + // pub media_proxy: Option, + // pub proxy_remote_files: bool, + // pub twa: Option, + // pub reserved_usernames: Vec, + // pub max_user_signups: Option, + // pub is_managed_hosting: bool, + // pub deepl: Option, + // pub libre_translate: Option, + // pub email: Option, + // pub object_storage: Option, + // pub summaly_proxy_url: Option, +} + +#[derive(Debug, PartialEq, Deserialize)] +#[serde(rename = "lowercase")] +pub enum IdGenerator { + AId, + MeId, + ULId, + ObjectID, +} + +/// database config +pub mod db { + use super::*; + #[derive(Debug, PartialEq, Deserialize)] + #[serde(rename = "camelCase")] + pub struct DbConfig { + pub host: Host, + pub port: Port, + pub db: String, + pub user: String, + pub pass: String, + #[serde(default = "true_fn")] + pub disable_cache: bool, + #[serde(default)] + pub extra: Extra, + } + + #[derive(Debug, PartialEq, Deserialize)] + pub struct Extra { + #[serde(default = "true_fn")] + pub ssl: bool, + } + + impl Default for Extra { + fn default() -> Self { + Self { ssl: true } + } + } +} + +/// redis config +pub mod redis { + use super::*; + + #[derive(Debug, PartialEq, Deserialize)] + #[serde(rename = "camelCase")] + pub struct RedisConfig { + pub host: Host, + pub port: Port, + #[serde(default)] + pub family: IpFamily, + pub pass: Option, + pub prefix: Option, + #[serde(default)] + pub db: u8, + } +} + +/// sonic search config +pub mod sonic { + use super::*; + + #[derive(Debug, PartialEq, Deserialize)] + pub struct SonicConfig { + pub host: Host, + pub port: Port, + #[serde(default)] + pub auth: Option, + #[serde(default)] + pub collection: Option, + #[serde(default)] + pub bucket: Option, + } +} + +/// elasticsearch config +pub mod elasticsearch { + use super::*; + + #[derive(Debug, PartialEq, Deserialize)] + pub struct ElasticsearchConfig { + pub host: Host, + pub port: Port, + #[serde(default)] + pub ssl: bool, + pub user: Option, + pub pass: Option, + pub index: Option, + } +} + +/// syslog configuration +pub mod syslog { + use super::*; + + #[derive(Debug, PartialEq, Deserialize)] + pub struct SyslogConfig { + host: Host, + port: Port, + } +} + +/// TWA configuration +pub mod twa { + use super::*; + + #[derive(Debug, PartialEq, Deserialize)] + pub struct TWAConfig {} +} + +impl Default for MaxNoteLength { + fn default() -> Self { + Self(3000) + } +} + +impl Default for MaxCommentLength { + fn default() -> Self { + Self(1500) + } +} + +fn true_fn() -> bool { + true +} diff --git a/packages/backend/src/config.rs b/packages/backend/crates/config/src/lib.rs similarity index 66% rename from packages/backend/src/config.rs rename to packages/backend/crates/config/src/lib.rs index 510bd1d62e..fa79f8d34e 100644 --- a/packages/backend/src/config.rs +++ b/packages/backend/crates/config/src/lib.rs @@ -5,9 +5,9 @@ use std::path::Path; use once_cell::sync::OnceCell; -mod cfg; +mod data; -pub use cfg::*; +pub use data::*; // Config Errors #[derive(Debug)] @@ -76,18 +76,6 @@ mod tests { path::PathBuf, }; - struct Guard(PathBuf); - - impl Drop for Guard { - fn drop(&mut self) { - println!("removing temp file..."); - match remove_file(&self.0) { - Ok(_) => println!("Successfully removed file"), - Err(e) => println!("Could not remove file: {}", e), - } - } - } - use super::*; #[test] @@ -97,11 +85,37 @@ mod tests { #[test] fn parses_test_config() { + struct Guard(PathBuf); + + impl Drop for Guard { + fn drop(&mut self) { + println!("removing temp file..."); + match remove_file(&self.0) { + Ok(_) => println!("Successfully removed file"), + Err(e) => println!("Could not remove file: {}", e), + } + } + } + // setup test temp config let mut temp_file = std::env::temp_dir(); temp_file.push(Path::new("calckey.test.config")); - let err = File::create(&temp_file).unwrap().write_all(br""); + let err = File::create(&temp_file).unwrap().write_all( + br" +url: https://example.tld/ +port: 3000 +db: + host: localhost + port: 5432 + db: calckey + user: example-calckey-user + pass: example-calckey-pass +redis: + host: localhost + port: 6379 +", + ); let _g = Guard(temp_file.clone()); @@ -112,20 +126,31 @@ mod tests { assert_eq!( config, Config { - url: Host("https://example.tld/".into()), + url: Host(Some("https".into()), "example.tld".into()), port: 3000, db: db::DbConfig { - host: String::from("localhost"), + host: Host(None, "localhost".into()), port: 5432, db: String::from("calckey"), user: String::from("example-calckey-user"), pass: String::from("example-calckey-pass"), disable_cache: true, - extra: db::Extra { extra: true } - } + extra: db::Extra { ssl: true } + }, + repository_url: None, + feedback_url: None, + redis: redis::RedisConfig { + host: Host(None, "localhost".into()), + port: 6379, + family: IpFamily::Both, + pass: None, + prefix: None, + db: 0, + }, + max_note_length: MaxNoteLength(3000), + max_caption_length: MaxCommentLength(1500), + cluster_limit: None } ); - - remove_file(&temp_file).unwrap(); } } diff --git a/packages/backend/crates/logging/Cargo.toml b/packages/backend/crates/logging/Cargo.toml new file mode 100644 index 0000000000..50bc5b1d25 --- /dev/null +++ b/packages/backend/crates/logging/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "logging" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +config = { path = "../config" } diff --git a/packages/backend/crates/logging/src/lib.rs b/packages/backend/crates/logging/src/lib.rs new file mode 100644 index 0000000000..99a9e00e88 --- /dev/null +++ b/packages/backend/crates/logging/src/lib.rs @@ -0,0 +1,7 @@ + + +pub fn idk() { + config::get_config(); +} + + diff --git a/packages/backend/package.json b/packages/backend/package.json index 8e3c70e1ab..8db84c21e0 100644 --- a/packages/backend/package.json +++ b/packages/backend/package.json @@ -3,11 +3,11 @@ "private": true, "type": "module", "scripts": { - "start": "cargo run --profile $( [ ${NODE_ENV:=dev} = 'production' ] && echo 'release' || echo $NODE_ENV )", + "start": "cargo run --profile ${NODE_ENV:=development}", "start:test": "NODE_ENV=test pnpm node ./built/index.js", "check": "cargo check", "migrate": "cargo run --bin migrate", - "build": "cargo build --profile $( [ ${NODE_ENV:=dev} = 'production' ] && echo 'release' || echo $NODE_ENV )", + "build": "cargo build --profile ${NODE_ENV:=development}", "revertmigration": "typeorm migration:revert -d ormconfig.js", "lint": "cargo check", "test": "cargo test --workspace" diff --git a/packages/backend/src/config/cfg.rs b/packages/backend/src/config/cfg.rs deleted file mode 100644 index 619fe44744..0000000000 --- a/packages/backend/src/config/cfg.rs +++ /dev/null @@ -1,133 +0,0 @@ -use serde::Deserialize; - -type Port = u16; - -#[derive(Debug, PartialEq, Deserialize)] -pub struct MaxNoteLength(pub u16); - -#[derive(Debug, PartialEq, Deserialize)] -pub enum IpFamily { - Both = 0, - IPv4 = 4, - IPv6 = 6, -} - -#[derive(Debug, PartialEq, Deserialize, Default)] -pub struct Host(pub String); - -#[derive(Debug, PartialEq, Deserialize)] -#[serde(rename = "camelCase")] -pub struct Config { - pub url: Host, - pub port: Port, - pub db: db::DbConfig, - pub redis: redis::RedisConfig, - pub sonic: sonic::SonicConfig, - pub elasticsearch: elasticsearch::ElasticsearchConfig, - pub id: IdGenerator, - pub max_note_length: MaxNoteLength, - pub max_caption_length: MaxNoteLength, - pub disable_hsts: bool, - pub cluster_limit: u16, - pub deliver_job_concurrency: u16, - pub inbox_job_concurrency: u16, - pub deliver_job_per_sec: u16, - pub inbox_job_per_sec: u16, - pub deliver_job_max_attempts: u16, - pub inbox_job_max_attempts: u16, - pub outgoing_address_family: IpFamily, - pub syslog: syslog::SyslogConfig, - pub proxy: Host, -} - -#[derive(Debug, PartialEq, Deserialize)] -pub enum IdGenerator { - AId, - MeId, - ULId, - ObjectID, -} - -pub mod db { - use super::*; - #[derive(Debug, PartialEq, Deserialize)] - #[serde(rename = "camelCase")] - pub struct DbConfig { - pub host: String, - pub port: Port, - pub db: String, - pub user: String, - pub pass: String, - #[serde(default = "true_fn")] - pub disable_cache: bool, - pub extra: Extra, - } - - #[derive(Debug, PartialEq, Deserialize)] - pub struct Extra { - #[serde(default = "true_fn")] - pub extra: bool, - } -} - -pub mod redis { - use super::*; - - #[derive(Debug, PartialEq, Deserialize)] - #[serde(rename = "camelCase")] - pub struct RedisConfig { - pub host: String, - pub port: Port, - #[serde(default = "ip_family_fn")] - pub family: IpFamily, - #[serde(default)] - pub pass: Option, - #[serde(default)] - pub prefix: Option, - #[serde(default)] - pub db: Option, - } -} - -pub mod sonic { - use super::*; - - #[derive(Debug, PartialEq, Deserialize)] - pub struct SonicConfig { - pub host: Host, - pub port: Port, - pub auth: String, - pub collection: String, - pub bucket: Option, - } -} - -pub mod elasticsearch { - use super::*; - - #[derive(Debug, PartialEq, Deserialize)] - pub struct ElasticsearchConfig { - pub host: Host, - pub port: Port, - #[serde(default)] - pub ssl: bool, - pub user: Option, - pub pass: Option, - } -} - -pub mod syslog { - use super::*; - - #[derive(Debug, PartialEq, Deserialize)] - pub struct SyslogConfig {} -} - - -fn true_fn() -> bool { - true -} - -fn ip_family_fn() -> IpFamily { - IpFamily::Both -} diff --git a/packages/backend/src/main.rs b/packages/backend/src/main.rs index 1acb88d966..174211ee14 100644 --- a/packages/backend/src/main.rs +++ b/packages/backend/src/main.rs @@ -1,14 +1,17 @@ -use std::path::Path; +use std::{path::Path, error::Error}; -mod config; +#[cfg(debug_assertions)] +extern crate config; -fn main() { +fn main() -> Result<(), Box> { // bootstrap // ENV // get config - config::init_config(Path::new("")); + config::init_config(Path::new(""))?; + + Ok(()) }