From 310ce493149a0e073cab667bea2b5bb230ff3b22 Mon Sep 17 00:00:00 2001 From: mdecimus Date: Mon, 11 Mar 2024 11:30:12 +0100 Subject: [PATCH] Updated settings REST API --- Cargo.lock | 2 + crates/cli/src/modules/database.rs | 8 +- crates/directory/src/backend/imap/config.rs | 4 +- crates/directory/src/backend/internal/mod.rs | 10 +- crates/directory/src/backend/ldap/config.rs | 4 +- crates/directory/src/backend/memory/config.rs | 2 +- crates/directory/src/backend/smtp/config.rs | 4 +- crates/directory/src/backend/sql/config.rs | 2 +- crates/directory/src/core/config.rs | 4 - crates/jmap/src/api/admin.rs | 288 +++++++++++++----- crates/jmap/src/services/housekeeper.rs | 12 +- crates/smtp/src/core/management.rs | 43 +-- crates/store/src/backend/foundationdb/main.rs | 6 +- crates/store/src/backend/mysql/main.rs | 2 +- crates/store/src/backend/postgres/main.rs | 2 +- crates/store/src/backend/redis/mod.rs | 88 +++--- crates/store/src/dispatch/blocked.rs | 12 +- crates/store/src/dispatch/config.rs | 24 +- crates/store/src/write/bitmap.rs | 1 - crates/utils/Cargo.toml | 2 + crates/utils/src/config/mod.rs | 4 +- crates/utils/src/lib.rs | 1 + crates/utils/src/url_params.rs | 58 ++++ resources/config/directory/imap.toml | 4 +- resources/config/directory/ldap.toml | 4 +- resources/config/directory/lmtp.toml | 4 +- resources/config/directory/memory.toml | 12 +- resources/config/directory/sql.toml | 2 +- resources/config/imap/settings.toml | 3 - resources/config/smtp/listener.toml | 6 +- resources/config/store/foundationdb.toml | 8 +- resources/config/store/mysql.toml | 4 +- resources/config/store/postgresql.toml | 4 +- resources/config/store/redis.toml | 14 +- 34 files changed, 420 insertions(+), 228 deletions(-) create mode 100644 crates/utils/src/url_params.rs diff --git a/Cargo.lock b/Cargo.lock index ad084f5d..a1bf24f1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6520,7 +6520,9 @@ dependencies = [ "blake3", "chrono", "dashmap", + "form_urlencoded", "futures", + "http-body-util", "lru-cache", "mail-auth", "mail-send", diff --git a/crates/cli/src/modules/database.rs b/crates/cli/src/modules/database.rs index 7a3c0f27..d900d7aa 100644 --- a/crates/cli/src/modules/database.rs +++ b/crates/cli/src/modules/database.rs @@ -44,7 +44,7 @@ impl ServerCommands { } ServerCommands::ReloadConfig {} => { client - .http_request::(Method::GET, "/api/reload/config", None) + .http_request::(Method::GET, "/api/reload/settings", None) .await; eprintln!("Success."); } @@ -52,7 +52,7 @@ impl ServerCommands { client .http_request::( Method::POST, - "/api/config", + "/api/settings", Some(vec![(key.clone(), value.unwrap_or_default())]), ) .await; @@ -62,7 +62,7 @@ impl ServerCommands { client .http_request::( Method::DELETE, - &format!("/api/config/{key}"), + &format!("/api/settings/{key}"), None, ) .await; @@ -72,7 +72,7 @@ impl ServerCommands { let results = client .http_request::, String>( Method::GET, - &format!("/api/config/{}", prefix.unwrap_or_default()), + &format!("/api/settings/{}", prefix.unwrap_or_default()), None, ) .await; diff --git a/crates/directory/src/backend/imap/config.rs b/crates/directory/src/backend/imap/config.rs index 81db4e96..40d85410 100644 --- a/crates/directory/src/backend/imap/config.rs +++ b/crates/directory/src/backend/imap/config.rs @@ -36,8 +36,8 @@ impl ImapDirectory { data_store: Store, ) -> utils::config::Result { let prefix = prefix.as_key(); - let address = config.value_require((&prefix, "address"))?; - let tls_implicit: bool = config.property_or_static((&prefix, "tls.implicit"), "false")?; + let address = config.value_require((&prefix, "host"))?; + let tls_implicit: bool = config.property_or_static((&prefix, "tls.enable"), "false")?; let port: u16 = config .property_or_static((&prefix, "port"), if tls_implicit { "993" } else { "143" })?; diff --git a/crates/directory/src/backend/internal/mod.rs b/crates/directory/src/backend/internal/mod.rs index 5c67f6cc..21c4f68b 100644 --- a/crates/directory/src/backend/internal/mod.rs +++ b/crates/directory/src/backend/internal/mod.rs @@ -24,7 +24,7 @@ pub mod lookup; pub mod manage; -use std::{fmt::Display, slice::Iter}; +use std::{fmt::Display, slice::Iter, str::FromStr}; use store::{write::key::KeySerializer, Deserialize, Serialize, U32_LEN}; use utils::codec::leb128::Leb128Iterator; @@ -269,3 +269,11 @@ impl Type { } } } + +impl FromStr for Type { + type Err = (); + + fn from_str(s: &str) -> Result { + Type::parse(s).ok_or(()) + } +} diff --git a/crates/directory/src/backend/ldap/config.rs b/crates/directory/src/backend/ldap/config.rs index d0eda70c..2abb08fe 100644 --- a/crates/directory/src/backend/ldap/config.rs +++ b/crates/directory/src/backend/ldap/config.rs @@ -47,7 +47,7 @@ impl LdapDirectory { }; let manager = LdapConnectionManager::new( - config.value_require((&prefix, "address"))?.to_string(), + config.value_require((&prefix, "url"))?.to_string(), LdapConnSettings::new() .set_conn_timeout(config.property_or_static((&prefix, "timeout"), "30s")?) .set_starttls(config.property_or_static((&prefix, "tls.enable"), "false")?) @@ -73,7 +73,7 @@ impl LdapDirectory { .map(|(_, v)| v.to_string()) .collect(), attr_type: config - .values((&prefix, "attributes.type")) + .values((&prefix, "attributes.class")) .map(|(_, v)| v.to_string()) .collect(), attr_description: config diff --git a/crates/directory/src/backend/memory/config.rs b/crates/directory/src/backend/memory/config.rs index c446ca8e..333eed43 100644 --- a/crates/directory/src/backend/memory/config.rs +++ b/crates/directory/src/backend/memory/config.rs @@ -46,7 +46,7 @@ impl MemoryDirectory { let name = config .value_require((prefix.as_str(), "principals", lookup_id, "name"))? .to_string(); - let typ = match config.value((prefix.as_str(), "principals", lookup_id, "type")) { + let typ = match config.value((prefix.as_str(), "principals", lookup_id, "class")) { Some("individual") => Type::Individual, Some("admin") => Type::Superuser, Some("group") => Type::Group, diff --git a/crates/directory/src/backend/smtp/config.rs b/crates/directory/src/backend/smtp/config.rs index a417be08..de66bf43 100644 --- a/crates/directory/src/backend/smtp/config.rs +++ b/crates/directory/src/backend/smtp/config.rs @@ -37,8 +37,8 @@ impl SmtpDirectory { data_store: Store, ) -> utils::config::Result { let prefix = prefix.as_key(); - let address = config.value_require((&prefix, "address"))?; - let tls_implicit: bool = config.property_or_static((&prefix, "tls.implicit"), "false")?; + let address = config.value_require((&prefix, "host"))?; + let tls_implicit: bool = config.property_or_static((&prefix, "tls.enable"), "false")?; let port: u16 = config .property_or_static((&prefix, "port"), if tls_implicit { "465" } else { "25" })?; diff --git a/crates/directory/src/backend/sql/config.rs b/crates/directory/src/backend/sql/config.rs index 1da5510b..af55e9ed 100644 --- a/crates/directory/src/backend/sql/config.rs +++ b/crates/directory/src/backend/sql/config.rs @@ -57,7 +57,7 @@ impl SqlDirectory { .unwrap_or_default() .to_string(), column_type: config - .value((&prefix, "columns.type")) + .value((&prefix, "columns.class")) .unwrap_or_default() .to_string(), ..Default::default() diff --git a/crates/directory/src/core/config.rs b/crates/directory/src/core/config.rs index 1b8744b3..c942edca 100644 --- a/crates/directory/src/core/config.rs +++ b/crates/directory/src/core/config.rs @@ -70,10 +70,6 @@ impl ConfigDirectory for Config { )); for id in self.sub_keys("directory", ".type") { - if id.ends_with(".columns") || id.ends_with(".attributes") || id.contains(".principals") - { - continue; - } // Parse directory if self.property_or_static::(("directory", id, "disable"), "false")? { tracing::debug!("Skipping disabled directory {id:?}."); diff --git a/crates/jmap/src/api/admin.rs b/crates/jmap/src/api/admin.rs index 3544a735..7b51e4b6 100644 --- a/crates/jmap/src/api/admin.rs +++ b/crates/jmap/src/api/admin.rs @@ -31,7 +31,8 @@ use http_body_util::combinators::BoxBody; use hyper::{body::Bytes, Method, StatusCode}; use jmap_proto::error::request::RequestError; use serde_json::json; -use utils::config::ConfigKey; +use store::ahash::AHashMap; +use utils::{config::ConfigKey, url_params::UrlParams}; use crate::{ auth::{oauth::OAuthCodeRequest, AccessToken}, @@ -67,6 +68,21 @@ pub struct PrincipalResponse { pub description: Option, } +#[derive(Debug, serde::Serialize, serde::Deserialize)] +#[serde(tag = "type")] +pub enum UpdateSettings { + Delete { + keys: Vec, + }, + Clear { + prefix: String, + }, + Insert { + prefix: String, + values: Vec<(String, String)>, + }, +} + impl JMAP { pub async fn handle_api_manage_request( &self, @@ -118,32 +134,13 @@ impl JMAP { } ("principal", None, &Method::GET) => { // List principal ids - let mut filter = None; - let mut typ = None; - let mut page: usize = 0; - let mut limit: usize = 0; + let params = UrlParams::new(req.uri().query()); + let filter = params.get("filter"); + let typ = params.parse("type"); + let page: usize = params.parse("page").unwrap_or(0); + let limit: usize = params.parse("limit").unwrap_or(0); - if let Some(query) = req.uri().query() { - for (key, value) in form_urlencoded::parse(query.as_bytes()) { - match key.as_ref() { - "limit" => { - limit = value.parse().unwrap_or_default(); - } - "page" => { - page = value.parse().unwrap_or_default(); - } - "type" => { - typ = Type::parse(value.as_ref()); - } - "filter" => { - filter = value.into(); - } - _ => {} - } - } - } - - match self.store.list_accounts(filter.as_deref(), typ).await { + match self.store.list_accounts(filter, typ).await { Ok(accounts) => { let (total, accounts) = if limit > 0 { let offset = page.saturating_sub(1) * limit; @@ -280,28 +277,12 @@ impl JMAP { } ("domain", None, &Method::GET) => { // List domains - let mut filter = None; - let mut page: usize = 0; - let mut limit: usize = 0; + let params = UrlParams::new(req.uri().query()); + let filter = params.get("filter"); + let page: usize = params.parse("page").unwrap_or(0); + let limit: usize = params.parse("limit").unwrap_or(0); - if let Some(query) = req.uri().query() { - for (key, value) in form_urlencoded::parse(query.as_bytes()) { - match key.as_ref() { - "limit" => { - limit = value.parse().unwrap_or_default(); - } - "page" => { - page = value.parse().unwrap_or_default(); - } - "filter" => { - filter = value.into(); - } - _ => {} - } - } - } - - match self.store.list_domains(filter.as_deref()).await { + match self.store.list_domains(filter).await { Ok(domains) => { let (total, domains) = if limit > 0 { let offset = page.saturating_sub(1) * limit; @@ -366,7 +347,7 @@ impl JMAP { .into_http_response(), } } - ("reload", Some("config"), &Method::GET) => { + ("reload", Some("settings"), &Method::GET) => { let _ = self .housekeeper_tx .send(housekeeper::Event::ReloadConfig) @@ -388,11 +369,137 @@ impl JMAP { })) .into_http_response() } - ("config", key, &Method::GET) => { - match self.store.config_list(key.unwrap_or_default()).await { - Ok(config) => JsonResponse::new(json!({ - "data": config.keys.into_iter().collect::>(), - })) + ("settings", None, &Method::GET) => { + // List settings + let params = UrlParams::new(req.uri().query()); + let prefix = params + .get("prefix") + .map(|p| { + if !p.ends_with('.') { + format!("{p}.") + } else { + p.to_string() + } + }) + .unwrap_or_default(); + let groupby = params + .get("groupby") + .map(|s| { + if !s.starts_with('.') { + format!(".{s}") + } else { + s.to_string() + } + }) + .unwrap_or_default(); + let filter = params.get("filter").unwrap_or_default(); + let limit: usize = params.parse("limit").unwrap_or(0); + let mut offset = + params.parse::("page").unwrap_or(0).saturating_sub(1) * limit; + let has_filter = !filter.is_empty(); + + match self.store.config_list(&prefix).await { + Ok(settings) => if groupby.len() > 1 && !settings.is_empty() { + // Obtain record ids + let mut total = 0; + let mut ids = Vec::new(); + for (key, _) in &settings { + if let Some(id) = key.strip_suffix(&groupby) { + if !id.is_empty() { + if !has_filter { + if offset == 0 { + if limit == 0 || ids.len() < limit { + ids.push(id); + } + } else { + offset -= 1; + } + total += 1; + } else { + ids.push(id); + } + } + } + } + + // Group settings by record id + let mut records = Vec::new(); + for id in ids { + let mut record = AHashMap::new(); + let prefix = format!("{id}."); + record.insert("_id".to_string(), id.to_string()); + for (k, v) in &settings { + if let Some(k) = k.strip_prefix(&prefix) { + record.insert(k.to_string(), v.to_string()); + } else if record.len() > 1 { + break; + } + } + + if has_filter { + if record.iter().any(|(_, v)| v.contains(filter)) { + if offset == 0 { + if limit == 0 || records.len() < limit { + records.push(record); + } + } else { + offset -= 1; + } + total += 1; + } + } else { + records.push(record); + } + } + + JsonResponse::new(json!({ + "data": { + "total": total, + "items": records, + }, + })) + } else if !groupby.is_empty() { + // groupby=. + let total = settings.len(); + let items = settings + .into_iter() + .filter_map(|(k, v)| { + if filter.is_empty() || k.contains(filter) || v.contains(filter) { + let k = + k.strip_prefix(&prefix).map(|k| k.to_string()).unwrap_or(k); + Some(json!({ + "_id": k, + "_value": v, + })) + } else { + None + } + }) + .skip(offset) + .take(if limit == 0 { total } else { limit }) + .collect::>(); + + JsonResponse::new(json!({ + "data": { + "total": total, + "items": items, + }, + })) + } else { + let total = settings.len(); + let items = settings + .into_iter() + .skip(offset) + .take(if limit == 0 { total } else { limit }) + .collect::>(); + + JsonResponse::new(json!({ + "data": { + "total": total, + "items": items, + }, + })) + } .into_http_response(), Err(err) => RequestError::blank( StatusCode::INTERNAL_SERVER_ERROR.as_u16(), @@ -402,14 +509,20 @@ impl JMAP { .into_http_response(), } } - ("config", Some(prefix), &Method::DELETE) if !prefix.is_empty() => { - let result = match prefix.strip_suffix('.') { - Some(prefix) if !prefix.is_empty() => { - self.store.config_clear_prefix(prefix).await - } - _ => self.store.config_clear(prefix).await, - }; - match result { + ("settings", Some(key), &Method::GET) => match self.store.config_get(key).await { + Ok(value) => JsonResponse::new(json!({ + "data": value, + })) + .into_http_response(), + Err(err) => RequestError::blank( + StatusCode::INTERNAL_SERVER_ERROR.as_u16(), + "Config fetch failed", + err.to_string(), + ) + .into_http_response(), + }, + ("settings", Some(prefix), &Method::DELETE) if !prefix.is_empty() => { + match self.store.config_clear(prefix).await { Ok(_) => JsonResponse::new(json!({ "data": (), })) @@ -422,19 +535,48 @@ impl JMAP { .into_http_response(), } } - ("config", None, &Method::POST) => { - if let Some(changes) = body - .and_then(|body| serde_json::from_slice::>(&body).ok()) + ("settings", None, &Method::POST) => { + if let Some(changes) = + body.and_then(|body| serde_json::from_slice::>(&body).ok()) { - match self - .store - .config_set( - changes - .into_iter() - .map(|(key, value)| ConfigKey { key, value }), - ) - .await - { + let mut result = Ok(()); + + 'next: for change in changes { + match change { + UpdateSettings::Delete { keys } => { + for key in keys { + result = self.store.config_clear(key).await; + if result.is_err() { + break 'next; + } + } + } + UpdateSettings::Clear { prefix } => { + result = self.store.config_clear_prefix(&prefix).await; + if result.is_err() { + break; + } + } + UpdateSettings::Insert { prefix, values } => { + result = self + .store + .config_set(values.into_iter().map(|(key, value)| ConfigKey { + key: if !prefix.is_empty() { + format!("{prefix}.{key}") + } else { + key + }, + value, + })) + .await; + if result.is_err() { + break; + } + } + } + } + + match result { Ok(_) => JsonResponse::new(json!({ "data": (), })) diff --git a/crates/jmap/src/services/housekeeper.rs b/crates/jmap/src/services/housekeeper.rs index 5b00e5ff..4999e430 100644 --- a/crates/jmap/src/services/housekeeper.rs +++ b/crates/jmap/src/services/housekeeper.rs @@ -23,7 +23,7 @@ use std::sync::Arc; -use store::dispatch::blocked::BLOCKED_IP_KEY; +use store::dispatch::blocked::BLOCKED_IP_PREFIX; use tokio::sync::mpsc; use utils::{ config::{cron::SimpleCron, Config, Servers}, @@ -110,10 +110,12 @@ pub fn spawn_housekeeper( // for now, we just reload the blocked IP addresses let core = core.clone(); tokio::spawn(async move { - match core.store.config_list(BLOCKED_IP_KEY).await { - Ok(config) => { - if let Err(err) = - core.directory.blocked_ips.reload_blocked_ips(&config) + match core.store.config_list(BLOCKED_IP_PREFIX).await { + Ok(settings) => { + if let Err(err) = core + .directory + .blocked_ips + .reload_blocked_ips(settings.iter().map(|(k, _)| k)) { tracing::error!( context = "store", diff --git a/crates/smtp/src/core/management.rs b/crates/smtp/src/core/management.rs index cc62ac88..6eac089b 100644 --- a/crates/smtp/src/core/management.rs +++ b/crates/smtp/src/core/management.rs @@ -21,7 +21,7 @@ * for more details. */ -use std::{borrow::Cow, collections::HashMap, net::IpAddr, str::FromStr, sync::Arc}; +use std::{net::IpAddr, str::FromStr, sync::Arc}; use directory::{AuthResult, Type}; use http_body_util::{combinators::BoxBody, BodyExt, Empty, Full}; @@ -54,7 +54,10 @@ use store::{ Deserialize, IterateParams, ValueKey, U64_LEN, }; -use utils::listener::{limiter::InFlight, SessionData, SessionManager, SessionStream}; +use utils::{ + listener::{limiter::InFlight, SessionData, SessionManager, SessionStream}, + url_params::UrlParams, +}; use crate::{ queue::{self, ErrorDetails, HostResponse, QueueId, Status}, @@ -345,7 +348,7 @@ impl SMTP { path_2: &str, path_3: Option<&str>, ) -> hyper::Response> { - let params = UrlParams::new(uri); + let params = UrlParams::new(uri.query()); let (status, response) = match (method, path_1, path_2, path_3) { (&Method::GET, "queue", "messages", None) => { @@ -927,40 +930,6 @@ fn not_found() -> (StatusCode, String) { ) } -#[derive(Default)] -struct UrlParams<'x> { - params: HashMap, Cow<'x, str>>, -} - -impl<'x> UrlParams<'x> { - pub fn new(uri: &'x Uri) -> Self { - if let Some(query) = uri.query() { - Self { - params: form_urlencoded::parse(query.as_bytes()) - .filter(|(_, value)| !value.is_empty()) - .collect(), - } - } else { - Self::default() - } - } - - pub fn get(&self, key: &str) -> Option<&str> { - self.params.get(key).map(|v| v.as_ref()) - } - - pub fn has_key(&self, key: &str) -> bool { - self.params.contains_key(key) - } - - pub fn parse(&self, key: &str) -> Option - where - T: std::str::FromStr, - { - self.get(key).and_then(|v| v.parse().ok()) - } -} - enum ReportType { Dmarc, Tls, diff --git a/crates/store/src/backend/foundationdb/main.rs b/crates/store/src/backend/foundationdb/main.rs index e2f77be0..e27a1724 100644 --- a/crates/store/src/backend/foundationdb/main.rs +++ b/crates/store/src/backend/foundationdb/main.rs @@ -33,7 +33,7 @@ impl FdbStore { let prefix = prefix.as_key(); let guard = unsafe { foundationdb::boot() }; - let db = Database::new(config.value((&prefix, "path")))?; + let db = Database::new(config.value((&prefix, "cluster-file")))?; if let Some(value) = config.property::((&prefix, "transaction.timeout"))? { db.set_option(DatabaseOption::TransactionTimeout(value.as_millis() as i32))?; } @@ -47,10 +47,10 @@ impl FdbStore { value.as_millis() as i32 ))?; } - if let Some(value) = config.property((&prefix, "transaction.machine-id"))? { + if let Some(value) = config.property((&prefix, "ids.machine"))? { db.set_option(DatabaseOption::MachineId(value))?; } - if let Some(value) = config.property((&prefix, "transaction.datacenter-id"))? { + if let Some(value) = config.property((&prefix, "ids.datacenter"))? { db.set_option(DatabaseOption::DatacenterId(value))?; } diff --git a/crates/store/src/backend/mysql/main.rs b/crates/store/src/backend/mysql/main.rs index 2d1baf14..3e1b625f 100644 --- a/crates/store/src/backend/mysql/main.rs +++ b/crates/store/src/backend/mysql/main.rs @@ -49,7 +49,7 @@ impl MysqlStore { .max_allowed_packet(config.property((&prefix, "max-allowed-packet"))?) .wait_timeout( config - .property::((&prefix, "timeout.wait"))? + .property::((&prefix, "timeout"))? .map(|t| t.as_secs() as usize), ); if let Some(port) = config.property((&prefix, "port"))? { diff --git a/crates/store/src/backend/postgres/main.rs b/crates/store/src/backend/postgres/main.rs index 3757f4c4..267ba264 100644 --- a/crates/store/src/backend/postgres/main.rs +++ b/crates/store/src/backend/postgres/main.rs @@ -46,7 +46,7 @@ impl PostgresStore { cfg.user = config.value((&prefix, "user")).map(|s| s.to_string()); cfg.password = config.value((&prefix, "password")).map(|s| s.to_string()); cfg.port = config.property((&prefix, "port"))?; - cfg.connect_timeout = config.property((&prefix, "timeout.connect"))?; + cfg.connect_timeout = config.property((&prefix, "timeout"))?; cfg.manager = Some(ManagerConfig { recycling_method: RecyclingMethod::Fast, }); diff --git a/crates/store/src/backend/redis/mod.rs b/crates/store/src/backend/redis/mod.rs index c313320f..6f542163 100644 --- a/crates/store/src/backend/redis/mod.rs +++ b/crates/store/src/backend/redis/mod.rs @@ -58,60 +58,64 @@ enum RedisPool { impl RedisStore { pub async fn open(config: &Config, prefix: impl AsKey) -> crate::Result { let prefix = prefix.as_key(); + let urls = config + .values((&prefix, "urls")) + .map(|(_, v)| v.to_string()) + .collect::>(); + if urls.is_empty() { + return Err(crate::Error::InternalError(format!( + "No Redis URLs specified for {prefix:?}" + ))); + } - let db = if let Some(url) = config.value((&prefix, "url")) { - Self { + Ok(match config.value_require((&prefix, "redis-type"))? { + "single" => Self { pool: RedisPool::Single(build_pool( config, &prefix, RedisConnectionManager { - client: Client::open(url)?, + client: Client::open(urls.into_iter().next().unwrap())?, timeout: config.property_or_static((&prefix, "timeout"), "10s")?, }, )?), + }, + "cluster" => { + let mut builder = ClusterClientBuilder::new(urls.into_iter()); + if let Some(value) = config.property((&prefix, "user"))? { + builder = builder.username(value); + } + if let Some(value) = config.property((&prefix, "password"))? { + builder = builder.password(value); + } + if let Some(value) = config.property((&prefix, "retry.total"))? { + builder = builder.retries(value); + } + if let Some(value) = config.property::((&prefix, "retry.max-wait"))? { + builder = builder.max_retry_wait(value.as_millis() as u64); + } + if let Some(value) = config.property::((&prefix, "retry.min-wait"))? { + builder = builder.min_retry_wait(value.as_millis() as u64); + } + if let Some(true) = config.property::((&prefix, "read-from-replicas"))? { + builder = builder.read_from_replicas(); + } + Self { + pool: RedisPool::Cluster(build_pool( + config, + &prefix, + RedisClusterConnectionManager { + client: builder.build()?, + timeout: config.property_or_static((&prefix, "timeout"), "10s")?, + }, + )?), + } } - } else { - let addresses = config - .values((&prefix, "urls")) - .map(|(_, v)| v.to_string()) - .collect::>(); - if addresses.is_empty() { + invalid => { return Err(crate::Error::InternalError(format!( - "No Redis cluster URLs specified for {prefix:?}" + "Invalid Redis type {invalid:?} for {prefix:?}" ))); } - let mut builder = ClusterClientBuilder::new(addresses.into_iter()); - if let Some(value) = config.property((&prefix, "username"))? { - builder = builder.username(value); - } - if let Some(value) = config.property((&prefix, "password"))? { - builder = builder.password(value); - } - if let Some(value) = config.property((&prefix, "retries"))? { - builder = builder.retries(value); - } - if let Some(value) = config.property::((&prefix, "max-retry-wait"))? { - builder = builder.max_retry_wait(value.as_millis() as u64); - } - if let Some(value) = config.property::((&prefix, "min-retry-wait"))? { - builder = builder.min_retry_wait(value.as_millis() as u64); - } - if let Some(true) = config.property::((&prefix, "read-from-replicas"))? { - builder = builder.read_from_replicas(); - } - Self { - pool: RedisPool::Cluster(build_pool( - config, - &prefix, - RedisClusterConnectionManager { - client: builder.build()?, - timeout: config.property_or_static((&prefix, "timeout"), "10s")?, - }, - )?), - } - }; - - Ok(db) + }) } } diff --git a/crates/store/src/dispatch/blocked.rs b/crates/store/src/dispatch/blocked.rs index f34d1a7f..8716016a 100644 --- a/crates/store/src/dispatch/blocked.rs +++ b/crates/store/src/dispatch/blocked.rs @@ -46,6 +46,7 @@ pub struct BlockedIps { } pub const BLOCKED_IP_KEY: &str = "server.security.blocked-networks"; +pub const BLOCKED_IP_PREFIX: &str = "server.security.blocked-networks."; impl BlockedIps { pub fn new(store: LookupStore) -> Self { @@ -64,14 +65,19 @@ impl BlockedIps { .property::("server.security.fail2ban")? .map(Arc::new), ); - self.reload_blocked_ips(config) + self.reload_blocked_ips(config.set_values(BLOCKED_IP_KEY)) } - pub fn reload_blocked_ips(&self, config: &Config) -> utils::config::Result<()> { + pub fn reload_blocked_ips(&self, ips: I) -> utils::config::Result<()> + where + T: AsRef, + I: IntoIterator, + { let mut ip_addresses = AHashSet::new(); let mut ip_networks = Vec::new(); - for ip in config.set_values(BLOCKED_IP_KEY) { + for ip in ips { + let ip = ip.as_ref(); if ip.contains('/') { ip_networks.push(ip.parse_key(BLOCKED_IP_KEY)?); } else { diff --git a/crates/store/src/dispatch/config.rs b/crates/store/src/dispatch/config.rs index 774f2bdb..bdb01e6f 100644 --- a/crates/store/src/dispatch/config.rs +++ b/crates/store/src/dispatch/config.rs @@ -21,7 +21,7 @@ * for more details. */ -use utils::config::{Config, ConfigKey}; +use utils::config::ConfigKey; use crate::{ write::{BatchBuilder, ValueClass}, @@ -34,8 +34,8 @@ impl Store { .await } - pub async fn config_list(&self, key: impl AsRef) -> crate::Result { - let key = key.as_ref().as_bytes(); + pub async fn config_list(&self, prefix: &str) -> crate::Result> { + let key = prefix.as_bytes(); let from_key = ValueKey::from(ValueClass::Config(key.to_vec())); let to_key = ValueKey::from(ValueClass::Config( key.iter() @@ -43,20 +43,26 @@ impl Store { .chain([u8::MAX, u8::MAX, u8::MAX, u8::MAX, u8::MAX]) .collect::>(), )); - let mut config = Config::default(); + let mut results = Vec::new(); self.iterate( IterateParams::new(from_key, to_key).ascending(), |key, value| { - config.keys.insert( - String::deserialize(key.get(1..).unwrap_or_default())?, - String::deserialize(value)?, - ); + let mut key = + std::str::from_utf8(key.get(1..).unwrap_or_default()).map_err(|_| { + crate::Error::InternalError("Failed to deserialize config key".to_string()) + })?; + if !prefix.is_empty() { + key = key.strip_prefix(prefix).unwrap_or(key); + } + + results.push((key.to_string(), String::deserialize(value)?)); + Ok(true) }, ) .await?; - Ok(config) + Ok(results) } pub async fn config_set(&self, keys: impl Iterator) -> crate::Result<()> { diff --git a/crates/store/src/write/bitmap.rs b/crates/store/src/write/bitmap.rs index 1c7190d7..7e3fccbf 100644 --- a/crates/store/src/write/bitmap.rs +++ b/crates/store/src/write/bitmap.rs @@ -74,7 +74,6 @@ impl DenseBitmap { pub fn block_index(index: u32) -> u32 { index & BITS_MASK_L } - } pub trait DeserializeBlock { diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index 76a1f2b2..487375db 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -41,6 +41,8 @@ proxy-header = { version = "0.1.0", features = ["tokio"] } regex = "1.7.0" blake3 = "1.3.3" lru-cache = "0.1.2" +http-body-util = "0.1.0" +form_urlencoded = "1.1.0" [target.'cfg(unix)'.dependencies] privdrop = "0.5.3" diff --git a/crates/utils/src/config/mod.rs b/crates/utils/src/config/mod.rs index fc346af2..23a5804f 100644 --- a/crates/utils/src/config/mod.rs +++ b/crates/utils/src/config/mod.rs @@ -187,8 +187,8 @@ impl Config { config } - pub fn update(&mut self, config: Self) { - self.keys.extend(config.keys); + pub fn update(&mut self, settings: Vec<(String, String)>) { + self.keys.extend(settings); } } diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index d0842f40..d5183d5a 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -35,6 +35,7 @@ pub mod lru_cache; pub mod map; pub mod snowflake; pub mod suffixlist; +pub mod url_params; use opentelemetry::KeyValue; use opentelemetry_otlp::WithExportConfig; diff --git a/crates/utils/src/url_params.rs b/crates/utils/src/url_params.rs new file mode 100644 index 00000000..e9f78101 --- /dev/null +++ b/crates/utils/src/url_params.rs @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{borrow::Cow, collections::HashMap}; + +#[derive(Default)] +pub struct UrlParams<'x> { + params: HashMap, Cow<'x, str>>, +} + +impl<'x> UrlParams<'x> { + pub fn new(query: Option<&'x str>) -> Self { + if let Some(query) = query { + Self { + params: form_urlencoded::parse(query.as_bytes()) + .filter(|(_, value)| !value.is_empty()) + .collect(), + } + } else { + Self::default() + } + } + + pub fn get(&self, key: &str) -> Option<&str> { + self.params.get(key).map(|v| v.as_ref()) + } + + pub fn has_key(&self, key: &str) -> bool { + self.params.contains_key(key) + } + + pub fn parse(&self, key: &str) -> Option + where + T: std::str::FromStr, + { + self.get(key).and_then(|v| v.parse().ok()) + } +} diff --git a/resources/config/directory/imap.toml b/resources/config/directory/imap.toml index 73c7c231..912906a5 100644 --- a/resources/config/directory/imap.toml +++ b/resources/config/directory/imap.toml @@ -4,7 +4,7 @@ [directory."imap"] type = "imap" -address = "127.0.0.1" +host = "127.0.0.1" port = 993 disable = true @@ -17,7 +17,7 @@ wait = "30s" recycle = "30s" [directory."imap".tls] -implicit = true +enable = true allow-invalid-certs = true [directory."imap".cache] diff --git a/resources/config/directory/ldap.toml b/resources/config/directory/ldap.toml index bb55c5e8..6e91a31a 100644 --- a/resources/config/directory/ldap.toml +++ b/resources/config/directory/ldap.toml @@ -4,7 +4,7 @@ [directory."ldap"] type = "ldap" -address = "ldap://localhost:389" +url = "ldap://localhost:389" base-dn = "dc=example,dc=org" timeout = "30s" disable = true @@ -50,7 +50,7 @@ domains = "(&(|(objectClass=posixAccount)(objectClass=posixGroup))(|(mail=*@?)(m [directory."ldap".attributes] name = "uid" -type = "objectClass" +class = "objectClass" description = ["principalName", "description"] secret = "userPassword" groups = ["memberOf", "otherGroups"] diff --git a/resources/config/directory/lmtp.toml b/resources/config/directory/lmtp.toml index 2732be78..bff3ba4f 100644 --- a/resources/config/directory/lmtp.toml +++ b/resources/config/directory/lmtp.toml @@ -4,7 +4,7 @@ [directory."lmtp"] type = "lmtp" -address = "127.0.0.1" +host = "127.0.0.1" port = 11200 disable = true @@ -21,7 +21,7 @@ wait = "30s" recycle = "30s" [directory."lmtp".tls] -implicit = false +enable = false allow-invalid-certs = true [directory."lmtp".cache] diff --git a/resources/config/directory/memory.toml b/resources/config/directory/memory.toml index cb5bbb88..d484d999 100644 --- a/resources/config/directory/memory.toml +++ b/resources/config/directory/memory.toml @@ -16,14 +16,14 @@ subaddressing = true [[directory."memory".principals]] name = "admin" -type = "admin" +class = "admin" description = "Superuser" secret = "changeme" email = ["postmaster@%{DEFAULT_DOMAIN}%"] [[directory."memory".principals]] name = "john" -type = "individual" +class = "individual" description = "John Doe" secret = "12345" email = ["john@%{DEFAULT_DOMAIN}%", "jdoe@%{DEFAULT_DOMAIN}%", "john.doe@%{DEFAULT_DOMAIN}%"] @@ -32,7 +32,7 @@ member-of = ["sales"] [[directory."memory".principals]] name = "jane" -type = "individual" +class = "individual" description = "Jane Doe" secret = "abcde" email = ["jane@%{DEFAULT_DOMAIN}%", "jane.doe@%{DEFAULT_DOMAIN}%"] @@ -41,7 +41,7 @@ member-of = ["sales", "support"] [[directory."memory".principals]] name = "bill" -type = "individual" +class = "individual" description = "Bill Foobar" secret = "$2y$05$bvIG6Nmid91Mu9RcmmWZfO5HJIMCT8riNW0hEp8f6/FuA2/mHZFpe" quota = 50000000 @@ -50,10 +50,10 @@ email-list = ["info@%{DEFAULT_DOMAIN}%"] [[directory."memory".principals]] name = "sales" -type = "group" +class = "group" description = "Sales Team" [[directory."memory".principals]] name = "support" -type = "group" +class = "group" description = "Support Team" diff --git a/resources/config/directory/sql.toml b/resources/config/directory/sql.toml index 49f0f06d..04a8ea23 100644 --- a/resources/config/directory/sql.toml +++ b/resources/config/directory/sql.toml @@ -20,7 +20,7 @@ entries = 500 ttl = {positive = '1h', negative = '10m'} [directory."sql".columns] -type = "type" +class = "type" secret = "secret" description = "description" quota = "quota" diff --git a/resources/config/imap/settings.toml b/resources/config/imap/settings.toml index 6ffecc6a..873ff42a 100644 --- a/resources/config/imap/settings.toml +++ b/resources/config/imap/settings.toml @@ -20,6 +20,3 @@ idle = "30m" [imap.rate-limit] requests = "2000/1m" concurrent = 6 - -[imap.protocol] -uidplus = false diff --git a/resources/config/smtp/listener.toml b/resources/config/smtp/listener.toml index fc4aa45d..f385fb7f 100644 --- a/resources/config/smtp/listener.toml +++ b/resources/config/smtp/listener.toml @@ -16,6 +16,6 @@ bind = ["[::]:465"] protocol = "smtp" tls.implicit = true -[server.listener."management"] -bind = ["127.0.0.1:8080"] -protocol = "http" +#[server.listener."management"] +#bind = ["127.0.0.1:8080"] +#protocol = "http" diff --git a/resources/config/store/foundationdb.toml b/resources/config/store/foundationdb.toml index c7f1b300..b52705c8 100644 --- a/resources/config/store/foundationdb.toml +++ b/resources/config/store/foundationdb.toml @@ -4,15 +4,17 @@ [store."foundationdb"] type = "foundationdb" -#path = "/etc/foundationdb/fdb.cluster" +#cluster-file = "/etc/foundationdb/fdb.cluster" disable = true #[store."foundationdb".transaction] #timeout = "5s" #retry-limit = 10 #max-retry-delay = "1s" -#machine-id = "stalwart" -#data-center-id = "my-datacenter" + +#[store."foundationdb".ids] +#machine = "stalwart" +#data-center = "my-datacenter" [store."foundationdb".purge] frequency = "0 3 *" diff --git a/resources/config/store/mysql.toml b/resources/config/store/mysql.toml index 1b291daf..b0fa130d 100644 --- a/resources/config/store/mysql.toml +++ b/resources/config/store/mysql.toml @@ -11,9 +11,7 @@ user = "root" password = "password" disable = true #max-allowed-packet = 1073741824 - -#[store."mysql".timeout] -#wait = "15s" +timeout = "15s" #[store."mysql".pool] #max-connections = 10 diff --git a/resources/config/store/postgresql.toml b/resources/config/store/postgresql.toml index 4eedfa73..bf0d2fef 100644 --- a/resources/config/store/postgresql.toml +++ b/resources/config/store/postgresql.toml @@ -9,11 +9,9 @@ port = 5432 database = "stalwart" user = "postgres" password = "mysecretpassword" +timeout = "15s" disable = true -[store."postgresql".timeout] -connect = "15s" - [store."postgresql".tls] enable = false allow-invalid-certs = false diff --git a/resources/config/store/redis.toml b/resources/config/store/redis.toml index 39e87224..92f741b8 100644 --- a/resources/config/store/redis.toml +++ b/resources/config/store/redis.toml @@ -4,13 +4,15 @@ [store."redis"] type = "redis" -url = "redis://127.0.0.1" -#urls = ["redis://192.168.1.1", "redis://192.168.1.1"] # for Redis cluster -username = "my_username" +redis-type = "single" +urls = ["redis://127.0.0.1"] +user = "my_username" password = "secretpassword" timeout = "10s" -#retries = 3 -#max-retry-wait = "1s" -#min-retry-wait = "500ms" #read-from-replicas = false disable = true + +#[store."redis".retry] +#total = 3 +#max-wait = "1s" +#min-wait = "500ms"