diff --git a/Cargo.lock b/Cargo.lock index 04a6bade..3cfbb566 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1504,7 +1504,7 @@ dependencies = [ [[package]] name = "directory" -version = "0.1.0" +version = "0.6.0" dependencies = [ "ahash 0.8.11", "argon2", diff --git a/crates/common/src/addresses.rs b/crates/common/src/addresses.rs index 3abce130..1f54c864 100644 --- a/crates/common/src/addresses.rs +++ b/crates/common/src/addresses.rs @@ -4,7 +4,7 @@ use directory::Directory; use utils::config::{utils::AsKey, Config}; use crate::{ - config::smtp::session::AddressMapping, + config::smtp::{session::AddressMapping, V_RECIPIENT}, expr::{functions::ResolveVariable, if_block::IfBlock, tokenizer::TokenMap, Variable}, Core, }; @@ -130,7 +130,11 @@ impl AddressMapping { } else if let Some(if_block) = IfBlock::try_parse( config, key, - &TokenMap::default().with_variables([("address", 1), ("email", 1), ("rcpt", 1)]), + &TokenMap::default().with_variables([ + ("address", V_RECIPIENT), + ("email", V_RECIPIENT), + ("rcpt", V_RECIPIENT), + ]), ) { AddressMapping::Custom(if_block) } else { @@ -148,7 +152,11 @@ impl ResolveVariable for Address<'_> { } impl AddressMapping { - async fn to_subaddress<'x, 'y: 'x>(&'x self, core: &Core, address: &'y str) -> Cow<'x, str> { + pub async fn to_subaddress<'x, 'y: 'x>( + &'x self, + core: &Core, + address: &'y str, + ) -> Cow<'x, str> { match self { AddressMapping::Enable => { if let Some((local_part, domain_part)) = address.rsplit_once('@') { @@ -172,7 +180,7 @@ impl AddressMapping { address.into() } - async fn to_catch_all<'x, 'y: 'x>( + pub async fn to_catch_all<'x, 'y: 'x>( &'x self, core: &Core, address: &'y str, diff --git a/crates/common/src/config/imap.rs b/crates/common/src/config/imap.rs index 9d07fa23..f24c20fb 100644 --- a/crates/common/src/config/imap.rs +++ b/crates/common/src/config/imap.rs @@ -2,6 +2,7 @@ use std::time::Duration; use utils::config::{Config, Rate}; +#[derive(Default)] pub struct ImapConfig { pub max_request_size: usize, pub max_auth_failures: u32, @@ -20,28 +21,28 @@ impl ImapConfig { pub fn parse(config: &mut Config) -> Self { ImapConfig { max_request_size: config - .property_or_default_("imap.request.max-size", "52428800") + .property_or_default("imap.request.max-size", "52428800") .unwrap_or(52428800), max_auth_failures: config - .property_or_default_("imap.auth.max-failures", "3") + .property_or_default("imap.auth.max-failures", "3") .unwrap_or(3), name_shared: config .value("imap.folders.name.shared") .unwrap_or("Shared Folders") .to_string(), timeout_auth: config - .property_or_default_("imap.timeout.authenticated", "30m") + .property_or_default("imap.timeout.authenticated", "30m") .unwrap_or_else(|| Duration::from_secs(1800)), timeout_unauth: config - .property_or_default_("imap.timeout.anonymous", "1m") + .property_or_default("imap.timeout.anonymous", "1m") .unwrap_or_else(|| Duration::from_secs(60)), timeout_idle: config - .property_or_default_("imap.timeout.idle", "30m") + .property_or_default("imap.timeout.idle", "30m") .unwrap_or_else(|| Duration::from_secs(1800)), - rate_requests: config.property_or_default_("imap.rate-limit.requests", "2000/1m"), - rate_concurrent: config.property_("imap.rate-limit.concurrent"), + rate_requests: config.property_or_default("imap.rate-limit.requests", "2000/1m"), + rate_concurrent: config.property("imap.rate-limit.concurrent"), allow_plain_auth: config - .property_or_default_("imap.auth.allow-plain-text", "false") + .property_or_default("imap.auth.allow-plain-text", "false") .unwrap_or(false), } } diff --git a/crates/common/src/config/jmap/capabilities.rs b/crates/common/src/config/jmap/capabilities.rs index 3b1cc958..8e21ae76 100644 --- a/crates/common/src/config/jmap/capabilities.rs +++ b/crates/common/src/config/jmap/capabilities.rs @@ -124,11 +124,11 @@ impl JmapConfig { Capabilities::SieveAccount(SieveAccountCapabilities { max_script_name: self.sieve_max_script_name, max_script_size: config - .property_("sieve.untrusted.max-script-size") + .property("sieve.untrusted.max-script-size") .unwrap_or(1024 * 1024), max_scripts: self.sieve_max_scripts, max_redirects: config - .property_("sieve.untrusted.max-redirects") + .property("sieve.untrusted.max-redirects") .unwrap_or(1), extensions, notification_methods: if !notification_methods.is_empty() { diff --git a/crates/common/src/config/jmap/settings.rs b/crates/common/src/config/jmap/settings.rs index d8d1a397..9ba9cbe6 100644 --- a/crates/common/src/config/jmap/settings.rs +++ b/crates/common/src/config/jmap/settings.rs @@ -6,6 +6,7 @@ use nlp::language::Language; use store::rand::{distributions::Alphanumeric, thread_rng, Rng}; use utils::config::{cron::SimpleCron, utils::ParseValue, Config, Rate}; +#[derive(Default)] pub struct JmapConfig { pub default_language: Language, pub query_max_results: usize, @@ -85,68 +86,67 @@ impl JmapConfig { ) .unwrap_or(Language::English), query_max_results: config - .property_("jmap.protocol.query.max-results") + .property("jmap.protocol.query.max-results") .unwrap_or(5000), changes_max_results: config - .property_("jmap.protocol.changes.max-results") + .property("jmap.protocol.changes.max-results") .unwrap_or(5000), snippet_max_results: config - .property_("jmap.protocol.search-snippet.max-results") + .property("jmap.protocol.search-snippet.max-results") .unwrap_or(100), request_max_size: config - .property_("jmap.protocol.request.max-size") + .property("jmap.protocol.request.max-size") .unwrap_or(10000000), request_max_calls: config - .property_("jmap.protocol.request.max-calls") + .property("jmap.protocol.request.max-calls") .unwrap_or(16), request_max_concurrent: config - .property_("jmap.protocol.request.max-concurrent") + .property("jmap.protocol.request.max-concurrent") .unwrap_or(4), get_max_objects: config - .property_("jmap.protocol.get.max-objects") + .property("jmap.protocol.get.max-objects") .unwrap_or(500), set_max_objects: config - .property_("jmap.protocol.set.max-objects") + .property("jmap.protocol.set.max-objects") .unwrap_or(500), upload_max_size: config - .property_("jmap.protocol.upload.max-size") + .property("jmap.protocol.upload.max-size") .unwrap_or(50000000), upload_max_concurrent: config - .property_("jmap.protocol.upload.max-concurrent") + .property("jmap.protocol.upload.max-concurrent") .unwrap_or(4), upload_tmp_quota_size: config - .property_("jmap.protocol.upload.quota.size") + .property("jmap.protocol.upload.quota.size") .unwrap_or(50000000), upload_tmp_quota_amount: config - .property_("jmap.protocol.upload.quota.files") + .property("jmap.protocol.upload.quota.files") .unwrap_or(1000), upload_tmp_ttl: config - .property_or_default_::("jmap.protocol.upload.ttl", "1h") + .property_or_default::("jmap.protocol.upload.ttl", "1h") .unwrap_or_else(|| Duration::from_secs(3600)) .as_secs(), - mailbox_max_depth: config.property_("jmap.mailbox.max-depth").unwrap_or(10), + mailbox_max_depth: config.property("jmap.mailbox.max-depth").unwrap_or(10), mailbox_name_max_len: config - .property_("jmap.mailbox.max-name-length") + .property("jmap.mailbox.max-name-length") .unwrap_or(255), mail_attachments_max_size: config - .property_("jmap.email.max-attachment-size") + .property("jmap.email.max-attachment-size") .unwrap_or(50000000), - mail_max_size: config.property_("jmap.email.max-size").unwrap_or(75000000), - mail_parse_max_items: config.property_("jmap.email.parse.max-items").unwrap_or(10), + mail_max_size: config.property("jmap.email.max-size").unwrap_or(75000000), + mail_parse_max_items: config.property("jmap.email.parse.max-items").unwrap_or(10), sieve_max_script_name: config - .property_("sieve.untrusted.limits.name-length") + .property("sieve.untrusted.limits.name-length") .unwrap_or(512), sieve_max_scripts: config - .property_("sieve.untrusted.limits.max-scripts") + .property("sieve.untrusted.limits.max-scripts") .unwrap_or(256), capabilities: BaseCapabilities::default(), session_cache_ttl: config - .property_("cache.session.ttl") + .property("cache.session.ttl") .unwrap_or(Duration::from_secs(3600)), - rate_authenticated: config.property_or_default_("jmap.rate-limit.account", "1000/1m"), - rate_authenticate_req: config - .property_or_default_("authentication.rate-limit", "10/1m"), - rate_anonymous: config.property_or_default_("jmap.rate-limit.anonymous", "100/1m"), + rate_authenticated: config.property_or_default("jmap.rate-limit.account", "1000/1m"), + rate_authenticate_req: config.property_or_default("authentication.rate-limit", "10/1m"), + rate_anonymous: config.property_or_default("jmap.rate-limit.anonymous", "100/1m"), oauth_key: config .value("oauth.key") .map(|s| s.to_string()) @@ -158,51 +158,51 @@ impl JmapConfig { .collect::() }), oauth_expiry_user_code: config - .property_or_default_::("oauth.expiry.user-code", "30m") + .property_or_default::("oauth.expiry.user-code", "30m") .unwrap_or_else(|| Duration::from_secs(30 * 60)) .as_secs(), oauth_expiry_auth_code: config - .property_or_default_::("oauth.expiry.auth-code", "10m") + .property_or_default::("oauth.expiry.auth-code", "10m") .unwrap_or_else(|| Duration::from_secs(10 * 60)) .as_secs(), oauth_expiry_token: config - .property_or_default_::("oauth.expiry.token", "1h") + .property_or_default::("oauth.expiry.token", "1h") .unwrap_or_else(|| Duration::from_secs(60 * 60)) .as_secs(), oauth_expiry_refresh_token: config - .property_or_default_::("oauth.expiry.refresh-token", "30d") + .property_or_default::("oauth.expiry.refresh-token", "30d") .unwrap_or_else(|| Duration::from_secs(30 * 24 * 60 * 60)) .as_secs(), oauth_expiry_refresh_token_renew: config - .property_or_default_::("oauth.expiry.refresh-token-renew", "4d") + .property_or_default::("oauth.expiry.refresh-token-renew", "4d") .unwrap_or_else(|| Duration::from_secs(4 * 24 * 60 * 60)) .as_secs(), oauth_max_auth_attempts: config - .property_or_default_("oauth.auth.max-attempts", "3") + .property_or_default("oauth.auth.max-attempts", "3") .unwrap_or(10), event_source_throttle: config - .property_or_default_("jmap.event-source.throttle", "1s") + .property_or_default("jmap.event-source.throttle", "1s") .unwrap_or_else(|| Duration::from_secs(1)), web_socket_throttle: config - .property_or_default_("jmap.web-socket.throttle", "1s") + .property_or_default("jmap.web-socket.throttle", "1s") .unwrap_or_else(|| Duration::from_secs(1)), web_socket_timeout: config - .property_or_default_("jmap.web-socket.timeout", "10m") + .property_or_default("jmap.web-socket.timeout", "10m") .unwrap_or_else(|| Duration::from_secs(10 * 60)), web_socket_heartbeat: config - .property_or_default_("jmap.web-socket.heartbeat", "1m") + .property_or_default("jmap.web-socket.heartbeat", "1m") .unwrap_or_else(|| Duration::from_secs(60)), push_max_total: config - .property_or_default_("jmap.push.max-total", "100") + .property_or_default("jmap.push.max-total", "100") .unwrap_or(100), principal_allow_lookups: config - .property_("jmap.principal.allow-lookups") + .property("jmap.principal.allow-lookups") .unwrap_or(true), encrypt: config - .property_or_default_("storage.encryption.enable", "true") + .property_or_default("storage.encryption.enable", "true") .unwrap_or(true), encrypt_append: config - .property_or_default_("storage.encryption.append", "false") + .property_or_default("storage.encryption.append", "false") .unwrap_or(false), spam_header: config.value("spam.header.is-spam").and_then(|v| { v.split_once(':').map(|(k, v)| { @@ -213,7 +213,7 @@ impl JmapConfig { }) }), http_use_forwarded: config - .property_("server.http.use-x-forwarded") + .property("server.http.use-x-forwarded") .unwrap_or(false), http_headers: config .values("server.http.headers") @@ -244,25 +244,25 @@ impl JmapConfig { .map_err(|e| config.new_parse_error("server.http.headers", e)) .unwrap_or_default(), push_attempt_interval: config - .property_or_default_("jmap.push.attempts.interval", "1m") + .property_or_default("jmap.push.attempts.interval", "1m") .unwrap_or_else(|| Duration::from_secs(60)), push_attempts_max: config - .property_or_default_("jmap.push.attempts.max", "3") + .property_or_default("jmap.push.attempts.max", "3") .unwrap_or(3), push_retry_interval: config - .property_or_default_("jmap.push.retry.interval", "1s") + .property_or_default("jmap.push.retry.interval", "1s") .unwrap_or_else(|| Duration::from_secs(1)), push_timeout: config - .property_or_default_("jmap.push.timeout.request", "10s") + .property_or_default("jmap.push.timeout.request", "10s") .unwrap_or_else(|| Duration::from_secs(10)), push_verify_timeout: config - .property_or_default_("jmap.push.timeout.verify", "1m") + .property_or_default("jmap.push.timeout.verify", "1m") .unwrap_or_else(|| Duration::from_secs(60)), push_throttle: config - .property_or_default_("jmap.push.throttle", "1s") + .property_or_default("jmap.push.throttle", "1s") .unwrap_or_else(|| Duration::from_secs(1)), session_purge_frequency: config - .property_or_default_::("jmap.session.purge.frequency", "15 * *") + .property_or_default::("jmap.session.purge.frequency", "15 * *") .unwrap_or_else(|| SimpleCron::parse_value("15 * *", "").unwrap()), }; diff --git a/crates/common/src/config/mod.rs b/crates/common/src/config/mod.rs index 4e13b20c..9b5accbf 100644 --- a/crates/common/src/config/mod.rs +++ b/crates/common/src/config/mod.rs @@ -24,7 +24,7 @@ pub mod tracers; impl Core { pub async fn parse(config: &mut Config, stores: Stores) -> Self { let mut data = config - .value_require_("storage.data") + .value_require("storage.data") .map(|id| id.to_string()) .and_then(|id| { if let Some(store) = stores.stores.get(&id) { @@ -36,7 +36,7 @@ impl Core { }) .unwrap_or_default(); let mut blob = config - .value_require_("storage.blob") + .value_require("storage.blob") .map(|id| id.to_string()) .and_then(|id| { if let Some(store) = stores.blob_stores.get(&id) { @@ -48,7 +48,7 @@ impl Core { }) .unwrap_or_default(); let mut lookup = config - .value_require_("storage.lookup") + .value_require("storage.lookup") .map(|id| id.to_string()) .and_then(|id| { if let Some(store) = stores.lookup_stores.get(&id) { @@ -63,7 +63,7 @@ impl Core { }) .unwrap_or_default(); let mut fts = config - .value_require_("storage.fts") + .value_require("storage.fts") .map(|id| id.to_string()) .and_then(|id| { if let Some(store) = stores.fts_stores.get(&id) { @@ -79,7 +79,7 @@ impl Core { .unwrap_or_default(); let directories = Directories::parse(config, &stores, data.clone()).await; let directory = config - .value_require_("storage.directory") + .value_require("storage.directory") .map(|id| id.to_string()) .and_then(|id| { if let Some(directory) = directories.directories.get(&id) { diff --git a/crates/common/src/config/scripts.rs b/crates/common/src/config/scripts.rs index 356af515..13cacb2e 100644 --- a/crates/common/src/config/scripts.rs +++ b/crates/common/src/config/scripts.rs @@ -39,52 +39,52 @@ impl Scripting { let untrusted_compiler = Compiler::new() .with_max_script_size( config - .property_("sieve.untrusted.limits.script-size") + .property("sieve.untrusted.limits.script-size") .unwrap_or(1024 * 1024), ) .with_max_string_size( config - .property_("sieve.untrusted.limits.string-length") + .property("sieve.untrusted.limits.string-length") .unwrap_or(4096), ) .with_max_variable_name_size( config - .property_("sieve.untrusted.limits.variable-name-length") + .property("sieve.untrusted.limits.variable-name-length") .unwrap_or(32), ) .with_max_nested_blocks( config - .property_("sieve.untrusted.limits.nested-blocks") + .property("sieve.untrusted.limits.nested-blocks") .unwrap_or(15), ) .with_max_nested_tests( config - .property_("sieve.untrusted.limits.nested-tests") + .property("sieve.untrusted.limits.nested-tests") .unwrap_or(15), ) .with_max_nested_foreverypart( config - .property_("sieve.untrusted.limits.nested-foreverypart") + .property("sieve.untrusted.limits.nested-foreverypart") .unwrap_or(3), ) .with_max_match_variables( config - .property_("sieve.untrusted.limits.match-variables") + .property("sieve.untrusted.limits.match-variables") .unwrap_or(30), ) .with_max_local_variables( config - .property_("sieve.untrusted.limits.local-variables") + .property("sieve.untrusted.limits.local-variables") .unwrap_or(128), ) .with_max_header_size( config - .property_("sieve.untrusted.limits.header-size") + .property("sieve.untrusted.limits.header-size") .unwrap_or(1024), ) .with_max_includes( config - .property_("sieve.untrusted.limits.includes") + .property("sieve.untrusted.limits.includes") .unwrap_or(3), ); @@ -92,48 +92,48 @@ impl Scripting { let untrusted_runtime = Runtime::new() .with_max_nested_includes( config - .property_("sieve.untrusted.limits.nested-includes") + .property("sieve.untrusted.limits.nested-includes") .unwrap_or(3), ) .with_cpu_limit( config - .property_("sieve.untrusted.limits.cpu") + .property("sieve.untrusted.limits.cpu") .unwrap_or(5000), ) .with_max_variable_size( config - .property_("sieve.untrusted.limits.variable-size") + .property("sieve.untrusted.limits.variable-size") .unwrap_or(4096), ) .with_max_redirects( config - .property_("sieve.untrusted.limits.redirects") + .property("sieve.untrusted.limits.redirects") .unwrap_or(1), ) .with_max_received_headers( config - .property_("sieve.untrusted.limits.received-headers") + .property("sieve.untrusted.limits.received-headers") .unwrap_or(10), ) .with_max_header_size( config - .property_("sieve.untrusted.limits.header-size") + .property("sieve.untrusted.limits.header-size") .unwrap_or(1024), ) .with_max_out_messages( config - .property_("sieve.untrusted.limits.outgoing-messages") + .property("sieve.untrusted.limits.outgoing-messages") .unwrap_or(3), ) .with_default_vacation_expiry( config - .property_::("sieve.untrusted.default-expiry.vacation") + .property::("sieve.untrusted.default-expiry.vacation") .unwrap_or(Duration::from_secs(30 * 86400)) .as_secs(), ) .with_default_duplicate_expiry( config - .property_::("sieve.untrusted.default-expiry.duplicate") + .property::("sieve.untrusted.default-expiry.duplicate") .unwrap_or(Duration::from_secs(7 * 86400)) .as_secs(), ) @@ -201,7 +201,7 @@ impl Scripting { .with_max_includes(10) .with_no_capability_check( config - .property_or_default_("sieve.trusted.no-capability-check", "true") + .property_or_default("sieve.trusted.no-capability-check", "true") .unwrap_or(true), ) .register_functions(&mut fnc_map); @@ -223,7 +223,7 @@ impl Scripting { .with_capability(Capability::While) .with_max_variable_size( config - .property_or_default_("sieve.trusted.limits.variable-size", "52428800") + .property_or_default("sieve.trusted.limits.variable-size", "52428800") .unwrap_or(52428800), ) .with_max_header_size(10240) @@ -231,22 +231,22 @@ impl Scripting { .with_valid_ext_lists(stores.lookup_stores.keys().map(|k| k.to_string())) .with_functions(&mut fnc_map); - if let Some(value) = config.property_("sieve.trusted.limits.redirects") { + if let Some(value) = config.property("sieve.trusted.limits.redirects") { trusted_runtime.set_max_redirects(value); } - if let Some(value) = config.property_("sieve.trusted.limits.out-messages") { + if let Some(value) = config.property("sieve.trusted.limits.out-messages") { trusted_runtime.set_max_out_messages(value); } - if let Some(value) = config.property_("sieve.trusted.limits.cpu") { + if let Some(value) = config.property("sieve.trusted.limits.cpu") { trusted_runtime.set_cpu_limit(value); } - if let Some(value) = config.property_("sieve.trusted.limits.nested-includes") { + if let Some(value) = config.property("sieve.trusted.limits.nested-includes") { trusted_runtime.set_max_nested_includes(value); } - if let Some(value) = config.property_("sieve.trusted.limits.received-headers") { + if let Some(value) = config.property("sieve.trusted.limits.received-headers") { trusted_runtime.set_max_received_headers(value); } - if let Some(value) = config.property_::("sieve.trusted.limits.duplicate-expiry") { + if let Some(value) = config.property::("sieve.trusted.limits.duplicate-expiry") { trusted_runtime.set_default_duplicate_expiry(value.as_secs()); } let hostname = if let Some(hostname) = config.value("sieve.trusted.hostname") { @@ -267,7 +267,7 @@ impl Scripting { { // Skip sub-scripts if config - .property_(("sieve.trusted.scripts", id.as_str(), "snippet")) + .property(("sieve.trusted.scripts", id.as_str(), "snippet")) .unwrap_or(false) { continue; @@ -311,16 +311,37 @@ impl Scripting { scripts, bayes_cache: BayesTokenCache::new( config - .property_or_default_("cache.bayes.capacity", "8192") + .property_or_default("cache.bayes.capacity", "8192") .unwrap_or(8192), config - .property_or_default_("cache.bayes.ttl.positive", "1h") + .property_or_default("cache.bayes.ttl.positive", "1h") .unwrap_or_else(|| Duration::from_secs(3600)), config - .property_or_default_("cache.bayes.ttl.negative", "1h") + .property_or_default("cache.bayes.ttl.negative", "1h") .unwrap_or_else(|| Duration::from_secs(3600)), ), remote_lists: Default::default(), } } } + +impl Default for Scripting { + fn default() -> Self { + Scripting { + untrusted_compiler: Compiler::new(), + untrusted_runtime: Runtime::new(), + trusted_runtime: Runtime::new(), + from_addr: "MAILER-DAEMON@localhost".to_string(), + from_name: "Mailer Daemon".to_string(), + return_path: "".to_string(), + sign: Vec::new(), + scripts: AHashMap::new(), + bayes_cache: BayesTokenCache::new( + 8192, + Duration::from_secs(3600), + Duration::from_secs(3600), + ), + remote_lists: Default::default(), + } + } +} diff --git a/crates/common/src/config/server/listener.rs b/crates/common/src/config/server/listener.rs index 90558662..d16d3ce9 100644 --- a/crates/common/src/config/server/listener.rs +++ b/crates/common/src/config/server/listener.rs @@ -65,7 +65,7 @@ impl Servers { // Parse protocol let id = id_.as_str(); let protocol = - if let Some(protocol) = config.property_require_(("server.listener", id, "protocol")) { + if let Some(protocol) = config.property_require(("server.listener", id, "protocol")) { protocol } else { return; @@ -73,7 +73,7 @@ impl Servers { // Build listeners let mut listeners = Vec::new(); - for (_, addr) in config.properties_::(("server.listener", id, "bind")) { + for (_, addr) in config.properties::(("server.listener", id, "bind")) { // Parse bind address and build socket let socket = match if addr.is_ipv4() { TcpSocket::new_v4() @@ -144,17 +144,17 @@ impl Servers { socket, addr, ttl: config - .property_or_else_(("server.listener", id, "socket.ttl"), "server.socket.ttl"), - backlog: config.property_or_else_( + .property_or_else(("server.listener", id, "socket.ttl"), "server.socket.ttl"), + backlog: config.property_or_else( ("server.listener", id, "socket.backlog"), "server.socket.backlog", ), - linger: config.property_or_else_( + linger: config.property_or_else( ("server.listener", id, "socket.linger"), "server.socket.linger", ), nodelay: config - .property_or_else_( + .property_or_else( ("server.listener", id, "socket.nodelay"), "server.socket.nodelay", ) @@ -172,7 +172,7 @@ impl Servers { // Build TLS config let (acceptor, tls_implicit) = if config - .property_or_else_(("server.listener", id, "tls.enable"), "server.tls.enable") + .property_or_else(("server.listener", id, "tls.enable"), "server.tls.enable") .unwrap_or(false) { // Parse protocol versions @@ -203,7 +203,7 @@ impl Servers { } else { "server.tls.disable-ciphers".as_key() }; - for (_, protocol) in config.properties_::(cipher_keys) { + for (_, protocol) in config.properties::(cipher_keys) { disabled_ciphers.push(protocol); } @@ -225,7 +225,7 @@ impl Servers { // Check if this port is used to receive ACME challenges let port_key = ("acme", acme_id, "port").as_key(); let acme_port = config - .property_or_default_::(port_key, "443") + .property_or_default::(port_key, "443") .unwrap_or(443); if listeners.iter().any(|l| l.addr.port() == acme_port) { acme_acceptor = Some(acme.clone()); @@ -284,7 +284,7 @@ impl Servers { }; server_config.ignore_client_order = config - .property_or_else_( + .property_or_else( ("server.listener", id, "tls.ignore-client-order"), "server.tls.ignore-client-order", ) @@ -308,7 +308,7 @@ impl Servers { ( acceptor, config - .property_or_else_( + .property_or_else( ("server.listener", id, "tls.implicit"), "server.tls.implicit", ) @@ -325,13 +325,13 @@ impl Servers { } else { "server.proxy.trusted-networks".as_key() }; - for (_, network) in config.properties_(proxy_keys) { + for (_, network) in config.properties(proxy_keys) { proxy_networks.push(network); } self.servers.push(Server { max_connections: config - .property_or_else_( + .property_or_else( ("server.listener", id, "max-connections"), "server.max-connections", ) diff --git a/crates/common/src/config/server/tls.rs b/crates/common/src/config/server/tls.rs index 0543c989..32ecf772 100644 --- a/crates/common/src/config/server/tls.rs +++ b/crates/common/src/config/server/tls.rs @@ -59,9 +59,9 @@ impl Servers { let key_pk = ("certificate", cert_id, "private-key"); let cert = config - .value_require_(key_cert) + .value_require(key_cert) .map(|s| s.as_bytes().to_vec()); - let pk = config.value_require_(key_pk).map(|s| s.as_bytes().to_vec()); + let pk = config.value_require(key_pk).map(|s| s.as_bytes().to_vec()); if let (Some(cert), Some(pk)) = (cert, pk) { match build_certified_key(cert, pk) { @@ -135,7 +135,7 @@ impl Servers { }) .collect::>(); let renew_before: Duration = config - .property_or_default_(("acme", acme_id.as_str(), "renew-before"), "30d") + .property_or_default(("acme", acme_id.as_str(), "renew-before"), "30d") .unwrap_or_else(|| Duration::from_secs(30 * 24 * 60 * 60)); if directory.is_empty() { diff --git a/crates/common/src/config/smtp/auth.rs b/crates/common/src/config/smtp/auth.rs index 3fd8c8f9..ac008c10 100644 --- a/crates/common/src/config/smtp/auth.rs +++ b/crates/common/src/config/smtp/auth.rs @@ -166,10 +166,10 @@ impl MailAuthConfig { } fn build_signature(config: &mut Config, id: &str) -> Option<(DkimSigner, ArcSealer)> { - match config.property_require_::(("signature", id, "algorithm"))? { + match config.property_require::(("signature", id, "algorithm"))? { Algorithm::RsaSha256 => { let pk = config - .value_require_(("signature", id, "private-key"))? + .value_require(("signature", id, "private-key"))? .to_string(); let key = RsaKey::::from_rsa_pem(&pk) .or_else(|_| RsaKey::::from_pkcs8_pem(&pk)) @@ -200,7 +200,7 @@ fn build_signature(config: &mut Config, id: &str) -> Option<(DkimSigner, ArcSeal (("signature", id, "public-key"), &mut public_key), (("signature", id, "private-key"), &mut private_key), ] { - let mut contents = config.value_require_(key)?.as_bytes().iter().copied(); + let mut contents = config.value_require(key)?.as_bytes().iter().copied(); let mut base64 = vec![]; 'outer: while let Some(ch) = contents.next() { @@ -281,10 +281,10 @@ fn parse_signature>( mail_auth::arc::ArcSealer, )> { let domain = config - .value_require_(("signature", id, "domain"))? + .value_require(("signature", id, "domain"))? .to_string(); let selector = config - .value_require_(("signature", id, "selector"))? + .value_require(("signature", id, "selector"))? .to_string(); let mut headers = config .values(("signature", id, "headers")) @@ -321,7 +321,7 @@ fn parse_signature>( .selector(selector) .headers(headers); - if let Some(c) = config.property_::(("signature", id, "canonicalization")) + if let Some(c) = config.property::(("signature", id, "canonicalization")) { signer = signer .body_canonicalization(c.body) @@ -331,29 +331,29 @@ fn parse_signature>( .header_canonicalization(c.headers); } - if let Some(c) = config.property_::(("signature", id, "expire")) { + if let Some(c) = config.property::(("signature", id, "expire")) { signer = signer.expiration(c.as_secs()); sealer = sealer.expiration(c.as_secs()); } - if let Some(true) = config.property_::(("signature", id, "set-body-length")) { + if let Some(true) = config.property::(("signature", id, "set-body-length")) { signer = signer.body_length(true); sealer = sealer.body_length(true); } - if let Some(true) = config.property_::(("signature", id, "report")) { + if let Some(true) = config.property::(("signature", id, "report")) { signer = signer.reporting(true); } - if let Some(auid) = config.property_::(("signature", id, "auid")) { + if let Some(auid) = config.property::(("signature", id, "auid")) { signer = signer.agent_user_identifier(auid); } - if let Some(atps) = config.property_::(("signature", id, "third-party")) { + if let Some(atps) = config.property::(("signature", id, "third-party")) { signer = signer.atps(atps); } - if let Some(atpsh) = config.property_::(("signature", id, "third-party-algo")) { + if let Some(atpsh) = config.property::(("signature", id, "third-party-algo")) { signer = signer.atpsh(atpsh); } diff --git a/crates/common/src/config/smtp/mod.rs b/crates/common/src/config/smtp/mod.rs index 1ccc909b..ce280e77 100644 --- a/crates/common/src/config/smtp/mod.rs +++ b/crates/common/src/config/smtp/mod.rs @@ -14,6 +14,7 @@ use self::{ session::SessionConfig, }; +#[derive(Default)] pub struct SmtpConfig { pub session: SessionConfig, pub queue: QueueConfig, diff --git a/crates/common/src/config/smtp/queue.rs b/crates/common/src/config/smtp/queue.rs index 3672ad96..0eedf402 100644 --- a/crates/common/src/config/smtp/queue.rs +++ b/crates/common/src/config/smtp/queue.rs @@ -325,12 +325,12 @@ impl QueueConfig { fn parse_relay_host(config: &mut Config, id: &str) -> Option { Some(RelayHost { - address: config.property_require_(("remote", id, "address"))?, + address: config.property_require(("remote", id, "address"))?, port: config - .property_require_(("remote", id, "port")) + .property_require(("remote", id, "port")) .unwrap_or(25), protocol: config - .property_require_(("remote", id, "protocol")) + .property_require(("remote", id, "protocol")) .unwrap_or(ServerProtocol::Smtp), auth: if let (Some(username), Some(secret)) = ( config.value(("remote", id, "auth.username")), @@ -341,10 +341,10 @@ fn parse_relay_host(config: &mut Config, id: &str) -> Option { None }, tls_implicit: config - .property_(("remote", id, "tls.implicit")) + .property(("remote", id, "tls.implicit")) .unwrap_or(true), tls_allow_invalid_certs: config - .property_(("remote", id, "tls.allow-invalid-certs")) + .property(("remote", id, "tls.allow-invalid-certs")) .unwrap_or(false), }) } @@ -481,10 +481,10 @@ fn parse_queue_quota_item(config: &mut Config, prefix: impl AsKey) -> Option((prefix.as_str(), "size")) + .property::((prefix.as_str(), "size")) .filter(|&v| v > 0), messages: config - .property_::((prefix.as_str(), "messages")) + .property::((prefix.as_str(), "messages")) .filter(|&v| v > 0), }; diff --git a/crates/common/src/config/smtp/report.rs b/crates/common/src/config/smtp/report.rs index 24507fdd..cb927bdc 100644 --- a/crates/common/src/config/smtp/report.rs +++ b/crates/common/src/config/smtp/report.rs @@ -99,12 +99,12 @@ impl ReportConfig { }), analysis: ReportAnalysis { addresses: config - .properties_::("report.analysis.addresses") + .properties::("report.analysis.addresses") .into_iter() .map(|(_, m)| m) .collect(), - forward: config.property_("report.analysis.forward").unwrap_or(true), - store: config.property_("report.analysis.store"), + forward: config.property("report.analysis.forward").unwrap_or(true), + store: config.property("report.analysis.store"), }, dkim: Report::parse(config, "dkim", &default_hostname, &sender_vars), spf: Report::parse(config, "spf", &default_hostname, &sender_vars), diff --git a/crates/common/src/config/smtp/resolver.rs b/crates/common/src/config/smtp/resolver.rs index 8abdf6dd..ac289afb 100644 --- a/crates/common/src/config/smtp/resolver.rs +++ b/crates/common/src/config/smtp/resolver.rs @@ -69,7 +69,7 @@ pub struct Policy { impl Resolvers { pub async fn parse(config: &mut Config) -> Self { let (resolver_config, mut opts) = match config - .value_require_("resolver.type") + .value_require("resolver.type") .unwrap_or("system") { "cloudflare" => (ResolverConfig::cloudflare(), ResolverOpts::default()), @@ -157,19 +157,19 @@ impl Resolvers { (ResolverConfig::cloudflare(), ResolverOpts::default()) } }; - if let Some(concurrency) = config.property_("resolver.concurrency") { + if let Some(concurrency) = config.property("resolver.concurrency") { opts.num_concurrent_reqs = concurrency; } - if let Some(timeout) = config.property_("resolver.timeout") { + if let Some(timeout) = config.property("resolver.timeout") { opts.timeout = timeout; } - if let Some(preserve) = config.property_("resolver.preserve-intermediates") { + if let Some(preserve) = config.property("resolver.preserve-intermediates") { opts.preserve_intermediates = preserve; } - if let Some(try_tcp_on_error) = config.property_("resolver.try-tcp-on-error") { + if let Some(try_tcp_on_error) = config.property("resolver.try-tcp-on-error") { opts.try_tcp_on_error = try_tcp_on_error; } - if let Some(attempts) = config.property_("resolver.attempts") { + if let Some(attempts) = config.property("resolver.attempts") { opts.attempts = attempts; } @@ -180,7 +180,7 @@ impl Resolvers { let mut capacities = [1024usize; 5]; for (pos, key) in ["txt", "mx", "ipv4", "ipv6", "ptr"].into_iter().enumerate() { - if let Some(capacity) = config.property_(("cache.resolver", key)) { + if let Some(capacity) = config.property(("cache.resolver", key)) { capacities[pos] = capacity; } } @@ -201,11 +201,11 @@ impl Resolvers { }, cache: DnsRecordCache { tlsa: LruCache::with_capacity( - config.property_("cache.resolver.tlsa.size").unwrap_or(1024), + config.property("cache.resolver.tlsa.size").unwrap_or(1024), ), mta_sts: LruCache::with_capacity( config - .property_("cache.resolver.mta-sts.size") + .property("cache.resolver.mta-sts.size") .unwrap_or(1024), ), }, diff --git a/crates/common/src/config/smtp/session.rs b/crates/common/src/config/smtp/session.rs index 86575516..67e26653 100644 --- a/crates/common/src/config/smtp/session.rs +++ b/crates/common/src/config/smtp/session.rs @@ -30,7 +30,7 @@ pub struct SessionConfig { pub extensions: Extensions, } -#[derive(Default)] +#[derive(Default, Debug)] pub struct SessionThrottle { pub connect: Vec, pub mail_from: Vec, @@ -302,7 +302,7 @@ impl SessionConfig { ), ( &mut session.auth.errors_max, - "session.auth.errors.max", + "session.auth.errors.total", &has_ehlo_hars, ), ( @@ -347,7 +347,7 @@ impl SessionConfig { ), ( &mut session.rcpt.errors_max, - "session.rcpt.errors.max", + "session.rcpt.errors.total", &has_sender_vars, ), ( @@ -499,9 +499,9 @@ fn parse_pipe(config: &mut Config, id: &str, token_map: &TokenMap) -> Option Option { let hostname = config - .value_require_(("session.data.milter", id, "hostname"))? + .value_require(("session.data.milter", id, "hostname"))? .to_string(); - let port = config.property_require_(("session.data.milter", id, "port"))?; + let port = config.property_require(("session.data.milter", id, "port"))?; Some(Milter { enable: IfBlock::try_parse(config, ("session.data.milter", id, "enable"), token_map) .unwrap_or_default(), @@ -518,28 +518,28 @@ fn parse_milter(config: &mut Config, id: &str, token_map: &TokenMap) -> Option Option((prefix.as_str(), "concurrency")) + .property::((prefix.as_str(), "concurrency")) .filter(|&v| v > 0), rate: config - .property_::((prefix.as_str(), "rate")) + .property::((prefix.as_str(), "rate")) .filter(|v| v.requests > 0), }; diff --git a/crates/common/src/config/storage.rs b/crates/common/src/config/storage.rs index 14818fd7..46742419 100644 --- a/crates/common/src/config/storage.rs +++ b/crates/common/src/config/storage.rs @@ -4,6 +4,7 @@ use ahash::AHashMap; use directory::Directory; use store::{write::purge::PurgeSchedule, BlobStore, FtsStore, LookupStore, Store}; +#[derive(Default)] pub struct Storage { pub data: Store, pub blob: BlobStore, diff --git a/crates/common/src/config/tracers.rs b/crates/common/src/config/tracers.rs index 9880a899..04245f17 100644 --- a/crates/common/src/config/tracers.rs +++ b/crates/common/src/config/tracers.rs @@ -59,7 +59,7 @@ impl Tracers { { "log" => { if let Some(path) = config - .value_require_(("tracer", id, "path")) + .value_require(("tracer", id, "path")) .map(|s| s.to_string()) { let prefix = config.value(("tracer", id, "prefix")).unwrap_or("stalwart"); @@ -80,7 +80,7 @@ impl Tracers { level, appender, ansi: config - .property_or_default_(("tracer", id, "ansi"), "true") + .property_or_default(("tracer", id, "ansi"), "true") .unwrap_or(true), }); } @@ -89,13 +89,13 @@ impl Tracers { tracers.push(Tracer::Stdout { level, ansi: config - .property_or_default_(("tracer", id, "ansi"), "true") + .property_or_default(("tracer", id, "ansi"), "true") .unwrap_or(true), }); } "otel" | "open-telemetry" => { match config - .value_require_(("tracer", id, "transport")) + .value_require(("tracer", id, "transport")) .unwrap_or_default() { "gprc" => { @@ -110,7 +110,7 @@ impl Tracers { } "http" => { if let Some(endpoint) = config - .value_require_(("tracer", id, "endpoint")) + .value_require(("tracer", id, "endpoint")) .map(|s| s.to_string()) { let mut headers = HashMap::new(); diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 8e450227..619b34c4 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -42,6 +42,7 @@ pub static DAEMON_NAME: &str = concat!("Stalwart Mail Server v", env!("CARGO_PKG pub type SharedCore = Arc>; +#[derive(Default)] pub struct Core { pub storage: Storage, pub sieve: Scripting, diff --git a/crates/common/src/listener/blocked.rs b/crates/common/src/listener/blocked.rs index 154824a7..e6c633be 100644 --- a/crates/common/src/listener/blocked.rs +++ b/crates/common/src/listener/blocked.rs @@ -34,7 +34,7 @@ use utils::config::{ use crate::Core; pub struct BlockedIps { - ip_addresses: RwLock>, + pub ip_addresses: RwLock>, ip_networks: Vec, has_networks: bool, limiter_rate: Option, @@ -71,7 +71,7 @@ impl BlockedIps { ip_addresses: RwLock::new(ip_addresses), has_networks: !ip_networks.is_empty(), ip_networks, - limiter_rate: config.property_::("authentication.fail2ban"), + limiter_rate: config.property::("authentication.fail2ban"), } } } diff --git a/crates/directory/Cargo.toml b/crates/directory/Cargo.toml index 2e56eab8..5a702a01 100644 --- a/crates/directory/Cargo.toml +++ b/crates/directory/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "directory" -version = "0.1.0" +version = "0.6.0" edition = "2021" resolver = "2" @@ -37,3 +37,6 @@ serde = { version = "1.0", features = ["derive"]} [dev-dependencies] tokio = { version = "1.23", features = ["full"] } + +[features] +test_mode = [] diff --git a/crates/directory/src/backend/imap/config.rs b/crates/directory/src/backend/imap/config.rs index 412a2e27..4a88f9de 100644 --- a/crates/directory/src/backend/imap/config.rs +++ b/crates/directory/src/backend/imap/config.rs @@ -33,22 +33,22 @@ use super::{ImapConnectionManager, ImapDirectory}; impl ImapDirectory { pub fn from_config(config: &mut Config, prefix: impl AsKey) -> Option { let prefix = prefix.as_key(); - let address = config.value_require_((&prefix, "host"))?.to_string(); + let address = config.value_require((&prefix, "host"))?.to_string(); let tls_implicit: bool = config - .property_or_default_((&prefix, "tls.enable"), "false") + .property_or_default((&prefix, "tls.enable"), "false") .unwrap_or_default(); let port: u16 = config - .property_or_default_((&prefix, "port"), if tls_implicit { "993" } else { "143" }) + .property_or_default((&prefix, "port"), if tls_implicit { "993" } else { "143" }) .unwrap_or(if tls_implicit { 993 } else { 143 }); let manager = ImapConnectionManager { addr: format!("{address}:{port}"), timeout: config - .property_or_default_((&prefix, "timeout"), "30s") + .property_or_default((&prefix, "timeout"), "30s") .unwrap_or_else(|| Duration::from_secs(30)), tls_connector: build_tls_connector( config - .property_or_default_((&prefix, "tls.allow-invalid-certs"), "false") + .property_or_default((&prefix, "tls.allow-invalid-certs"), "false") .unwrap_or_default(), ), tls_hostname: address.to_string(), diff --git a/crates/directory/src/backend/ldap/config.rs b/crates/directory/src/backend/ldap/config.rs index 51ab85f9..f27deb79 100644 --- a/crates/directory/src/backend/ldap/config.rs +++ b/crates/directory/src/backend/ldap/config.rs @@ -37,7 +37,7 @@ impl LdapDirectory { let bind_dn = if let Some(dn) = config.value((&prefix, "bind.dn")) { Bind::new( dn.to_string(), - config.value_require_((&prefix, "bind.secret"))?.to_string(), + config.value_require((&prefix, "bind.secret"))?.to_string(), ) .into() } else { @@ -45,28 +45,28 @@ impl LdapDirectory { }; let manager = LdapConnectionManager::new( - config.value_require_((&prefix, "url"))?.to_string(), + config.value_require((&prefix, "url"))?.to_string(), LdapConnSettings::new() .set_conn_timeout( config - .property_or_default_((&prefix, "timeout"), "30s") + .property_or_default((&prefix, "timeout"), "30s") .unwrap_or_else(|| Duration::from_secs(30)), ) .set_starttls( config - .property_or_default_((&prefix, "tls.enable"), "false") + .property_or_default((&prefix, "tls.enable"), "false") .unwrap_or_default(), ) .set_no_tls_verify( config - .property_or_default_((&prefix, "tls.allow-invalid-certs"), "false") + .property_or_default((&prefix, "tls.allow-invalid-certs"), "false") .unwrap_or_default(), ), bind_dn, ); let mut mappings = LdapMappings { - base_dn: config.value_require_((&prefix, "base-dn"))?.to_string(), + base_dn: config.value_require((&prefix, "base-dn"))?.to_string(), filter_name: LdapFilter::from_config(config, (&prefix, "filter.name")), filter_email: LdapFilter::from_config(config, (&prefix, "filter.email")), filter_verify: LdapFilter::from_config(config, (&prefix, "filter.verify")), @@ -121,7 +121,7 @@ impl LdapDirectory { } let auth_bind = if config - .property_or_default_::((&prefix, "bind.auth.enable"), "false") + .property_or_default::((&prefix, "bind.auth.enable"), "false") .unwrap_or_default() { LdapFilter::from_config(config, (&prefix, "bind.auth.dn")).into() diff --git a/crates/directory/src/backend/memory/config.rs b/crates/directory/src/backend/memory/config.rs index 986afebd..44e2a64e 100644 --- a/crates/directory/src/backend/memory/config.rs +++ b/crates/directory/src/backend/memory/config.rs @@ -49,7 +49,7 @@ impl MemoryDirectory { { let lookup_id = lookup_id.as_str(); let name = config - .value_require_((prefix.as_str(), "principals", lookup_id, "name"))? + .value_require((prefix.as_str(), "principals", lookup_id, "name"))? .to_string(); let typ = match config.value((prefix.as_str(), "principals", lookup_id, "class")) { Some("individual") => Type::Individual, @@ -147,7 +147,7 @@ impl MemoryDirectory { .value((prefix.as_str(), "principals", lookup_id, "description")) .map(|v| v.to_string()), quota: config - .property_((prefix.as_str(), "principals", lookup_id, "quota")) + .property((prefix.as_str(), "principals", lookup_id, "quota")) .unwrap_or(0), member_of, id, diff --git a/crates/directory/src/backend/smtp/config.rs b/crates/directory/src/backend/smtp/config.rs index 297c7094..5ef8cd45 100644 --- a/crates/directory/src/backend/smtp/config.rs +++ b/crates/directory/src/backend/smtp/config.rs @@ -33,23 +33,23 @@ use super::{SmtpConnectionManager, SmtpDirectory}; impl SmtpDirectory { pub fn from_config(config: &mut Config, prefix: impl AsKey, is_lmtp: bool) -> Option { let prefix = prefix.as_key(); - let address = config.value_require_((&prefix, "host"))?.to_string(); + let address = config.value_require((&prefix, "host"))?.to_string(); let tls_implicit: bool = config - .property_or_default_((&prefix, "tls.enable"), "false") + .property_or_default((&prefix, "tls.enable"), "false") .unwrap_or_default(); let port: u16 = config - .property_or_default_((&prefix, "port"), if tls_implicit { "465" } else { "25" }) + .property_or_default((&prefix, "port"), if tls_implicit { "465" } else { "25" }) .unwrap_or(if tls_implicit { 465 } else { 25 }); let manager = SmtpConnectionManager { builder: SmtpClientBuilder { addr: format!("{address}:{port}"), timeout: config - .property_or_default_((&prefix, "timeout"), "30s") + .property_or_default((&prefix, "timeout"), "30s") .unwrap_or_else(|| Duration::from_secs(30)), tls_connector: build_tls_connector( config - .property_or_default_((&prefix, "tls.allow-invalid-certs"), "false") + .property_or_default((&prefix, "tls.allow-invalid-certs"), "false") .unwrap_or_default(), ), tls_hostname: address.to_string(), @@ -63,10 +63,10 @@ impl SmtpDirectory { say_ehlo: false, }, max_rcpt: config - .property_or_default_((&prefix, "limits.rcpt"), "10") + .property_or_default((&prefix, "limits.rcpt"), "10") .unwrap_or(10), max_auth_errors: config - .property_or_default_((&prefix, "limits.auth-errors"), "3") + .property_or_default((&prefix, "limits.auth-errors"), "3") .unwrap_or(10), }; diff --git a/crates/directory/src/backend/sql/config.rs b/crates/directory/src/backend/sql/config.rs index 23dc9116..d2273766 100644 --- a/crates/directory/src/backend/sql/config.rs +++ b/crates/directory/src/backend/sql/config.rs @@ -34,7 +34,7 @@ impl SqlDirectory { data_store: Store, ) -> Option { let prefix = prefix.as_key(); - let store_id = config.value_require_((&prefix, "store"))?.to_string(); + let store_id = config.value_require((&prefix, "store"))?.to_string(); let store = if let Some(store) = stores.lookup_stores.get(&store_id) { store.clone() } else { diff --git a/crates/directory/src/core/cache.rs b/crates/directory/src/core/cache.rs index f12004be..c82e475e 100644 --- a/crates/directory/src/core/cache.rs +++ b/crates/directory/src/core/cache.rs @@ -47,12 +47,12 @@ pub struct LookupCache { impl CachedDirectory { pub fn try_from_config(config: &mut Config, prefix: impl AsKey) -> Option { let prefix = prefix.as_key(); - let cached_entries = config.property_((&prefix, "cache.entries"))?; + let cached_entries = config.property((&prefix, "cache.entries"))?; let cache_ttl_positive = config - .property_((&prefix, "cache.ttl.positive")) + .property((&prefix, "cache.ttl.positive")) .unwrap_or(Duration::from_secs(86400)); let cache_ttl_negative = config - .property_((&prefix, "cache.ttl.positive")) + .property((&prefix, "cache.ttl.positive")) .unwrap_or_else(|| Duration::from_secs(3600)); Some(CachedDirectory { diff --git a/crates/directory/src/core/config.rs b/crates/directory/src/core/config.rs index e82db8d1..413106f4 100644 --- a/crates/directory/src/core/config.rs +++ b/crates/directory/src/core/config.rs @@ -58,18 +58,18 @@ impl Directories { #[cfg(feature = "test_mode")] { if config - .property_or_default_::(("directory", id, "disable"), "false") + .property_or_default::(("directory", id, "disable"), "false") .unwrap_or(false) { tracing::debug!("Skipping disabled directory {id:?}."); continue; } } - let protocol = config.value_require_(("directory", id, "type")).unwrap(); + let protocol = config.value_require(("directory", id, "type")).unwrap(); let prefix = ("directory", id); let store = match protocol { "internal" => Some(DirectoryInner::Internal( - if let Some(store_id) = config.value_require_(("directory", id, "store")) { + if let Some(store_id) = config.value_require(("directory", id, "store")) { if let Some(data) = stores.stores.get(store_id) { match data.clone().init().await { Ok(data) => data, @@ -128,99 +128,6 @@ impl Directories { } } -#[allow(async_fn_in_trait)] -pub trait ConfigDirectory { - async fn parse_directory( - &mut self, - stores: &Stores, - data_store: Store, - ) -> utils::config::Result; -} - -impl ConfigDirectory for Config { - async fn parse_directory( - &mut self, - stores: &Stores, - data_store: Store, - ) -> utils::config::Result { - let mut config = Directories { - directories: AHashMap::new(), - }; - - for id in self - .sub_keys("directory", ".type") - .map(|s| s.to_string()) - .collect::>() - { - // Parse directory - let id = id.as_str(); - if self.property_or_default::(("directory", id, "disable"), "false")? { - tracing::debug!("Skipping disabled directory {id:?}."); - continue; - } - let protocol = self.value_require(("directory", id, "type"))?; - let prefix = ("directory", id); - let store = match protocol { - "internal" => DirectoryInner::Internal( - stores - .stores - .get(self.value_require(("directory", id, "store"))?) - .cloned() - .ok_or_else(|| { - format!( - "Failed to find store {:?} for directory {:?}.", - self.value_require(("directory", id, "store")).unwrap(), - id - ) - })? - .init() - .await - .map_err(|err| { - format!( - "Failed to initialize store {:?} for directory {:?}: {:?}.", - self.value_require(("directory", id, "store")).unwrap(), - id, - err - ) - })?, - ), - "ldap" => DirectoryInner::Ldap( - LdapDirectory::from_config(self, prefix, data_store.clone()).unwrap(), - ), - "sql" => DirectoryInner::Sql( - SqlDirectory::from_config(self, prefix, stores, data_store.clone()).unwrap(), - ), - "imap" => DirectoryInner::Imap(ImapDirectory::from_config(self, prefix).unwrap()), - "smtp" => { - DirectoryInner::Smtp(SmtpDirectory::from_config(self, prefix, false).unwrap()) - } - "lmtp" => { - DirectoryInner::Smtp(SmtpDirectory::from_config(self, prefix, true).unwrap()) - } - "memory" => DirectoryInner::Memory( - MemoryDirectory::from_config(self, prefix, data_store.clone()) - .await - .unwrap(), - ), - unknown => { - return Err(format!("Unknown directory type: {unknown:?}")); - } - }; - - // Build directory - let directory = Arc::new(Directory { - store, - cache: CachedDirectory::try_from_config(self, ("directory", id)), - }); - - // Add directory - config.directories.insert(id.to_string(), directory); - } - - Ok(config) - } -} - pub(crate) fn build_pool( config: &mut Config, prefix: &str, @@ -230,17 +137,17 @@ pub(crate) fn build_pool( .runtime(Runtime::Tokio1) .max_size( config - .property_or_default_((prefix, "pool.max-connections"), "10") + .property_or_default((prefix, "pool.max-connections"), "10") .unwrap_or(10), ) .create_timeout( config - .property_or_default_::((prefix, "pool.timeout.create"), "30s") + .property_or_default::((prefix, "pool.timeout.create"), "30s") .unwrap_or_else(|| Duration::from_secs(30)) .into(), ) - .wait_timeout(config.property_or_default_((prefix, "pool.timeout.wait"), "30s")) - .recycle_timeout(config.property_or_default_((prefix, "pool.timeout.recycle"), "30s")) + .wait_timeout(config.property_or_default((prefix, "pool.timeout.wait"), "30s")) + .recycle_timeout(config.property_or_default((prefix, "pool.timeout.recycle"), "30s")) .build() .map_err(|err| { format!( diff --git a/crates/directory/src/core/dispatch.rs b/crates/directory/src/core/dispatch.rs index 9cb77b7f..d5478215 100644 --- a/crates/directory/src/core/dispatch.rs +++ b/crates/directory/src/core/dispatch.rs @@ -94,11 +94,9 @@ impl Directory { DirectoryInner::Memory(store) => store.rcpt(email).await, }?; - if result { - // Update cache - if let Some(cache) = &self.cache { - cache.set_rcpt(email, true); - } + // Update cache + if let Some(cache) = &self.cache { + cache.set_rcpt(email, result); } Ok(result) diff --git a/crates/imap/src/lib.rs b/crates/imap/src/lib.rs index 3140a845..bad28308 100644 --- a/crates/imap/src/lib.rs +++ b/crates/imap/src/lib.rs @@ -40,10 +40,10 @@ static SERVER_GREETING: &str = "Stalwart IMAP4rev2 at your service."; impl IMAP { pub async fn init(config: &mut Config, jmap_instance: JmapInstance) -> ImapInstance { let shard_amount = config - .property_::("cache.shard") + .property::("cache.shard") .unwrap_or(32) .next_power_of_two() as usize; - let capacity = config.property_("cache.capacity").unwrap_or(100); + let capacity = config.property("cache.capacity").unwrap_or(100); let inner = Inner { greeting_plain: StatusResponse::ok(SERVER_GREETING) @@ -62,10 +62,10 @@ impl IMAP { shard_amount, ), cache_account: LruCache::with_capacity( - config.property_("cache.account.size").unwrap_or(2048), + config.property("cache.account.size").unwrap_or(2048), ), cache_mailbox: LruCache::with_capacity( - config.property_("cache.mailbox.size").unwrap_or(2048), + config.property("cache.mailbox.size").unwrap_or(2048), ), }; diff --git a/crates/jmap/src/lib.rs b/crates/jmap/src/lib.rs index 9f7e194f..2d672224 100644 --- a/crates/jmap/src/lib.rs +++ b/crates/jmap/src/lib.rs @@ -125,16 +125,16 @@ impl JMAP { let (state_tx, state_rx) = init_state_manager(); let (housekeeper_tx, housekeeper_rx) = init_housekeeper(); let shard_amount = config - .property_::("cache.shard") + .property::("cache.shard") .unwrap_or(32) .next_power_of_two() as usize; - let capacity = config.property_("cache.capacity").unwrap_or(100); + let capacity = config.property("cache.capacity").unwrap_or(100); let inner = Inner { sessions: TtlDashMap::with_capacity(capacity, shard_amount), access_tokens: TtlDashMap::with_capacity(capacity, shard_amount), snowflake_id: config - .property_::("cluster.node-id") + .property::("cluster.node-id") .map(SnowflakeIdGenerator::with_node_id) .unwrap_or_default(), concurrency_limiter: DashMap::with_capacity_and_hasher_and_shard_amount( @@ -146,7 +146,7 @@ impl JMAP { state_tx, housekeeper_tx, cache_threads: LruCache::with_capacity( - config.property_("cache.thread.size").unwrap_or(2048), + config.property("cache.thread.size").unwrap_or(2048), ), }; diff --git a/crates/jmap/src/services/index.rs b/crates/jmap/src/services/index.rs index c7d6d67b..350c2c49 100644 --- a/crates/jmap/src/services/index.rs +++ b/crates/jmap/src/services/index.rs @@ -201,6 +201,7 @@ impl JMAP { .write( BatchBuilder::new() .with_account_id(key.account_id) + .with_collection(Collection::Email) .update_document(key.document_id) .clear(ValueClass::IndexEmail(key.seq)) .build_batch(), diff --git a/crates/nlp/src/language/mod.rs b/crates/nlp/src/language/mod.rs index 6b578d50..30f0a05b 100644 --- a/crates/nlp/src/language/mod.rs +++ b/crates/nlp/src/language/mod.rs @@ -56,9 +56,12 @@ impl Language { } } -#[derive(Debug, PartialEq, Clone, Copy, Hash, Eq, serde::Serialize, serde::Deserialize)] +#[derive( + Debug, PartialEq, Clone, Copy, Hash, Eq, serde::Serialize, serde::Deserialize, Default, +)] pub enum Language { Esperanto = 0, + #[default] English = 1, Russian = 2, Mandarin = 3, diff --git a/crates/smtp/src/core/mod.rs b/crates/smtp/src/core/mod.rs index aaf91f28..5821452d 100644 --- a/crates/smtp/src/core/mod.rs +++ b/crates/smtp/src/core/mod.rs @@ -437,3 +437,25 @@ impl SessionAddress { } } } + +#[cfg(feature = "test_mode")] +impl Default for Inner { + fn default() -> Self { + Self { + worker_pool: rayon::ThreadPoolBuilder::new() + .num_threads(num_cpus::get()) + .build() + .unwrap(), + session_throttle: Default::default(), + queue_throttle: Default::default(), + queue_tx: mpsc::channel(1).0, + report_tx: mpsc::channel(1).0, + snowflake_id: Default::default(), + connectors: TlsConnectors { + pki_verify: mail_send::smtp::tls::build_tls_connector(false), + dummy_verify: mail_send::smtp::tls::build_tls_connector(true), + }, + delivery_tx: mpsc::channel(1).0, + } + } +} diff --git a/crates/smtp/src/inbound/spawn.rs b/crates/smtp/src/inbound/spawn.rs index 92172a4b..1e0f3993 100644 --- a/crates/smtp/src/inbound/spawn.rs +++ b/crates/smtp/src/inbound/spawn.rs @@ -133,7 +133,9 @@ impl Session { .core .eval_if::(&self.core.core.smtp.session.connect.greeting, self) .await - .unwrap_or_else(|| "Stalwart ESMTP at your service".to_string()); + .filter(|g| !g.is_empty()) + .map(|g| format!("220 {}\r\n", g)) + .unwrap_or_else(|| "220 Stalwart ESMTP at your service.\r\n".to_string()); if self.write(greeting.as_bytes()).await.is_err() { return false; diff --git a/crates/smtp/src/lib.rs b/crates/smtp/src/lib.rs index b2b17bae..2fa99e33 100644 --- a/crates/smtp/src/lib.rs +++ b/crates/smtp/src/lib.rs @@ -49,9 +49,9 @@ impl SMTP { #[cfg(feature = "local_delivery")] delivery_tx: mpsc::Sender, ) -> SmtpInstance { // Build inner - let capacity = config.property_("cache.capacity").unwrap_or(2); + let capacity = config.property("cache.capacity").unwrap_or(2); let shard = config - .property_::("cache.shard") + .property::("cache.shard") .unwrap_or(32) .next_power_of_two() as usize; let (queue_tx, queue_rx) = mpsc::channel(1024); @@ -60,7 +60,7 @@ impl SMTP { worker_pool: rayon::ThreadPoolBuilder::new() .num_threads(std::cmp::max( config - .property_::("global.thread-pool") + .property::("global.thread-pool") .filter(|v| *v > 0) .unwrap_or_else(num_cpus::get), 4, @@ -80,7 +80,7 @@ impl SMTP { queue_tx, report_tx, snowflake_id: config - .property_::("cluster.node-id") + .property::("cluster.node-id") .map(SnowflakeIdGenerator::with_node_id) .unwrap_or_default(), connectors: TlsConnectors { diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index 12868bcd..f66749ac 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -8,7 +8,7 @@ resolver = "2" utils = { path = "../utils" } nlp = { path = "../nlp" } rocksdb = { version = "0.22", optional = true, features = ["multi-threaded-cf"] } -foundationdb = { version = "0.9.0", features = ["embedded-fdb-include", "fdb-7_3"], optional = true } +foundationdb = { version = "0.9.0", features = ["embedded-fdb-include", "fdb-7_1"], optional = true } rusqlite = { version = "0.31.0", features = ["bundled"], optional = true } rust-s3 = { version = "0.33.0", default-features = false, features = ["tokio-rustls-tls", "no-verify-ssl"], optional = true } tokio = { version = "1.23", features = ["sync", "fs", "io-util"] } diff --git a/crates/store/src/backend/elastic/mod.rs b/crates/store/src/backend/elastic/mod.rs index 42563161..86dcdc46 100644 --- a/crates/store/src/backend/elastic/mod.rs +++ b/crates/store/src/backend/elastic/mod.rs @@ -49,7 +49,7 @@ impl ElasticSearchStore { let credentials = if let Some(user) = config.value((&prefix, "user")) { let user = user.to_string(); let password = config - .value_require_((&prefix, "password")) + .value_require((&prefix, "password")) .unwrap_or_default(); Some(Credentials::Basic(user, password.to_string())) } else { @@ -66,7 +66,7 @@ impl ElasticSearchStore { builder = builder.auth(credentials); } if config - .property_or_default_::((&prefix, "tls.allow-invalid-certs"), "false") + .property_or_default::((&prefix, "tls.allow-invalid-certs"), "false") .unwrap_or(false) { builder = builder.cert_validation(CertificateValidation::None); @@ -105,10 +105,10 @@ impl ElasticSearchStore { es.create_index( config - .property_or_default_((&prefix, "index.shards"), "3") + .property_or_default((&prefix, "index.shards"), "3") .unwrap_or(3), config - .property_or_default_((&prefix, "index.replicas"), "0") + .property_or_default((&prefix, "index.replicas"), "0") .unwrap_or(0), ) .await diff --git a/crates/store/src/backend/foundationdb/main.rs b/crates/store/src/backend/foundationdb/main.rs index fcb909e9..871f2836 100644 --- a/crates/store/src/backend/foundationdb/main.rs +++ b/crates/store/src/backend/foundationdb/main.rs @@ -23,7 +23,7 @@ use std::time::Duration; -use foundationdb::{options::DatabaseOption, Database}; +use foundationdb::{api, options::DatabaseOption, Database}; use utils::config::{utils::AsKey, Config}; use super::FdbStore; @@ -31,15 +31,36 @@ use super::FdbStore; impl FdbStore { pub async fn open(config: &mut Config, prefix: impl AsKey) -> Option { let prefix = prefix.as_key(); - let guard = unsafe { foundationdb::boot() }; + let guard = unsafe { + api::FdbApiBuilder::default() + .build() + .map_err(|err| { + config.new_build_error( + prefix.as_str(), + format!("Failed to boot FoundationDB: {err:?}"), + ) + }) + .ok()? + .boot() + .map_err(|err| { + config.new_build_error( + prefix.as_str(), + format!("Failed to boot FoundationDB: {err:?}"), + ) + }) + .ok()? + }; let db = Database::new(config.value((&prefix, "cluster-file"))) .map_err(|err| { - config.new_build_error(prefix.as_str(), format!("Failed to open database: {err:?}")) + config.new_build_error( + prefix.as_str(), + format!("Failed to create FoundationDB database: {err:?}"), + ) }) .ok()?; - if let Some(value) = config.property_::((&prefix, "transaction.timeout")) { + if let Some(value) = config.property::((&prefix, "transaction.timeout")) { db.set_option(DatabaseOption::TransactionTimeout(value.as_millis() as i32)) .map_err(|err| { config.new_build_error( @@ -49,7 +70,7 @@ impl FdbStore { }) .ok()?; } - if let Some(value) = config.property_((&prefix, "transaction.retry-limit")) { + if let Some(value) = config.property((&prefix, "transaction.retry-limit")) { db.set_option(DatabaseOption::TransactionRetryLimit(value)) .map_err(|err| { config.new_build_error( @@ -59,8 +80,7 @@ impl FdbStore { }) .ok()?; } - if let Some(value) = config.property_::((&prefix, "transaction.max-retry-delay")) - { + if let Some(value) = config.property::((&prefix, "transaction.max-retry-delay")) { db.set_option(DatabaseOption::TransactionMaxRetryDelay( value.as_millis() as i32 )) @@ -72,7 +92,7 @@ impl FdbStore { }) .ok()?; } - if let Some(value) = config.property_((&prefix, "ids.machine")) { + if let Some(value) = config.property((&prefix, "ids.machine")) { db.set_option(DatabaseOption::MachineId(value)) .map_err(|err| { config.new_build_error( @@ -82,7 +102,7 @@ impl FdbStore { }) .ok()?; } - if let Some(value) = config.property_((&prefix, "ids.datacenter")) { + if let Some(value) = config.property((&prefix, "ids.datacenter")) { db.set_option(DatabaseOption::DatacenterId(value)) .map_err(|err| { config.new_build_error( diff --git a/crates/store/src/backend/fs/mod.rs b/crates/store/src/backend/fs/mod.rs index 7df5ec07..abeff842 100644 --- a/crates/store/src/backend/fs/mod.rs +++ b/crates/store/src/backend/fs/mod.rs @@ -40,7 +40,7 @@ pub struct FsStore { impl FsStore { pub async fn open(config: &mut Config, prefix: impl AsKey) -> Option { let prefix = prefix.as_key(); - let path = PathBuf::from(config.value_require_((&prefix, "path"))?); + let path = PathBuf::from(config.value_require((&prefix, "path"))?); if !path.exists() { fs::create_dir_all(&path) .await @@ -57,7 +57,7 @@ impl FsStore { path, hash_levels: std::cmp::min( config - .property_or_default_((&prefix, "depth"), "2") + .property_or_default((&prefix, "depth"), "2") .unwrap_or(2), 5, ), diff --git a/crates/store/src/backend/mysql/main.rs b/crates/store/src/backend/mysql/main.rs index 45fa3829..27a2cd5a 100644 --- a/crates/store/src/backend/mysql/main.rs +++ b/crates/store/src/backend/mysql/main.rs @@ -37,27 +37,27 @@ impl MysqlStore { pub async fn open(config: &mut Config, prefix: impl AsKey) -> Option { let prefix = prefix.as_key(); let mut opts = OptsBuilder::default() - .ip_or_hostname(config.value_require_((&prefix, "host"))?.to_string()) + .ip_or_hostname(config.value_require((&prefix, "host"))?.to_string()) .user(config.value((&prefix, "user")).map(|s| s.to_string())) .pass(config.value((&prefix, "password")).map(|s| s.to_string())) .db_name( config - .value_require_((&prefix, "database"))? + .value_require((&prefix, "database"))? .to_string() .into(), ) - .max_allowed_packet(config.property_((&prefix, "max-allowed-packet"))) + .max_allowed_packet(config.property((&prefix, "max-allowed-packet"))) .wait_timeout( config - .property_::((&prefix, "timeout")) + .property::((&prefix, "timeout")) .map(|t| t.as_secs() as usize), ); - if let Some(port) = config.property_((&prefix, "port")) { + if let Some(port) = config.property((&prefix, "port")) { opts = opts.tcp_port(port); } if config - .property_or_default_::((&prefix, "tls.allow-invalid-certs"), "false") + .property_or_default::((&prefix, "tls.allow-invalid-certs"), "false") .unwrap_or_default() { opts = opts.ssl_opts(Some( @@ -68,10 +68,10 @@ impl MysqlStore { // Configure connection pool let mut pool_min = PoolConstraints::default().min(); let mut pool_max = PoolConstraints::default().max(); - if let Some(n_size) = config.property_::((&prefix, "pool.min-connections")) { + if let Some(n_size) = config.property::((&prefix, "pool.min-connections")) { pool_min = n_size; } - if let Some(n_size) = config.property_::((&prefix, "pool.max-connections")) { + if let Some(n_size) = config.property::((&prefix, "pool.max-connections")) { pool_max = n_size; } opts = opts.pool_opts( diff --git a/crates/store/src/backend/postgres/main.rs b/crates/store/src/backend/postgres/main.rs index d225505f..fc2ef964 100644 --- a/crates/store/src/backend/postgres/main.rs +++ b/crates/store/src/backend/postgres/main.rs @@ -39,30 +39,30 @@ impl PostgresStore { let prefix = prefix.as_key(); let mut cfg = Config::new(); cfg.dbname = config - .value_require_((&prefix, "database"))? + .value_require((&prefix, "database"))? .to_string() .into(); cfg.host = config.value((&prefix, "host")).map(|s| s.to_string()); cfg.user = config.value((&prefix, "user")).map(|s| s.to_string()); cfg.password = config.value((&prefix, "password")).map(|s| s.to_string()); - cfg.port = config.property_((&prefix, "port")); - cfg.connect_timeout = config.property_((&prefix, "timeout")); + cfg.port = config.property((&prefix, "port")); + cfg.connect_timeout = config.property((&prefix, "timeout")); cfg.manager = Some(ManagerConfig { recycling_method: RecyclingMethod::Fast, }); - if let Some(max_conn) = config.property_::((&prefix, "pool.max-connections")) { + if let Some(max_conn) = config.property::((&prefix, "pool.max-connections")) { cfg.pool = PoolConfig::new(max_conn).into(); } let db = Self { conn_pool: if config - .property_or_default_::((&prefix, "tls.enable"), "false") + .property_or_default::((&prefix, "tls.enable"), "false") .unwrap_or_default() { cfg.create_pool( Some(Runtime::Tokio1), MakeRustlsConnect::new(rustls_client_config( config - .property_or_default_((&prefix, "tls.allow-invalid-certs"), "false") + .property_or_default((&prefix, "tls.allow-invalid-certs"), "false") .unwrap_or_default(), )), ) diff --git a/crates/store/src/backend/redis/mod.rs b/crates/store/src/backend/redis/mod.rs index b3f800d6..9d80aad4 100644 --- a/crates/store/src/backend/redis/mod.rs +++ b/crates/store/src/backend/redis/mod.rs @@ -67,7 +67,7 @@ impl RedisStore { return None; } - Some(match config.value_require_((&prefix, "redis-type"))? { + Some(match config.value_require((&prefix, "redis-type"))? { "single" => { let client = Client::open(urls.into_iter().next().unwrap()) .map_err(|err| { @@ -78,7 +78,7 @@ impl RedisStore { }) .ok()?; let timeout = config - .property_or_default_((&prefix, "timeout"), "10s") + .property_or_default((&prefix, "timeout"), "10s") .unwrap_or_else(|| Duration::from_secs(10)); Self { @@ -96,22 +96,22 @@ impl RedisStore { } "cluster" => { let mut builder = ClusterClientBuilder::new(urls.into_iter()); - if let Some(value) = config.property_((&prefix, "user")) { + if let Some(value) = config.property((&prefix, "user")) { builder = builder.username(value); } - if let Some(value) = config.property_((&prefix, "password")) { + if let Some(value) = config.property((&prefix, "password")) { builder = builder.password(value); } - if let Some(value) = config.property_((&prefix, "retry.total")) { + if let Some(value) = config.property((&prefix, "retry.total")) { builder = builder.retries(value); } - if let Some(value) = config.property_::((&prefix, "retry.max-wait")) { + if let Some(value) = config.property::((&prefix, "retry.max-wait")) { builder = builder.max_retry_wait(value.as_millis() as u64); } - if let Some(value) = config.property_::((&prefix, "retry.min-wait")) { + if let Some(value) = config.property::((&prefix, "retry.min-wait")) { builder = builder.min_retry_wait(value.as_millis() as u64); } - if let Some(true) = config.property_::((&prefix, "read-from-replicas")) { + if let Some(true) = config.property::((&prefix, "read-from-replicas")) { builder = builder.read_from_replicas(); } @@ -125,7 +125,7 @@ impl RedisStore { }) .ok()?; let timeout = config - .property_or_default_((&prefix, "timeout"), "10s") + .property_or_default((&prefix, "timeout"), "10s") .unwrap_or_else(|| Duration::from_secs(10)); Self { @@ -163,17 +163,17 @@ fn build_pool( .runtime(Runtime::Tokio1) .max_size( config - .property_or_default_((prefix, "pool.max-connections"), "10") + .property_or_default((prefix, "pool.max-connections"), "10") .unwrap_or(10), ) .create_timeout( config - .property_or_default_::((prefix, "pool.create-timeout"), "30s") + .property_or_default::((prefix, "pool.create-timeout"), "30s") .unwrap_or_else(|| Duration::from_secs(30)) .into(), ) - .wait_timeout(config.property_or_default_((prefix, "pool.wait-timeout"), "30s")) - .recycle_timeout(config.property_or_default_((prefix, "pool.recycle-timeout"), "30s")) + .wait_timeout(config.property_or_default((prefix, "pool.wait-timeout"), "30s")) + .recycle_timeout(config.property_or_default((prefix, "pool.recycle-timeout"), "30s")) .build() .map_err(|err| { format!( diff --git a/crates/store/src/backend/rocksdb/main.rs b/crates/store/src/backend/rocksdb/main.rs index 7afa264c..158affad 100644 --- a/crates/store/src/backend/rocksdb/main.rs +++ b/crates/store/src/backend/rocksdb/main.rs @@ -40,7 +40,7 @@ impl RocksDbStore { pub async fn open(config: &mut Config, prefix: impl AsKey) -> Option { let prefix = prefix.as_key(); // Create the database directory if it doesn't exist - let idx_path: PathBuf = PathBuf::from(config.value_require_((&prefix, "path"))?); + let idx_path: PathBuf = PathBuf::from(config.value_require((&prefix, "path"))?); std::fs::create_dir_all(&idx_path) .map_err(|err| { config.new_build_error( @@ -73,7 +73,7 @@ impl RocksDbStore { cf_opts.set_enable_blob_files(true); cf_opts.set_min_blob_size( config - .property_or_default_((&prefix, "min-blob-size"), "16834") + .property_or_default((&prefix, "min-blob-size"), "16834") .unwrap_or(16834), ); cfs.push(ColumnFamilyDescriptor::new(CF_BLOBS, cf_opts)); @@ -95,7 +95,7 @@ impl RocksDbStore { //db_opts.set_max_successive_merges(100); db_opts.set_write_buffer_size( config - .property_or_default_((&prefix, "write-buffer-size"), "134217728") + .property_or_default((&prefix, "write-buffer-size"), "134217728") .unwrap_or(134217728), ); @@ -112,7 +112,7 @@ impl RocksDbStore { worker_pool: rayon::ThreadPoolBuilder::new() .num_threads( config - .property_::((&prefix, "pool.workers")) + .property::((&prefix, "pool.workers")) .filter(|v| *v > 0) .unwrap_or_else(|| num_cpus::get() * 4), ) diff --git a/crates/store/src/backend/rocksdb/write.rs b/crates/store/src/backend/rocksdb/write.rs index 133475ac..777cc686 100644 --- a/crates/store/src/backend/rocksdb/write.rs +++ b/crates/store/src/backend/rocksdb/write.rs @@ -41,8 +41,7 @@ use super::{ use crate::{ backend::deserialize_i64_le, write::{ - Batch, BitmapClass, LookupClass, Operation, ValueClass, ValueOp, MAX_COMMIT_ATTEMPTS, - MAX_COMMIT_TIME, + Batch, BitmapClass, Operation, ValueClass, ValueOp, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, }, BitmapKey, Deserialize, IndexKey, Key, LogKey, ValueKey, SUBSPACE_COUNTERS, WITHOUT_BLOCK_NUM, }; @@ -211,6 +210,7 @@ impl<'x> RocksDBTransaction<'x> { document_id, class, }; + let is_counter = key.is_counter(); let key = key.serialize(0); @@ -373,20 +373,6 @@ impl<'x> RocksDBTransaction<'x> { } => { document_id = *document_id_; } - Operation::Value { - class, - op: ValueOp::AtomicAdd(by), - } => { - let key = ValueKey { - account_id, - collection, - document_id, - class, - } - .serialize(0); - - wb.merge_cf(&self.cf_counters, &key, &by.to_le_bytes()[..]); - } Operation::Value { class, op } => { let key = ValueKey { account_id, @@ -394,19 +380,28 @@ impl<'x> RocksDBTransaction<'x> { document_id, class, }; + + let is_counter = key.is_counter(); let key = key.serialize(0); - if let ValueOp::Set(value) = op { - wb.put_cf(&self.cf_values, &key, value); - } else { - wb.delete_cf( - if matches!(class, ValueClass::Lookup(LookupClass::Counter(_))) { - &self.cf_counters - } else { - &self.cf_values - }, - &key, - ); + match op { + ValueOp::Set(value) => { + wb.put_cf(&self.cf_values, &key, value); + } + ValueOp::AtomicAdd(by) => { + wb.merge_cf(&self.cf_counters, &key, &by.to_le_bytes()[..]); + } + ValueOp::Clear => { + wb.delete_cf( + if is_counter { + &self.cf_counters + } else { + &self.cf_values + }, + &key, + ); + } + ValueOp::AddAndGet(_) => unreachable!(), } } Operation::Index { field, key, set } => { diff --git a/crates/store/src/backend/s3/mod.rs b/crates/store/src/backend/s3/mod.rs index 2ede55a0..28dd07d9 100644 --- a/crates/store/src/backend/s3/mod.rs +++ b/crates/store/src/backend/s3/mod.rs @@ -42,7 +42,7 @@ impl S3Store { pub async fn open(config: &mut Config, prefix: impl AsKey) -> Option { // Obtain region and endpoint from config let prefix = prefix.as_key(); - let region = config.value_require_((&prefix, "region"))?.to_string(); + let region = config.value_require((&prefix, "region"))?.to_string(); let region = if let Some(endpoint) = config.value((&prefix, "endpoint")) { Region::Custom { region: region.to_string(), @@ -66,12 +66,12 @@ impl S3Store { }) .ok()?; let timeout = config - .property_or_default_::((&prefix, "timeout"), "30s") + .property_or_default::((&prefix, "timeout"), "30s") .unwrap_or_else(|| Duration::from_secs(30)); Some(S3Store { bucket: Bucket::new( - config.value_require_((&prefix, "bucket"))?, + config.value_require((&prefix, "bucket"))?, region, credentials, ) diff --git a/crates/store/src/backend/sqlite/main.rs b/crates/store/src/backend/sqlite/main.rs index d2aa95a8..a19366ee 100644 --- a/crates/store/src/backend/sqlite/main.rs +++ b/crates/store/src/backend/sqlite/main.rs @@ -39,11 +39,11 @@ impl SqliteStore { conn_pool: Pool::builder() .max_size( config - .property_((&prefix, "pool.max-connections")) + .property((&prefix, "pool.max-connections")) .unwrap_or_else(|| (num_cpus::get() * 4) as u32), ) .build( - SqliteConnectionManager::file(config.value_require_((&prefix, "path"))?) + SqliteConnectionManager::file(config.value_require((&prefix, "path"))?) .with_init(|c| { c.execute_batch(concat!( "PRAGMA journal_mode = WAL; ", @@ -63,7 +63,7 @@ impl SqliteStore { worker_pool: rayon::ThreadPoolBuilder::new() .num_threads( config - .property_::((&prefix, "pool.workers")) + .property::((&prefix, "pool.workers")) .filter(|v| *v > 0) .unwrap_or_else(num_cpus::get), ) diff --git a/crates/store/src/config.rs b/crates/store/src/config.rs index cee7a73e..47bb9bd9 100644 --- a/crates/store/src/config.rs +++ b/crates/store/src/config.rs @@ -68,14 +68,14 @@ impl Stores { #[cfg(feature = "test_mode")] { if config - .property_or_default_::(("store", id, "disable"), "false") + .property_or_default::(("store", id, "disable"), "false") .unwrap_or(false) { tracing::debug!("Skipping disabled store {id:?}."); continue; } } - let protocol = if let Some(protocol) = config.value_require_(("store", id, "type")) { + let protocol = if let Some(protocol) = config.value_require(("store", id, "type")) { protocol.to_ascii_lowercase() } else { continue; @@ -83,8 +83,8 @@ impl Stores { let prefix = ("store", id); let store_id = id.to_string(); let compression_algo = config - .property_or_default_::(("store", id, "compression"), "lz4") - .unwrap_or(CompressionAlgo::Lz4); + .property_or_default::(("store", id, "compression"), "none") + .unwrap_or(CompressionAlgo::None); let lookup_store: Store = match protocol.as_str() { #[cfg(feature = "rocks")] @@ -245,7 +245,7 @@ impl Stores { { let store_id = config.value("storage.data").unwrap().to_string(); if let Some(cron) = - config.property_::(("store", store_id.as_str(), "purge.frequency")) + config.property::(("store", store_id.as_str(), "purge.frequency")) { stores.purge_schedules.push(PurgeSchedule { cron, @@ -260,7 +260,7 @@ impl Stores { { let store_id = config.value("storage.blob").unwrap().to_string(); if let Some(cron) = - config.property_::(("store", store_id.as_str(), "purge.frequency")) + config.property::(("store", store_id.as_str(), "purge.frequency")) { stores.purge_schedules.push(PurgeSchedule { cron, @@ -275,7 +275,7 @@ impl Stores { } for (store_id, store) in &stores.lookup_stores { if let Some(cron) = - config.property_::(("store", store_id.as_str(), "purge.frequency")) + config.property::(("store", store_id.as_str(), "purge.frequency")) { stores.purge_schedules.push(PurgeSchedule { cron, diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index e57cc39e..dfa7d751 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -689,73 +689,3 @@ impl std::fmt::Debug for Store { } } } - -impl Stores { - pub fn get_store( - &self, - config: &utils::config::Config, - key: &str, - ) -> utils::config::Result { - self.stores - .get(config.value_require(key)?) - .cloned() - .ok_or_else(|| { - format!( - "Unable to find data store '{}' defined in key '{}'", - config.value_require(key).unwrap(), - key - ) - }) - } - - pub fn get_blob_store( - &self, - config: &utils::config::Config, - key: &str, - ) -> utils::config::Result { - self.blob_stores - .get(config.value_require(key)?) - .cloned() - .ok_or_else(|| { - format!( - "Unable to find blob store '{}' defined in key '{}'", - config.value_require(key).unwrap(), - key - ) - }) - } - - pub fn get_lookup_store( - &self, - config: &utils::config::Config, - key: &str, - ) -> utils::config::Result { - self.lookup_stores - .get(config.value_require(key)?) - .cloned() - .ok_or_else(|| { - format!( - "Unable to find Lookup store '{}' defined in key '{}'", - config.value_require(key).unwrap(), - key - ) - }) - } - - pub fn get_fts_store( - &self, - config: &utils::config::Config, - key: &str, - ) -> utils::config::Result { - self.fts_stores - .get(config.value_require(key)?) - .cloned() - .ok_or_else(|| { - format!( - "Unable to find FTS store '{}' defined in key '{}'", - config.value_require(key).unwrap(), - key - ) - }) - } -} diff --git a/crates/utils/src/config/cron.rs b/crates/utils/src/config/cron.rs index 98ca6aa2..abd09d96 100644 --- a/crates/utils/src/config/cron.rs +++ b/crates/utils/src/config/cron.rs @@ -139,3 +139,9 @@ impl ParseValue for SimpleCron { Err(format!("Invalid cron key {key:?}: parse cron expression.")) } } + +impl Default for SimpleCron { + fn default() -> Self { + SimpleCron::Hour { minute: 0 } + } +} diff --git a/crates/utils/src/config/parser.rs b/crates/utils/src/config/parser.rs index b7da30f6..c0bfbe57 100644 --- a/crates/utils/src/config/parser.rs +++ b/crates/utils/src/config/parser.rs @@ -34,9 +34,9 @@ const MAX_NEST_LEVEL: usize = 10; // Simple TOML parser for Stalwart Mail Server configuration files. impl Config { - pub fn new(toml: &str) -> Result { + pub fn new(toml: impl AsRef) -> Result { let mut config = Config::default(); - config.parse(toml)?; + config.parse(toml.as_ref())?; Ok(config) } diff --git a/crates/utils/src/config/utils.rs b/crates/utils/src/config/utils.rs index 65d77700..3048ad35 100644 --- a/crates/utils/src/config/utils.rs +++ b/crates/utils/src/config/utils.rs @@ -37,16 +37,7 @@ use smtp_proto::MtPriority; use super::{Config, ConfigError, Rate}; impl Config { - pub fn property(&self, key: impl AsKey) -> super::Result> { - let key = key.as_key(); - if let Some(value) = self.keys.get(&key) { - T::parse_value(key, value).map(Some) - } else { - Ok(None) - } - } - - pub fn property_(&mut self, key: impl AsKey) -> Option { + pub fn property(&mut self, key: impl AsKey) -> Option { let key = key.as_key(); if let Some(value) = self.keys.get(&key) { match T::parse_value(key.as_str(), value) { @@ -62,16 +53,6 @@ impl Config { } pub fn property_or_default( - &self, - key: impl AsKey, - default: &str, - ) -> super::Result { - let key = key.as_key(); - let value = self.keys.get(&key).map_or(default, |v| v.as_str()); - T::parse_value(key, value) - } - - pub fn property_or_default_( &mut self, key: impl AsKey, default: &str, @@ -94,17 +75,6 @@ impl Config { } pub fn property_or_else( - &self, - key: impl AsKey, - default: impl AsKey, - ) -> super::Result> { - match self.property(key) { - Ok(None) => self.property(default), - result => result, - } - } - - pub fn property_or_else_( &mut self, key: impl AsKey, default: impl AsKey, @@ -127,15 +97,7 @@ impl Config { } } - pub fn property_require(&self, key: impl AsKey) -> super::Result { - match self.property(key.clone()) { - Ok(Some(result)) => Ok(result), - Ok(None) => Err(format!("Missing property {:?}.", key.as_key())), - Err(err) => Err(err), - } - } - - pub fn property_require_(&mut self, key: impl AsKey) -> Option { + pub fn property_require(&mut self, key: impl AsKey) -> Option { let key = key.as_key(); if let Some(value) = self.keys.get(&key) { match T::parse_value(key.as_str(), value) { @@ -200,25 +162,7 @@ impl Config { }) } - pub fn properties( - &self, - prefix: impl AsKey, - ) -> impl Iterator> { - let full_prefix = prefix.as_key(); - let prefix = prefix.as_prefix(); - - self.keys.iter().filter_map(move |(key, value)| { - if key.starts_with(&prefix) || key == &full_prefix { - T::parse_value(key.as_str(), value) - .map(|value| (key.as_str(), value)) - .into() - } else { - None - } - }) - } - - pub fn properties_(&mut self, prefix: impl AsKey) -> Vec<(String, T)> { + pub fn properties(&mut self, prefix: impl AsKey) -> Vec<(String, T)> { let full_prefix = prefix.as_key(); let prefix = prefix.as_prefix(); let mut results = Vec::new(); @@ -247,14 +191,7 @@ impl Config { self.keys.contains_key(&key.as_key()) } - pub fn value_require(&self, key: impl AsKey) -> super::Result<&str> { - self.keys - .get(&key.as_key()) - .map(|s| s.as_str()) - .ok_or_else(|| format!("Missing property {:?}.", key.as_key())) - } - - pub fn value_require_(&mut self, key: impl AsKey) -> Option<&str> { + pub fn value_require(&mut self, key: impl AsKey) -> Option<&str> { let key = key.as_key(); if let Some(value) = self.keys.get(&key) { Some(value.as_str()) @@ -821,14 +758,12 @@ ip = "a:b::1:1" assert_eq!( config .property::("servers.my relay.transaction.auth.limits.0001.idle") - .unwrap() .unwrap(), 20 ); assert_eq!( config .property::(("servers", "submissions", "ip")) - .unwrap() .unwrap(), "a:b::1:1".parse::().unwrap() ); diff --git a/crates/utils/src/suffixlist.rs b/crates/utils/src/suffixlist.rs index 55b41fc4..fddb8a00 100644 --- a/crates/utils/src/suffixlist.rs +++ b/crates/utils/src/suffixlist.rs @@ -64,6 +64,7 @@ impl From<&str> for PublicSuffix { } impl PublicSuffix { + #[allow(unused_variables)] pub async fn parse(config: &mut Config, key: &str) -> PublicSuffix { let values = config .values(key) @@ -155,6 +156,7 @@ impl PublicSuffix { } } + #[cfg(not(feature = "test_mode"))] config.new_build_error( key, if has_values { diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 8b887d83..e914c65c 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -20,7 +20,7 @@ redis = ["store/redis"] [dev-dependencies] store = { path = "../crates/store", features = ["test_mode"] } nlp = { path = "../crates/nlp" } -directory = { path = "../crates/directory" } +directory = { path = "../crates/directory", features = ["test_mode"] } jmap = { path = "../crates/jmap", features = ["test_mode"] } jmap_proto = { path = "../crates/jmap-proto" } imap = { path = "../crates/imap", features = ["test_mode"] } diff --git a/tests/resources/smtp/config/servers.toml b/tests/resources/smtp/config/servers.toml index dede2d1e..1eac1447 100644 --- a/tests/resources/smtp/config/servers.toml +++ b/tests/resources/smtp/config/servers.toml @@ -45,9 +45,9 @@ linger = 1 tos = 1 [certificate."default"] -cert = "file://{CERT}" -private-key = "file://{PK}" +cert = "%{file:{CERT}}%" +private-key = "%{file:{PK}}%" [certificate."other"] -cert = "file://{CERT}" -private-key = "file://{PK}" +cert = "%{file:{CERT}}%" +private-key = "%{file:{PK}}%" diff --git a/tests/resources/smtp/config/throttle.toml b/tests/resources/smtp/config/throttle.toml index c0b0dc42..5c8adf03 100644 --- a/tests/resources/smtp/config/throttle.toml +++ b/tests/resources/smtp/config/throttle.toml @@ -3,8 +3,10 @@ match = "remote_ip == '127.0.0.1'" key = ["remote_ip", "authenticated_as"] concurrency = 100 rate = "50/30s" +enable = true [[throttle]] key = "sender_domain" concurrency = 10000 +enable = true diff --git a/tests/resources/smtp/sieve/stage_ehlo.sieve b/tests/resources/smtp/sieve/stage_ehlo.sieve index baee2b56..309f2ea1 100644 --- a/tests/resources/smtp/sieve/stage_ehlo.sieve +++ b/tests/resources/smtp/sieve/stage_ehlo.sieve @@ -1,5 +1,5 @@ require ["variables", "extlists", "reject"]; -if string :list "${env.helo_domain}" "local/invalid-ehlos" { - reject "551 5.1.1 Your domain '${env.helo_domain}' has been blacklisted."; +if eval "contains(['spammer.org', 'spammer.net'], env.helo_domain)" { + reject "551 5.1.1 Your domain '${env.helo_domain}' has been blocklisted."; } diff --git a/tests/src/directory/ldap.rs b/tests/src/directory/ldap.rs index c606f1b4..929225c5 100644 --- a/tests/src/directory/ldap.rs +++ b/tests/src/directory/ldap.rs @@ -42,6 +42,7 @@ async fn ldap_directory() { let mut config = DirectoryTest::new("sqlite".into()).await; let handle = config.directories.directories.remove("ldap").unwrap(); let base_store = config.stores.stores.get("sqlite").unwrap(); + let core = config.core; // Test authentication assert_eq!( @@ -150,27 +151,39 @@ async fn ldap_directory() { // Ids by email compare_sorted( - handle.email_to_ids("jane@example.org").await.unwrap(), + core.email_to_ids(&handle, "jane@example.org") + .await + .unwrap(), map_account_ids(base_store, vec!["jane"]).await, ); compare_sorted( - handle.email_to_ids("jane+alias@example.org").await.unwrap(), + core.email_to_ids(&handle, "jane+alias@example.org") + .await + .unwrap(), map_account_ids(base_store, vec!["jane"]).await, ); compare_sorted( - handle.email_to_ids("info@example.org").await.unwrap(), + core.email_to_ids(&handle, "info@example.org") + .await + .unwrap(), map_account_ids(base_store, vec!["bill", "jane", "john"]).await, ); compare_sorted( - handle.email_to_ids("info+alias@example.org").await.unwrap(), + core.email_to_ids(&handle, "info+alias@example.org") + .await + .unwrap(), map_account_ids(base_store, vec!["bill", "jane", "john"]).await, ); compare_sorted( - handle.email_to_ids("unknown@example.org").await.unwrap(), + core.email_to_ids(&handle, "unknown@example.org") + .await + .unwrap(), Vec::::new(), ); assert_eq!( - handle.email_to_ids("anything@catchall.org").await.unwrap(), + core.email_to_ids(&handle, "anything@catchall.org") + .await + .unwrap(), map_account_ids(base_store, vec!["robert"]).await ); @@ -179,32 +192,41 @@ async fn ldap_directory() { assert!(!handle.is_local_domain("other.org").await.unwrap()); // RCPT TO - assert!(handle.rcpt("jane@example.org").await.unwrap()); - assert!(handle.rcpt("info@example.org").await.unwrap()); - assert!(handle.rcpt("jane+alias@example.org").await.unwrap()); - assert!(handle.rcpt("info+alias@example.org").await.unwrap()); - assert!(handle.rcpt("random_user@catchall.org").await.unwrap()); - assert!(!handle.rcpt("invalid@example.org").await.unwrap()); + assert!(core.rcpt(&handle, "jane@example.org").await.unwrap()); + assert!(core.rcpt(&handle, "info@example.org").await.unwrap()); + assert!(core.rcpt(&handle, "jane+alias@example.org").await.unwrap()); + assert!(core.rcpt(&handle, "info+alias@example.org").await.unwrap()); + assert!(core + .rcpt(&handle, "random_user@catchall.org") + .await + .unwrap()); + assert!(!core.rcpt(&handle, "invalid@example.org").await.unwrap()); // VRFY compare_sorted( - handle.vrfy("jane").await.unwrap(), + core.vrfy(&handle, "jane").await.unwrap(), vec!["jane@example.org".to_string()], ); compare_sorted( - handle.vrfy("john").await.unwrap(), + core.vrfy(&handle, "john").await.unwrap(), vec!["john@example.org".to_string()], ); compare_sorted( - handle.vrfy("jane+alias@example").await.unwrap(), + core.vrfy(&handle, "jane+alias@example").await.unwrap(), vec!["jane@example.org".to_string()], ); - compare_sorted(handle.vrfy("info").await.unwrap(), Vec::::new()); - compare_sorted(handle.vrfy("invalid").await.unwrap(), Vec::::new()); + compare_sorted( + core.vrfy(&handle, "info").await.unwrap(), + Vec::::new(), + ); + compare_sorted( + core.vrfy(&handle, "invalid").await.unwrap(), + Vec::::new(), + ); // EXPN compare_sorted( - handle.expn("info@example.org").await.unwrap(), + core.expn(&handle, "info@example.org").await.unwrap(), vec![ "bill@example.org".to_string(), "jane@example.org".to_string(), @@ -212,7 +234,7 @@ async fn ldap_directory() { ], ); compare_sorted( - handle.expn("john@example.org").await.unwrap(), + core.expn(&handle, "john@example.org").await.unwrap(), Vec::::new(), ); } diff --git a/tests/src/directory/mod.rs b/tests/src/directory/mod.rs index 60183ec1..1442ec1e 100644 --- a/tests/src/directory/mod.rs +++ b/tests/src/directory/mod.rs @@ -27,53 +27,38 @@ pub mod ldap; pub mod smtp; pub mod sql; -use common::config::smtp::session::AddressMapping; -use directory::{ - backend::internal::manage::ManageDirectory, core::config::ConfigDirectory, Directories, - Principal, -}; +use common::{config::smtp::session::AddressMapping, Core}; +use directory::{backend::internal::manage::ManageDirectory, Directories, Principal}; use mail_send::Credentials; use rustls::ServerConfig; use rustls_pemfile::{certs, pkcs8_private_keys}; use rustls_pki_types::PrivateKeyDer; -use std::{borrow::Cow, io::BufReader, path::PathBuf, sync::Arc}; +use std::{borrow::Cow, io::BufReader, sync::Arc}; use store::{LookupStore, Store, Stores}; use tokio_rustls::TlsAcceptor; -use crate::store::TempDir; +use crate::{store::TempDir, AssertConfig}; const CONFIG: &str = r#" [directory."rocksdb"] type = "internal" store = "rocksdb" -[directory."rocksdb".options] -catch-all = true -subaddressing = true - [directory."foundationdb"] type = "internal" store = "foundationdb" -[directory."foundationdb".options] -catch-all = true -subaddressing = true - [directory."sqlite"] type = "sql" store = "sqlite" -[directory."sqlite".options] -catch-all = true -subaddressing = true - [directory."sqlite".columns] name = "name" description = "description" secret = "secret" email = "address" quota = "quota" -type = "type" +class = "type" [store."rocksdb"] type = "rocksdb" @@ -104,17 +89,13 @@ lookup = "sqlite" type = "sql" store = "postgresql" -[directory."postgresql".options] -catch-all = true -subaddressing = true - [directory."postgresql".columns] name = "name" description = "description" secret = "secret" email = "address" quota = "quota" -type = "type" +class = "type" [store."postgresql"] type = "postgresql" @@ -139,17 +120,13 @@ domains = "SELECT 1 FROM emails WHERE address LIKE '%@' || $1 LIMIT 1" type = "sql" store = "mysql" -[directory."mysql".options] -catch-all = true -subaddressing = true - [directory."mysql".columns] name = "name" description = "description" secret = "secret" email = "address" quota = "quota" -type = "type" +class = "type" [store."mysql"] type = "mysql" @@ -172,7 +149,7 @@ domains = "SELECT 1 FROM emails WHERE address LIKE CONCAT('%@', ?) LIMIT 1" [directory."ldap"] type = "ldap" -address = "ldap://localhost:3893" +url = "ldap://localhost:3893" base-dn = "dc=example,dc=org" [directory."ldap".bind] @@ -183,10 +160,6 @@ secret = "mysecret" enable = false dn = "cn=?,ou=svcaccts,dc=example,dc=org" -[directory."ldap".options] -catch-all = true -subaddressing = true - [directory."ldap".filter] name = "(&(|(objectClass=posixAccount)(objectClass=posixGroup))(uid=?))" email = "(&(|(objectClass=posixAccount)(objectClass=posixGroup))(|(mail=?)(givenName=?)(sn=?)))" @@ -205,27 +178,27 @@ groups = ["memberOf", "otherGroups"] email = "mail" email-alias = "givenName" quota = "diskQuota" -type = "objectClass" +class = "objectClass" ############################################################################## [directory."imap"] type = "imap" -address = "127.0.0.1" +host = "127.0.0.1" port = 9198 [directory."imap".pool] max-connections = 5 [directory."imap".tls] -implicit = true +enable = true allow-invalid-certs = true ############################################################################## [directory."smtp"] type = "lmtp" -address = "127.0.0.1" +host = "127.0.0.1" port = 9199 [directory."smtp".limits] @@ -236,7 +209,7 @@ rcpt = 5 max-connections = 5 [directory."smtp".tls] -implicit = true +enable = true allow-invalid-certs = true [directory."smtp".cache] @@ -248,13 +221,9 @@ ttl = {positive = '10s', negative = '5s'} [directory."local"] type = "memory" -[directory."local".options] -catch-all = true -subaddressing = true - [[directory."local".principals]] name = "john" -type = "individual" +class = "individual" description = "John Doe" secret = "12345" email = ["john@example.org", "jdoe@example.org", "john.doe@example.org"] @@ -263,7 +232,7 @@ member-of = ["sales"] [[directory."local".principals]] name = "jane" -type = "individual" +class = "individual" description = "Jane Doe" secret = "abcde" email = "jane@example.org" @@ -272,7 +241,7 @@ member-of = ["sales", "support"] [[directory."local".principals]] name = "bill" -type = "individual" +class = "individual" description = "Bill Foobar" secret = "$2y$05$bvIG6Nmid91Mu9RcmmWZfO5HJIMCT8riNW0hEp8f6/FuA2/mHZFpe" quota = 500000 @@ -281,12 +250,12 @@ email-list = ["info@example.org"] [[directory."local".principals]] name = "sales" -type = "group" +class = "group" description = "Sales Team" [[directory."local".principals]] name = "support" -type = "group" +class = "group" description = "Support Team" "#; @@ -299,6 +268,7 @@ pub struct DirectoryTest { pub directories: Directories, pub stores: Stores, pub temp_dir: TempDir, + pub core: Core, } impl DirectoryTest { @@ -308,24 +278,41 @@ impl DirectoryTest { if id_store.is_some() { // Disable foundationdb store for SQL tests (the fdb select api version can only be run once per process) config_file = config_file - .replace("type = \"foundationdb\"", "type = \"ignore\"") - .replace("store = \"foundationdb\"", "disable = true"); + .replace( + "type = \"foundationdb\"", + "type = \"foundationdb\"\ndisable = true", + ) + .replace( + "store = \"foundationdb\"", + "store = \"foundationdb\"\ndisable = true", + ) + } else { + // Disable internal store + config_file = + config_file.replace("type = \"memory\"", "type = \"memory\"\ndisable = true") } - let config = utils::config::Config::new(&config_file).unwrap(); - let stores = config.parse_stores().await.unwrap(); + let mut config = utils::config::Config::new(&config_file).unwrap(); + let stores = Stores::parse(&mut config).await; + let directories = Directories::parse( + &mut config, + &stores, + id_store + .map(|id| stores.stores.get(id).unwrap().clone()) + .unwrap_or_default(), + ) + .await; + config.assert_no_errors(); + + // Enable catch-all and subaddressing + let mut core = Core::default(); + core.smtp.session.rcpt.catch_all = AddressMapping::Enable; + core.smtp.session.rcpt.subaddressing = AddressMapping::Enable; DirectoryTest { - directories: config - .parse_directory( - &stores, - id_store - .map(|id| stores.stores.get(id).unwrap().clone()) - .unwrap_or_default(), - ) - .await - .unwrap(), + directories, stores, temp_dir, + core, } } } @@ -530,6 +517,9 @@ impl core::fmt::Debug for Item { } } +/* + +// DEPRECATED - TODO: Remove #[tokio::test(flavor = "multi_thread")] #[ignore] async fn lookup_local() { @@ -539,12 +529,12 @@ async fn lookup_local() { format = "regex" values = ["^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$", "^(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$"] - + [store."local/glob"] type = "memory" format = "glob" values = ["*@example.org", "test@*", "localhost", "*+*@*.domain.net"] - + [store."local/list"] type = "memory" format = "list" @@ -564,7 +554,7 @@ async fn lookup_local() { ) .unwrap();*/ - let lookups = utils::config::Config::new( + let mut config = utils::config::Config::new( &LOOKUP_CONFIG.replace( "%PATH%", PathBuf::from(env!("CARGO_MANIFEST_DIR")) @@ -578,11 +568,9 @@ async fn lookup_local() { .unwrap(), ), ) - .unwrap() - .parse_stores() - .await - .unwrap() - .lookup_stores; + .unwrap(); + + let lookups = Stores::parse(&mut config).await.lookup_stores; for (lookup, item, expect) in [ ("glob", "user@example.org", true), @@ -613,6 +601,7 @@ async fn lookup_local() { ); } } +*/ #[tokio::test] async fn address_mappings() { @@ -639,22 +628,23 @@ async fn address_mappings() { expected-catch = "info@example.org" "#; - let config = utils::config::Config::new(MAPPINGS).unwrap(); + let mut config = utils::config::Config::new(MAPPINGS).unwrap(); const ADDR: &str = "john.doe+alias@example.org"; const ADDR_NO_MATCH: &str = "jane@example.org"; + let core = Core::default(); for test in ["enable", "disable", "custom"] { - let catch_all = AddressMapping::from_config(&config, (test, "catch-all")).unwrap(); - let subaddressing = AddressMapping::from_config(&config, (test, "subaddressing")).unwrap(); + let catch_all = AddressMapping::parse(&mut config, (test, "catch-all")); + let subaddressing = AddressMapping::parse(&mut config, (test, "subaddressing")); assert_eq!( - subaddressing.to_subaddress(ADDR).await, + subaddressing.to_subaddress(&core, ADDR).await, config.value_require((test, "expected-sub")).unwrap(), "failed subaddress for {test:?}" ); assert_eq!( - subaddressing.to_subaddress(ADDR_NO_MATCH).await, + subaddressing.to_subaddress(&core, ADDR_NO_MATCH).await, config .value_require((test, "expected-sub-nomatch")) .unwrap(), @@ -662,7 +652,7 @@ async fn address_mappings() { ); assert_eq!( - catch_all.to_catch_all(ADDR).await, + catch_all.to_catch_all(&core, ADDR).await, config .property_require::>((test, "expected-catch")) .unwrap() diff --git a/tests/src/directory/smtp.rs b/tests/src/directory/smtp.rs index f3ead856..b2b130c6 100644 --- a/tests/src/directory/smtp.rs +++ b/tests/src/directory/smtp.rs @@ -39,7 +39,7 @@ use crate::directory::{DirectoryTest, Item, LookupResult}; use super::dummy_tls_acceptor; #[tokio::test] -async fn smtp_directory() { +async fn lmtp_directory() { // Spawn mock LMTP server let shutdown = spawn_mock_lmtp_server(5); tokio::time::sleep(std::time::Duration::from_millis(100)).await; @@ -47,6 +47,7 @@ async fn smtp_directory() { // Obtain directory handle let mut config = DirectoryTest::new(None).await; let handle = config.directories.directories.remove("smtp").unwrap(); + let core = config.core; // Basic lookup let tests = vec![ @@ -94,19 +95,19 @@ async fn smtp_directory() { for (item, expected) in &tests { let result: LookupResult = match item { - Item::IsAccount(v) => handle.rcpt(v).await.unwrap().into(), + Item::IsAccount(v) => core.rcpt(&handle, v).await.unwrap().into(), Item::Authenticate(v) => handle .query(QueryBy::Credentials(v), true) .await .unwrap() .is_some() .into(), - Item::Verify(v) => match handle.vrfy(v).await { + Item::Verify(v) => match core.vrfy(&handle, v).await { Ok(v) => v.into(), Err(DirectoryError::Unsupported) => LookupResult::False, Err(e) => panic!("Unexpected error: {e:?}"), }, - Item::Expand(v) => match handle.expn(v).await { + Item::Expand(v) => match core.expn(&handle, v).await { Ok(v) => v.into(), Err(DirectoryError::Unsupported) => LookupResult::False, Err(e) => panic!("Unexpected error: {e:?}"), @@ -118,27 +119,29 @@ async fn smtp_directory() { // Concurrent requests let mut requests = Vec::new(); + let core = Arc::new(core); for n in 0..100 { let (item, expected) = &tests[n % tests.len()]; let item = item.append(n); let item_clone = item.clone(); let handle = handle.clone(); + let core = core.clone(); requests.push(( tokio::spawn(async move { let result: LookupResult = match &item { - Item::IsAccount(v) => handle.rcpt(v).await.unwrap().into(), + Item::IsAccount(v) => core.rcpt(&handle, v).await.unwrap().into(), Item::Authenticate(v) => handle .query(QueryBy::Credentials(v), true) .await .unwrap() .is_some() .into(), - Item::Verify(v) => match handle.vrfy(v).await { + Item::Verify(v) => match core.vrfy(&handle, v).await { Ok(v) => v.into(), Err(DirectoryError::Unsupported) => LookupResult::False, Err(e) => panic!("Unexpected error: {e:?}"), }, - Item::Expand(v) => match handle.expn(v).await { + Item::Expand(v) => match core.expn(&handle, v).await { Ok(v) => v.into(), Err(DirectoryError::Unsupported) => LookupResult::False, Err(e) => panic!("Unexpected error: {e:?}"), @@ -172,10 +175,11 @@ async fn smtp_directory() { let item = item.append(n); let item_clone = item.clone(); let handle = handle.clone(); + let core = core.clone(); requests.push(( tokio::spawn(async move { let result: LookupResult = match &item { - Item::IsAccount(v) => handle.rcpt(v).await.unwrap().into(), + Item::IsAccount(v) => core.rcpt(&handle, v).await.unwrap().into(), _ => unreachable!(), }; diff --git a/tests/src/directory/sql.rs b/tests/src/directory/sql.rs index 75bb61a1..acc6c1b9 100644 --- a/tests/src/directory/sql.rs +++ b/tests/src/directory/sql.rs @@ -50,6 +50,7 @@ async fn sql_directory() { store: config.stores.lookup_stores.remove(directory_id).unwrap(), }; let base_store = config.stores.stores.get(directory_id).unwrap(); + let core = config.core; // Create tables store.create_test_directory().await; @@ -216,27 +217,39 @@ async fn sql_directory() { // Ids by email assert_eq!( - handle.email_to_ids("jane@example.org").await.unwrap(), + core.email_to_ids(&handle, "jane@example.org") + .await + .unwrap(), map_account_ids(base_store, vec!["jane"]).await ); assert_eq!( - handle.email_to_ids("info@example.org").await.unwrap(), + core.email_to_ids(&handle, "info@example.org") + .await + .unwrap(), map_account_ids(base_store, vec!["bill", "jane", "john"]).await ); assert_eq!( - handle.email_to_ids("jane+alias@example.org").await.unwrap(), + core.email_to_ids(&handle, "jane+alias@example.org") + .await + .unwrap(), map_account_ids(base_store, vec!["jane"]).await ); assert_eq!( - handle.email_to_ids("info+alias@example.org").await.unwrap(), + core.email_to_ids(&handle, "info+alias@example.org") + .await + .unwrap(), map_account_ids(base_store, vec!["bill", "jane", "john"]).await ); assert_eq!( - handle.email_to_ids("unknown@example.org").await.unwrap(), + core.email_to_ids(&handle, "unknown@example.org") + .await + .unwrap(), Vec::::new() ); assert_eq!( - handle.email_to_ids("anything@catchall.org").await.unwrap(), + core.email_to_ids(&handle, "anything@catchall.org") + .await + .unwrap(), map_account_ids(base_store, vec!["robert"]).await ); @@ -245,32 +258,41 @@ async fn sql_directory() { assert!(!handle.is_local_domain("other.org").await.unwrap()); // RCPT TO - assert!(handle.rcpt("jane@example.org").await.unwrap()); - assert!(handle.rcpt("info@example.org").await.unwrap()); - assert!(handle.rcpt("jane+alias@example.org").await.unwrap()); - assert!(handle.rcpt("info+alias@example.org").await.unwrap()); - assert!(handle.rcpt("random_user@catchall.org").await.unwrap()); - assert!(!handle.rcpt("invalid@example.org").await.unwrap()); + assert!(core.rcpt(&handle, "jane@example.org").await.unwrap()); + assert!(core.rcpt(&handle, "info@example.org").await.unwrap()); + assert!(core.rcpt(&handle, "jane+alias@example.org").await.unwrap()); + assert!(core.rcpt(&handle, "info+alias@example.org").await.unwrap()); + assert!(core + .rcpt(&handle, "random_user@catchall.org") + .await + .unwrap()); + assert!(!core.rcpt(&handle, "invalid@example.org").await.unwrap()); // VRFY assert_eq!( - handle.vrfy("jane").await.unwrap(), + core.vrfy(&handle, "jane").await.unwrap(), vec!["jane@example.org".to_string()] ); assert_eq!( - handle.vrfy("john").await.unwrap(), + core.vrfy(&handle, "john").await.unwrap(), vec!["john@example.org".to_string()] ); assert_eq!( - handle.vrfy("jane+alias@example").await.unwrap(), + core.vrfy(&handle, "jane+alias@example").await.unwrap(), vec!["jane@example.org".to_string()] ); - assert_eq!(handle.vrfy("info").await.unwrap(), Vec::::new()); - assert_eq!(handle.vrfy("invalid").await.unwrap(), Vec::::new()); + assert_eq!( + core.vrfy(&handle, "info").await.unwrap(), + Vec::::new() + ); + assert_eq!( + core.vrfy(&handle, "invalid").await.unwrap(), + Vec::::new() + ); // EXPN assert_eq!( - handle.expn("info@example.org").await.unwrap(), + core.expn(&handle, "info@example.org").await.unwrap(), vec![ "bill@example.org".to_string(), "jane@example.org".to_string(), @@ -278,7 +300,7 @@ async fn sql_directory() { ] ); assert_eq!( - handle.expn("john@example.org").await.unwrap(), + core.expn(&handle, "john@example.org").await.unwrap(), Vec::::new() ); } diff --git a/tests/src/imap/mod.rs b/tests/src/imap/mod.rs index d6b97ab3..7b289428 100644 --- a/tests/src/imap/mod.rs +++ b/tests/src/imap/mod.rs @@ -38,11 +38,15 @@ pub mod thread; use std::{path::PathBuf, sync::Arc, time::Duration}; use ::managesieve::core::ManageSieveSessionManager; -use common::config::server::ServerProtocol; +use common::{ + config::server::{ServerProtocol, Servers}, + Core, +}; +use ::store::Stores; use ahash::AHashSet; -use directory::{backend::internal::manage::ManageDirectory, core::config::ConfigDirectory}; -use imap::core::{ImapSessionManager, IMAP}; +use directory::backend::internal::manage::ManageDirectory; +use imap::core::{ImapSessionManager, Inner, IMAP}; use imap_proto::ResponseType; use jmap::{api::JmapSessionManager, services::IPC_CHANNEL_BUFFER, JMAP}; use smtp::core::{SmtpSessionManager, SMTP}; @@ -51,13 +55,13 @@ use tokio::{ net::TcpStream, sync::{mpsc, watch}, }; -use utils::UnwrapFailure; +use utils::config::Config; -use crate::{add_test_certs, directory::DirectoryStore, store::TempDir}; +use crate::{add_test_certs, directory::DirectoryStore, store::TempDir, AssertConfig}; const SERVER: &str = r#" [server] -hostname = "imap.example.org" +hostname = "'imap.example.org'" [server.listener.imap] bind = ["127.0.0.1:9991"] @@ -114,8 +118,8 @@ hash = 64 type = "system" [queue.outbound] -next-hop = [ { if = "key_exists('local/domains', rcpt_domain)", then = "'local'" }, - { if = "key_exists('local/remote-domains', rcpt_domain)", then = "'mock-smtp'" }, +next-hop = [ { if = "rcpt_domain == 'example.com'", then = "'local'" }, + { if = "contains(['remote.org', 'foobar.com', 'test.com', 'other_domain.com'], rcpt_domain)", then = "'mock-smtp'" }, { else = false } ] [remote."mock-smtp"] @@ -124,7 +128,7 @@ port = 9999 protocol = "smtp" [remote."mock-smtp".tls] -implicit = false +enable = false allow-invalid-certs = true [session.extensions] @@ -169,8 +173,8 @@ disable = true allow-invalid-certs = true [certificate.default] -cert = "file://{CERT}" -private-key = "file://{PK}" +cert = "%{file:{CERT}}%" +private-key = "%{file:{PK}}%" [imap.protocol] uidplus = true @@ -235,17 +239,7 @@ description = "description" secret = "secret" email = "address" quota = "quota" -type = "type" - -[store."local/domains"] -type = "memory" -format = "list" -values = ["example.com"] - -[store."local/remote-domains"] -type = "memory" -format = "list" -values = ["remote.org", "foobar.com", "test.com", "other_domain.com"] +class = "type" [oauth] key = "parerga_und_paralipomena" @@ -262,7 +256,7 @@ refresh-token-renew = "2s" #[allow(dead_code)] pub struct IMAPTest { jmap: Arc, - imap: Arc, + imap: Arc, temp_dir: TempDir, shutdown_tx: watch::Sender, } @@ -270,64 +264,78 @@ pub struct IMAPTest { async fn init_imap_tests(store_id: &str, delete_if_exists: bool) -> IMAPTest { // Load and parse config let temp_dir = TempDir::new("imap_tests", delete_if_exists); - let config = utils::config::Config::new( - &add_test_certs(SERVER) + let mut config = Config::new( + add_test_certs(SERVER) .replace("{STORE}", store_id) .replace("{TMP}", &temp_dir.path.display().to_string()), ) .unwrap(); - let mut servers = config.parse_servers().unwrap(); - let stores = config.parse_stores().await.failed("Invalid configuration"); - let directory = config - .parse_directory( - &stores, - stores.core.storage.datas.get(store_id).unwrap().clone(), - ) - .await - .unwrap(); + config.resolve_macros().await; - // Start JMAP and SMTP servers - servers.bind(&config); + // Parse servers + let servers = Servers::parse(&mut config); + + // Bind ports and drop privileges + servers.bind_and_drop_priv(&mut config); + + // Build stores + let stores = Stores::parse(&mut config).await; + + // Parse core + let core = Core::parse(&mut config, stores).await; + let store = core.storage.data.clone(); + let shared_core = core.into_shared(); + + // Init servers let (delivery_tx, delivery_rx) = mpsc::channel(IPC_CHANNEL_BUFFER); - let smtp = SMTP::init(&config, &servers, &stores, &directory, delivery_tx) - .await - .failed("Invalid configuration file"); + let smtp = SMTP::init(&mut config, shared_core.clone(), delivery_tx).await; let jmap = JMAP::init( - &config, - &stores, - &directory, - &mut servers, + &mut config, delivery_rx, - smtp.clone(), + shared_core.clone(), + smtp.inner.clone(), ) - .await - .failed("Invalid configuration file"); - let imap: Arc = IMAP::init(&config) - .await - .failed("Invalid configuration file"); - let (shutdown_tx, _) = servers.spawn(|server, shutdown_rx| { - match &server.protocol { - ServerProtocol::Jmap => { - server.spawn(JmapSessionManager::new(jmap.clone()), shutdown_rx) - } - ServerProtocol::Imap => server.spawn( - ImapSessionManager::new(jmap.clone(), imap.clone()), - shutdown_rx, - ), - ServerProtocol::ManageSieve => server.spawn( - ManageSieveSessionManager::new(jmap.clone(), imap.clone()), - shutdown_rx, - ), - ServerProtocol::Smtp | ServerProtocol::Lmtp => { - server.spawn(SmtpSessionManager::new(smtp.clone()), shutdown_rx) - } - _ => unreachable!(), - }; - }); + .await; + let imap = IMAP::init(&mut config, jmap.clone()).await; + config.assert_no_errors(); + // Spawn servers + let shutdown_tx = servers.spawn( + |server, shutdown_rx| { + match &server.protocol { + ServerProtocol::Smtp | ServerProtocol::Lmtp => server.spawn( + SmtpSessionManager::new(smtp.clone()), + shared_core.clone(), + shutdown_rx, + ), + ServerProtocol::Http => server.spawn( + JmapSessionManager::new(jmap.clone()), + shared_core.clone(), + shutdown_rx, + ), + ServerProtocol::Imap => server.spawn( + ImapSessionManager::new(imap.clone()), + shared_core.clone(), + shutdown_rx, + ), + ServerProtocol::ManageSieve => server.spawn( + ManageSieveSessionManager::new(imap.clone()), + shared_core.clone(), + shutdown_rx, + ), + }; + }, + store.clone(), + ); // Create tables and test accounts let lookup = DirectoryStore { - store: stores.lookup_stores.get("auth").unwrap().clone(), + store: shared_core + .load() + .storage + .lookups + .get("auth") + .unwrap() + .clone(), }; lookup.create_test_directory().await; lookup @@ -350,20 +358,15 @@ async fn init_imap_tests(store_id: &str, delete_if_exists: bool) -> IMAPTest { .await; if delete_if_exists { - jmap.core.storage.data.destroy().await; + store.destroy().await; } // Assign Id 0 to admin (required for some tests) - jmap.core - .storage - .data - .get_or_create_account_id("admin") - .await - .unwrap(); + store.get_or_create_account_id("admin").await.unwrap(); IMAPTest { - jmap, - imap, + jmap: JMAP::from(jmap.clone()).into(), + imap: imap.imap_inner, temp_dir, shutdown_tx, } diff --git a/tests/src/jmap/auth_acl.rs b/tests/src/jmap/auth_acl.rs index d7e6163f..d687535c 100644 --- a/tests/src/jmap/auth_acl.rs +++ b/tests/src/jmap/auth_acl.rs @@ -49,26 +49,18 @@ pub async fn test(params: &mut JMAPTest) { let trash_id = Id::new(TRASH_ID as u64).to_string(); params - .core - .storage .directory .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") .await; params - .core - .storage .directory .create_test_user_with_email("jane.smith@example.com", "abcde", "Jane Smith") .await; params - .core - .storage .directory .create_test_user_with_email("bill@example.com", "098765", "Bill Foobar") .await; params - .core - .storage .directory .create_test_group_with_email("sales@example.com", "Sales Group") .await; @@ -692,8 +684,6 @@ pub async fn test(params: &mut JMAPTest) { // Add John and Jane to the Sales group for name in ["jdoe@example.com", "jane.smith@example.com"] { params - .core - .storage .directory .add_to_group(name, "sales@example.com") .await; @@ -793,8 +783,6 @@ pub async fn test(params: &mut JMAPTest) { // Remove John from the sales group params - .core - .storage .directory .remove_from_group("jdoe@example.com", "sales@example.com") .await; diff --git a/tests/src/jmap/auth_limits.rs b/tests/src/jmap/auth_limits.rs index 1c95caea..d5a1097b 100644 --- a/tests/src/jmap/auth_limits.rs +++ b/tests/src/jmap/auth_limits.rs @@ -21,12 +21,15 @@ * for more details. */ -use std::{sync::Arc, time::Duration}; +use std::{ + net::{IpAddr, Ipv4Addr}, + sync::Arc, + time::Duration, +}; use common::listener::blocked::BLOCKED_IP_KEY; use directory::backend::internal::manage::ManageDirectory; use imap_proto::ResponseType; -use jmap::services::housekeeper::Event; use jmap_client::{ client::{Client, Credentials}, core::set::{SetError, SetErrorType}, @@ -48,8 +51,6 @@ pub async fn test(params: &mut JMAPTest) { // Create test account let server = params.server.clone(); params - .core - .storage .directory .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") .await; @@ -64,14 +65,12 @@ pub async fn test(params: &mut JMAPTest) { ) .to_string(); params - .core - .storage .directory .link_test_address("jdoe@example.com", "john.doe@example.com", "alias") .await; // Reset rate limiters - server.concurrency_limiter.clear(); + server.inner.concurrency_limiter.clear(); // Wait until the beginning of the 5 seconds bucket const LIMIT: u64 = 5; @@ -170,10 +169,12 @@ pub async fn test(params: &mut JMAPTest) { .await .unwrap(); server - .housekeeper_tx - .send(Event::ReloadConfig) - .await - .unwrap(); + .core + .network + .blocked_ips + .ip_addresses + .write() + .remove(&IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1))); // Valid authentication requests should not be rate limited for _ in 0..110 { @@ -251,7 +252,7 @@ pub async fn test(params: &mut JMAPTest) { .unwrap(); }); } - tokio::time::sleep(Duration::from_millis(100)).await; + tokio::time::sleep(Duration::from_millis(500)).await; assert!(matches!( client .mailbox_query( @@ -262,7 +263,7 @@ pub async fn test(params: &mut JMAPTest) { Err(jmap_client::Error::Problem(err)) if err.status() == Some(400))); // Wait for sleep to be done - tokio::time::sleep(Duration::from_millis(1500)).await; + tokio::time::sleep(Duration::from_millis(1000)).await; // Concurrent upload test for _ in 0..4 { @@ -271,7 +272,7 @@ pub async fn test(params: &mut JMAPTest) { client_.upload(None, b"sleep".to_vec(), None).await.unwrap(); }); } - tokio::time::sleep(Duration::from_millis(100)).await; + tokio::time::sleep(Duration::from_millis(500)).await; assert!(matches!( client.upload(None, b"sleep".to_vec(), None).await, Err(jmap_client::Error::Problem(err)) if err.status() == Some(400))); diff --git a/tests/src/jmap/auth_oauth.rs b/tests/src/jmap/auth_oauth.rs index 92677eae..4647a830 100644 --- a/tests/src/jmap/auth_oauth.rs +++ b/tests/src/jmap/auth_oauth.rs @@ -45,8 +45,6 @@ pub async fn test(params: &mut JMAPTest) { // Create test account let server = params.server.clone(); params - .core - .storage .directory .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") .await; diff --git a/tests/src/jmap/blob.rs b/tests/src/jmap/blob.rs index 7b31f7e7..b9c34042 100644 --- a/tests/src/jmap/blob.rs +++ b/tests/src/jmap/blob.rs @@ -34,8 +34,6 @@ pub async fn test(params: &mut JMAPTest) { println!("Running blob tests..."); let server = params.server.clone(); params - .core - .storage .directory .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") .await; diff --git a/tests/src/jmap/crypto.rs b/tests/src/jmap/crypto.rs index 0694bf97..c3a51780 100644 --- a/tests/src/jmap/crypto.rs +++ b/tests/src/jmap/crypto.rs @@ -42,8 +42,6 @@ pub async fn test(params: &mut JMAPTest) { let server = params.server.clone(); let client = &mut params.client; params - .core - .storage .directory .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") .await; diff --git a/tests/src/jmap/delivery.rs b/tests/src/jmap/delivery.rs index 16367a78..552def56 100644 --- a/tests/src/jmap/delivery.rs +++ b/tests/src/jmap/delivery.rs @@ -42,20 +42,14 @@ pub async fn test(params: &mut JMAPTest) { // Create a domain name and a test account let server = params.server.clone(); params - .core - .storage .directory .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") .await; params - .core - .storage .directory .create_test_user_with_email("jane@example.com", "abcdef", "Jane Smith") .await; params - .core - .storage .directory .create_test_user_with_email("bill@example.com", "098765", "Bill Foobar") .await; @@ -90,28 +84,20 @@ pub async fn test(params: &mut JMAPTest) { ) .to_string(); params - .core - .storage .directory .link_test_address("jdoe@example.com", "john.doe@example.com", "alias") .await; // Create a mailing list params - .core - .storage .directory .link_test_address("jdoe@example.com", "members@example.com", "list") .await; params - .core - .storage .directory .link_test_address("jane@example.com", "members@example.com", "list") .await; params - .core - .storage .directory .link_test_address("bill@example.com", "members@example.com", "list") .await; @@ -255,8 +241,6 @@ pub async fn test(params: &mut JMAPTest) { // Removing members from the mailing list and chunked ingest params - .core - .storage .directory .remove_test_alias("jdoe@example.com", "members@example.com") .await; diff --git a/tests/src/jmap/email_submission.rs b/tests/src/jmap/email_submission.rs index b30c0702..6bf1fe4a 100644 --- a/tests/src/jmap/email_submission.rs +++ b/tests/src/jmap/email_submission.rs @@ -85,7 +85,7 @@ pub async fn test(params: &mut JMAPTest) { let server = params.server.clone(); let client = &mut params.client; let (mut smtp_rx, smtp_settings) = spawn_mock_smtp_server(); - server.smtp.resolvers.dns.ipv4_add( + server.core.smtp.resolvers.dns.ipv4_add( "localhost", vec!["127.0.0.1".parse().unwrap()], Instant::now() + std::time::Duration::from_secs(10), @@ -94,14 +94,10 @@ pub async fn test(params: &mut JMAPTest) { // Create a test account let server = params.server.clone(); params - .core - .storage .directory .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") .await; params - .core - .storage .directory .link_test_address("jdoe@example.com", "john.doe@example.com", "alias") .await; diff --git a/tests/src/jmap/event_source.rs b/tests/src/jmap/event_source.rs index ed6fe910..8f550a8c 100644 --- a/tests/src/jmap/event_source.rs +++ b/tests/src/jmap/event_source.rs @@ -43,8 +43,6 @@ pub async fn test(params: &mut JMAPTest) { // Create test account let server = params.server.clone(); params - .core - .storage .directory .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") .await; diff --git a/tests/src/jmap/mod.rs b/tests/src/jmap/mod.rs index 7e730b76..190b0bf3 100644 --- a/tests/src/jmap/mod.rs +++ b/tests/src/jmap/mod.rs @@ -24,8 +24,10 @@ use std::{sync::Arc, time::Duration}; use base64::{engine::general_purpose, Engine}; -use common::config::server::ServerProtocol; -use directory::core::config::ConfigDirectory; +use common::{ + config::server::{ServerProtocol, Servers}, + Core, +}; use imap::core::{ImapSessionManager, IMAP}; use jmap::{ api::JmapSessionManager, @@ -34,13 +36,15 @@ use jmap::{ }; use jmap_client::client::{Client, Credentials}; use jmap_proto::types::id::Id; +use managesieve::core::ManageSieveSessionManager; use reqwest::header; use smtp::core::{SmtpSessionManager, SMTP}; +use store::Stores; use tokio::sync::{mpsc, watch}; -use utils::UnwrapFailure; +use utils::config::Config; -use crate::{add_test_certs, directory::DirectoryStore, store::TempDir}; +use crate::{add_test_certs, directory::DirectoryStore, store::TempDir, AssertConfig}; pub mod auth_acl; pub mod auth_limits; @@ -70,12 +74,12 @@ pub mod websocket; const SERVER: &str = r#" [server] -hostname = "jmap.example.org" +hostname = "'jmap.example.org'" +url = "'https://127.0.0.1:8899'" [server.listener.jmap] bind = ["127.0.0.1:8899"] -url = "https://127.0.0.1:8899" -protocol = "jmap" +protocol = "http" max-connections = 81920 tls.implicit = true @@ -98,9 +102,9 @@ enable = true implicit = false certificate = "default" -[server.security] -blocked-networks = {} +[authentication] fail2ban = "101/5s" +rate-limit = "100/2s" [session.ehlo] reject-non-fqdn = false @@ -126,8 +130,8 @@ hash = 64 type = "system" [queue.outbound] -next-hop = [ { if = "key_exists('local/domains', rcpt_domain)", then = "'local'" }, - { if = "key_exists('local/remote-domains', rcpt_domain)", then = "'mock-smtp'" }, +next-hop = [ { if = "rcpt_domain == 'example.com'", then = "'local'" }, + { if = "contains(['remote.org', 'foobar.com', 'test.com', 'other_domain.com'], rcpt_domain)", then = "'mock-smtp'" }, { else = false } ] [remote."mock-smtp"] @@ -176,11 +180,11 @@ url = "https://localhost:9200" user = "elastic" password = "RtQ-Lu6+o4rxx=XJplVJ" allow-invalid-certs = true -disable = true # Elastic is disabled by default +disable = true [certificate.default] -cert = "file://{CERT}" -private-key = "file://{PK}" +cert = "%{file:{CERT}}%" +private-key = "%{file:{PK}}%" [storage] data = "{STORE}" @@ -212,7 +216,6 @@ size = 50000 [jmap.rate-limit] account = "1000/1m" -authentication = "100/2s" anonymous = "100/1m" [jmap.event-source] @@ -248,17 +251,7 @@ description = "description" secret = "secret" email = "address" quota = "quota" -type = "type" - -[store."local/domains"] -type = "memory" -format = "list" -values = ["example.com"] - -[store."local/remote-domains"] -type = "memory" -format = "list" -values = ["remote.org", "foobar.com", "test.com", "other_domain.com"] +class = "type" [oauth] key = "parerga_und_paralipomena" @@ -271,6 +264,11 @@ user-code = "1s" token = "1s" refresh-token = "3s" refresh-token-renew = "2s" + +[session.extensions] +expn = true +vrfy = true + "#; #[tokio::test(flavor = "multi_thread")] @@ -281,7 +279,7 @@ pub async fn jmap_tests() { .with_env_filter( tracing_subscriber::EnvFilter::builder() .parse( - format!("smtp={level},imap={level},jmap={level},store={level},utils={level},directory={level}"), + format!("smtp={level},imap={level},jmap={level},store={level},utils={level},directory={level},common={level}"), ) .unwrap(), ) @@ -369,6 +367,7 @@ pub async fn wait_for_index(server: &JMAP) { loop { let (tx, rx) = tokio::sync::oneshot::channel(); server + .inner .housekeeper_tx .send(Event::IndexIsActive(tx)) .await @@ -397,67 +396,79 @@ pub async fn assert_is_empty(server: Arc) { async fn init_jmap_tests(store_id: &str, delete_if_exists: bool) -> JMAPTest { // Load and parse config let temp_dir = TempDir::new("jmap_tests", delete_if_exists); - let config = utils::config::Config::new( - &add_test_certs(SERVER) + let mut config = Config::new( + add_test_certs(SERVER) .replace("{STORE}", store_id) .replace("{TMP}", &temp_dir.path.display().to_string()), ) .unwrap(); - let mut servers = config.parse_servers().unwrap(); - let stores = config.parse_stores().await.failed("Invalid configuration"); - let directory = config - .parse_directory( - &stores, - stores.core.storage.datas.get(store_id).unwrap().clone(), - ) - .await - .unwrap(); + config.resolve_macros().await; - // Start JMAP and SMTP servers - servers.bind(&config); + // Parse servers + let servers = Servers::parse(&mut config); + + // Bind ports and drop privileges + servers.bind_and_drop_priv(&mut config); + + // Build stores + let stores = Stores::parse(&mut config).await; + + // Parse core + let core = Core::parse(&mut config, stores).await; + let store = core.storage.data.clone(); + let shared_core = core.into_shared(); + + // Init servers let (delivery_tx, delivery_rx) = mpsc::channel(IPC_CHANNEL_BUFFER); - let smtp = SMTP::init(&config, &servers, &stores, &directory, delivery_tx) - .await - .failed("Invalid configuration file"); + let smtp = SMTP::init(&mut config, shared_core.clone(), delivery_tx).await; let jmap = JMAP::init( - &config, - &stores, - &directory, - &mut servers, + &mut config, delivery_rx, - smtp.clone(), + shared_core.clone(), + smtp.inner.clone(), ) - .await - .failed("Invalid configuration file"); - let imap: Arc = IMAP::init(&config) - .await - .failed("Invalid configuration file"); - jmap.core - .storage - .directory - .blocked_ips - .reload(&config) - .unwrap(); + .await; + let imap = IMAP::init(&mut config, jmap.clone()).await; + config.assert_no_errors(); - let (shutdown_tx, _) = servers.spawn(|server, shutdown_rx| { - match &server.protocol { - ServerProtocol::Smtp | ServerProtocol::Lmtp => { - server.spawn(SmtpSessionManager::new(smtp.clone()), shutdown_rx) - } - ServerProtocol::Jmap => { - server.spawn(JmapSessionManager::new(jmap.clone()), shutdown_rx) - } - ServerProtocol::Imap => server.spawn( - ImapSessionManager::new(jmap.clone(), imap.clone()), - shutdown_rx, - ), - _ => unreachable!(), - }; - }); + // Spawn servers + let shutdown_tx = servers.spawn( + |server, shutdown_rx| { + match &server.protocol { + ServerProtocol::Smtp | ServerProtocol::Lmtp => server.spawn( + SmtpSessionManager::new(smtp.clone()), + shared_core.clone(), + shutdown_rx, + ), + ServerProtocol::Http => server.spawn( + JmapSessionManager::new(jmap.clone()), + shared_core.clone(), + shutdown_rx, + ), + ServerProtocol::Imap => server.spawn( + ImapSessionManager::new(imap.clone()), + shared_core.clone(), + shutdown_rx, + ), + ServerProtocol::ManageSieve => server.spawn( + ManageSieveSessionManager::new(imap.clone()), + shared_core.clone(), + shutdown_rx, + ), + }; + }, + store.clone(), + ); // Create tables let directory = DirectoryStore { - store: stores.core.storage.lookups.get("auth").unwrap().clone(), + store: shared_core + .load() + .storage + .lookups + .get("auth") + .unwrap() + .clone(), }; directory.create_test_directory().await; directory @@ -465,7 +476,7 @@ async fn init_jmap_tests(store_id: &str, delete_if_exists: bool) -> JMAPTest { .await; if delete_if_exists { - jmap.core.storage.data.destroy().await; + store.destroy().await; } // Create client @@ -479,7 +490,7 @@ async fn init_jmap_tests(store_id: &str, delete_if_exists: bool) -> JMAPTest { client.set_default_account_id(Id::new(1)); JMAPTest { - server: jmap, + server: JMAP::from(jmap).into(), temp_dir, client, directory, diff --git a/tests/src/jmap/push_subscription.rs b/tests/src/jmap/push_subscription.rs index 3e0e639c..ced22134 100644 --- a/tests/src/jmap/push_subscription.rs +++ b/tests/src/jmap/push_subscription.rs @@ -30,7 +30,7 @@ use std::{ }; use base64::{engine::general_purpose, Engine}; -use common::listener::SessionData; +use common::{config::server::Servers, listener::SessionData, Core}; use directory::backend::internal::manage::ManageDirectory; use ece::EcKeyComponents; use hyper::{body, header::CONTENT_ENCODING, server::conn::http1, service::service_fn, StatusCode}; @@ -45,25 +45,27 @@ use jmap::{ }; use jmap_client::{mailbox::Role, push_subscription::Keys}; use jmap_proto::types::{id::Id, type_state::DataType}; -use store::ahash::AHashSet; +use store::{ahash::AHashSet, Store}; use tokio::sync::mpsc; +use utils::config::Config; use crate::{ add_test_certs, jmap::{assert_is_empty, mailbox::destroy_all_mailboxes, test_account_login}, + AssertConfig, }; use super::JMAPTest; -const SERVER: &str = " +const SERVER: &str = r#" [server] -hostname = 'jmap-push.example.org' +hostname = "'jmap-push.example.org'" +url = "'https://127.0.0.1:9000'" [server.listener.jmap] bind = ['127.0.0.1:9000'] -url = 'https://127.0.0.1:9000' -protocol = 'jmap' +protocol = 'http' [server.socket] reuse-addr = true @@ -74,9 +76,9 @@ implicit = false certificate = 'default' [certificate.default] -cert = 'file://{CERT}' -private-key = 'file://{PK}' -"; +cert = '%{file:{CERT}}%' +private-key = '%{file:{PK}}%' +"#; pub async fn test(params: &mut JMAPTest) { println!("Running Push Subscription tests..."); @@ -84,8 +86,6 @@ pub async fn test(params: &mut JMAPTest) { // Create test account let server = params.server.clone(); params - .core - .storage .directory .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") .await; @@ -117,15 +117,20 @@ pub async fn test(params: &mut JMAPTest) { }); // Start mock push server - let settings = utils::config::Config::new(&add_test_certs(SERVER)).unwrap(); - let servers = settings.parse_servers().unwrap(); + let mut settings = Config::new(&add_test_certs(SERVER)).unwrap(); + settings.resolve_macros().await; + let servers = Servers::parse(&mut settings); // Start JMAP server let manager = SessionManager::from(push_server.clone()); - servers.bind(&settings); - let _shutdown_tx = servers.spawn(|server, shutdown_rx| { - server.spawn(manager.clone(), shutdown_rx); - }); + servers.bind_and_drop_priv(&mut settings); + settings.assert_no_errors(); + let _shutdown_tx = servers.spawn( + |server, shutdown_rx| { + server.spawn(manager.clone(), Core::default().into_shared(), shutdown_rx); + }, + Store::default(), + ); // Register push notification (no encryption) let push_id = client diff --git a/tests/src/jmap/quota.rs b/tests/src/jmap/quota.rs index f5c15b7d..661e3b33 100644 --- a/tests/src/jmap/quota.rs +++ b/tests/src/jmap/quota.rs @@ -39,14 +39,10 @@ pub async fn test(params: &mut JMAPTest) { println!("Running quota tests..."); let server = params.server.clone(); params - .core - .storage .directory .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") .await; params - .core - .storage .directory .create_test_user_with_email("robert@example.com", "aabbcc", "Robert Foobar") .await; @@ -69,14 +65,10 @@ pub async fn test(params: &mut JMAPTest) { .unwrap(), ); params - .core - .storage .directory .set_test_quota("robert@example.com", 1024) .await; params - .core - .storage .directory .add_to_group("robert@example.com", "jdoe@example.com") .await; diff --git a/tests/src/jmap/sieve_script.rs b/tests/src/jmap/sieve_script.rs index 283b4371..7413e1a2 100644 --- a/tests/src/jmap/sieve_script.rs +++ b/tests/src/jmap/sieve_script.rs @@ -51,8 +51,6 @@ pub async fn test(params: &mut JMAPTest) { // Create test account params - .core - .storage .directory .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") .await; @@ -293,7 +291,7 @@ pub async fn test(params: &mut JMAPTest) { // Start mock SMTP server let (mut smtp_rx, smtp_settings) = spawn_mock_smtp_server(); - server.smtp.resolvers.dns.ipv4_add( + server.core.smtp.resolvers.dns.ipv4_add( "localhost", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(10), diff --git a/tests/src/jmap/vacation_response.rs b/tests/src/jmap/vacation_response.rs index b1c52743..7fbbe9b6 100644 --- a/tests/src/jmap/vacation_response.rs +++ b/tests/src/jmap/vacation_response.rs @@ -45,8 +45,6 @@ pub async fn test(params: &mut JMAPTest) { let server = params.server.clone(); let client = &mut params.client; params - .core - .storage .directory .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") .await; @@ -64,7 +62,7 @@ pub async fn test(params: &mut JMAPTest) { // Start mock SMTP server let (mut smtp_rx, smtp_settings) = spawn_mock_smtp_server(); - server.smtp.resolvers.dns.ipv4_add( + server.core.smtp.resolvers.dns.ipv4_add( "localhost", vec!["127.0.0.1".parse().unwrap()], Instant::now() + std::time::Duration::from_secs(10), diff --git a/tests/src/jmap/websocket.rs b/tests/src/jmap/websocket.rs index 4c29f145..4b247750 100644 --- a/tests/src/jmap/websocket.rs +++ b/tests/src/jmap/websocket.rs @@ -47,8 +47,6 @@ pub async fn test(params: &mut JMAPTest) { // Authenticate all accounts params - .core - .storage .directory .create_test_user_with_email("jdoe@example.com", "12345", "John Doe") .await; diff --git a/tests/src/lib.rs b/tests/src/lib.rs index e5b24e5d..569ae81a 100644 --- a/tests/src/lib.rs +++ b/tests/src/lib.rs @@ -53,3 +53,26 @@ pub fn add_test_certs(config: &str) -> String { .replace("{CERT}", cert.as_path().to_str().unwrap()) .replace("{PK}", pk.as_path().to_str().unwrap()) } + +#[cfg(test)] +pub trait AssertConfig { + fn assert_no_errors(self) -> Self; + fn assert_no_warnings(self) -> Self; +} + +#[cfg(test)] +impl AssertConfig for utils::config::Config { + fn assert_no_errors(self) -> Self { + if !self.errors.is_empty() { + panic!("Errors: {:#?}", self.errors); + } + self + } + + fn assert_no_warnings(self) -> Self { + if !self.missing.is_empty() { + panic!("Warnings: {:#?}", self.missing); + } + self + } +} diff --git a/tests/src/smtp/config.rs b/tests/src/smtp/config.rs index 5f8bb48b..f8ecac38 100644 --- a/tests/src/smtp/config.rs +++ b/tests/src/smtp/config.rs @@ -25,11 +25,12 @@ use std::{fs, net::IpAddr, path::PathBuf, time::Duration}; use common::{ config::{ - server::{Listener, Server, ServerProtocol}, - smtp::*, + server::{Listener, Server, ServerProtocol, Servers}, + smtp::{throttle::parse_throttle, *}, }, - expr::{functions::ResolveVariable, if_block::*, *}, + expr::{functions::ResolveVariable, if_block::*, tokenizer::TokenMap, *}, listener::TcpAcceptor, + Core, }; use tokio::net::TcpSocket; @@ -59,10 +60,11 @@ fn parse_if_blocks() { file.push("config"); file.push("if-blocks.toml"); - let config = Config::new(&fs::read_to_string(file).unwrap()).unwrap(); + let mut config = Config::new(fs::read_to_string(file).unwrap()).unwrap(); // Create context and add some conditions - let available_keys = vec![ + + let token_map = TokenMap::default().with_smtp_variables(&[ V_RECIPIENT, V_RECIPIENT_DOMAIN, V_SENDER, @@ -72,15 +74,10 @@ fn parse_if_blocks() { V_REMOTE_IP, V_LOCAL_IP, V_PRIORITY, - ]; + ]); assert_eq!( - config - .parse_if_block("durations", |name| { - map_expr_token::(name, &available_keys) - }) - .unwrap() - .unwrap(), + IfBlock::try_parse(&mut config, "durations", &token_map).unwrap(), IfBlock { key: "durations".to_string(), if_then: vec![ @@ -125,12 +122,7 @@ fn parse_if_blocks() { ); assert_eq!( - config - .parse_if_block("string-list", |name| { - map_expr_token::(name, &available_keys) - }) - .unwrap() - .unwrap(), + IfBlock::try_parse(&mut config, "string-list", &token_map).unwrap(), IfBlock { key: "string-list".to_string(), if_then: vec![ @@ -182,12 +174,7 @@ fn parse_if_blocks() { ); assert_eq!( - config - .parse_if_block("string-list-bis", |name| { - map_expr_token::(name, &available_keys) - }) - .unwrap() - .unwrap(), + IfBlock::try_parse(&mut config, "string-list-bis", &token_map).unwrap(), IfBlock { key: "string-list-bis".to_string(), if_then: vec![ @@ -240,12 +227,7 @@ fn parse_if_blocks() { ); assert_eq!( - config - .parse_if_block("single-value", |name| { - map_expr_token::(name, &available_keys) - }) - .unwrap() - .unwrap(), + IfBlock::try_parse(&mut config, "single-value", &token_map).unwrap(), IfBlock { key: "single-value".to_string(), if_then: vec![], @@ -262,38 +244,37 @@ fn parse_if_blocks() { "bad-if-without-else", "bad-multiple-else", ] { - if let Ok(value) = config.parse_if_block(bad_rule, |name| { - map_expr_token::(name, &available_keys) - }) { + if let Some(value) = IfBlock::try_parse(&mut config, bad_rule, &token_map) { panic!("Condition {bad_rule:?} had unexpected result {value:?}"); } } } #[test] -fn parse_throttle() { +fn parse_throttles() { let mut file = PathBuf::from(env!("CARGO_MANIFEST_DIR")); file.push("resources"); file.push("smtp"); file.push("config"); file.push("throttle.toml"); - let available_keys = vec![ - V_RECIPIENT, - V_RECIPIENT_DOMAIN, - V_SENDER, - V_SENDER_DOMAIN, - V_AUTHENTICATED_AS, - V_LISTENER, - V_REMOTE_IP, - V_LOCAL_IP, - V_PRIORITY, - ]; - - let config = Config::new(&fs::read_to_string(file).unwrap()).unwrap(); - let throttle = config - .parse_throttle("throttle", &available_keys, u16::MAX) - .unwrap(); + let mut config = Config::new(fs::read_to_string(file).unwrap()).unwrap(); + let throttle = parse_throttle( + &mut config, + "throttle", + &TokenMap::default().with_smtp_variables(&[ + V_RECIPIENT, + V_RECIPIENT_DOMAIN, + V_SENDER, + V_SENDER_DOMAIN, + V_AUTHENTICATED_AS, + V_LISTENER, + V_REMOTE_IP, + V_LOCAL_IP, + V_PRIORITY, + ]), + u16::MAX, + ); assert_eq!( throttle, @@ -335,8 +316,8 @@ fn parse_servers() { let toml = add_test_certs(&fs::read_to_string(file).unwrap()); // Parse servers - let config = Config::new(&toml).unwrap(); - let servers = config.parse_servers().unwrap().inner; + let mut config = Config::new(toml).unwrap(); + let servers = Servers::parse(&mut config).servers; let expected_servers = vec![ Server { id: "smtp".to_string(), @@ -404,21 +385,6 @@ fn parse_servers() { "failed for {}", expected_server.id ); - assert_eq!( - server.internal_id, expected_server.internal_id, - "failed for {}", - expected_server.id - ); - assert_eq!( - server.hostname, expected_server.hostname, - "failed for {}", - expected_server.id - ); - assert_eq!( - server.data, expected_server.data, - "failed for {}", - expected_server.id - ); assert_eq!( server.protocol, expected_server.protocol, "failed for {}", @@ -459,23 +425,23 @@ async fn eval_if() { file.push("config"); file.push("rules-eval.toml"); - let config = Config::new(&fs::read_to_string(file).unwrap()).unwrap(); - let servers = vec![ - Server { - id: "smtp".to_string(), - ..Default::default() - }, - Server { - id: "smtps".to_string(), - ..Default::default() - }, - ]; - let mut context = ConfigContext::new(); - context.stores = config.parse_stores().await.unwrap(); + let mut config = Config::new(fs::read_to_string(file).unwrap()).unwrap(); + let envelope = TestEnvelope::from_config(&mut config); + let token_map = TokenMap::default().with_smtp_variables(&[ + V_RECIPIENT, + V_RECIPIENT_DOMAIN, + V_SENDER, + V_SENDER_DOMAIN, + V_AUTHENTICATED_AS, + V_LISTENER, + V_REMOTE_IP, + V_LOCAL_IP, + V_PRIORITY, + V_MX, + ]); + let core = Core::default(); - let envelope = TestEnvelope::from_config(&config); - - for (key, expr) in &config.keys { + for (key, _) in config.keys.clone() { if !key.starts_with("rule.") { continue; } @@ -486,32 +452,12 @@ async fn eval_if() { IfBlock { key: key.to_string(), if_then: vec![IfThen { - expr: Expression::parse(key.as_str(), expr, |name| { - map_expr_token::( - name, - &[ - V_RECIPIENT, - V_RECIPIENT_DOMAIN, - V_SENDER, - V_SENDER_DOMAIN, - V_AUTHENTICATED_AS, - V_LISTENER, - V_REMOTE_IP, - V_LOCAL_IP, - V_PRIORITY, - V_MX, - ], - ) - }) - .unwrap(), + expr: Expression::try_parse(&mut config, key.as_str(), &token_map).unwrap(), then: Expression::from(true), }], default: Expression::from(false), } - .eval( - |name| { envelope.resolve_variable(name) }, - |_, _| async { Default::default() } - ) + .eval(&envelope, &core, &key) .await .to_bool(), expected_result.parse::().unwrap(), @@ -528,48 +474,40 @@ async fn eval_dynvalue() { file.push("config"); file.push("rules-dynvalue.toml"); - let config = Config::new(&fs::read_to_string(file).unwrap()).unwrap(); - let mut context = ConfigContext::new(); - context.stores = config.parse_stores().await.unwrap(); + let mut config = Config::new(fs::read_to_string(file).unwrap()).unwrap(); + let envelope = TestEnvelope::from_config(&mut config); + let token_map = TokenMap::default().with_smtp_variables(&[ + V_RECIPIENT, + V_RECIPIENT_DOMAIN, + V_SENDER, + V_SENDER_DOMAIN, + V_AUTHENTICATED_AS, + V_LISTENER, + V_REMOTE_IP, + V_LOCAL_IP, + V_PRIORITY, + V_MX, + ]); + let core = Core::default(); - let envelope = TestEnvelope::from_config(&config); - - for test_name in config.sub_keys("eval", "") { + for test_name in config + .sub_keys("eval", "") + .map(|s| s.to_string()) + .collect::>() + { //println!("============= Testing {:?} ==================", key); - let if_block = config - .parse_if_block(("eval", test_name, "test"), |name| { - map_expr_token::( - name, - &[ - V_RECIPIENT, - V_RECIPIENT_DOMAIN, - V_SENDER, - V_SENDER_DOMAIN, - V_AUTHENTICATED_AS, - V_LISTENER, - V_REMOTE_IP, - V_LOCAL_IP, - V_PRIORITY, - V_MX, - ], - ) - }) - .unwrap() - .unwrap(); + let if_block = IfBlock::try_parse( + &mut config, + ("eval", test_name.as_str(), "test"), + &token_map, + ) + .unwrap(); let expected = config - .property_require::>(("eval", test_name, "expect")) - .unwrap(); + .property_require::>(("eval", test_name.as_str(), "expect")) + .unwrap_or_else(|| panic!("Missing expect for test {test_name:?}")); assert_eq!( - String::try_from( - if_block - .eval( - |name| { envelope.resolve_variable(name) }, - |_, _| async { Default::default() } - ) - .await - ) - .ok(), + String::try_from(if_block.eval(&envelope, &core, test_name.as_str()).await).ok(), expected, "failed for test {test_name:?}" ); @@ -596,7 +534,7 @@ impl ResolveVariable for TestEnvelope { } impl TestEnvelope { - pub fn from_config(config: &Config) -> Self { + pub fn from_config(config: &mut Config) -> Self { Self { local_ip: config.property_require("envelope.local-ip").unwrap(), remote_ip: config.property_require("envelope.remote-ip").unwrap(), diff --git a/tests/src/smtp/inbound/antispam.rs b/tests/src/smtp/inbound/antispam.rs index e596ad43..b99841d9 100644 --- a/tests/src/smtp/inbound/antispam.rs +++ b/tests/src/smtp/inbound/antispam.rs @@ -7,7 +7,6 @@ use std::{ time::{Duration, Instant}, }; -use crate::smtp::session::TestSession; use ahash::AHashMap; use common::{ expr::if_block::IfBlock, @@ -15,18 +14,20 @@ use common::{ functions::html::{get_attribute, html_attr_tokens, html_img_area, html_to_tokens}, ScriptModification, }, + Core, }; use mail_auth::{dmarc::Policy, DkimResult, DmarcResult, IprevResult, SpfResult, MX}; use sieve::runtime::Variable; use smtp::{ - core::{Session, SessionAddress, SMTP}, + core::{Inner, Session, SessionAddress, SMTP}, inbound::AuthResult, scripts::ScriptResult, }; +use store::Stores; use tokio::runtime::Handle; use utils::config::Config; -use crate::smtp::{TestConfig, TestSMTP}; +use crate::smtp::{build_smtp, session::TestSession, TestSMTP}; const CONFIG: &str = r#" [sieve.trusted] @@ -165,8 +166,8 @@ async fn antispam() { "reputation", "pyzor", ]; - let mut core = SMTP::test(); - let qr = core.init_test_queue("smtp_antispam_test"); + let mut inner = Inner::default(); + let qr = inner.init_test_queue("smtp_antispam_test"); let base_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .parent() .unwrap() @@ -227,14 +228,12 @@ async fn antispam() { config.push_str(&format!("combined = '''{all_scripts}\n'''\n")); // Parse config - let config = Config::new(&config).unwrap(); - let mut ctx = ConfigContext::new(); - ctx.stores = config.parse_stores().await.unwrap(); - core.sieve = config.parse_sieve(&mut ctx).unwrap(); - core.core.storage.lookups = ctx.stores.lookups.clone(); - core.core.storage.scripts = ctx.scripts.clone(); - let config = &mut core.core.smtp.session; + let mut config = Config::new(&config).unwrap(); + let stores = Stores::parse(&mut config).await; + let mut core = Core::parse(&mut config, stores).await; + let config = &mut core.smtp.session; config.rcpt.relay = IfBlock::new(true); + qr.set_core_stores(&mut core); // Add mock DNS entries for (domain, ip) in [ @@ -269,7 +268,7 @@ async fn antispam() { "127.0.0.8", ), ] { - core.core.smtp.resolvers.dns.ipv4_add( + core.smtp.resolvers.dns.ipv4_add( domain, vec![ip.parse().unwrap()], Instant::now() + Duration::from_secs(100), @@ -281,7 +280,7 @@ async fn antispam() { "gmail.com", "custom.disposable.org", ] { - core.core.smtp.resolvers.dns.mx_add( + core.smtp.resolvers.dns.mx_add( mx, vec![MX { exchanges: vec!["127.0.0.1".parse().unwrap()], @@ -291,7 +290,7 @@ async fn antispam() { ); } - let core = Arc::new(core); + let core = build_smtp(core, Inner::default()); // Run tests let base_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) @@ -304,7 +303,7 @@ async fn antispam() { continue; }*/ println!("===== {test_name} ====="); - let script = ctx.scripts.remove(test_name).unwrap(); + let script = core.core.sieve.scripts.get(test_name).cloned().unwrap(); let contents = fs::read_to_string(base_path.join(format!("{test_name}.test"))).unwrap(); let mut lines = contents.lines(); diff --git a/tests/src/smtp/inbound/auth.rs b/tests/src/smtp/inbound/auth.rs index 9cae4672..a5376aee 100644 --- a/tests/src/smtp/inbound/auth.rs +++ b/tests/src/smtp/inbound/auth.rs @@ -21,21 +21,28 @@ * for more details. */ -use common::{config::smtp::session::Mechanism, expr::if_block::IfBlock}; -use directory::core::config::ConfigDirectory; -use store::Store; +use common::Core; + +use store::Stores; use utils::config::Config; use crate::smtp::{ - inbound::dummy_stores, + build_smtp, session::{TestSession, VerifyResponse}, - ParseTestConfig, TestConfig, + TempDir, }; -use smtp::core::{Session, State, SMTP}; +use smtp::core::{Inner, Session, State}; -const DIRECTORY: &str = r#" +const CONFIG: &str = r#" [storage] -lookup = "dummy" +data = "sqlite" +lookup = "sqlite" +blob = "sqlite" +fts = "sqlite" + +[store."sqlite"] +type = "sqlite" +path = "{TMP}/queue.db" [directory."local"] type = "memory" @@ -55,41 +62,43 @@ secret = "p4ssw0rd" email = "jane@example.org" email-list = ["info@example.org"] member-of = ["sales", "support"] + +[session.auth] +require = [{if = "remote_ip = '10.0.0.1'", then = true}, + {else = false}] +mechanisms = [{if = "remote_ip = '10.0.0.1'", then = "[plain, login]"}, + {else = 0}] +directory = [{if = "remote_ip = '10.0.0.1'", then = "'local'"}, + {else = false}] +must-match-sender = true + +[session.auth.errors] +total = [{if = "remote_ip = '10.0.0.1'", then = 2}, + {else = 3}] +wait = "100ms" + +[session.extensions] +future-release = [{if = '!is_empty(authenticated_as)', then = '1d'}, + {else = false}] "#; #[tokio::test] async fn auth() { - let mut core = SMTP::test(); - core.core.storage.directories = Config::new(DIRECTORY) - .unwrap() - .parse_directory(&dummy_stores(), Store::default()) - .await - .unwrap() - .directories; + // Enable logging + /*tracing::subscriber::set_global_default( + tracing_subscriber::FmtSubscriber::builder() + .with_max_level(tracing::Level::TRACE) + .finish(), + ) + .unwrap();*/ - let config = &mut core.core.smtp.session.auth; - - config.require = r#"[{if = "remote_ip = '10.0.0.1'", then = true}, - {else = false}]"# - .parse_if(); - config.directory = r#"[{if = "remote_ip = '10.0.0.1'", then = "'local'"}, - {else = false}]"# - .parse_if(); - config.errors_max = r#"[{if = "remote_ip = '10.0.0.1'", then = 2}, - {else = 3}]"# - .parse_if(); - config.errors_wait = "'100ms'".parse_if(); - config.mechanisms = r#"[{if = "remote_ip = '10.0.0.1'", then = "[plain, login]"}, - {else = 0}]"# - .parse_if_constant::(); - config.must_match_sender = IfBlock::new(true); - core.core.smtp.session.extensions.future_release = - r"[{if = '!is_empty(authenticated_as)', then = '1d'}, - {else = false}]" - .parse_if(); + let tmp_dir = TempDir::new("smtp_auth_test", true); + let mut config = Config::new(tmp_dir.update_config(CONFIG)).unwrap(); + let stores = Stores::parse(&mut config).await; + let core = Core::parse(&mut config, stores).await; // EHLO should not advertise plain text auth without TLS - let mut session = Session::test(core); + let mut session = Session::test(build_smtp(core, Inner::default())); session.data.remote_ip_str = "10.0.0.1".to_string(); session.eval_session_params().await; session.stream.tls = false; diff --git a/tests/src/smtp/inbound/basic.rs b/tests/src/smtp/inbound/basic.rs index ec88e85e..da6b5189 100644 --- a/tests/src/smtp/inbound/basic.rs +++ b/tests/src/smtp/inbound/basic.rs @@ -21,15 +21,17 @@ * for more details. */ +use common::Core; +use smtp::core::{Inner, Session}; + use crate::smtp::{ + build_smtp, session::{TestSession, VerifyResponse}, - TestConfig, }; -use smtp::core::{Session, SMTP}; #[tokio::test] async fn basic_commands() { - let mut session = Session::test(SMTP::test()); + let mut session = Session::test(build_smtp(Core::default(), Inner::default())); // STARTTLS should be available on clear text connections session.stream.tls = false; diff --git a/tests/src/smtp/inbound/data.rs b/tests/src/smtp/inbound/data.rs index 48beb418..072918a7 100644 --- a/tests/src/smtp/inbound/data.rs +++ b/tests/src/smtp/inbound/data.rs @@ -21,23 +21,28 @@ * for more details. */ -use std::sync::Arc; - -use common::expr::if_block::IfBlock; -use directory::core::config::ConfigDirectory; -use store::Store; +use common::Core; +use store::Stores; use utils::config::Config; use crate::smtp::{ - inbound::{dummy_stores, TestMessage}, + build_smtp, + inbound::TestMessage, session::{load_test_message, TestSession, VerifyResponse}, - ParseTestConfig, TestConfig, TestSMTP, + TempDir, TestSMTP, }; -use smtp::core::{Session, SMTP}; +use smtp::core::{Inner, Session}; -const DIRECTORY: &str = r#" +const CONFIG: &str = r#" [storage] -lookup = "dummy" +data = "sqlite" +lookup = "sqlite" +blob = "sqlite" +fts = "sqlite" + +[store."sqlite"] +type = "sqlite" +path = "{TMP}/queue.db" [directory."local"] type = "memory" @@ -66,6 +71,45 @@ description = "Mike Foobar" secret = "p4ssw0rd" email = "mike@test.com" +[session.rcpt] +directory = "'local'" + +[[queue.quota]] +match = "sender = 'john@doe.org'" +key = ['sender'] +messages = 1 + +[session.data.limits] +messages = [{if = "remote_ip = '10.0.0.1'", then = 1}, + {else = 100}] +received-headers = 3 + +[session.data.add-headers] +received = [{if = "remote_ip = '10.0.0.3'", then = true}, + {else = false}] +received-spf = [{if = "remote_ip = '10.0.0.3'", then = true}, + {else = false}] +auth-results = [{if = "remote_ip = '10.0.0.3'", then = true}, + {else = false}] +message-id = [{if = "remote_ip = '10.0.0.3'", then = true}, + {else = false}] +date = [{if = "remote_ip = '10.0.0.3'", then = true}, + {else = false}] +return-path = [{if = "remote_ip = '10.0.0.3'", then = true}, + {else = false}] + +[[queue.quota]] +match = "rcpt_domain = 'foobar.org'" +key = ['rcpt_domain'] +size = 450 +enable = true + +[[queue.quota]] +match = "rcpt = 'jane@domain.net'" +key = ['rcpt'] +size = 450 +enable = true + "#; #[tokio::test] @@ -77,52 +121,17 @@ async fn data() { .finish(), ) .unwrap();*/ - let mut core = SMTP::test(); // Create temp dir for queue - let mut qr = core.init_test_queue("smtp_data_test"); - core.core.storage.directories = Config::new(DIRECTORY) - .unwrap() - .parse_directory(&dummy_stores(), Store::default()) - .await - .unwrap() - .directories; - let config = &mut core.core.smtp.session.rcpt; - config.directory = IfBlock::new("local".to_string()); - - let config = &mut core.core.smtp.session; - config.data.add_auth_results = r#"[{if = "remote_ip = '10.0.0.3'", then = true}, - {else = false}]"# - .parse_if(); - config.data.add_date = config.data.add_auth_results.clone(); - config.data.add_message_id = config.data.add_auth_results.clone(); - config.data.add_received = config.data.add_auth_results.clone(); - config.data.add_return_path = config.data.add_auth_results.clone(); - config.data.add_received_spf = config.data.add_auth_results.clone(); - config.data.max_received_headers = IfBlock::new(3); - config.data.max_messages = r#"[{if = "remote_ip = '10.0.0.1'", then = 1}, - {else = 100}]"# - .parse_if(); - - core.core.smtp.queue.quota = r#"[[queue.quota]] - match = "sender = 'john@doe.org'" - key = ['sender'] - messages = 1 - - [[queue.quota]] - match = "rcpt_domain = 'foobar.org'" - key = ['rcpt_domain'] - size = 450 - - [[queue.quota]] - match = "rcpt = 'jane@domain.net'" - key = ['rcpt'] - size = 450 - "# - .parse_quota(); + let mut inner = Inner::default(); + let tmp_dir = TempDir::new("smtp_data_test", true); + let mut config = Config::new(tmp_dir.update_config(CONFIG)).unwrap(); + let stores = Stores::parse(&mut config).await; + let core = Core::parse(&mut config, stores).await; + let mut qr = inner.init_test_queue(&core); // Test queue message builder - let core = Arc::new(core); + let core = build_smtp(core, inner); let mut session = Session::test(core.clone()); session.data.remote_ip_str = "10.0.0.1".to_string(); session.eval_session_params().await; diff --git a/tests/src/smtp/inbound/dmarc.rs b/tests/src/smtp/inbound/dmarc.rs index 9e0d11e5..d56d434b 100644 --- a/tests/src/smtp/inbound/dmarc.rs +++ b/tests/src/smtp/inbound/dmarc.rs @@ -21,16 +21,10 @@ * for more details. */ -use std::{ - sync::Arc, - time::{Duration, Instant}, -}; +use std::time::{Duration, Instant}; + +use common::{config::smtp::report::AggregateFrequency, Core}; -use common::{ - config::smtp::{auth::VerifyStrategy, report::AggregateFrequency}, - expr::if_block::IfBlock, -}; -use directory::core::config::ConfigDirectory; use mail_auth::{ common::{parse::TxtRecordParser, verify::DomainKey}, dkim::DomainKeyReport, @@ -38,19 +32,27 @@ use mail_auth::{ report::DmarcResult, spf::Spf, }; -use store::Store; +use store::Stores; use utils::config::Config; use crate::smtp::{ - inbound::{dummy_stores, sign::TextConfigContext, TestMessage, TestReportingEvent}, + build_smtp, + inbound::{sign::SIGNATURES, TestMessage, TestReportingEvent}, session::{TestSession, VerifyResponse}, - ParseTestConfig, TestConfig, TestSMTP, + TempDir, TestSMTP, }; -use smtp::core::{Session, SMTP}; +use smtp::core::{Inner, Session}; -const DIRECTORY: &str = r#" +const CONFIG: &str = r#" [storage] -lookup = "dummy" +data = "sqlite" +lookup = "sqlite" +blob = "sqlite" +fts = "sqlite" + +[store."sqlite"] +type = "sqlite" +path = "{TMP}/queue.db" [directory."local"] type = "memory" @@ -61,33 +63,78 @@ description = "John Doe" secret = "secret" email = ["jdoe@example.com"] +[session.rcpt] +directory = "'local'" + +[session.data.add-headers] +received = true +received-spf = true +auth-results = true +message-id = true +date = true +return-path = false + +[report.dkim] +send = "[1, 1s]" +sign = "['rsa']" + +[report.spf] +send = "[1, 1s]" +sign = "['rsa']" + +[report.dmarc] +send = "[1, 1s]" +sign = "['rsa']" + +[report.dmarc.aggregate] +send = "daily" + +[auth.spf.verify] +ehlo = [{if = "remote_ip = '10.0.0.2'", then = 'strict'}, + { else = 'relaxed' }] +mail-from = [{if = "remote_ip = '10.0.0.2'", then = 'strict'}, + { else = 'relaxed' }] + +[auth.dmarc] +verify = "strict" + +[auth.arc] +verify = "strict" + +[auth.dkim] +verify = [{if = "sender_domain = 'test.net'", then = 'relaxed'}, + { else = 'strict' }] + "#; #[tokio::test] async fn dmarc() { - let mut core = SMTP::test(); - core.core.storage.signers = ConfigContext::new().parse_signatures().signers; + let mut inner = Inner::default(); + let tmp_dir = TempDir::new("smtp_dmarc_test", true); + let mut config = Config::new(tmp_dir.update_config(CONFIG.to_string() + SIGNATURES)).unwrap(); + let stores = Stores::parse(&mut config).await; + let core = Core::parse(&mut config, stores).await; // Create temp dir for queue - let mut qr = core.init_test_queue("smtp_dmarc_test"); + let mut qr = inner.init_test_queue(&core); // Add SPF, DKIM and DMARC records - core.core.smtp.resolvers.dns.txt_add( + core.smtp.resolvers.dns.txt_add( "mx.example.com", Spf::parse(b"v=spf1 ip4:10.0.0.1 ip4:10.0.0.2 -all").unwrap(), Instant::now() + Duration::from_secs(5), ); - core.core.smtp.resolvers.dns.txt_add( + core.smtp.resolvers.dns.txt_add( "example.com", Spf::parse(b"v=spf1 ip4:10.0.0.1 -all ra=spf-failures rr=e:f:s:n").unwrap(), Instant::now() + Duration::from_secs(5), ); - core.core.smtp.resolvers.dns.txt_add( + core.smtp.resolvers.dns.txt_add( "foobar.com", Spf::parse(b"v=spf1 ip4:10.0.0.1 -all").unwrap(), Instant::now() + Duration::from_secs(5), ); - core.core.smtp.resolvers.dns.txt_add( + core.smtp.resolvers.dns.txt_add( "ed._domainkey.example.com", DomainKey::parse( concat!( @@ -99,7 +146,7 @@ async fn dmarc() { .unwrap(), Instant::now() + Duration::from_secs(5), ); - core.core.smtp.resolvers.dns.txt_add( + core.smtp.resolvers.dns.txt_add( "default._domainkey.example.com", DomainKey::parse( concat!( @@ -114,12 +161,12 @@ async fn dmarc() { .unwrap(), Instant::now() + Duration::from_secs(5), ); - core.core.smtp.resolvers.dns.txt_add( + core.smtp.resolvers.dns.txt_add( "_report._domainkey.example.com", DomainKeyReport::parse(b"ra=dkim-failures; rp=100; rr=d:o:p:s:u:v:x;").unwrap(), Instant::now() + Duration::from_secs(5), ); - core.core.smtp.resolvers.dns.txt_add( + core.smtp.resolvers.dns.txt_add( "_dmarc.example.com", Dmarc::parse( concat!( @@ -134,48 +181,10 @@ async fn dmarc() { ); // Create report channels - let mut rr = core.init_test_report(); - core.core.storage.directories = Config::new(DIRECTORY) - .unwrap() - .parse_directory(&dummy_stores(), Store::default()) - .await - .unwrap() - .directories; - let config = &mut core.core.smtp.session.rcpt; - config.directory = IfBlock::new("local".to_string()); - - let config = &mut core.core.smtp.session; - config.data.add_auth_results = IfBlock::new(true); - config.data.add_date = IfBlock::new(true); - config.data.add_message_id = IfBlock::new(true); - config.data.add_received = IfBlock::new(true); - config.data.add_return_path = IfBlock::new(true); - config.data.add_received_spf = IfBlock::new(true); - - let config = &mut core.core.smtp.report; - config.dkim.send = "\"[1, 1s]\"".parse_if(); - config.dmarc.send = config.dkim.send.clone(); - config.spf.send = config.dkim.send.clone(); - config.dmarc_aggregate.send = IfBlock::new(AggregateFrequency::Daily); - - let config = &mut core.mail_auth; - config.spf.verify_ehlo = r#"[{if = "remote_ip = '10.0.0.2'", then = 'strict'}, - { else = 'relaxed' }]"# - .parse_if_constant::(); - config.spf.verify_mail_from = config.spf.verify_ehlo.clone(); - config.dmarc.verify = IfBlock::new(VerifyStrategy::Strict); - config.arc.verify = config.dmarc.verify.clone(); - config.dkim.verify = r#"[{if = "sender_domain = 'test.net'", then = 'relaxed'}, - { else = 'strict' }]"# - .parse_if_constant::(); - - let config = &mut core.core.smtp.report; - config.spf.sign = "\"['rsa']\"".parse_if(); - config.dmarc.sign = "\"['rsa']\"".parse_if(); - config.dkim.sign = "\"['rsa']\"".parse_if(); + let mut rr = inner.init_test_report(); // SPF must pass - let core = Arc::new(core); + let core = build_smtp(core, inner); let mut session = Session::test(core.clone()); session.data.remote_ip_str = "10.0.0.2".to_string(); session.data.remote_ip = session.data.remote_ip_str.parse().unwrap(); diff --git a/tests/src/smtp/inbound/ehlo.rs b/tests/src/smtp/inbound/ehlo.rs index d2f719b5..54760c49 100644 --- a/tests/src/smtp/inbound/ehlo.rs +++ b/tests/src/smtp/inbound/ehlo.rs @@ -23,47 +23,53 @@ use std::time::{Duration, Instant}; -use common::{config::smtp::auth::VerifyStrategy, expr::if_block::IfBlock}; +use common::Core; use mail_auth::{common::parse::TxtRecordParser, spf::Spf, SpfResult}; -use smtp_proto::MtPriority; + +use smtp::core::{Inner, Session}; +use utils::config::Config; use crate::smtp::{ + build_smtp, session::{TestSession, VerifyResponse}, - ParseTestConfig, TestConfig, }; -use smtp::core::{Session, SMTP}; + +const CONFIG: &str = r#" +[session.data.limits] +size = [{if = "remote_ip = '10.0.0.1'", then = 1024}, + {else = 2048}] + +[session.extensions] +future-release = [{if = "remote_ip = '10.0.0.1'", then = '1h'}, + {else = false}] +mt-priority = [{if = "remote_ip = '10.0.0.1'", then = 'nsep'}, + {else = false}] + +[session.ehlo] +reject-non-fqdn = true + +[auth.spf.verify] +ehlo = [{if = "remote_ip = '10.0.0.2'", then = 'strict'}, + {else = 'relaxed'}] +"#; #[tokio::test] async fn ehlo() { - let mut core = SMTP::test(); - core.core.smtp.resolvers.dns.txt_add( + let mut config = Config::new(CONFIG).unwrap(); + let core = Core::parse(&mut config, Default::default()).await; + core.smtp.resolvers.dns.txt_add( "mx1.foobar.org", Spf::parse(b"v=spf1 ip4:10.0.0.1 -all").unwrap(), Instant::now() + Duration::from_secs(5), ); - core.core.smtp.resolvers.dns.txt_add( + core.smtp.resolvers.dns.txt_add( "mx2.foobar.org", Spf::parse(b"v=spf1 ip4:10.0.0.2 -all").unwrap(), Instant::now() + Duration::from_secs(5), ); - let config = &mut core.core.smtp.session; - config.data.max_message_size = r#"[{if = "remote_ip = '10.0.0.1'", then = 1024}, - {else = 2048}]"# - .parse_if(); - config.extensions.future_release = r#"[{if = "remote_ip = '10.0.0.1'", then = '1h'}, - {else = false}]"# - .parse_if(); - config.extensions.mt_priority = r#"[{if = "remote_ip = '10.0.0.1'", then = 'nsep'}, - {else = false}]"# - .parse_if_constant::(); - core.mail_auth.spf.verify_ehlo = r#"[{if = "remote_ip = '10.0.0.2'", then = 'strict'}, - {else = 'relaxed'}]"# - .parse_if_constant::(); - config.ehlo.reject_non_fqdn = IfBlock::new(true); - // Reject non-FQDN domains - let mut session = Session::test(core); + let mut session = Session::test(build_smtp(core, Inner::default())); session.data.remote_ip_str = "10.0.0.1".to_string(); session.data.remote_ip = session.data.remote_ip_str.parse().unwrap(); session.stream.tls = false; diff --git a/tests/src/smtp/inbound/limits.rs b/tests/src/smtp/inbound/limits.rs index 37273827..af475b30 100644 --- a/tests/src/smtp/inbound/limits.rs +++ b/tests/src/smtp/inbound/limits.rs @@ -23,31 +23,36 @@ use std::time::{Duration, Instant}; +use common::Core; use tokio::sync::watch; +use smtp::core::{Inner, Session}; +use utils::config::Config; + use crate::smtp::{ + build_smtp, session::{TestSession, VerifyResponse}, - ParseTestConfig, TestConfig, }; -use smtp::core::{Session, SMTP}; + +const CONFIG: &str = r#" +[session] +transfer-limit = [{if = "remote_ip = '10.0.0.1'", then = 10}, + {else = 1024}] +timeout = [{if = "remote_ip = '10.0.0.2'", then = '500ms'}, + {else = '30m'}] +duration = [{if = "remote_ip = '10.0.0.3'", then = '500ms'}, + {else = '60m'}] +"#; #[tokio::test] async fn limits() { - let mut core = SMTP::test(); - let config = &mut core.core.smtp.session; - config.transfer_limit = r#"[{if = "remote_ip = '10.0.0.1'", then = 10}, - {else = 1024}]"# - .parse_if(); - config.timeout = r#"[{if = "remote_ip = '10.0.0.2'", then = '500ms'}, - {else = '30m'}]"# - .parse_if(); - config.duration = r#"[{if = "remote_ip = '10.0.0.3'", then = '500ms'}, - {else = '60m'}]"# - .parse_if(); + let mut config = Config::new(CONFIG).unwrap(); + let core = Core::parse(&mut config, Default::default()).await; + let (_tx, rx) = watch::channel(true); // Exceed max line length - let mut session = Session::test_with_shutdown(core, rx); + let mut session = Session::test_with_shutdown(build_smtp(core, Inner::default()), rx); session.data.remote_ip_str = "10.0.0.1".to_string(); let mut buf = vec![b'A'; 2049]; session.ingest(&buf).await.unwrap(); diff --git a/tests/src/smtp/inbound/mail.rs b/tests/src/smtp/inbound/mail.rs index 0dcb0ab8..a0842342 100644 --- a/tests/src/smtp/inbound/mail.rs +++ b/tests/src/smtp/inbound/mail.rs @@ -26,80 +26,100 @@ use std::{ time::{Duration, Instant, SystemTime}, }; -use common::{config::smtp::auth::VerifyStrategy, expr::if_block::IfBlock}; +use common::Core; use mail_auth::{common::parse::TxtRecordParser, spf::Spf, IprevResult, SpfResult}; -use smtp_proto::{MtPriority, MAIL_BY_NOTIFY, MAIL_BY_RETURN, MAIL_REQUIRETLS}; +use smtp_proto::{MAIL_BY_NOTIFY, MAIL_BY_RETURN, MAIL_REQUIRETLS}; + +use smtp::core::{Inner, Session}; +use store::Stores; +use utils::config::Config; use crate::smtp::{ + build_smtp, session::{TestSession, VerifyResponse}, - ParseTestConfig, TestConfig, + TempDir, }; -use smtp::core::{Session, SMTP}; + +const CONFIG: &str = r#" +[storage] +data = "sqlite" +lookup = "sqlite" +blob = "sqlite" +fts = "sqlite" + +[store."sqlite"] +type = "sqlite" +path = "{TMP}/data.db" + +[session.ehlo] +require = true + +[auth.spf.verify] +ehlo = 'relaxed' +mail-from = [{if = "remote_ip = '10.0.0.2'", then = 'strict'}, + {else = 'relaxed'}] + +[auth.iprev] +verify = [{if = "remote_ip = '10.0.0.2'", then = 'strict'}, + {else = 'relaxed'}] + +[session.extensions] +future-release = [{if = "remote_ip = '10.0.0.2'", then = '1d'}, + {else = false}] +deliver-by = [{if = "remote_ip = '10.0.0.2'", then = '1d'}, + {else = false}] +requiretls = [{if = "remote_ip = '10.0.0.2'", then = true}, + {else = false}] +mt-priority = [{if = "remote_ip = '10.0.0.2'", then = 'nsep'}, + {else = false}] + +[session.data.limits] +size = [{if = "remote_ip = '10.0.0.2'", then = 2048}, + {else = 1024}] + +[[session.throttle]] +match = "remote_ip = '10.0.0.1'" +key = 'sender' +rate = '2/1s' +enable = true + +"#; #[tokio::test] async fn mail() { - let mut core = SMTP::test(); - core.core.smtp.resolvers.dns.txt_add( + let tmp_dir = TempDir::new("smtp_mail_test", true); + let mut config = Config::new(tmp_dir.update_config(CONFIG)).unwrap(); + let stores = Stores::parse(&mut config).await; + let core = Core::parse(&mut config, stores).await; + core.smtp.resolvers.dns.txt_add( "foobar.org", Spf::parse(b"v=spf1 ip4:10.0.0.1 -all").unwrap(), Instant::now() + Duration::from_secs(5), ); - core.core.smtp.resolvers.dns.txt_add( + core.smtp.resolvers.dns.txt_add( "mx1.foobar.org", Spf::parse(b"v=spf1 ip4:10.0.0.1 -all").unwrap(), Instant::now() + Duration::from_secs(5), ); - core.core.smtp.resolvers.dns.ptr_add( + core.smtp.resolvers.dns.ptr_add( "10.0.0.1".parse().unwrap(), vec!["mx1.foobar.org.".to_string()], Instant::now() + Duration::from_secs(5), ); - core.core.smtp.resolvers.dns.ipv4_add( + core.smtp.resolvers.dns.ipv4_add( "mx1.foobar.org.", vec!["10.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(5), ); - core.core.smtp.resolvers.dns.ptr_add( + core.smtp.resolvers.dns.ptr_add( "10.0.0.2".parse().unwrap(), vec!["mx2.foobar.org.".to_string()], Instant::now() + Duration::from_secs(5), ); - let config = &mut core.core.smtp.session; - config.ehlo.require = IfBlock::new(true); - core.mail_auth.spf.verify_ehlo = IfBlock::new(VerifyStrategy::Relaxed); - core.mail_auth.spf.verify_mail_from = r#"[{if = "remote_ip = '10.0.0.2'", then = 'strict'}, - {else = 'relaxed'}]"# - .parse_if_constant::(); - core.mail_auth.iprev.verify = r#"[{if = "remote_ip = '10.0.0.2'", then = 'strict'}, - {else = 'relaxed'}]"# - .parse_if_constant::(); - config.extensions.future_release = r#"[{if = "remote_ip = '10.0.0.2'", then = '1d'}, - {else = false}]"# - .parse_if(); - config.extensions.deliver_by = r#"[{if = "remote_ip = '10.0.0.2'", then = '1d'}, - {else = false}]"# - .parse_if(); - config.extensions.requiretls = r#"[{if = "remote_ip = '10.0.0.2'", then = true}, - {else = false}]"# - .parse_if(); - config.extensions.mt_priority = r#"[{if = "remote_ip = '10.0.0.2'", then = 'nsep'}, - {else = false}]"# - .parse_if_constant::(); - config.data.max_message_size = r#"[{if = "remote_ip = '10.0.0.2'", then = 2048}, - {else = 1024}]"# - .parse_if(); - - config.throttle.mail_from = r#"[[throttle]] - match = "remote_ip = '10.0.0.1'" - key = 'sender' - rate = '2/1s' - "# - .parse_throttle(); - // Be rude and do not say EHLO let core = Arc::new(core); - let mut session = Session::test(core.clone()); + let mut session = Session::test(build_smtp(core.clone(), Inner::default())); session.data.remote_ip_str = "10.0.0.1".to_string(); session.data.remote_ip = session.data.remote_ip_str.parse().unwrap(); session.eval_session_params().await; @@ -182,7 +202,7 @@ async fn mail() { .unwrap(); session.response().assert_code("550 5.7.25"); session.data.iprev = None; - core.core.smtp.resolvers.dns.ipv4_add( + core.smtp.resolvers.dns.ipv4_add( "mx2.foobar.org.", vec!["10.0.0.2".parse().unwrap()], Instant::now() + Duration::from_secs(5), @@ -194,7 +214,7 @@ async fn mail() { .await .unwrap(); session.response().assert_code("550 5.7.23"); - core.core.smtp.resolvers.dns.txt_add( + core.smtp.resolvers.dns.txt_add( "foobar.org", Spf::parse(b"v=spf1 ip4:10.0.0.1 ip4:10.0.0.2 -all").unwrap(), Instant::now() + Duration::from_secs(5), diff --git a/tests/src/smtp/inbound/milter.rs b/tests/src/smtp/inbound/milter.rs index d10573e1..ddc8d9b7 100644 --- a/tests/src/smtp/inbound/milter.rs +++ b/tests/src/smtp/inbound/milter.rs @@ -26,27 +26,31 @@ use std::{fs, net::SocketAddr, path::PathBuf, sync::Arc, time::Duration}; use common::{ config::smtp::session::{Milter, MilterVersion}, expr::if_block::IfBlock, + Core, }; use mail_auth::AuthenticatedMessage; use mail_parser::MessageParser; use serde::Deserialize; use smtp::{ - core::{Session, SessionData, SMTP}, + core::{Inner, Session, SessionData}, inbound::milter::{ receiver::{FrameResult, Receiver}, Action, Command, Macros, MilterClient, Modification, Options, Response, }, }; +use store::Stores; use tokio::{ io::{AsyncReadExt, AsyncWriteExt}, net::{TcpListener, TcpStream}, sync::watch, }; +use utils::config::Config; use crate::smtp::{ + build_smtp, inbound::TestMessage, session::{load_test_message, TestSession, VerifyResponse}, - ParseTestConfig, TestConfig, TestSMTP, + TempDir, TestSMTP, }; #[derive(Debug, Deserialize)] @@ -55,6 +59,31 @@ struct HeaderTest { result: String, } +const CONFIG: &str = r#" +[storage] +data = "sqlite" +lookup = "sqlite" +blob = "sqlite" +fts = "sqlite" + +[store."sqlite"] +type = "sqlite" +path = "{TMP}/queue.db" + +[session.rcpt] +relay = true + +[[session.data.milter]] +hostname = "127.0.0.1" +port = 9332 +#port = 11332 +#port = 7357 +enable = true +options.version = 6 +tls = false + +"#; + #[tokio::test] async fn milter_session() { // Enable logging @@ -67,25 +96,17 @@ async fn milter_session() { .unwrap();*/ // Configure tests + let tmp_dir = TempDir::new("smtp_milter_test", true); + let mut config = Config::new(tmp_dir.update_config(CONFIG)).unwrap(); + let stores = Stores::parse(&mut config).await; + let core = Core::parse(&mut config, stores).await; let _rx = spawn_mock_milter_server(); tokio::time::sleep(Duration::from_millis(100)).await; - let mut core = SMTP::test(); - let mut qr = core.init_test_queue("smtp_milter_test"); - let config = &mut core.core.smtp.session; - config.rcpt.relay = IfBlock::new(true); - config.data.milters = r#"[[session.data.milter]] - hostname = "127.0.0.1" - port = 9332 - #port = 11332 - #port = 7357 - enable = true - options.version = 6 - tls = false - "# - .parse_milters(); + let mut inner = Inner::default(); + let mut qr = inner.init_test_queue(&core); // Build session - let mut session = Session::test(core); + let mut session = Session::test(build_smtp(core, inner)); session.data.remote_ip_str = "10.0.0.1".to_string(); session.eval_session_params().await; session.ehlo("mx.doe.org").await; diff --git a/tests/src/smtp/inbound/mod.rs b/tests/src/smtp/inbound/mod.rs index e422078a..20b90c97 100644 --- a/tests/src/smtp/inbound/mod.rs +++ b/tests/src/smtp/inbound/mod.rs @@ -25,7 +25,7 @@ use std::time::Duration; use store::{ write::{key::DeserializeBigEndian, Bincode, QueueClass, QueueEvent, ReportEvent, ValueClass}, - Deserialize, IterateParams, Store, Stores, ValueKey, U64_LEN, + Deserialize, IterateParams, ValueKey, U64_LEN, }; use tokio::sync::mpsc::error::TryRecvError; @@ -37,7 +37,7 @@ use smtp::{ use super::{QueueReceiver, ReportReceiver}; -pub mod antispam; +//pub mod antispam; pub mod auth; pub mod basic; pub mod data; @@ -85,6 +85,7 @@ impl QueueReceiver { pub async fn assert_report_is_empty(&self) { assert_eq!(self.read_report_events().await, vec![]); + let todo = "fix antispam"; for (from_key, to_key) in [ ( @@ -378,17 +379,3 @@ impl TestMessage for Message { .collect() } } - -pub fn dummy_stores() -> Stores { - let mut stores = Stores::default(); - let store = Store::default(); - stores.stores.insert("dummy".to_string(), store.clone()); - stores - .lookups - .insert("dummy".to_string(), store.clone().into()); - stores - .fts_stores - .insert("dummy".to_string(), store.clone().into()); - stores.blob_stores.insert("dummy".to_string(), store.into()); - stores -} diff --git a/tests/src/smtp/inbound/rcpt.rs b/tests/src/smtp/inbound/rcpt.rs index dfc6ea71..ad91a1d0 100644 --- a/tests/src/smtp/inbound/rcpt.rs +++ b/tests/src/smtp/inbound/rcpt.rs @@ -23,22 +23,30 @@ use std::time::Duration; -use common::expr::if_block::IfBlock; -use directory::core::config::ConfigDirectory; +use common::Core; + use smtp_proto::{RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_SUCCESS}; -use store::Store; +use store::Stores; use utils::config::Config; -use crate::smtp::{ - inbound::dummy_stores, - session::{TestSession, VerifyResponse}, - ParseTestConfig, TestConfig, -}; -use smtp::core::{Session, State, SMTP}; +use smtp::core::{Inner, Session, State}; -const DIRECTORY: &str = r#" +use crate::smtp::{ + build_smtp, + session::{TestSession, VerifyResponse}, + TempDir, +}; + +const CONFIG: &str = r#" [storage] -lookup = "dummy" +data = "sqlite" +lookup = "sqlite" +blob = "sqlite" +fts = "sqlite" + +[store."sqlite"] +type = "sqlite" +path = "{TMP}/queue.db" [directory."local"] type = "memory" @@ -67,45 +75,48 @@ description = "Mike Foobar" secret = "p4ssw0rd" email = "mike@foobar.org" +[session.rcpt] +directory = "'local'" +max-recipients = [{if = "remote_ip = '10.0.0.1'", then = 3}, + {else = 5}] +relay = [{if = "remote_ip = '10.0.0.1'", then = false}, + {else = true}] + +[session.rcpt.errors] +total = [{if = "remote_ip = '10.0.0.1'", then = 3}, + {else = 100}] +wait = [{if = "remote_ip = '10.0.0.1'", then = '5ms'}, + {else = '1s'}] + +[session.extensions] +dsn = [{if = "remote_ip = '10.0.0.1'", then = false}, + {else = true}] + +[[session.throttle]] +match = "remote_ip = '10.0.0.1' && !is_empty(rcpt)" +key = 'sender' +rate = '2/1s' +enable = true + "#; #[tokio::test] async fn rcpt() { - let mut core = SMTP::test(); - - let config_ext = &mut core.core.smtp.session.extensions; - core.core.storage.directories = Config::new(DIRECTORY) - .unwrap() - .parse_directory(&dummy_stores(), Store::default()) - .await - .unwrap() - .directories; - let config = &mut core.core.smtp.session.rcpt; - config.directory = IfBlock::new("local".to_string()); - config.max_recipients = r#"[{if = "remote_ip = '10.0.0.1'", then = 3}, - {else = 5}]"# - .parse_if(); - config.relay = r#"[{if = "remote_ip = '10.0.0.1'", then = false}, - {else = true}]"# - .parse_if(); - config_ext.dsn = r#"[{if = "remote_ip = '10.0.0.1'", then = false}, - {else = true}]"# - .parse_if(); - config.errors_max = r#"[{if = "remote_ip = '10.0.0.1'", then = 3}, - {else = 100}]"# - .parse_if(); - config.errors_wait = r#"[{if = "remote_ip = '10.0.0.1'", then = '5ms'}, - {else = '1s'}]"# - .parse_if(); - core.core.smtp.session.throttle.rcpt_to = r#"[[throttle]] - match = "remote_ip = '10.0.0.1'" - key = 'sender' - rate = '2/1s' - "# - .parse_throttle(); + // Enable logging + /* + tracing::subscriber::set_global_default( + tracing_subscriber::FmtSubscriber::builder() + .with_max_level(tracing::Level::TRACE) + .finish(), + ) + .unwrap();*/ + let tmp_dir = TempDir::new("smtp_rcpt_test", true); + let mut config = Config::new(tmp_dir.update_config(CONFIG)).unwrap(); + let stores = Stores::parse(&mut config).await; + let core = Core::parse(&mut config, stores).await; // RCPT without MAIL FROM - let mut session = Session::test(core); + let mut session = Session::test(build_smtp(core, Inner::default())); session.data.remote_ip_str = "10.0.0.1".to_string(); session.eval_session_params().await; session.ehlo("mx1.foobar.org").await; diff --git a/tests/src/smtp/inbound/rewrite.rs b/tests/src/smtp/inbound/rewrite.rs index 2c878a25..20a5803c 100644 --- a/tests/src/smtp/inbound/rewrite.rs +++ b/tests/src/smtp/inbound/rewrite.rs @@ -21,21 +21,14 @@ * for more details. */ -use crate::smtp::{ - inbound::{dummy_stores, sign::TextConfigContext}, - session::TestSession, - TestConfig, -}; -use common::{config::smtp::*, expr::if_block::IfBlock}; -use directory::core::config::ConfigDirectory; -use smtp::core::{Session, SMTP}; -use store::Store; +use common::Core; + +use smtp::core::{Inner, Session}; use utils::config::Config; -const CONFIG: &str = r#" -[storage] -lookup = "dummy" +use crate::smtp::{build_smtp, session::TestSession}; +const CONFIG: &str = r#" [session.mail] rewrite = [ { if = "ends_with(sender_domain, '.foobar.net') & matches('^([^.]+)@([^.]+)\.(.+)$', sender)", then = "$1 + '+' + $2 + '@' + $3"}, { else = false } ] @@ -47,6 +40,7 @@ rewrite = [ { if = "rcpt_domain = 'foobar.net' & matches('^([^.]+)\\.([^.]+)@(.+ { else = false } ] script = [ { if = "rcpt_domain = 'foobar.org'", then = "'rcpt'" }, { else = false } ] +relay = true [sieve.trusted] from-name = "Sieve Daemon" @@ -62,8 +56,8 @@ cpu = 10000 nested-includes = 5 duplicate-expiry = "7d" -[sieve.trusted.scripts] -mail = ''' +[sieve.trusted.scripts."mail"] +contents = ''' require ["variables", "envelope"]; if allof( envelope :domain :is "from" "foobar.org", @@ -73,7 +67,8 @@ if allof( envelope :domain :is "from" "foobar.org", ''' -rcpt = ''' +[sieve.trusted.scripts."rcpt"] +contents = ''' require ["variables", "envelope", "regex"]; if allof( envelope :localpart :contains "to" ".", @@ -96,45 +91,11 @@ async fn address_rewrite() { .unwrap();*/ // Prepare config - let available_keys = &[V_SENDER, V_SENDER_DOMAIN, V_RECIPIENT, V_RECIPIENT_DOMAIN]; - let mut core = SMTP::test(); - let mut ctx = ConfigContext::new().parse_signatures(); - let settings = Config::new(CONFIG).unwrap(); - ctx.directory = settings - .parse_directory(&dummy_stores(), Store::default()) - .await - .unwrap(); - core.sieve = settings.parse_sieve(&mut ctx).unwrap(); - core.core.storage.scripts = ctx.scripts; - let config = &mut core.core.smtp.session; - config.mail.script = settings - .parse_if_block("session.mail.script", |name| { - map_expr_token::(name, available_keys) - }) - .unwrap() - .unwrap_or_default(); - config.mail.rewrite = settings - .parse_if_block("session.mail.rewrite", |name| { - map_expr_token::(name, available_keys) - }) - .unwrap() - .unwrap_or_default(); - config.rcpt.script = settings - .parse_if_block("session.rcpt.script", |name| { - map_expr_token::(name, available_keys) - }) - .unwrap() - .unwrap_or_default(); - config.rcpt.rewrite = settings - .parse_if_block("session.rcpt.rewrite", |name| { - map_expr_token::(name, available_keys) - }) - .unwrap() - .unwrap_or_default(); - config.rcpt.relay = IfBlock::new(true); + let mut config = Config::new(CONFIG).unwrap(); + let core = Core::parse(&mut config, Default::default()).await; // Init session - let mut session = Session::test(core); + let mut session = Session::test(build_smtp(core, Inner::default())); session.data.remote_ip_str = "10.0.0.1".to_string(); session.eval_session_params().await; session.ehlo("mx.doe.org").await; diff --git a/tests/src/smtp/inbound/scripts.rs b/tests/src/smtp/inbound/scripts.rs index a556c727..3b4347d3 100644 --- a/tests/src/smtp/inbound/scripts.rs +++ b/tests/src/smtp/inbound/scripts.rs @@ -22,45 +22,44 @@ */ use core::panic; -use std::{fmt::Write, fs, path::PathBuf, sync::Arc}; +use std::{fmt::Write, fs, path::PathBuf}; use crate::smtp::{ - inbound::{sign::TextConfigContext, TestMessage, TestQueueEvent}, + build_smtp, + inbound::{sign::SIGNATURES, TestMessage, TestQueueEvent}, session::{TestSession, VerifyResponse}, - TestConfig, TestSMTP, + TempDir, TestSMTP, }; -use common::{config::smtp::V_REMOTE_IP, expr::if_block::IfBlock}; -use directory::core::config::ConfigDirectory; +use common::Core; + use smtp::{ - core::{Session, SMTP}, + core::{Inner, Session}, scripts::ScriptResult, }; -use store::Store; +use store::Stores; use tokio::runtime::Handle; use utils::config::Config; const CONFIG: &str = r#" [storage] +data = "sql" lookup = "sql" +blob = "sql" +fts = "sql" [store."sql"] type = "sqlite" -path = "%PATH%/smtp_sieve.db" +path = "{TMP}/smtp_sieve.db" [store."sql".pool] max-connections = 10 min-connections = 0 idle-timeout = "5m" -[store."local/invalid-ehlos"] -type = "memory" -format = "list" -values = ["spammer.org", "spammer.net"] - [session.data.pipe."test"] command = [ { if = "remote_ip = '10.0.0.123'", then = "'/bin/bash'" }, { else = false } ] -arguments = "['%CFG_PATH%/pipe_me.sh', 'hello', 'world']" +arguments = "['{CFG_PATH}/pipe_me.sh', 'hello', 'world']" timeout = "10s" [sieve.trusted] @@ -78,7 +77,23 @@ cpu = 10000 nested-includes = 5 duplicate-expiry = "7d" -[sieve.trusted.scripts] +[session.connect] +script = "'stage_connect'" +greeting = "'mx.example.org at your service'" + +[session.ehlo] +script = "'stage_ehlo'" + +[session.mail] +script = "'stage_mail'" + +[session.rcpt] +script = "'stage_rcpt'" +relay = true + +[session.data] +script = "'stage_data'" + "#; #[tokio::test] @@ -91,7 +106,7 @@ async fn sieve_scripts() { .unwrap();*/ // Add test scripts - let mut config = CONFIG.to_string(); + let mut config = CONFIG.to_string() + SIGNATURES; for entry in fs::read_dir( PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("resources") @@ -103,7 +118,7 @@ async fn sieve_scripts() { let entry = entry.unwrap(); writeln!( &mut config, - "{} = \"file://{}\"", + "[sieve.trusted.scripts.{}]\ncontents = \"%{{file:{}}}%\"", entry .file_name() .to_str() @@ -117,14 +132,12 @@ async fn sieve_scripts() { } // Prepare config - let mut core = SMTP::test(); - let mut qr = core.init_test_queue("smtp_sieve_test"); - let mut ctx = ConfigContext::new().parse_signatures(); - let config = Config::new( - &config - .replace("%PATH%", qr._temp_dir.temp_dir.as_path().to_str().unwrap()) - .replace( - "%CFG_PATH%", + let mut inner = Inner::default(); + let tmp_dir = TempDir::new("smtp_sieve_test", true); + let mut config = Config::new( + tmp_dir.update_config( + config.replace( + "{CFG_PATH}", PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("resources") .join("smtp") @@ -133,30 +146,16 @@ async fn sieve_scripts() { .to_str() .unwrap(), ), + ), ) .unwrap(); - ctx.stores = config.parse_stores().await.unwrap(); - core.core.storage.lookups = ctx.stores.lookups.clone(); - core.core.storage.directories = config - .parse_directory(&ctx.stores, Store::default()) - .await - .unwrap() - .directories; - let pipes = config.parse_pipes(&[V_REMOTE_IP]).unwrap(); - core.sieve = config.parse_sieve(&mut ctx).unwrap(); - core.core.storage.signers = ctx.signers; - core.core.storage.scripts = ctx.scripts.clone(); - let config = &mut core.core.smtp.session; - config.connect.script = IfBlock::new("stage_connect".to_string()); - config.ehlo.script = IfBlock::new("stage_ehlo".to_string()); - config.mail.script = IfBlock::new("stage_mail".to_string()); - config.rcpt.script = IfBlock::new("stage_rcpt".to_string()); - config.data.script = IfBlock::new("stage_data".to_string()); - config.rcpt.relay = IfBlock::new(true); - config.data.pipe_commands = pipes; - let core = Arc::new(core); + config.resolve_macros().await; + let stores = Stores::parse(&mut config).await; + let core = Core::parse(&mut config, stores).await; + let mut qr = inner.init_test_queue(&core); // Build session + let core = build_smtp(core, inner); let mut session = Session::test(core.clone()); session.data.remote_ip_str = "10.0.0.88".parse().unwrap(); session.data.remote_ip = session.data.remote_ip_str.parse().unwrap(); @@ -164,7 +163,7 @@ async fn sieve_scripts() { // Run tests let span = tracing::info_span!("sieve_scripts"); - for (name, script) in &ctx.scripts { + for (name, script) in &core.core.sieve.scripts { if name.starts_with("stage_") || name.ends_with("_include") { continue; } @@ -203,7 +202,7 @@ async fn sieve_scripts() { session .cmd( "EHLO spammer.org", - "551 5.1.1 Your domain 'spammer.org' has been blacklisted", + "551 5.1.1 Your domain 'spammer.org' has been blocklisted", ) .await; session.cmd("EHLO foobar.net", "250").await; diff --git a/tests/src/smtp/inbound/sign.rs b/tests/src/smtp/inbound/sign.rs index 4c3d1941..5ce03156 100644 --- a/tests/src/smtp/inbound/sign.rs +++ b/tests/src/smtp/inbound/sign.rs @@ -23,23 +23,24 @@ use std::time::{Duration, Instant}; -use common::{config::smtp::auth::VerifyStrategy, expr::if_block::IfBlock}; -use directory::core::config::ConfigDirectory; +use common::Core; + use mail_auth::{ common::{parse::TxtRecordParser, verify::DomainKey}, spf::Spf, }; -use store::Store; +use store::Stores; use utils::config::Config; use crate::smtp::{ - inbound::{dummy_stores, TestMessage}, + build_smtp, + inbound::TestMessage, session::{TestSession, VerifyResponse}, - ParseTestConfig, TestConfig, TestSMTP, + TempDir, TestSMTP, }; -use smtp::core::{Session, SMTP}; +use smtp::core::{Inner, Session}; -const SIGNATURES: &str = " +pub const SIGNATURES: &str = " [signature.rsa] private-key = ''' -----BEGIN RSA PRIVATE KEY----- @@ -91,9 +92,16 @@ canonicalization = 'relaxed/simple' set-body-length = false "; -const DIRECTORY: &str = r#" +const CONFIG: &str = r#" [storage] -lookup = "dummy" +data = "sqlite" +lookup = "sqlite" +blob = "sqlite" +fts = "sqlite" + +[store."sqlite"] +type = "sqlite" +path = "{TMP}/queue.db" [directory."local"] type = "memory" @@ -103,27 +111,58 @@ name = "john" description = "John Doe" secret = "secret" email = ["jdoe@example.com"] + +[session.rcpt] +directory = "'local'" + +[session.data.add-headers] +received = true +received-spf = true +auth-results = true +message-id = true +date = true +return-path = false + +[auth.spf.verify] +ehlo = "relaxed" +mail-from = "relaxed" + +[auth.dkim] +verify = "relaxed" +sign = "['rsa']" + +[auth.arc] +verify = "relaxed" +seal = "'ed'" + +[auth.dmarc] +verify = "relaxed" + "#; #[tokio::test] async fn sign_and_seal() { - let mut core = SMTP::test(); + let tmp_dir = TempDir::new("smtp_sign_test", true); + let mut config = Config::new(tmp_dir.update_config(CONFIG.to_string() + SIGNATURES)).unwrap(); + let stores = Stores::parse(&mut config).await; + let core = Core::parse(&mut config, stores).await; + let mut inner = Inner::default(); // Create temp dir for queue - let mut qr = core.init_test_queue("smtp_sign_test"); + let mut qr = inner.init_test_queue(&core); // Add SPF, DKIM and DMARC records - core.core.smtp.resolvers.dns.txt_add( + core.smtp.resolvers.dns.txt_add( "mx.example.com", Spf::parse(b"v=spf1 ip4:10.0.0.1 ip4:10.0.0.2 -all").unwrap(), Instant::now() + Duration::from_secs(5), ); - core.core.smtp.resolvers.dns.txt_add( + core.smtp.resolvers.dns.txt_add( "example.com", Spf::parse(b"v=spf1 ip4:10.0.0.1 -all").unwrap(), Instant::now() + Duration::from_secs(5), ); - core.core.smtp.resolvers.dns.txt_add( + core.smtp.resolvers.dns.txt_add( "ed._domainkey.scamorza.org", DomainKey::parse( concat!( @@ -135,7 +174,7 @@ async fn sign_and_seal() { .unwrap(), Instant::now() + Duration::from_secs(5), ); - core.core.smtp.resolvers.dns.txt_add( + core.smtp.resolvers.dns.txt_add( "rsa._domainkey.manchego.org", DomainKey::parse( concat!( @@ -151,37 +190,8 @@ async fn sign_and_seal() { Instant::now() + Duration::from_secs(5), ); - core.core.storage.directories = Config::new(DIRECTORY) - .unwrap() - .parse_directory(&dummy_stores(), Store::default()) - .await - .unwrap() - .directories; - let config = &mut core.core.smtp.session.rcpt; - config.directory = IfBlock::new("local".to_string()); - - let config = &mut core.core.smtp.session; - config.data.add_auth_results = IfBlock::new(true); - config.data.add_date = IfBlock::new(true); - config.data.add_message_id = IfBlock::new(true); - config.data.add_received = IfBlock::new(true); - config.data.add_return_path = IfBlock::new(true); - config.data.add_received_spf = IfBlock::new(true); - - let config = &mut core.mail_auth; - let ctx = ConfigContext::new().parse_signatures(); - core.core.storage.signers = ctx.signers; - core.core.storage.sealers = ctx.sealers; - config.spf.verify_ehlo = IfBlock::new(VerifyStrategy::Relaxed); - config.spf.verify_mail_from = config.spf.verify_ehlo.clone(); - config.dkim.verify = config.spf.verify_ehlo.clone(); - config.arc.verify = config.spf.verify_ehlo.clone(); - config.dmarc.verify = config.spf.verify_ehlo.clone(); - config.dkim.sign = "\"['rsa']\"".parse_if(); - config.arc.seal = "\"'ed'\"".parse_if(); - // Test DKIM signing - let mut session = Session::test(core); + let mut session = Session::test(build_smtp(core, inner)); session.data.remote_ip_str = "10.0.0.2".to_string(); session.eval_session_params().await; session.ehlo("mx.example.com").await; @@ -214,17 +224,3 @@ async fn sign_and_seal() { "ARC-Message-Signature: i=3; a=ed25519-sha256; s=ed; d=example.com; c=relaxed/simple;", ); } - -pub trait TextConfigContext<'x> { - fn parse_signatures(self) -> ConfigContext; -} - -impl<'x> TextConfigContext<'x> for ConfigContext { - fn parse_signatures(mut self) -> Self { - Config::new(SIGNATURES) - .unwrap() - .parse_signatures(&mut self) - .unwrap(); - self - } -} diff --git a/tests/src/smtp/inbound/throttle.rs b/tests/src/smtp/inbound/throttle.rs index becc09a6..f6959810 100644 --- a/tests/src/smtp/inbound/throttle.rs +++ b/tests/src/smtp/inbound/throttle.rs @@ -23,8 +23,41 @@ use std::time::Duration; -use crate::smtp::{session::TestSession, ParseTestConfig, TestConfig, TestSMTP}; -use smtp::core::{Session, SessionAddress, SMTP}; +use crate::smtp::{build_smtp, session::TestSession, TempDir}; +use common::Core; +use smtp::core::{Inner, Session, SessionAddress}; +use store::Stores; +use utils::config::Config; + +const CONFIG: &str = r#" +[storage] +data = "sqlite" +lookup = "sqlite" +blob = "sqlite" +fts = "sqlite" + +[store."sqlite"] +type = "sqlite" +path = "{TMP}/data.db" + +[[session.throttle]] +match = "remote_ip = '10.0.0.1'" +key = 'remote_ip' +concurrency = 2 +rate = '3/1s' +enable = true + +[[session.throttle]] +key = 'sender' +rate = '2/1s' +enable = true + +[[session.throttle]] +key = ['remote_ip', 'rcpt'] +rate = '2/1s' +enable = true + +"#; #[tokio::test] async fn throttle_inbound() { @@ -37,29 +70,14 @@ async fn throttle_inbound() { ) .unwrap();*/ - let mut core = SMTP::test(); - let _qr = core.init_test_queue("smtp_inbound_throttle"); - let config = &mut core.core.smtp.session; - config.throttle.connect = r#"[[throttle]] - match = "remote_ip = '10.0.0.1'" - key = 'remote_ip' - concurrency = 2 - rate = '3/1s' - "# - .parse_throttle(); - config.throttle.mail_from = r#"[[throttle]] - key = 'sender' - rate = '2/1s' - "# - .parse_throttle(); - config.throttle.rcpt_to = r#"[[throttle]] - key = ['remote_ip', 'rcpt'] - rate = '2/1s' - "# - .parse_throttle(); + let tmp_dir = TempDir::new("smtp_inbound_throttle", true); + let mut config = Config::new(tmp_dir.update_config(CONFIG)).unwrap(); + let stores = Stores::parse(&mut config).await; + let core = Core::parse(&mut config, stores).await; + let inner = Inner::default(); // Test connection concurrency limit - let mut session = Session::test(core); + let mut session = Session::test(build_smtp(core, inner)); session.data.remote_ip_str = "10.0.0.1".to_string(); assert!( session.is_allowed().await, diff --git a/tests/src/smtp/inbound/vrfy.rs b/tests/src/smtp/inbound/vrfy.rs index 98bb0d92..7aeff185 100644 --- a/tests/src/smtp/inbound/vrfy.rs +++ b/tests/src/smtp/inbound/vrfy.rs @@ -21,21 +21,29 @@ * for more details. */ -use common::expr::if_block::IfBlock; -use directory::core::config::ConfigDirectory; -use store::Store; +use common::Core; + +use store::Stores; use utils::config::Config; -use crate::smtp::{ - inbound::dummy_stores, - session::{TestSession, VerifyResponse}, - ParseTestConfig, TestConfig, -}; -use smtp::core::{Session, SMTP}; +use smtp::core::{Inner, Session}; -const DIRECTORY: &str = r#" +use crate::smtp::{ + build_smtp, + session::{TestSession, VerifyResponse}, + TempDir, +}; + +const CONFIG: &str = r#" [storage] -lookup = "dummy" +data = "sqlite" +lookup = "sqlite" +blob = "sqlite" +fts = "sqlite" + +[store."sqlite"] +type = "sqlite" +path = "{TMP}/data.db" [directory."local"] type = "memory" @@ -61,31 +69,26 @@ secret = "p4ssw0rd" email = "bill@foobar.org" email-list = ["sales@foobar.org"] +[session.rcpt] +directory = "'local'" + +[session.extensions] +vrfy = [{if = "remote_ip = '10.0.0.1'", then = true}, + {else = false}] +expn = [{if = "remote_ip = '10.0.0.1'", then = true}, + {else = false}] + "#; #[tokio::test] async fn vrfy_expn() { - let mut core = SMTP::test(); - - core.core.storage.directories = Config::new(DIRECTORY) - .unwrap() - .parse_directory(&dummy_stores(), Store::default()) - .await - .unwrap() - .directories; - let config = &mut core.core.smtp.session.rcpt; - config.directory = IfBlock::new("local".to_string()); - - let config = &mut core.core.smtp.session.extensions; - config.vrfy = r#"[{if = "remote_ip = '10.0.0.1'", then = true}, - {else = false}]"# - .parse_if(); - config.expn = r#"[{if = "remote_ip = '10.0.0.1'", then = true}, - {else = false}]"# - .parse_if(); + let tmp_dir = TempDir::new("smtp_vrfy_test", true); + let mut config = Config::new(tmp_dir.update_config(CONFIG)).unwrap(); + let stores = Stores::parse(&mut config).await; + let core = Core::parse(&mut config, stores).await; // EHLO should not advertise VRFY/EXPN to 10.0.0.2 - let mut session = Session::test(core); + let mut session = Session::test(build_smtp(core, Inner::default())); session.data.remote_ip_str = "10.0.0.2".to_string(); session.eval_session_params().await; session diff --git a/tests/src/smtp/lookup/sql.rs b/tests/src/smtp/lookup/sql.rs index d88e5096..fffbb853 100644 --- a/tests/src/smtp/lookup/sql.rs +++ b/tests/src/smtp/lookup/sql.rs @@ -24,30 +24,33 @@ use std::time::{Duration, Instant}; use common::{ - config::smtp::{session::Mechanism, *}, - expr::{if_block::IfBlock, Expression}, + config::smtp::*, + expr::{tokenizer::TokenMap, Expression}, + Core, }; -use directory::core::config::ConfigDirectory; + use mail_auth::MX; -use smtp_proto::{AUTH_LOGIN, AUTH_PLAIN}; -use store::Store; +use store::Stores; use utils::config::Config; use crate::{ directory::DirectoryStore, smtp::{ + build_smtp, session::{TestSession, VerifyResponse}, - ParseTestConfig, TestConfig, + TempDir, }, - store::TempDir, }; use smtp::{ - core::{Session, SMTP}, + core::{Inner, Session}, queue::RecipientDomain, }; const CONFIG: &str = r#" [storage] +data = "sql" +blob = "sql" +fts = "sql" lookup = "sql" [store."sql"] @@ -74,7 +77,39 @@ description = "description" secret = "secret" email = "address" quota = "quota" -type = "type" +class = "type" + +[session.auth] +directory = "'sql'" +mechanisms = "[plain, login]" +errors.wait = "5ms" + +[session.rcpt] +directory = "'sql'" +relay = false +errors.wait = "5ms" + +[session.extensions] +requiretls = [{if = "key_exists('sql/is_ip_allowed', remote_ip)", then = true}, + {else = false}] +expn = true +vrfy = true + +[test."sql"] +expr = "sql_query('sql', 'SELECT description FROM domains WHERE name = ?', 'foobar.org')" +expect = "Main domain" + +[test."dns"] +expr = "dns_query(rcpt_domain, 'mx')[0]" +expect = "mx.foobar.org" + +[test."key_get"] +expr = "key_get('sql', 'hello') + '-' + key_exists('sql', 'hello') + '-' + key_set('sql', 'hello', 'world') + '-' + key_get('sql', 'hello') + '-' + key_exists('sql', 'hello')" +expect = "0-0-1-world-1" + +[test."counter_get"] +expr = "counter_get('sql', 'county') + '-' + counter_incr('sql', 'county', 1) + '-' + counter_incr('sql', 'county', 1) + '-' + counter_get('sql', 'county')" +expect = "0-1-2-2" "#; @@ -91,18 +126,13 @@ async fn lookup_sql() { // Parse settings let temp_dir = TempDir::new("smtp_lookup_tests", true); - let config_file = CONFIG.replace("{TMP}", &temp_dir.path.to_string_lossy()); - let mut core = SMTP::test(); - let mut ctx = ConfigContext::new(); - let config = Config::new(&config_file).unwrap(); - ctx.stores = config.parse_stores().await.unwrap(); - core.core.storage.lookups = ctx.stores.lookups.clone(); - core.core.storage.directories = config - .parse_directory(&ctx.stores, Store::default()) - .await - .unwrap() - .directories; - core.core.smtp.resolvers.dns.mx_add( + let mut config = Config::new(temp_dir.update_config(CONFIG)).unwrap(); + let stores = Stores::parse(&mut config).await; + + let inner = Inner::default(); + let core = Core::parse(&mut config, stores).await; + + core.smtp.resolvers.dns.mx_add( "test.org", vec![MX { exchanges: vec!["mx.foobar.org".to_string()], @@ -113,7 +143,7 @@ async fn lookup_sql() { // Obtain directory handle let handle = DirectoryStore { - store: ctx.stores.lookups.get("sql").unwrap().clone(), + store: core.storage.lookups.get("sql").unwrap().clone(), }; // Create tables @@ -160,76 +190,33 @@ async fn lookup_sql() { } // Test expression functions - for (expr, expected) in [ - ( - "sql_query('sql', 'SELECT description FROM domains WHERE name = ?', 'foobar.org')", - "Main domain", - ), - ("dns_query(rcpt_domain, 'mx')[0]", "mx.foobar.org"), - ( - concat!( - "key_get('sql', 'hello') + '-' + key_exists('sql', 'hello') + '-' + ", - "key_set('sql', 'hello', 'world') + '-' + key_get('sql', 'hello') + ", - "'-' + key_exists('sql', 'hello')" - ), - "0-0-1-world-1", - ), - ( - concat!( - "counter_get('sql', 'county') + '-' + counter_incr('sql', 'county', 1) + '-' ", - "+ counter_incr('sql', 'county', 1) + '-' + counter_get('sql', 'county')" - ), - "0-1-2-2", - ), - ] { - let e = Expression::parse("test", expr, |name| { - map_expr_token::( - name, - &[ - V_RECIPIENT, - V_RECIPIENT_DOMAIN, - V_SENDER, - V_SENDER_DOMAIN, - V_MX, - V_HELO_DOMAIN, - V_AUTHENTICATED_AS, - V_LISTENER, - V_REMOTE_IP, - V_LOCAL_IP, - V_PRIORITY, - ], - ) - }) - .unwrap(); + let token_map = TokenMap::default().with_smtp_variables(&[ + V_RECIPIENT, + V_RECIPIENT_DOMAIN, + V_SENDER, + V_SENDER_DOMAIN, + V_MX, + V_HELO_DOMAIN, + V_AUTHENTICATED_AS, + V_LISTENER, + V_REMOTE_IP, + V_LOCAL_IP, + V_PRIORITY, + ]); + for test_name in ["sql", "dns", "key_get", "counter_get"] { + let e = + Expression::try_parse(&mut config, ("test", test_name, "expr"), &token_map).unwrap(); assert_eq!( - core.core - .eval_expr::(&e, &RecipientDomain::new("test.org"), "text") + core.eval_expr::(&e, &RecipientDomain::new("test.org"), "text") .await .unwrap(), - expected, + config.value(("test", test_name, "expect")).unwrap(), "failed for '{}'", - expr + test_name ); } - // Enable AUTH - let config = &mut core.core.smtp.session.auth; - config.directory = "\"'sql'\"".parse_if(); - config.mechanisms = IfBlock::new(Mechanism::from(AUTH_PLAIN | AUTH_LOGIN)); - config.errors_wait = IfBlock::new(Duration::from_millis(5)); - - // Enable VRFY/EXPN/RCPT - let config = &mut core.core.smtp.session.rcpt; - config.directory = "\"'sql'\"".parse_if(); - config.relay = IfBlock::new(false); - config.errors_wait = IfBlock::new(Duration::from_millis(5)); - - // Enable REQUIRETLS based on SQL lookup - core.core.smtp.session.extensions.requiretls = - r#"[{if = "key_exists('sql/is_ip_allowed', remote_ip)", then = true}, - {else = false}]"# - .parse_if(); - let mut session = Session::test(core); + let mut session = Session::test(build_smtp(core, inner)); session.data.remote_ip_str = "10.0.0.50".parse().unwrap(); session.eval_session_params().await; session.stream.tls = true; diff --git a/tests/src/smtp/lookup/utils.rs b/tests/src/smtp/lookup/utils.rs index 81648dbb..ef077e23 100644 --- a/tests/src/smtp/lookup/utils.rs +++ b/tests/src/smtp/lookup/utils.rs @@ -28,15 +28,41 @@ use common::{ report::AggregateFrequency, resolver::{Mode, MxPattern, Policy}, }, - expr::if_block::IfBlock, + Core, }; -use mail_auth::{IpLookupStrategy, MX}; +use mail_auth::MX; -use ::smtp::{core::SMTP, outbound::NextHop}; +use ::smtp::outbound::NextHop; use mail_parser::DateTime; -use smtp::{outbound::lookup::ToNextHop, queue::RecipientDomain}; +use smtp::{ + core::Inner, + outbound::{lookup::ToNextHop, mta_sts::parse::ParsePolicy}, + queue::RecipientDomain, + reporting::AggregateTimestamp, +}; +use utils::config::Config; -use crate::smtp::{ParseTestConfig, TestConfig}; +use crate::smtp::build_smtp; + +const CONFIG_V4: &str = r#" +[queue.outbound.source-ip] +v4 = "['10.0.0.1', '10.0.0.2', '10.0.0.3', '10.0.0.4']" +v6 = "['a:b::1', 'a:b::2', 'a:b::3', 'a:b::4']" + +[queue.outbound] +ip-strategy = "ipv4_then_ipv6" + +"#; + +const CONFIG_V6: &str = r#" +[queue.outbound.source-ip] +v4 = "['10.0.0.1', '10.0.0.2', '10.0.0.3', '10.0.0.4']" +v6 = "['a:b::1', 'a:b::2', 'a:b::3', 'a:b::4']" + +[queue.outbound] +ip-strategy = "ipv6_then_ipv4" + +"#; #[tokio::test] async fn lookup_ip() { @@ -52,25 +78,11 @@ async fn lookup_ip() { "10.0.0.3".parse().unwrap(), "10.0.0.4".parse().unwrap(), ]; - let mut core = SMTP::test(); - core.core.smtp.queue.source_ip.ipv4 = format!( - "\"[{}]\"", - ipv4.iter() - .map(|ip| format!("'{}'", ip)) - .collect::>() - .join(",") - ) - .as_str() - .parse_if(); - core.core.smtp.queue.source_ip.ipv6 = format!( - "\"[{}]\"", - ipv6.iter() - .map(|ip| format!("'{}'", ip)) - .collect::>() - .join(",") - ) - .as_str() - .parse_if(); + let mut config = Config::new(CONFIG_V4).unwrap(); + let core = build_smtp( + Core::parse(&mut config, Default::default()).await, + Inner::default(), + ); core.core.smtp.resolvers.dns.ipv4_add( "mx.foobar.org", vec![ @@ -86,7 +98,6 @@ async fn lookup_ip() { ); // Ipv4 strategy - core.core.smtp.queue.ip_strategy = IfBlock::new(IpLookupStrategy::Ipv4thenIpv6); let resolve_result = core .resolve_host( &NextHop::MX("mx.foobar.org"), @@ -104,7 +115,24 @@ async fn lookup_ip() { .contains(&"172.168.0.100".parse().unwrap())); // Ipv6 strategy - core.core.smtp.queue.ip_strategy = IfBlock::new(IpLookupStrategy::Ipv6thenIpv4); + let mut config = Config::new(CONFIG_V6).unwrap(); + let core = build_smtp( + Core::parse(&mut config, Default::default()).await, + Inner::default(), + ); + core.core.smtp.resolvers.dns.ipv4_add( + "mx.foobar.org", + vec![ + "172.168.0.100".parse().unwrap(), + "172.168.0.101".parse().unwrap(), + ], + Instant::now() + Duration::from_secs(10), + ); + core.core.smtp.resolvers.dns.ipv6_add( + "mx.foobar.org", + vec!["e:f::a".parse().unwrap(), "e:f::b".parse().unwrap()], + Instant::now() + Duration::from_secs(10), + ); let resolve_result = core .resolve_host( &NextHop::MX("mx.foobar.org"), diff --git a/tests/src/smtp/management/queue.rs b/tests/src/smtp/management/queue.rs index 1bc8ae73..398976d2 100644 --- a/tests/src/smtp/management/queue.rs +++ b/tests/src/smtp/management/queue.rs @@ -28,17 +28,14 @@ use std::{ use ahash::{AHashMap, HashMap, HashSet}; use common::{config::server::ServerProtocol, expr::if_block::IfBlock}; -use directory::core::config::ConfigDirectory; + use mail_auth::MX; use mail_parser::DateTime; use reqwest::{header::AUTHORIZATION, Method, StatusCode}; use store::Store; use utils::config::Config; -use crate::smtp::{ - inbound::dummy_stores, management::send_manage_request, outbound::start_test_server, - session::TestSession, TestConfig, TestSMTP, -}; +use crate::smtp::{management::send_manage_request, session::TestSession, TestSMTP}; use smtp::{ core::{management::Message, Session, SMTP}, queue::{manager::SpawnQueue, QueueId, Status}, @@ -78,15 +75,17 @@ async fn manage_queue() { .unwrap();*/ // Start remote test server - let mut core = SMTP::test(); - core.core.smtp.session.rcpt.relay = IfBlock::new(true); + let mut inner = Inner::default(); + let mut core = Core::default(); + core.smtp.session.rcpt.relay = IfBlock::new(true); let mut remote_qr = core.init_test_queue("smtp_manage_queue_remote"); let remote_core = Arc::new(core); let _rx_remote = start_test_server(remote_core.clone(), &[ServerProtocol::Smtp]); // Add mock DNS entries - let mut core = SMTP::test(); - core.core.smtp.resolvers.dns.mx_add( + let mut inner = Inner::default(); + let mut core = Core::default(); + core.smtp.resolvers.dns.mx_add( "foobar.org", vec![MX { exchanges: vec!["mx1.foobar.org".to_string()], @@ -95,7 +94,7 @@ async fn manage_queue() { Instant::now() + Duration::from_secs(10), ); - core.core.smtp.resolvers.dns.ipv4_add( + core.smtp.resolvers.dns.ipv4_add( "mx1.foobar.org", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(10), @@ -107,14 +106,14 @@ async fn manage_queue() { .parse_directory(&dummy_stores(), Store::default()) .await .unwrap(); - core.core.storage.directory = directory.directories.get("local").unwrap().clone(); - core.core.smtp.session.rcpt.relay = IfBlock::new(true); - core.core.smtp.session.rcpt.max_recipients = IfBlock::new(100); - core.core.smtp.session.extensions.future_release = IfBlock::new(Duration::from_secs(86400)); - core.core.smtp.session.extensions.dsn = IfBlock::new(true); - core.core.smtp.queue.retry = IfBlock::new(Duration::from_secs(1000)); - core.core.smtp.queue.notify = IfBlock::new(Duration::from_secs(2000)); - core.core.smtp.queue.expire = IfBlock::new(Duration::from_secs(3000)); + core.storage.directory = directory.directories.get("local").unwrap().clone(); + core.smtp.session.rcpt.relay = IfBlock::new(true); + core.smtp.session.rcpt.max_recipients = IfBlock::new(100); + core.smtp.session.extensions.future_release = IfBlock::new(Duration::from_secs(86400)); + core.smtp.session.extensions.dsn = IfBlock::new(true); + core.smtp.queue.retry = IfBlock::new(Duration::from_secs(1000)); + core.smtp.queue.notify = IfBlock::new(Duration::from_secs(2000)); + core.smtp.queue.expire = IfBlock::new(Duration::from_secs(3000)); let local_qr = core.init_test_queue("smtp_manage_queue_local"); let core = Arc::new(core); local_qr.queue_rx.spawn(core.clone()); @@ -157,7 +156,7 @@ async fn manage_queue() { ("e", ("bill5@foobar.net", vec!["john@foobar.org"])), ("f", ("", vec!["success@foobar.org", "delay@foobar.org"])), ]); - let mut session = Session::test(core.clone()); + let mut session = Session::test(build_smtp(core, Inner::default())); session.data.remote_ip_str = "10.0.0.1".to_string(); session.eval_session_params().await; session.ehlo("foobar.net").await; diff --git a/tests/src/smtp/management/report.rs b/tests/src/smtp/management/report.rs index 0593fb64..8f051bc6 100644 --- a/tests/src/smtp/management/report.rs +++ b/tests/src/smtp/management/report.rs @@ -28,7 +28,7 @@ use common::{ config::{server::ServerProtocol, smtp::report::AggregateFrequency}, expr::if_block::IfBlock, }; -use directory::core::config::ConfigDirectory; + use mail_auth::{ common::parse::TxtRecordParser, dmarc::Dmarc, @@ -43,12 +43,7 @@ use store::Store; use tokio::sync::mpsc; use utils::config::Config; -use crate::smtp::{ - inbound::dummy_stores, - management::{queue::List, send_manage_request}, - outbound::start_test_server, - TestConfig, -}; +use crate::smtp::management::{queue::List, send_manage_request}; use smtp::{ core::{management::Report, SMTP}, reporting::{scheduler::SpawnReport, DmarcEvent, TlsEvent}, @@ -81,8 +76,9 @@ async fn manage_reports() { .unwrap();*/ // Start reporting service - let mut core = SMTP::test(); - let config = &mut core.core.smtp.report; + let mut inner = Inner::default(); + let mut core = Core::default(); + let config = &mut core.smtp.report; config.dmarc_aggregate.max_size = IfBlock::new(1024); config.tls.max_size = IfBlock::new(1024); let directory = Config::new(DIRECTORY) @@ -90,7 +86,7 @@ async fn manage_reports() { .parse_directory(&dummy_stores(), Store::default()) .await .unwrap(); - core.core.storage.directory = directory.directories.get("local").unwrap().clone(); + core.storage.directory = directory.directories.get("local").unwrap().clone(); let (report_tx, report_rx) = mpsc::channel(1024); core.report.tx = report_tx; let core = Arc::new(core); diff --git a/tests/src/smtp/mod.rs b/tests/src/smtp/mod.rs index 8b3b1f94..166fd366 100644 --- a/tests/src/smtp/mod.rs +++ b/tests/src/smtp/mod.rs @@ -21,434 +21,48 @@ * for more details. */ -use std::{path::PathBuf, sync::Arc, time::Duration}; +use std::{path::PathBuf, sync::Arc}; -use common::{config::smtp::*, expr::if_block::IfBlock}; -use dashmap::DashMap; -use directory::{Directory, DirectoryInner}; -use mail_auth::{ - common::lru::{DnsCache, LruCache}, - hickory_resolver::config::{ResolverConfig, ResolverOpts}, - IpLookupStrategy, Resolver, -}; -use mail_send::smtp::tls::build_tls_connector; -use sieve::Runtime; -use smtp_proto::{AUTH_LOGIN, AUTH_PLAIN}; -use store::{backend::sqlite::SqliteStore, BlobStore, LookupStore, Store}; +use common::Core; + +use smtp::core::{Inner, SMTP}; +use store::{BlobStore, Store}; use tokio::sync::mpsc; -use utils::{config::Config, snowflake::SnowflakeIdGenerator}; - pub mod config; pub mod inbound; -pub mod lookup; +/* pub mod management; -pub mod outbound; pub mod queue; -pub mod reporting; +pub mod reporting;*/ +pub mod lookup; +pub mod outbound; pub mod session; -pub trait ParseTestConfig { - fn parse_if(&self) -> IfBlock; - fn parse_if_constant(&self) -> IfBlock; - fn parse_throttle(&self) -> Vec; - fn parse_quota(&self) -> QueueQuotas; - fn parse_queue_throttle(&self) -> QueueThrottle; - fn parse_milters(&self) -> Vec; -} - -impl ParseTestConfig for &str { - fn parse_if(&self) -> IfBlock { - self.parse_if_constant::() - } - - fn parse_if_constant(&self) -> IfBlock { - Config::new(&format!("test = {self}\n")) - .unwrap_or_else(|err| panic!("Failed to parse if {}: {}", self, err)) - .parse_if_block("test", |name| { - map_expr_token::( - name, - &[ - V_RECIPIENT, - V_RECIPIENT_DOMAIN, - V_SENDER, - V_SENDER_DOMAIN, - V_MX, - V_HELO_DOMAIN, - V_AUTHENTICATED_AS, - V_LISTENER, - V_REMOTE_IP, - V_LOCAL_IP, - V_PRIORITY, - ], - ) - }) - .unwrap_or_else(|err| panic!("Failed to parse if {}: {}", self, err)) - .unwrap() - } - - fn parse_throttle(&self) -> Vec { - Config::new(self) - .unwrap() - .parse_throttle( - "throttle", - &[ - V_RECIPIENT, - V_RECIPIENT_DOMAIN, - V_SENDER, - V_SENDER_DOMAIN, - V_MX, - V_HELO_DOMAIN, - V_AUTHENTICATED_AS, - V_LISTENER, - V_REMOTE_IP, - V_LOCAL_IP, - V_PRIORITY, - ], - u16::MAX, - ) - .unwrap() - } - - fn parse_quota(&self) -> QueueQuotas { - Config::new(self).unwrap().parse_queue_quota().unwrap() - } - - fn parse_queue_throttle(&self) -> QueueThrottle { - Config::new(self).unwrap().parse_queue_throttle().unwrap() - } - - fn parse_milters(&self) -> Vec { - Config::new(self) - .unwrap() - .parse_milters(&[ - V_RECIPIENT, - V_RECIPIENT_DOMAIN, - V_SENDER, - V_SENDER_DOMAIN, - V_MX, - V_HELO_DOMAIN, - V_AUTHENTICATED_AS, - V_LISTENER, - V_REMOTE_IP, - V_LOCAL_IP, - V_PRIORITY, - ]) - .unwrap() - } -} - -pub trait TestConfig { - fn test() -> Self; -} - -impl TestConfig for SMTP { - fn test() -> Self { - let store = Store::default(); - SMTP { - worker_pool: rayon::ThreadPoolBuilder::new() - .num_threads(num_cpus::get()) - .build() - .unwrap(), - session: SessionCore::test(), - queue: QueueCore::test(), - resolvers: Resolvers { - dns: Resolver::new_system_conf().unwrap(), - dnssec: DnssecResolver::with_capacity( - ResolverConfig::cloudflare(), - ResolverOpts::default(), - ) - .unwrap(), - cache: smtp::core::DnsCache { - tlsa: LruCache::with_capacity(100), - mta_sts: LruCache::with_capacity(100), - }, - }, - mail_auth: MailAuthConfig::test(), - report: ReportCore::test(), - sieve: SieveCore::test(), - delivery_tx: mpsc::channel(1).0, - shared: Shared { - scripts: Default::default(), - signers: Default::default(), - sealers: Default::default(), - directories: Default::default(), - lookups: Default::default(), - relay_hosts: Default::default(), - directory: Arc::new(Directory { - store: DirectoryInner::Internal(store.clone()), - catch_all: AddressMapping::Disable, - subaddressing: AddressMapping::Disable, - cache: None, - blocked_ips: Arc::new(BlockedIps::new(store.clone().into())), - }), - lookup: LookupStore::Store(store.clone()), - blob: store.clone().into(), - data: store, - }, - } - } -} - -impl TestConfig for SessionCore { - fn test() -> Self { - SessionCore { - config: SessionConfig::test(), - throttle: DashMap::with_capacity_and_hasher_and_shard_amount( - 10, - ThrottleKeyHasherBuilder::default(), - 16, - ), - } - } -} - -impl TestConfig for SessionConfig { - fn test() -> Self { - Self { - timeout: IfBlock::new(Duration::from_secs(10)), - duration: IfBlock::new(Duration::from_secs(10)), - transfer_limit: IfBlock::new(1024 * 1024), - throttle: SessionThrottle { - connect: vec![], - mail_from: vec![], - rcpt_to: vec![], - }, - connect: Connect { - script: IfBlock::default(), - }, - ehlo: Ehlo { - script: IfBlock::default(), - require: IfBlock::new(true), - reject_non_fqdn: IfBlock::new(false), - }, - extensions: Extensions { - pipelining: IfBlock::new(true), - chunking: IfBlock::new(true), - requiretls: IfBlock::new(true), - no_soliciting: IfBlock::new("domain.org".to_string()), - future_release: IfBlock::default(), - deliver_by: IfBlock::default(), - mt_priority: IfBlock::default(), - dsn: IfBlock::new(true), - expn: IfBlock::new(true), - vrfy: IfBlock::new(true), - }, - auth: Auth { - directory: IfBlock::default(), - mechanisms: IfBlock::new(Mechanism::from(AUTH_PLAIN | AUTH_LOGIN)), - require: IfBlock::new(false), - errors_max: IfBlock::new(10), - errors_wait: IfBlock::new(Duration::from_secs(1)), - allow_plain_text: IfBlock::new(false), - must_match_sender: IfBlock::new(false), - }, - mail: Mail { - script: IfBlock::default(), - rewrite: IfBlock::default(), - }, - rcpt: Rcpt { - script: IfBlock::default(), - relay: IfBlock::new(false), - directory: IfBlock::default(), - errors_max: IfBlock::new(3), - errors_wait: IfBlock::new(Duration::from_secs(1)), - max_recipients: IfBlock::new(3), - rewrite: IfBlock::default(), - }, - data: Data { - script: IfBlock::default(), - max_messages: IfBlock::new(10), - max_message_size: IfBlock::new(1024 * 1024), - max_received_headers: IfBlock::new(10), - add_received: IfBlock::new(true), - add_received_spf: IfBlock::new(true), - add_return_path: IfBlock::new(true), - add_auth_results: IfBlock::new(true), - add_message_id: IfBlock::new(true), - add_date: IfBlock::new(true), - pipe_commands: vec![], - milters: vec![], - }, - } - } -} - -impl TestConfig for QueueCore { - fn test() -> Self { - Self { - config: QueueConfig::test(), - throttle: DashMap::with_capacity_and_hasher_and_shard_amount( - 10, - ThrottleKeyHasherBuilder::default(), - 16, - ), - tx: mpsc::channel(1024).0, - snowflake_id: SnowflakeIdGenerator::new(), - connectors: TlsConnectors { - pki_verify: build_tls_connector(false), - dummy_verify: build_tls_connector(true), - }, - } - } -} - -impl TestConfig for QueueConfig { - fn test() -> Self { - Self { - retry: IfBlock::new(Duration::from_secs(10)), - notify: IfBlock::new(Duration::from_secs(20)), - expire: IfBlock::new(Duration::from_secs(10)), - hostname: IfBlock::new("mx.example.org".to_string()), - next_hop: Default::default(), - max_mx: IfBlock::new(5), - max_multihomed: IfBlock::new(5), - source_ip: QueueOutboundSourceIp { - ipv4: IfBlock::default(), - ipv6: IfBlock::default(), - }, - ip_strategy: IfBlock::new(IpLookupStrategy::Ipv4thenIpv6), - tls: QueueOutboundTls { - dane: IfBlock::new(smtp::config::RequireOptional::Optional), - mta_sts: IfBlock::new(smtp::config::RequireOptional::Optional), - start: IfBlock::new(smtp::config::RequireOptional::Optional), - invalid_certs: IfBlock::new(false), - }, - dsn: Dsn { - name: IfBlock::new("Mail Delivery Subsystem".to_string()), - address: IfBlock::new("MAILER-DAEMON@example.org".to_string()), - sign: IfBlock::default(), - }, - timeout: QueueOutboundTimeout { - connect: IfBlock::new(Duration::from_secs(1)), - greeting: IfBlock::new(Duration::from_secs(1)), - tls: IfBlock::new(Duration::from_secs(1)), - ehlo: IfBlock::new(Duration::from_secs(1)), - mail: IfBlock::new(Duration::from_secs(1)), - rcpt: IfBlock::new(Duration::from_secs(1)), - data: IfBlock::new(Duration::from_secs(1)), - mta_sts: IfBlock::new(Duration::from_secs(1)), - }, - throttle: QueueThrottle { - sender: vec![], - rcpt: vec![], - host: vec![], - }, - quota: QueueQuotas { - sender: vec![], - rcpt: vec![], - rcpt_domain: vec![], - }, - } - } -} - -impl TestConfig for MailAuthConfig { - fn test() -> Self { - Self { - dkim: DkimAuthConfig { - verify: IfBlock::new(VerifyStrategy::Relaxed), - sign: IfBlock::default(), - }, - arc: ArcAuthConfig { - verify: IfBlock::new(VerifyStrategy::Relaxed), - seal: IfBlock::default(), - }, - spf: SpfAuthConfig { - verify_ehlo: IfBlock::new(VerifyStrategy::Relaxed), - verify_mail_from: IfBlock::new(VerifyStrategy::Relaxed), - }, - dmarc: DmarcAuthConfig { - verify: IfBlock::new(VerifyStrategy::Relaxed), - }, - iprev: IpRevAuthConfig { - verify: IfBlock::new(VerifyStrategy::Relaxed), - }, - } - } -} - -impl TestConfig for ReportCore { - fn test() -> Self { - Self { - config: ReportConfig::test(), - tx: mpsc::channel(1024).0, - } - } -} - -impl TestConfig for ReportConfig { - fn test() -> Self { - Self { - submitter: IfBlock::new("example.org".to_string()), - analysis: ReportAnalysis { - addresses: vec![], - forward: true, - store: None, - report_id: SnowflakeIdGenerator::new(), - }, - dkim: Report::test(), - spf: Report::test(), - dmarc: Report::test(), - dmarc_aggregate: AggregateReport::test(), - tls: AggregateReport::test(), - } - } -} - -impl TestConfig for Report { - fn test() -> Self { - Self { - name: IfBlock::default(), - address: IfBlock::default(), - subject: IfBlock::default(), - sign: IfBlock::default(), - send: IfBlock::default(), - } - } -} - -impl TestConfig for AggregateReport { - fn test() -> Self { - Self { - name: IfBlock::default(), - address: IfBlock::default(), - org_name: IfBlock::default(), - contact_info: IfBlock::default(), - send: IfBlock::default(), - sign: IfBlock::default(), - max_size: IfBlock::default(), - } - } -} - -impl TestConfig for SieveCore { - fn test() -> Self { - SieveCore { - runtime: Runtime::new_with_context(SieveContext::default()), - from_addr: "MAILER-DAEMON@example.org".to_string(), - from_name: "Mailer Daemon".to_string(), - return_path: "".to_string(), - sign: vec![], - } - } -} - pub struct TempDir { pub temp_dir: PathBuf, pub delete: bool, } -pub fn make_temp_dir(name: &str, delete: bool) -> TempDir { - let mut temp_dir = std::env::temp_dir(); - temp_dir.push(name); - if !temp_dir.exists() { - let _ = std::fs::create_dir(&temp_dir); - } else if delete { - let _ = std::fs::remove_dir_all(&temp_dir); - let _ = std::fs::create_dir(&temp_dir); +impl TempDir { + pub fn new(name: &str, delete: bool) -> TempDir { + let todo = "make sure all includes are there"; + let mut temp_dir = std::env::temp_dir(); + temp_dir.push(name); + if !temp_dir.exists() { + let _ = std::fs::create_dir(&temp_dir); + } else if delete { + let _ = std::fs::remove_dir_all(&temp_dir); + let _ = std::fs::create_dir(&temp_dir); + } + TempDir { temp_dir, delete } + } + + pub fn update_config(&self, config: impl AsRef) -> String { + config + .as_ref() + .replace("{TMP}", self.temp_dir.to_str().unwrap()) } - TempDir { temp_dir, delete } } impl Drop for TempDir { @@ -475,7 +89,6 @@ pub fn add_test_certs(config: &str) -> String { } pub struct QueueReceiver { - _temp_dir: TempDir, store: Store, blob_store: BlobStore, pub queue_rx: mpsc::Receiver, @@ -486,39 +99,32 @@ pub struct ReportReceiver { } pub trait TestSMTP { - fn init_test_queue(&mut self, test_name: &str) -> QueueReceiver; + fn init_test_queue(&mut self, core: &Core) -> QueueReceiver; fn init_test_report(&mut self) -> ReportReceiver; } -const QUEUE_STORE_CONFIG: &str = r#"[store."sqlite"] -type = "sqlite" -path = "{TMP}/queue.db" -"#; - -impl TestSMTP for SMTP { - fn init_test_queue(&mut self, test_name: &str) -> QueueReceiver { - let _temp_dir = make_temp_dir(test_name, true); - let config = - Config::new(&QUEUE_STORE_CONFIG.replace("{TMP}", _temp_dir.temp_dir.to_str().unwrap())) - .unwrap(); - let store = Store::SQLite(SqliteStore::open(&config, "store.sqlite").unwrap().into()); - self.core.storage.data = store.clone(); - self.core.storage.blob = store.clone().into(); - self.core.storage.lookup = store.clone().into(); +impl TestSMTP for Inner { + fn init_test_queue(&mut self, core: &Core) -> QueueReceiver { let (queue_tx, queue_rx) = mpsc::channel(128); - self.inner.queue_tx = queue_tx; + self.queue_tx = queue_tx; QueueReceiver { - blob_store: store.clone().into(), - store, + blob_store: core.storage.blob.clone(), + store: core.storage.data.clone(), queue_rx, - _temp_dir, } } fn init_test_report(&mut self) -> ReportReceiver { let (report_tx, report_rx) = mpsc::channel(128); - self.inner.report_tx = report_tx; + self.report_tx = report_tx; ReportReceiver { report_rx } } } + +fn build_smtp(core: impl Into>, inner: impl Into>) -> SMTP { + SMTP { + core: core.into(), + inner: inner.into(), + } +} diff --git a/tests/src/smtp/outbound/dane.rs b/tests/src/smtp/outbound/dane.rs index 4d2b7f2b..643fcd18 100644 --- a/tests/src/smtp/outbound/dane.rs +++ b/tests/src/smtp/outbound/dane.rs @@ -34,13 +34,9 @@ use std::{ use common::{ config::{ server::ServerProtocol, - smtp::{ - queue::RequireOptional, - report::AggregateFrequency, - resolver::{DnsRecordCache, DnssecResolver, Resolvers, Tlsa, TlsaEntry}, - }, + smtp::resolver::{DnsRecordCache, DnssecResolver, Resolvers, Tlsa, TlsaEntry}, }, - expr::if_block::IfBlock, + Core, }; use mail_auth::{ common::{ @@ -60,16 +56,35 @@ use utils::suffixlist::PublicSuffix; use crate::smtp::{ inbound::{TestMessage, TestQueueEvent, TestReportingEvent}, - outbound::start_test_server, + outbound::TestServer, session::{TestSession, VerifyResponse}, - TestConfig, TestSMTP, }; +use smtp::outbound::dane::verify::TlsaVerify; use smtp::{ - core::{Session, SMTP}, + core::SMTP, queue::{Error, ErrorDetails, Status}, reporting::PolicyType, }; +const LOCAL: &str = r#" +[session.rcpt] +relay = true + +[report.tls.aggregate] +send = "weekly" + +[queue.outbound.tls] +dane = "require" +"#; + +const REMOTE: &str = " +[session.ehlo] +reject-non-fqdn = false + +[session.rcpt] +relay = true +"; + #[tokio::test] #[serial_test::serial] async fn dane_verify() { @@ -81,13 +96,14 @@ async fn dane_verify() { .unwrap();*/ // Start test server - let mut core = SMTP::test(); - core.core.smtp.session.rcpt.relay = IfBlock::new(true); - let mut remote_qr = core.init_test_queue("smtp_dane_remote"); - let _rx = start_test_server(core.into(), &[ServerProtocol::Smtp]); + let mut remote = TestServer::new("smtp_dane_remote", REMOTE, true).await; + let _rx = remote.start(&[ServerProtocol::Smtp]).await; + + // Fail on missing TLSA record + let mut local = TestServer::new("smtp_dane_local", LOCAL, true).await; // Add mock DNS entries - let mut core = SMTP::test(); + let core = local.build_smtp(); core.core.smtp.resolvers.dns.mx_add( "foobar.org", vec![MX { @@ -107,38 +123,32 @@ async fn dane_verify() { Instant::now() + Duration::from_secs(10), ); - // Fail on missing TLSA record - let mut local_qr = core.init_test_queue("smtp_dane_local"); - let mut rr = core.init_test_report(); - core.core.smtp.session.rcpt.relay = IfBlock::new(true); - core.core.smtp.queue.tls.dane = IfBlock::new(RequireOptional::Require); - core.core.smtp.report.tls.send = IfBlock::new(AggregateFrequency::Weekly); - - let core = Arc::new(core); - let mut session = Session::test(core.clone()); + let mut session = local.new_session(); session.data.remote_ip_str = "10.0.0.1".to_string(); session.eval_session_params().await; session.ehlo("mx.test.org").await; session .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; - local_qr + local + .qr .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; - local_qr + local + .qr .expect_message() .await - .read_lines(&local_qr) + .read_lines(&local.qr) .await .assert_contains(" (DANE failed to authenticate") .assert_contains("No TLSA records found"); - local_qr.read_event().await.assert_reload(); - local_qr.assert_no_events(); + local.qr.read_event().await.assert_reload(); + local.qr.assert_no_events(); // Expect TLS failure report - let report = rr.read_report().await.unwrap_tls(); + let report = local.rr.read_report().await.unwrap_tls(); assert_eq!(report.domain, "foobar.org"); assert_eq!(report.policy, PolicyType::Tlsa(None)); assert_eq!( @@ -165,7 +175,7 @@ async fn dane_verify() { has_end_entities: true, has_intermediates: false, }); - core.core.smtp.resolvers.tlsa_add( + core.tlsa_add( "_25._tcp.mx.foobar.org", tlsa.clone(), Instant::now() + Duration::from_secs(10), @@ -173,29 +183,31 @@ async fn dane_verify() { session .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; - local_qr + local + .qr .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; - local_qr + local + .qr .expect_message() .await - .read_lines(&local_qr) + .read_lines(&local.qr) .await .assert_contains(" (DANE failed to authenticate") .assert_contains("No matching certificates found"); - local_qr.read_event().await.assert_reload(); - local_qr.assert_no_events(); + local.qr.read_event().await.assert_reload(); + local.qr.assert_no_events(); // Expect TLS failure report - let report = rr.read_report().await.unwrap_tls(); + let report = local.rr.read_report().await.unwrap_tls(); assert_eq!(report.policy, PolicyType::Tlsa(tlsa.into())); assert_eq!( report.failure.as_ref().unwrap().result_type, ResultType::ValidationFailure ); - remote_qr.assert_no_events(); + remote.qr.assert_no_events(); // DANE successful delivery let tlsa = Arc::new(Tlsa { @@ -211,7 +223,7 @@ async fn dane_verify() { has_end_entities: true, has_intermediates: false, }); - core.core.smtp.resolvers.tlsa_add( + core.tlsa_add( "_25._tcp.mx.foobar.org", tlsa.clone(), Instant::now() + Duration::from_secs(10), @@ -219,22 +231,24 @@ async fn dane_verify() { session .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; - local_qr + local + .qr .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; - local_qr.read_event().await.assert_reload(); - local_qr.assert_no_events(); - remote_qr + local.qr.read_event().await.assert_reload(); + local.qr.assert_no_events(); + remote + .qr .expect_message() .await - .read_lines(&remote_qr) + .read_lines(&remote.qr) .await .assert_contains("using TLSv1.3 with cipher"); // Expect TLS success report - let report = rr.read_report().await.unwrap_tls(); + let report = local.rr.read_report().await.unwrap_tls(); assert_eq!(report.policy, PolicyType::Tlsa(tlsa.into())); assert!(report.failure.is_none()); } @@ -246,7 +260,8 @@ async fn dane_test() { opts.validate = true; opts.try_tcp_on_error = true; - let r = Resolvers { + let mut core = Core::default(); + core.smtp.resolvers = Resolvers { dns: Resolver::new_cloudflare().unwrap(), dnssec: DnssecResolver { resolver: AsyncResolver::tokio(conf, opts), @@ -257,6 +272,10 @@ async fn dane_test() { }, psl: PublicSuffix::default(), }; + let r = SMTP { + core: core.into(), + inner: Default::default(), + }; // Add dns entries let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); diff --git a/tests/src/smtp/outbound/extensions.rs b/tests/src/smtp/outbound/extensions.rs index e08fcb6f..92c5b9f4 100644 --- a/tests/src/smtp/outbound/extensions.rs +++ b/tests/src/smtp/outbound/extensions.rs @@ -21,22 +21,40 @@ * for more details. */ -use std::{ - sync::Arc, - time::{Duration, Instant}, -}; +use std::time::{Duration, Instant}; -use common::{config::server::ServerProtocol, expr::if_block::IfBlock}; +use common::config::server::ServerProtocol; use mail_auth::MX; use smtp_proto::{MAIL_REQUIRETLS, MAIL_RET_HDRS, MAIL_SMTPUTF8, RCPT_NOTIFY_NEVER}; use crate::smtp::{ inbound::{TestMessage, TestQueueEvent}, - outbound::start_test_server, + outbound::TestServer, session::{TestSession, VerifyResponse}, - TestConfig, TestSMTP, }; -use smtp::core::{Session, SMTP}; + +const LOCAL: &str = r#" +[session.rcpt] +relay = true + +[session.extensions] +dsn = true +"#; + +const REMOTE: &str = r#" +[session.ehlo] +reject-non-fqdn = false + +[session.rcpt] +relay = true + +[session.data.limits] +size = 1500 + +[session.extensions] +dsn = true +requiretls = true +"#; #[tokio::test] #[serial_test::serial] @@ -49,16 +67,14 @@ async fn extensions() { .unwrap();*/ // Start test server - let mut core = SMTP::test(); - core.core.smtp.session.rcpt.relay = IfBlock::new(true); - core.core.smtp.session.data.max_message_size = IfBlock::new(1500); - core.core.smtp.session.extensions.dsn = IfBlock::new(true); - core.core.smtp.session.extensions.requiretls = IfBlock::new(true); - let mut remote_qr = core.init_test_queue("smtp_ext_remote"); - let _rx = start_test_server(core.into(), &[ServerProtocol::Smtp]); + let mut remote = TestServer::new("smtp_ext_remote", REMOTE, true).await; + let _rx = remote.start(&[ServerProtocol::Smtp]).await; + + // Successful delivery with DSN + let mut local = TestServer::new("smtp_ext_local", LOCAL, true).await; // Add mock DNS entries - let mut core = SMTP::test(); + let core = local.build_smtp(); core.core.smtp.resolvers.dns.mx_add( "foobar.org", vec![MX { @@ -73,13 +89,7 @@ async fn extensions() { Instant::now() + Duration::from_secs(10), ); - // Successful delivery with DSN - let mut local_qr = core.init_test_queue("smtp_ext_local"); - core.core.smtp.session.rcpt.relay = IfBlock::new(true); - core.core.smtp.session.extensions.dsn = IfBlock::new(true); - let core = Arc::new(core); - //let mut queue = Queue::default(); - let mut session = Session::test(core.clone()); + let mut session = local.new_session(); session.data.remote_ip_str = "10.0.0.1".to_string(); session.eval_session_params().await; session.ehlo("mx.test.org").await; @@ -91,25 +101,28 @@ async fn extensions() { "250", ) .await; - local_qr + local + .qr .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; - local_qr + local + .qr .expect_message() .await - .read_lines(&local_qr) + .read_lines(&local.qr) .await .assert_contains(" (delivered to") .assert_contains("Final-Recipient: rfc822;bill@foobar.org") .assert_contains("Action: delivered"); - local_qr.read_event().await.assert_reload(); - remote_qr + local.qr.read_event().await.assert_reload(); + remote + .qr .expect_message() .await - .read_lines(&remote_qr) + .read_lines(&remote.qr) .await .assert_contains("using TLSv1.3 with cipher"); @@ -117,22 +130,24 @@ async fn extensions() { session .send_message("john@test.org", &["bill@foobar.org"], "test:arc", "250") .await; - local_qr + local + .qr .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; - local_qr + local + .qr .expect_message() .await - .read_lines(&local_qr) + .read_lines(&local.qr) .await .assert_contains(" (host 'mx.foobar.org' rejected command 'MAIL FROM:") .assert_contains("Action: failed") .assert_contains("Diagnostic-Code: smtp;552") .assert_contains("Status: 5.3.4"); - local_qr.read_event().await.assert_reload(); - remote_qr.assert_no_events(); + local.qr.read_event().await.assert_reload(); + remote.qr.assert_no_events(); // Test DSN, SMTPUTF8 and REQUIRETLS extensions session @@ -143,13 +158,14 @@ async fn extensions() { "250", ) .await; - local_qr + local + .qr .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; - local_qr.read_event().await.assert_reload(); - let message = remote_qr.expect_message().await; + local.qr.read_event().await.assert_reload(); + let message = remote.qr.expect_message().await; assert_eq!(message.env_id, Some("abc123".to_string())); assert!((message.flags & MAIL_RET_HDRS) != 0); assert!((message.flags & MAIL_REQUIRETLS) != 0); diff --git a/tests/src/smtp/outbound/ip_lookup.rs b/tests/src/smtp/outbound/ip_lookup.rs index 504d1b5c..c38596e8 100644 --- a/tests/src/smtp/outbound/ip_lookup.rs +++ b/tests/src/smtp/outbound/ip_lookup.rs @@ -21,16 +21,28 @@ * for more details. */ -use std::{ - sync::Arc, - time::{Duration, Instant}, -}; +use std::time::{Duration, Instant}; -use common::{config::server::ServerProtocol, expr::if_block::IfBlock}; +use common::config::server::ServerProtocol; use mail_auth::{IpLookupStrategy, MX}; -use crate::smtp::{outbound::start_test_server, session::TestSession, TestConfig, TestSMTP}; -use smtp::core::{Session, SMTP}; +use crate::smtp::{outbound::TestServer, session::TestSession}; + +const LOCAL: &str = r#" +[session.rcpt] +relay = true + +[queue.outbound] +ip-strategy = "ipv6_then_ipv4" +"#; + +const REMOTE: &str = r#" +[session.ehlo] +reject-non-fqdn = false + +[session.rcpt] +relay = true +"#; #[tokio::test] #[serial_test::serial] @@ -43,16 +55,14 @@ async fn ip_lookup_strategy() { .unwrap();*/ // Start test server - let mut core = SMTP::test(); - core.core.smtp.session.rcpt.relay = IfBlock::new(true); - let mut remote_qr = core.init_test_queue("smtp_iplookup_remote"); - let _rx = start_test_server(core.into(), &[ServerProtocol::Smtp]); + let mut remote = TestServer::new("smtp_iplookup_remote", REMOTE, true).await; + let _rx = remote.start(&[ServerProtocol::Smtp]).await; for strategy in [IpLookupStrategy::Ipv6Only, IpLookupStrategy::Ipv6thenIpv4] { //println!("-> Strategy: {:?}", strategy); // Add mock DNS entries - let mut core = SMTP::test(); - core.core.smtp.queue.ip_strategy = IfBlock::new(IpLookupStrategy::Ipv6thenIpv4); + let mut local = TestServer::new("smtp_iplookup_local", LOCAL, true).await; + let core = local.build_smtp(); core.core.smtp.resolvers.dns.mx_add( "foobar.org", vec![MX { @@ -75,27 +85,24 @@ async fn ip_lookup_strategy() { ); // Retry on failed STARTTLS - let mut local_qr = core.init_test_queue("smtp_iplookup_local"); - core.core.smtp.session.rcpt.relay = IfBlock::new(true); - - let core = Arc::new(core); - let mut session = Session::test(core.clone()); + let mut session = local.new_session(); session.data.remote_ip_str = "10.0.0.1".to_string(); session.eval_session_params().await; session.ehlo("mx.test.org").await; session .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; - local_qr + local + .qr .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; tokio::time::sleep(Duration::from_millis(100)).await; if matches!(strategy, IpLookupStrategy::Ipv6thenIpv4) { - remote_qr.expect_message().await; + remote.qr.expect_message().await; } else { - let message = local_qr.last_queued_message().await; + let message = local.qr.last_queued_message().await; let status = message.domains[0].status.to_string(); assert!( status.contains("Connection refused"), diff --git a/tests/src/smtp/outbound/lmtp.rs b/tests/src/smtp/outbound/lmtp.rs index 715c7d7c..779740ee 100644 --- a/tests/src/smtp/outbound/lmtp.rs +++ b/tests/src/smtp/outbound/lmtp.rs @@ -21,26 +21,50 @@ * for more details. */ -use std::{ - sync::Arc, - time::{Duration, Instant}, -}; +use std::time::{Duration, Instant}; use crate::smtp::{ inbound::TestMessage, - outbound::start_test_server, + outbound::TestServer, session::{TestSession, VerifyResponse}, - ParseTestConfig, TestConfig, TestSMTP, -}; -use common::{config::server::ServerProtocol, expr::if_block::IfBlock}; -use smtp::{ - core::{Session, SMTP}, - queue::{DeliveryAttempt, Event}, }; +use common::config::server::ServerProtocol; +use smtp::queue::{DeliveryAttempt, Event}; use store::write::now; -use utils::config::Config; const REMOTE: &str = " +[session.ehlo] +reject-non-fqdn = false + +[session.rcpt] +relay = true + +[session.extensions] +dsn = true +"; + +const LOCAL: &str = r#" +[queue.outbound] +next-hop = [{if = "rcpt_domain = 'foobar.org'", then = "'lmtp'"}, + {else = false}] + +[session.rcpt] +relay = true +max-recipients = 100 + +[session.extensions] +dsn = true + +[queue.schedule] +retry = "1s" +notify = [{if = "rcpt_domain = 'foobar.org'", then = "[1s, 2s]"}, + {else = [1s]}] +expire = [{if = "rcpt_domain = 'foobar.org'", then = "4s"}, + {else = "5s"}] + +[queue.outbound.timeouts] +data = "50ms" + [remote.lmtp] address = lmtp.foobar.org port = 9924 @@ -50,7 +74,7 @@ concurrency = 5 [remote.lmtp.tls] implicit = true allow-invalid-certs = true -"; +"#; #[tokio::test] #[serial_test::serial] @@ -63,44 +87,21 @@ async fn lmtp_delivery() { .unwrap();*/ // Start test server - let mut core = SMTP::test(); - core.core.smtp.session.rcpt.relay = IfBlock::new(true); - core.core.smtp.session.extensions.dsn = IfBlock::new(true); - let mut remote_qr = core.init_test_queue("lmtp_delivery_remote"); - let _rx = start_test_server(core.into(), &[ServerProtocol::Lmtp]); + let mut remote = TestServer::new("lmtp_delivery_remote", REMOTE, true).await; + let _rx = remote.start(&[ServerProtocol::Lmtp]).await; + + // Multiple delivery attempts + let mut local = TestServer::new("lmtp_delivery_local", LOCAL, true).await; // Add mock DNS entries - let mut core = SMTP::test(); + let core = local.build_smtp(); core.core.smtp.resolvers.dns.ipv4_add( "lmtp.foobar.org", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(10), ); - // Multiple delivery attempts - let mut local_qr = core.init_test_queue("lmtp_delivery_local"); - core.core.storage.relay_hosts.insert( - "lmtp".to_string(), - Config::new(REMOTE).unwrap().parse_host("lmtp").unwrap(), - ); - core.core.smtp.queue.next_hop = r#"[{if = "rcpt_domain = 'foobar.org'", then = "'lmtp'"}, - {else = false}]"# - .parse_if(); - core.core.smtp.session.rcpt.relay = IfBlock::new(true); - core.core.smtp.session.rcpt.max_recipients = IfBlock::new(100); - core.core.smtp.session.extensions.dsn = IfBlock::new(true); - let config = &mut core.core.smtp.queue; - config.retry = IfBlock::new(Duration::from_secs(1)); - config.notify = r#"[{if = "rcpt_domain = 'foobar.org'", then = "[1s, 2s]"}, - {else = [1s]}]"# - .parse_if(); - config.expire = r#"[{if = "rcpt_domain = 'foobar.org'", then = "4s"}, - {else = "5s"}]"# - .parse_if(); - config.timeout.data = IfBlock::new(Duration::from_millis(50)); - - let core = Arc::new(core); - let mut session = Session::test(core.clone()); + let mut session = local.new_session(); session.data.remote_ip_str = "10.0.0.1".to_string(); session.eval_session_params().await; session.ehlo("mx.test.org").await; @@ -119,14 +120,15 @@ async fn lmtp_delivery() { "250", ) .await; - local_qr + local + .qr .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; let mut dsn = Vec::new(); loop { - match local_qr.try_read_event().await { + match local.qr.try_read_event().await { Some(Event::Reload) => {} Some(Event::OnHold(_)) => unreachable!(), None | Some(Event::Stop) => break, @@ -152,14 +154,14 @@ async fn lmtp_delivery() { } } } - local_qr.assert_queue_is_empty().await; + local.qr.assert_queue_is_empty().await; assert_eq!(dsn.len(), 4); let mut dsn = dsn.into_iter(); dsn.next() .unwrap() - .read_lines(&local_qr) + .read_lines(&local.qr) .await .assert_contains(" (delivered to") .assert_contains(" (delivered to") @@ -169,27 +171,28 @@ async fn lmtp_delivery() { dsn.next() .unwrap() - .read_lines(&local_qr) + .read_lines(&local.qr) .await .assert_contains(" (host 'lmtp.foobar.org' rejected") .assert_contains("Action: delayed"); dsn.next() .unwrap() - .read_lines(&local_qr) + .read_lines(&local.qr) .await .assert_contains(" (host 'lmtp.foobar.org' rejected") .assert_contains("Action: delayed"); dsn.next() .unwrap() - .read_lines(&local_qr) + .read_lines(&local.qr) .await .assert_contains(" (host 'lmtp.foobar.org' rejected") .assert_contains("Action: failed"); assert_eq!( - remote_qr + remote + .qr .expect_message() .await .recipients @@ -202,5 +205,5 @@ async fn lmtp_delivery() { "john@foobar.org".to_string() ] ); - remote_qr.assert_no_events(); + remote.qr.assert_no_events(); } diff --git a/tests/src/smtp/outbound/mod.rs b/tests/src/smtp/outbound/mod.rs index 5e771a50..6b4e4303 100644 --- a/tests/src/smtp/outbound/mod.rs +++ b/tests/src/smtp/outbound/mod.rs @@ -21,15 +21,25 @@ * for more details. */ -use std::sync::Arc; +use common::{ + config::server::{ServerProtocol, Servers}, + Core, +}; +use store::{BlobStore, Store, Stores}; +use tokio::sync::{mpsc, watch}; -use common::config::server::ServerProtocol; -use tokio::sync::watch; - -use ::smtp::core::{SmtpAdminSessionManager, SmtpSessionManager, SMTP}; +use ::smtp::core::{ + Inner, Session, SmtpAdminSessionManager, SmtpInstance, SmtpSessionManager, SMTP, +}; use utils::config::Config; -use super::add_test_certs; +use crate::AssertConfig; + +use super::{ + add_test_certs, + session::{DummyIo, TestSession}, + QueueReceiver, ReportReceiver, TempDir, TestSMTP, +}; pub mod dane; pub mod extensions; @@ -68,35 +78,105 @@ implicit = false certificate = 'default' [certificate.default] -cert = 'file://{CERT}' -private-key = 'file://{PK}' +cert = '%{file:{CERT}}%' +private-key = '%{file:{PK}}%' "; -pub fn start_test_server(core: Arc, protocols: &[ServerProtocol]) -> watch::Sender { - // Spawn listeners - let config = Config::new(&add_test_certs(SERVER)).unwrap(); - let mut servers = config.parse_servers().unwrap(); +const STORES: &str = r#" +[storage] +data = "sqlite" +lookup = "sqlite" +blob = "sqlite" +fts = "sqlite" - // Filter out protocols - servers - .inner - .retain(|server| protocols.contains(&server.protocol)); +[store."sqlite"] +type = "sqlite" +path = "{TMP}/queue.db" - // Start servers - servers.bind(&config); - let smtp_manager = SmtpSessionManager::new(core.clone()); - let smtp_admin_manager = SmtpAdminSessionManager::new(core); - servers - .spawn(|server, shutdown_rx| { - match &server.protocol { - ServerProtocol::Smtp | ServerProtocol::Lmtp => { - server.spawn(smtp_manager.clone(), shutdown_rx) - } - ServerProtocol::Http => server.spawn(smtp_admin_manager.clone(), shutdown_rx), - ServerProtocol::Imap | ServerProtocol::ManageSieve => { - unreachable!() - } - }; - }) - .0 +"#; + +pub struct TestServer { + pub instance: SmtpInstance, + pub temp_dir: TempDir, + pub qr: QueueReceiver, + pub rr: ReportReceiver, +} + +impl TestServer { + pub async fn new(name: &str, config: impl AsRef, with_receiver: bool) -> TestServer { + let temp_dir = TempDir::new(name, true); + let mut config = + Config::new(temp_dir.update_config(STORES.to_string() + config.as_ref())).unwrap(); + let stores = Stores::parse(&mut config).await; + let core = Core::parse(&mut config, stores).await; + let mut inner = Inner::default(); + let qr = if with_receiver { + inner.init_test_queue(&core) + } else { + QueueReceiver { + store: Store::default(), + blob_store: BlobStore::default(), + queue_rx: mpsc::channel(1).1, + } + }; + let rr = if with_receiver { + inner.init_test_report() + } else { + ReportReceiver { + report_rx: mpsc::channel(1).1, + } + }; + + TestServer { + instance: SmtpInstance::new(core.into_shared(), inner), + temp_dir, + qr, + rr, + } + } + + pub async fn start(&self, protocols: &[ServerProtocol]) -> watch::Sender { + // Spawn listeners + let mut config = Config::new(add_test_certs(SERVER)).unwrap(); + config.resolve_macros().await; + let mut servers = Servers::parse(&mut config); + + // Filter out protocols + servers + .servers + .retain(|server| protocols.contains(&server.protocol)); + + // Start servers + servers.bind_and_drop_priv(&mut config); + config.assert_no_errors(); + let instance = self.instance.clone(); + let smtp_manager = SmtpSessionManager::new(instance.clone()); + let smtp_admin_manager = SmtpAdminSessionManager::new(instance.clone()); + servers.spawn( + |server, shutdown_rx| { + match &server.protocol { + ServerProtocol::Smtp | ServerProtocol::Lmtp => { + server.spawn(smtp_manager.clone(), instance.core.clone(), shutdown_rx) + } + ServerProtocol::Http => server.spawn( + smtp_admin_manager.clone(), + instance.core.clone(), + shutdown_rx, + ), + ServerProtocol::Imap | ServerProtocol::ManageSieve => { + unreachable!() + } + }; + }, + instance.core.load().storage.data.clone(), + ) + } + + pub fn new_session(&self) -> Session { + Session::test(self.build_smtp()) + } + + pub fn build_smtp(&self) -> SMTP { + SMTP::from(self.instance.clone()) + } } diff --git a/tests/src/smtp/outbound/mta_sts.rs b/tests/src/smtp/outbound/mta_sts.rs index 259f6821..ac8f4c53 100644 --- a/tests/src/smtp/outbound/mta_sts.rs +++ b/tests/src/smtp/outbound/mta_sts.rs @@ -26,13 +26,7 @@ use std::{ time::{Duration, Instant}, }; -use common::{ - config::{ - server::ServerProtocol, - smtp::{queue::RequireOptional, report::AggregateFrequency, resolver::Policy}, - }, - expr::if_block::IfBlock, -}; +use common::config::{server::ServerProtocol, smtp::resolver::Policy}; use mail_auth::{ common::parse::TxtRecordParser, mta_sts::{MtaSts, ReportUri, TlsRpt}, @@ -42,16 +36,34 @@ use mail_auth::{ use crate::smtp::{ inbound::{TestMessage, TestQueueEvent, TestReportingEvent}, - outbound::start_test_server, + outbound::TestServer, session::{TestSession, VerifyResponse}, - TestConfig, TestSMTP, }; use smtp::{ - core::{Session, SMTP}, - outbound::mta_sts::lookup::STS_TEST_POLICY, + outbound::mta_sts::{lookup::STS_TEST_POLICY, parse::ParsePolicy}, reporting::PolicyType, }; +const LOCAL: &str = r#" +[session.rcpt] +relay = true + +[queue.outbound.tls] +mta-sts = "require" + +[report.tls.aggregate] +send = "weekly" + +"#; + +const REMOTE: &str = r#" +[session.ehlo] +reject-non-fqdn = false + +[session.rcpt] +relay = true +"#; + #[tokio::test] #[serial_test::serial] async fn mta_sts_verify() { @@ -63,13 +75,14 @@ async fn mta_sts_verify() { .unwrap();*/ // Start test server - let mut core = SMTP::test(); - core.core.smtp.session.rcpt.relay = IfBlock::new(true); - let mut remote_qr = core.init_test_queue("smtp_mta_sts_remote"); - let _rx = start_test_server(core.into(), &[ServerProtocol::Smtp]); + let mut remote = TestServer::new("smtp_mta_sts_remote", REMOTE, true).await; + let _rx = remote.start(&[ServerProtocol::Smtp]).await; + + // Fail on missing MTA-STS record + let mut local = TestServer::new("smtp_mta_sts_local", LOCAL, true).await; // Add mock DNS entries - let mut core = SMTP::test(); + let core = local.build_smtp(); core.core.smtp.resolvers.dns.mx_add( "foobar.org", vec![MX { @@ -89,38 +102,31 @@ async fn mta_sts_verify() { Instant::now() + Duration::from_secs(10), ); - // Fail on missing MTA-STS record - let mut local_qr = core.init_test_queue("smtp_mta_sts_local"); - let mut rr = core.init_test_report(); - core.core.smtp.session.rcpt.relay = IfBlock::new(true); - core.core.smtp.queue.tls.mta_sts = IfBlock::new(RequireOptional::Require); - core.core.smtp.report.tls.send = IfBlock::new(AggregateFrequency::Weekly); - - let core = Arc::new(core); - //let mut queue = Queue::default(); - let mut session = Session::test(core.clone()); + let mut session = local.new_session(); session.data.remote_ip_str = "10.0.0.1".to_string(); session.eval_session_params().await; session.ehlo("mx.test.org").await; session .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; - local_qr + local + .qr .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; - local_qr + local + .qr .expect_message() .await - .read_lines(&local_qr) + .read_lines(&local.qr) .await .assert_contains(" (MTA-STS failed to authenticate") .assert_contains("Record not found"); - local_qr.read_event().await.assert_reload(); + local.qr.read_event().await.assert_reload(); // Expect TLS failure report - let report = rr.read_report().await.unwrap_tls(); + let report = local.rr.read_report().await.unwrap_tls(); assert_eq!(report.domain, "foobar.org"); assert_eq!(report.policy, PolicyType::Sts(None)); assert_eq!( @@ -141,22 +147,24 @@ async fn mta_sts_verify() { session .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; - local_qr + local + .qr .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; - local_qr + local + .qr .expect_message() .await - .read_lines(&local_qr) + .read_lines(&local.qr) .await .assert_contains(" (MTA-STS failed to authenticate") .assert_contains("No 'mx' entries found"); - local_qr.read_event().await.assert_reload(); + local.qr.read_event().await.assert_reload(); // Expect TLS failure report - let report = rr.read_report().await.unwrap_tls(); + let report = local.rr.read_report().await.unwrap_tls(); assert_eq!(report.policy, PolicyType::Sts(None)); assert_eq!( report.failure.as_ref().unwrap().result_type, @@ -174,22 +182,24 @@ async fn mta_sts_verify() { session .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; - local_qr + local + .qr .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; - local_qr + local + .qr .expect_message() .await - .read_lines(&local_qr) + .read_lines(&local.qr) .await .assert_contains(" (MTA-STS failed to authenticate") .assert_contains("not authorized by policy"); - local_qr.read_event().await.assert_reload(); + local.qr.read_event().await.assert_reload(); // Expect TLS failure report - let report = rr.read_report().await.unwrap_tls(); + let report = local.rr.read_report().await.unwrap_tls(); assert_eq!( report.policy, PolicyType::Sts( @@ -204,7 +214,7 @@ async fn mta_sts_verify() { report.failure.as_ref().unwrap().result_type, ResultType::ValidationFailure ); - remote_qr.assert_no_events(); + remote.qr.assert_no_events(); // MTA-STS successful validation core.core.smtp.resolvers.dns.txt_add( @@ -223,21 +233,23 @@ async fn mta_sts_verify() { session .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; - local_qr + local + .qr .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; - local_qr.read_event().await.assert_reload(); - remote_qr + local.qr.read_event().await.assert_reload(); + remote + .qr .expect_message() .await - .read_lines(&remote_qr) + .read_lines(&remote.qr) .await .assert_contains("using TLSv1.3 with cipher"); // Expect TLS success report - let report = rr.read_report().await.unwrap_tls(); + let report = local.rr.read_report().await.unwrap_tls(); assert_eq!( report.policy, PolicyType::Sts( diff --git a/tests/src/smtp/outbound/smtp.rs b/tests/src/smtp/outbound/smtp.rs index 66d1f971..0cb39e83 100644 --- a/tests/src/smtp/outbound/smtp.rs +++ b/tests/src/smtp/outbound/smtp.rs @@ -21,25 +21,47 @@ * for more details. */ -use std::{ - sync::Arc, - time::{Duration, Instant}, -}; +use std::time::{Duration, Instant}; -use common::{config::server::ServerProtocol, expr::if_block::IfBlock}; +use common::config::server::ServerProtocol; use mail_auth::MX; use store::write::now; use crate::smtp::{ inbound::{TestMessage, TestQueueEvent}, - outbound::start_test_server, + outbound::TestServer, session::{TestSession, VerifyResponse}, - ParseTestConfig, TestConfig, TestSMTP, -}; -use smtp::{ - core::{Session, SMTP}, - queue::{DeliveryAttempt, Event}, }; +use smtp::queue::{DeliveryAttempt, Event}; + +const LOCAL: &str = r#" +[session.rcpt] +relay = true +max-recipients = 100 + +[session.extensions] +dsn = true + +[queue.schedule] +retry = "1s" +notify = [{if = "rcpt_domain = 'foobar.org'", then = "[1s, 2s]"}, + {if = "rcpt_domain = 'foobar.com'", then = "[5s, 6s]"}, + {else = [1s]}] +expire = [{if = "rcpt_domain = 'foobar.org'", then = "6s"}, + {else = "7s"}] +"#; + +const REMOTE: &str = r#" +[session.ehlo] +reject-non-fqdn = false + +[session.rcpt] +relay = true + +[session.extensions] +dsn = true +chunking = false +"#; const SMUGGLER: &str = r#"From: Joe SixPack To: Suzie Q @@ -73,16 +95,15 @@ async fn smtp_delivery() { .unwrap();*/ // Start test server - let mut core = SMTP::test(); - core.core.smtp.session.rcpt.relay = IfBlock::new(true); - core.core.smtp.session.extensions.dsn = IfBlock::new(true); - core.core.smtp.session.extensions.chunking = IfBlock::new(false); - let mut remote_qr = core.init_test_queue("smtp_delivery_remote"); - let remote_core = Arc::new(core); - let _rx = start_test_server(remote_core.clone(), &[ServerProtocol::Smtp]); + let mut remote = TestServer::new("smtp_delivery_remote", REMOTE, true).await; + let _rx = remote.start(&[ServerProtocol::Smtp]).await; + let remote_core = remote.build_smtp(); + + // Multiple delivery attempts + let mut local = TestServer::new("smtp_delivery_local", LOCAL, true).await; // Add mock DNS entries - let mut core = SMTP::test(); + let core = local.build_smtp(); for domain in ["foobar.org", "foobar.net", "foobar.com"] { core.core.smtp.resolvers.dns.mx_add( domain, @@ -104,23 +125,7 @@ async fn smtp_delivery() { ); } - // Multiple delivery attempts - let mut local_qr = core.init_test_queue("smtp_delivery_local"); - core.core.smtp.session.rcpt.relay = IfBlock::new(true); - core.core.smtp.session.rcpt.max_recipients = IfBlock::new(100); - core.core.smtp.session.extensions.dsn = IfBlock::new(true); - let config = &mut core.core.smtp.queue; - config.retry = IfBlock::new(Duration::from_secs(1)); - config.notify = r#"[{if = "rcpt_domain = 'foobar.org'", then = "[1s, 2s]"}, - {if = "rcpt_domain = 'foobar.com'", then = "[5s, 6s]"}, - {else = [1s]}]"# - .parse_if(); - config.expire = r#"[{if = "rcpt_domain = 'foobar.org'", then = "6s"}, - {else = "7s"}]"# - .parse_if(); - - let core = Arc::new(core); - let mut session = Session::test(core.clone()); + let mut session = local.new_session(); session.data.remote_ip_str = "10.0.0.1".to_string(); session.eval_session_params().await; session.ehlo("mx.test.org").await; @@ -140,10 +145,11 @@ async fn smtp_delivery() { "250", ) .await; - let message = local_qr.expect_message().await; + let message = local.qr.expect_message().await; let num_domains = message.domains.len(); assert_eq!(num_domains, 3); - local_qr + local + .qr .delivery_attempt(message.id) .await .try_deliver(core.clone()) @@ -151,7 +157,7 @@ async fn smtp_delivery() { let mut dsn = Vec::new(); let mut domain_retries = vec![0; num_domains]; loop { - match local_qr.try_read_event().await { + match local.qr.try_read_event().await { Some(Event::Reload) => {} Some(Event::OnHold(_)) => unreachable!(), None | Some(Event::Stop) => break, @@ -188,14 +194,14 @@ async fn smtp_delivery() { "retries {domain_retries:?}" ); - local_qr.assert_queue_is_empty().await; + local.qr.assert_queue_is_empty().await; assert_eq!(dsn.len(), 5); let mut dsn = dsn.into_iter(); dsn.next() .unwrap() - .read_lines(&local_qr) + .read_lines(&local.qr) .await .assert_contains(" (delivered to") .assert_contains(" (delivered to") @@ -205,7 +211,7 @@ async fn smtp_delivery() { dsn.next() .unwrap() - .read_lines(&local_qr) + .read_lines(&local.qr) .await .assert_contains(" (host ") .assert_contains(" (host ") @@ -213,26 +219,27 @@ async fn smtp_delivery() { dsn.next() .unwrap() - .read_lines(&local_qr) + .read_lines(&local.qr) .await .assert_contains(" (host ") .assert_contains("Action: delayed"); dsn.next() .unwrap() - .read_lines(&local_qr) + .read_lines(&local.qr) .await .assert_contains(" (host "); dsn.next() .unwrap() - .read_lines(&local_qr) + .read_lines(&local.qr) .await .assert_contains(" (host ") .assert_contains("Action: failed"); assert_eq!( - remote_qr + remote + .qr .consume_message(&remote_core) .await .recipients @@ -242,7 +249,8 @@ async fn smtp_delivery() { vec!["ok@foobar.org".to_string()] ); assert_eq!( - remote_qr + remote + .qr .consume_message(&remote_core) .await .recipients @@ -252,7 +260,7 @@ async fn smtp_delivery() { vec!["ok@foobar.net".to_string()] ); - remote_qr.assert_no_events(); + remote.qr.assert_no_events(); // SMTP smuggling for separator in ["\n", "\r"].iter() { @@ -268,17 +276,19 @@ async fn smtp_delivery() { session .send_message("john@doe.org", &["bill@foobar.com"], &message, "250") .await; - local_qr + local + .qr .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; - local_qr.read_event().await.assert_reload(); + local.qr.read_event().await.assert_reload(); - let message = remote_qr + let message = remote + .qr .consume_message(&remote_core) .await - .read_message(&remote_qr) + .read_message(&remote.qr) .await; assert!( diff --git a/tests/src/smtp/outbound/throttle.rs b/tests/src/smtp/outbound/throttle.rs index 7fea4966..ae9c1994 100644 --- a/tests/src/smtp/outbound/throttle.rs +++ b/tests/src/smtp/outbound/throttle.rs @@ -23,55 +23,80 @@ use std::{ net::{IpAddr, Ipv4Addr}, - sync::Arc, time::{Duration, Instant}, }; -use common::expr::if_block::IfBlock; use mail_auth::MX; use store::write::now; -use crate::smtp::{ - inbound::TestQueueEvent, queue::manager::new_message, session::TestSession, ParseTestConfig, - TestConfig, TestSMTP, -}; -use smtp::{ - core::{Session, SMTP}, - queue::{Message, QueueEnvelope}, -}; +use crate::smtp::{inbound::TestQueueEvent, outbound::TestServer, session::TestSession}; +use smtp::queue::{Message, QueueEnvelope}; + +const CONFIG: &str = r#" +[session.rcpt] +relay = true + +[queue.schedule] +retry = "1h" +notify = "1h" +expire = "1h" -const THROTTLE: &str = r#" [[queue.throttle]] match = "sender_domain = 'foobar.org'" key = 'sender_domain' concurrency = 1 +enable = true [[queue.throttle]] match = "sender_domain = 'foobar.net'" key = 'sender_domain' rate = '1/30m' +enable = true [[queue.throttle]] match = "rcpt_domain = 'example.org'" key = 'rcpt_domain' concurrency = 1 +enable = true [[queue.throttle]] match = "rcpt_domain = 'example.net'" key = 'rcpt_domain' rate = '1/40m' +enable = true [[queue.throttle]] match = "mx = 'mx.test.org'" key = 'mx' concurrency = 1 +enable = true [[queue.throttle]] match = "mx = 'mx.test.net'" key = 'mx' rate = '1/50m' +enable = true "#; +pub fn new_message(id: u64) -> Message { + let todo = "remove"; + Message { + size: 0, + id, + created: 0, + return_path: "sender@foobar.org".to_string(), + return_path_lcase: "".to_string(), + return_path_domain: "foobar.org".to_string(), + recipients: vec![], + domains: vec![], + flags: 0, + env_id: None, + priority: 0, + quota_keys: vec![], + blob_hash: Default::default(), + } +} + #[tokio::test] async fn throttle_outbound() { /*tracing::subscriber::set_global_default( @@ -84,23 +109,18 @@ async fn throttle_outbound() { // Build test message let mut test_message = new_message(0); test_message.return_path_domain = "foobar.org".to_string(); - let mut core = SMTP::test(); - let mut local_qr = core.init_test_queue("smtp_throttle_outbound"); - core.core.smtp.session.rcpt.relay = IfBlock::new(true); - core.core.smtp.queue.throttle = THROTTLE.parse_queue_throttle(); - core.core.smtp.queue.retry = IfBlock::new(Duration::from_secs(86400)); - core.core.smtp.queue.notify = IfBlock::new(Duration::from_secs(86400)); - core.core.smtp.queue.expire = IfBlock::new(Duration::from_secs(86400)); - let core = Arc::new(core); - let mut session = Session::test(core.clone()); + let mut local = TestServer::new("smtp_throttle_outbound", CONFIG, true).await; + + let core = local.build_smtp(); + let mut session = local.new_session(); session.data.remote_ip_str = "10.0.0.1".to_string(); session.eval_session_params().await; session.ehlo("mx.test.org").await; session .send_message("john@foobar.org", &["bill@test.org"], "test:no_dkim", "250") .await; - assert_eq!(local_qr.last_queued_due().await as i64 - now() as i64, 0); + assert_eq!(local.qr.last_queued_due().await as i64 - now() as i64, 0); // Throttle sender let span = tracing::info_span!("test"); @@ -119,13 +139,14 @@ async fn throttle_outbound() { assert!(!in_flight.is_empty()); // Expect concurrency throttle for sender domain 'foobar.org' - local_qr + local + .qr .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; tokio::time::sleep(Duration::from_millis(100)).await; - local_qr.read_event().await.unwrap_on_hold(); + local.qr.read_event().await.unwrap_on_hold(); in_flight.clear(); // Expect rate limit throttle for sender domain 'foobar.net' @@ -144,14 +165,15 @@ async fn throttle_outbound() { session .send_message("john@foobar.net", &["bill@test.org"], "test:no_dkim", "250") .await; - local_qr + local + .qr .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; tokio::time::sleep(Duration::from_millis(100)).await; - local_qr.read_event().await.assert_reload(); - let due = local_qr.last_queued_due().await - now(); + local.qr.read_event().await.assert_reload(); + let due = local.qr.last_queued_due().await - now(); assert!(due > 0, "Due: {}", due); // Expect concurrency throttle for recipient domain 'example.org' @@ -175,13 +197,14 @@ async fn throttle_outbound() { "250", ) .await; - local_qr + local + .qr .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; tokio::time::sleep(Duration::from_millis(100)).await; - local_qr.read_event().await.unwrap_on_hold(); + local.qr.read_event().await.unwrap_on_hold(); in_flight.clear(); // Expect rate limit throttle for recipient domain 'example.org' @@ -204,14 +227,15 @@ async fn throttle_outbound() { "250", ) .await; - local_qr + local + .qr .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; tokio::time::sleep(Duration::from_millis(100)).await; - local_qr.read_event().await.assert_reload(); - let due = local_qr.last_queued_due().await - now(); + local.qr.read_event().await.assert_reload(); + let due = local.qr.last_queued_due().await - now(); assert!(due > 0, "Due: {}", due); // Expect concurrency throttle for mx 'mx.test.org' @@ -242,12 +266,13 @@ async fn throttle_outbound() { session .send_message("john@test.net", &["jane@test.org"], "test:no_dkim", "250") .await; - local_qr + local + .qr .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; - local_qr.read_event().await.unwrap_on_hold(); + local.qr.read_event().await.unwrap_on_hold(); in_flight.clear(); // Expect rate limit throttle for mx 'mx.test.net' @@ -278,15 +303,16 @@ async fn throttle_outbound() { session .send_message("john@test.net", &["jane@test.net"], "test:no_dkim", "250") .await; - local_qr + local + .qr .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; tokio::time::sleep(Duration::from_millis(100)).await; - local_qr.read_event().await.assert_reload(); - let due = local_qr.last_queued_due().await - now(); + local.qr.read_event().await.assert_reload(); + let due = local.qr.last_queued_due().await - now(); assert!(due > 0, "Due: {}", due); } diff --git a/tests/src/smtp/outbound/tls.rs b/tests/src/smtp/outbound/tls.rs index ee85437a..509f8326 100644 --- a/tests/src/smtp/outbound/tls.rs +++ b/tests/src/smtp/outbound/tls.rs @@ -21,25 +21,40 @@ * for more details. */ -use std::{ - sync::Arc, - time::{Duration, Instant}, -}; +use std::time::{Duration, Instant}; -use common::{ - config::{server::ServerProtocol, smtp::queue::RequireOptional}, - expr::if_block::IfBlock, -}; +use common::config::server::ServerProtocol; use mail_auth::MX; use store::write::now; use crate::smtp::{ inbound::TestMessage, - outbound::start_test_server, + outbound::TestServer, session::{TestSession, VerifyResponse}, - TestConfig, TestSMTP, }; -use smtp::core::{Session, SMTP}; + +const LOCAL: &str = r#" +[session.rcpt] +relay = true + +[queue.outbound] +hostname = "'badtls.foobar.org'" + +[queue.outbound.tls] +starttls = "optional" +"#; + +const REMOTE: &str = r#" +[session.rcpt] +relay = true + +[session.ehlo] +reject-non-fqdn = false + +[session.extensions] +dsn = true +chunking = false +"#; #[tokio::test] #[serial_test::serial] @@ -52,14 +67,14 @@ async fn starttls_optional() { .unwrap();*/ // Start test server - let mut core = SMTP::test(); - core.core.smtp.session.rcpt.relay = IfBlock::new(true); - let mut remote_qr = core.init_test_queue("smtp_starttls_remote"); - let _rx = start_test_server(core.into(), &[ServerProtocol::Smtp]); + let mut remote = TestServer::new("smtp_starttls_remote", REMOTE, true).await; + let _rx = remote.start(&[ServerProtocol::Smtp]).await; + + // Retry on failed STARTTLS + let mut local = TestServer::new("smtp_starttls_local", LOCAL, true).await; // Add mock DNS entries - let mut core = SMTP::test(); - core.core.smtp.queue.hostname = IfBlock::new("badtls.foobar.org".to_string()); + let core = local.build_smtp(); core.core.smtp.resolvers.dns.mx_add( "foobar.org", vec![MX { @@ -74,25 +89,20 @@ async fn starttls_optional() { Instant::now() + Duration::from_secs(10), ); - // Retry on failed STARTTLS - let mut local_qr = core.init_test_queue("smtp_starttls_local"); - core.core.smtp.session.rcpt.relay = IfBlock::new(true); - core.core.smtp.queue.tls.start = IfBlock::new(RequireOptional::Optional); - - let core = Arc::new(core); - let mut session = Session::test(core.clone()); + let mut session = local.new_session(); session.data.remote_ip_str = "10.0.0.1".to_string(); session.eval_session_params().await; session.ehlo("mx.test.org").await; session .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; - local_qr + local + .qr .expect_message_then_deliver() .await .try_deliver(core.clone()) .await; - let mut retry = local_qr.expect_message().await; + let mut retry = local.qr.expect_message().await; assert!(retry.domains[0].disable_tls); let prev_due = retry.domains[0].retry.due; let next_due = now(); @@ -101,16 +111,18 @@ async fn starttls_optional() { retry .save_changes(&core, prev_due.into(), next_due.into()) .await; - local_qr + local + .qr .delivery_attempt(queue_id) .await .try_deliver(core.clone()) .await; tokio::time::sleep(Duration::from_millis(100)).await; - remote_qr + remote + .qr .expect_message() .await - .read_lines(&remote_qr) + .read_lines(&remote.qr) .await .assert_not_contains("using TLSv1.3 with cipher"); } diff --git a/tests/src/smtp/queue/concurrent.rs b/tests/src/smtp/queue/concurrent.rs index 42385b85..5f1e0966 100644 --- a/tests/src/smtp/queue/concurrent.rs +++ b/tests/src/smtp/queue/concurrent.rs @@ -29,7 +29,7 @@ use std::{ use common::{config::server::ServerProtocol, expr::if_block::IfBlock}; use mail_auth::MX; -use crate::smtp::{outbound::start_test_server, session::TestSession, TestConfig, TestSMTP}; +use crate::smtp::{session::TestSession, TestSMTP}; use smtp::{ core::{Session, SMTP}, queue::manager::Queue, @@ -47,14 +47,16 @@ async fn concurrent_queue() { .unwrap();*/ // Start test server - let mut core = SMTP::test(); - core.core.smtp.session.rcpt.relay = IfBlock::new(true); + let mut inner = Inner::default(); + let mut core = Core::default(); + core.smtp.session.rcpt.relay = IfBlock::new(true); let remote_qr = core.init_test_queue("smtp_concurrent_queue_remote"); let _rx = start_test_server(core.into(), &[ServerProtocol::Smtp]); // Add mock DNS entries - let mut core = SMTP::test(); - core.core.smtp.resolvers.dns.mx_add( + let mut inner = Inner::default(); + let mut core = Core::default(); + core.smtp.resolvers.dns.mx_add( "foobar.org", vec![MX { exchanges: vec!["mx.foobar.org".to_string()], @@ -62,17 +64,17 @@ async fn concurrent_queue() { }], Instant::now() + Duration::from_secs(100), ); - core.core.smtp.resolvers.dns.ipv4_add( + core.smtp.resolvers.dns.ipv4_add( "mx.foobar.org", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(100), ); let local_qr = core.init_test_queue("smtp_concurrent_queue_local"); - core.core.smtp.session.rcpt.relay = IfBlock::new(true); - core.core.smtp.session.data.max_messages = IfBlock::new(200); + core.smtp.session.rcpt.relay = IfBlock::new(true); + core.smtp.session.data.max_messages = IfBlock::new(200); let core = Arc::new(core); - let mut session = Session::test(core.clone()); + let mut session = Session::test(build_smtp(core, Inner::default())); session.data.remote_ip_str = "10.0.0.1".to_string(); session.eval_session_params().await; session.ehlo("mx.test.org").await; @@ -105,9 +107,8 @@ async fn concurrent_queue() { assert_eq!(remote_messages.len(), 100); // Make sure local store is queue - core.core - .storage + core.storage .data - .assert_is_empty(core.core.storage.blob.clone()) + .assert_is_empty(core.storage.blob.clone()) .await; } diff --git a/tests/src/smtp/queue/dsn.rs b/tests/src/smtp/queue/dsn.rs index 84e73474..4ec7367b 100644 --- a/tests/src/smtp/queue/dsn.rs +++ b/tests/src/smtp/queue/dsn.rs @@ -27,9 +27,7 @@ use smtp_proto::{Response, RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_S use store::write::now; use utils::BlobHash; -use crate::smtp::{ - inbound::sign::TextConfigContext, ParseTestConfig, QueueReceiver, TestConfig, TestSMTP, -}; +use crate::smtp::{inbound::sign::TextConfigContext, QueueReceiver, TestSMTP}; use smtp::{ core::SMTP, queue::{Domain, Error, ErrorDetails, HostResponse, Message, Recipient, Schedule, Status}, @@ -93,9 +91,10 @@ async fn generate_dsn() { let span = tracing::span!(tracing::Level::INFO, "hi"); // Load config - let mut core = SMTP::test(); - core.core.storage.signers = ConfigContext::new().parse_signatures().signers; - let config = &mut core.core.smtp.queue.dsn; + let mut inner = Inner::default(); + let mut core = Core::default(); + core.storage.signers = ConfigContext::new().parse_signatures().signers; + let config = &mut core.smtp.queue.dsn; config.sign = "\"['rsa']\"".parse_if(); // Create temp dir for queue diff --git a/tests/src/smtp/queue/manager.rs b/tests/src/smtp/queue/manager.rs index 7b443877..7e952204 100644 --- a/tests/src/smtp/queue/manager.rs +++ b/tests/src/smtp/queue/manager.rs @@ -31,11 +31,12 @@ use smtp::{ }; use store::write::now; -use crate::smtp::{TestConfig, TestSMTP}; +use crate::smtp::TestSMTP; #[tokio::test] async fn queue_due() { - let mut core = SMTP::test(); + let mut inner = Inner::default(); + let mut core = Core::default(); let qr = core.init_test_queue("smtp_queue_due_test"); let core = Arc::new(core); diff --git a/tests/src/smtp/queue/retry.rs b/tests/src/smtp/queue/retry.rs index 1d8c8e35..9f79f450 100644 --- a/tests/src/smtp/queue/retry.rs +++ b/tests/src/smtp/queue/retry.rs @@ -26,7 +26,7 @@ use std::{sync::Arc, time::Duration}; use crate::smtp::{ inbound::{TestMessage, TestQueueEvent}, session::{TestSession, VerifyResponse}, - ParseTestConfig, TestConfig, TestSMTP, + TestSMTP, }; use common::expr::if_block::IfBlock; use smtp::{ @@ -44,17 +44,18 @@ async fn queue_retry() { ) .unwrap();*/ - let mut core = SMTP::test(); + let mut inner = Inner::default(); + let mut core = Core::default(); // Create temp dir for queue let mut qr = core.init_test_queue("smtp_queue_retry_test"); - let config = &mut core.core.smtp.session.rcpt; + let config = &mut core.smtp.session.rcpt; config.relay = IfBlock::new(true); - let config = &mut core.core.smtp.session.extensions; + let config = &mut core.smtp.session.extensions; config.deliver_by = IfBlock::new(Duration::from_secs(86400)); config.future_release = IfBlock::new(Duration::from_secs(86400)); - let config = &mut core.core.smtp.queue; + let config = &mut core.smtp.queue; config.retry = r#""[1s, 2s, 3s]""#.parse_if(); config.notify = r#"[{if = "sender_domain = 'test.org'", then = "[1s, 2s]"}, {else = ['15h', '22h']}]"# @@ -66,7 +67,7 @@ async fn queue_retry() { // Create test message let core = Arc::new(core); - let mut session = Session::test(core.clone()); + let mut session = Session::test(build_smtp(core, Inner::default())); session.data.remote_ip_str = "10.0.0.1".to_string(); session.eval_session_params().await; session.ehlo("mx.test.org").await; diff --git a/tests/src/smtp/reporting/analyze.rs b/tests/src/smtp/reporting/analyze.rs index 48692623..4e3bd5eb 100644 --- a/tests/src/smtp/reporting/analyze.rs +++ b/tests/src/smtp/reporting/analyze.rs @@ -23,7 +23,7 @@ use std::{sync::Arc, time::Duration}; -use crate::smtp::{inbound::TestQueueEvent, session::TestSession, TestConfig, TestSMTP}; +use crate::smtp::{inbound::TestQueueEvent, session::TestSession, TestSMTP}; use common::{config::smtp::report::AddressMatch, expr::if_block::IfBlock}; use smtp::core::{Session, SMTP}; use store::{ @@ -33,15 +33,16 @@ use store::{ #[tokio::test(flavor = "multi_thread")] async fn report_analyze() { - let mut core = SMTP::test(); + let mut inner = Inner::default(); + let mut core = Core::default(); // Create temp dir for queue let mut qr = core.init_test_queue("smtp_analyze_report_test"); - let config = &mut core.core.smtp.session.rcpt; + let config = &mut core.smtp.session.rcpt; config.relay = IfBlock::new(true); - let config = &mut core.core.smtp.session.data; + let config = &mut core.smtp.session.data; config.max_messages = IfBlock::new(1024); - let config = &mut core.core.smtp.report.analysis; + let config = &mut core.smtp.report.analysis; config.addresses = vec![ AddressMatch::StartsWith("reports@".to_string()), AddressMatch::EndsWith("@dmarc.foobar.org".to_string()), @@ -58,7 +59,7 @@ async fn report_analyze() { &[utils::config::ServerProtocol::Http], );*/ - let mut session = Session::test(core.clone()); + let mut session = Session::test(build_smtp(core, Inner::default())); session.data.remote_ip_str = "10.0.0.1".to_string(); session.eval_session_params().await; session.ehlo("mx.test.org").await; diff --git a/tests/src/smtp/reporting/dmarc.rs b/tests/src/smtp/reporting/dmarc.rs index ccc9a638..e9015ec1 100644 --- a/tests/src/smtp/reporting/dmarc.rs +++ b/tests/src/smtp/reporting/dmarc.rs @@ -38,7 +38,7 @@ use store::write::QueueClass; use crate::smtp::{ inbound::{sign::TextConfigContext, TestMessage}, session::VerifyResponse, - ParseTestConfig, TestConfig, TestSMTP, + TestSMTP, }; use smtp::{core::SMTP, reporting::DmarcEvent}; @@ -52,9 +52,10 @@ async fn report_dmarc() { .unwrap();*/ // Create scheduler - let mut core = SMTP::test(); - core.core.storage.signers = ConfigContext::new().parse_signatures().signers; - let config = &mut core.core.smtp.report; + let mut inner = Inner::default(); + let mut core = Core::default(); + core.storage.signers = ConfigContext::new().parse_signatures().signers; + let config = &mut core.smtp.report; config.dmarc_aggregate.sign = "\"['rsa']\"".parse_if(); config.dmarc_aggregate.max_size = IfBlock::new(4096); config.submitter = IfBlock::new("mx.example.org".to_string()); @@ -63,7 +64,7 @@ async fn report_dmarc() { config.dmarc_aggregate.contact_info = IfBlock::new("https://foobar.org/contact".to_string()); // Authorize external report for foobar.org - core.core.smtp.resolvers.dns.txt_add( + core.smtp.resolvers.dns.txt_add( "foobar.org._report._dmarc.foobar.net", Dmarc::parse(b"v=DMARC1;").unwrap(), Instant::now() + Duration::from_secs(10), diff --git a/tests/src/smtp/reporting/scheduler.rs b/tests/src/smtp/reporting/scheduler.rs index 14c701a6..de0911b0 100644 --- a/tests/src/smtp/reporting/scheduler.rs +++ b/tests/src/smtp/reporting/scheduler.rs @@ -32,7 +32,7 @@ use mail_auth::{ }; use store::write::QueueClass; -use crate::smtp::{TestConfig, TestSMTP}; +use crate::smtp::TestSMTP; use smtp::{ core::SMTP, reporting::{dmarc::DmarcFormat, DmarcEvent, PolicyType, TlsEvent}, @@ -48,9 +48,10 @@ async fn report_scheduler() { .unwrap();*/ // Create scheduler - let mut core = SMTP::test(); + let mut inner = Inner::default(); + let mut core = Core::default(); let qr = core.init_test_queue("smtp_report_queue_test"); - let config = &mut core.core.smtp.report; + let config = &mut core.smtp.report; config.dmarc_aggregate.max_size = IfBlock::new(500); config.tls.max_size = IfBlock::new(550); diff --git a/tests/src/smtp/reporting/tls.rs b/tests/src/smtp/reporting/tls.rs index bccaa7a9..8d85194a 100644 --- a/tests/src/smtp/reporting/tls.rs +++ b/tests/src/smtp/reporting/tls.rs @@ -35,7 +35,7 @@ use store::write::QueueClass; use crate::smtp::{ inbound::{sign::TextConfigContext, TestMessage}, session::VerifyResponse, - ParseTestConfig, TestConfig, TestSMTP, + TestSMTP, }; use smtp::{ core::SMTP, @@ -53,9 +53,10 @@ async fn report_tls() { .unwrap();*/ // Create scheduler - let mut core = SMTP::test(); - core.core.storage.signers = ConfigContext::new().parse_signatures().signers; - let config = &mut core.core.smtp.report; + let mut inner = Inner::default(); + let mut core = Core::default(); + core.storage.signers = ConfigContext::new().parse_signatures().signers; + let config = &mut core.smtp.report; config.tls.sign = "\"['rsa']\"".parse_if(); config.tls.max_size = IfBlock::new(1532); config.submitter = IfBlock::new("mx.example.org".to_string()); diff --git a/tests/src/smtp/session.rs b/tests/src/smtp/session.rs index 281d7730..66966172 100644 --- a/tests/src/smtp/session.rs +++ b/tests/src/smtp/session.rs @@ -36,8 +36,6 @@ use tokio::{ use smtp::core::{Session, SessionAddress, SessionData, SessionParameters, State, SMTP}; use tokio_rustls::TlsAcceptor; -use super::TestConfig; - pub struct DummyIo { pub tx_buf: Vec, pub rx_buf: Vec, @@ -99,8 +97,8 @@ impl Unpin for DummyIo {} #[allow(async_fn_in_trait)] pub trait TestSession { - fn test(core: impl Into>) -> Self; - fn test_with_shutdown(core: impl Into>, shutdown_rx: watch::Receiver) -> Self; + fn test(core: SMTP) -> Self; + fn test_with_shutdown(core: SMTP, shutdown_rx: watch::Receiver) -> Self; fn response(&mut self) -> Vec; fn write_rx(&mut self, data: &str); async fn rset(&mut self); @@ -114,11 +112,11 @@ pub trait TestSession { } impl TestSession for Session { - fn test_with_shutdown(core: impl Into>, shutdown_rx: watch::Receiver) -> Self { + fn test_with_shutdown(core: SMTP, shutdown_rx: watch::Receiver) -> Self { Self { state: State::default(), instance: Arc::new(ServerInstance::test_with_shutdown(shutdown_rx)), - core: core.into(), + core, span: tracing::info_span!("test"), stream: DummyIo { rx_buf: vec![], @@ -136,7 +134,7 @@ impl TestSession for Session { } } - fn test(core: impl Into>) -> Self { + fn test(core: SMTP) -> Self { Self::test_with_shutdown(core, watch::channel(false).1) } @@ -383,8 +381,6 @@ impl ResolvesServerCert for DummyCertResolver { } } -impl TestConfig for ServerInstance { - fn test() -> Self { - Self::test_with_shutdown(watch::channel(false).1) - } +pub fn test_server_instance() -> ServerInstance { + ServerInstance::test_with_shutdown(watch::channel(false).1) } diff --git a/tests/src/store/blob.rs b/tests/src/store/blob.rs index cd2b15cb..6d622f9f 100644 --- a/tests/src/store/blob.rs +++ b/tests/src/store/blob.rs @@ -24,7 +24,7 @@ use ahash::AHashMap; use store::{ write::{blob::BlobQuota, now, BatchBuilder, BlobOp}, - BlobClass, BlobStore, Serialize, + BlobClass, BlobStore, Serialize, Stores, }; use utils::{config::Config, BlobHash}; @@ -33,9 +33,9 @@ use crate::store::{TempDir, CONFIG}; #[tokio::test] pub async fn blob_tests() { let temp_dir = TempDir::new("blob_tests", true); - let config = - Config::new(&CONFIG.replace("{TMP}", temp_dir.path.as_path().to_str().unwrap())).unwrap(); - let stores = config.parse_stores().await.unwrap(); + let mut config = + Config::new(CONFIG.replace("{TMP}", temp_dir.path.as_path().to_str().unwrap())).unwrap(); + let stores = Stores::parse(&mut config).await; for (store_id, blob_store) in &stores.blob_stores { println!("Testing blob store {}...", store_id); diff --git a/tests/src/store/lookup.rs b/tests/src/store/lookup.rs index fef66f4f..d5d3e013 100644 --- a/tests/src/store/lookup.rs +++ b/tests/src/store/lookup.rs @@ -23,17 +23,22 @@ use std::time::Duration; -use store::LookupStore; +use store::{LookupStore, Stores}; use utils::config::{Config, Rate}; -use crate::store::{TempDir, CONFIG}; +use crate::{ + store::{TempDir, CONFIG}, + AssertConfig, +}; #[tokio::test] pub async fn lookup_tests() { let temp_dir = TempDir::new("lookup_tests", true); - let config = - Config::new(&CONFIG.replace("{TMP}", temp_dir.path.as_path().to_str().unwrap())).unwrap(); - let stores = config.parse_stores().await.unwrap(); + let mut config = + Config::new(CONFIG.replace("{TMP}", temp_dir.path.as_path().to_str().unwrap())) + .unwrap() + .assert_no_errors(); + let stores = Stores::parse(&mut config).await; let rate = Rate { requests: 1, period: Duration::from_secs(1), diff --git a/tests/src/store/mod.rs b/tests/src/store/mod.rs index 1f0ade4c..ea855be7 100644 --- a/tests/src/store/mod.rs +++ b/tests/src/store/mod.rs @@ -29,9 +29,11 @@ pub mod query; use std::io::Read; -use store::FtsStore; +use store::{FtsStore, Stores}; use utils::config::Config; +use crate::AssertConfig; + pub struct TempDir { pub path: std::path::PathBuf, } @@ -78,7 +80,8 @@ password = "password" [store."redis"] type = "redis" -url = "redis://127.0.0.1" +urls = "redis://127.0.0.1" +redis-type = "single" "#; @@ -86,8 +89,10 @@ url = "redis://127.0.0.1" pub async fn store_tests() { let insert = true; let temp_dir = TempDir::new("store_tests", insert); - let config = Config::new(&CONFIG.replace("{TMP}", &temp_dir.path.to_string_lossy())).unwrap(); - let stores = config.parse_stores().await.unwrap(); + let mut config = Config::new(CONFIG.replace("{TMP}", &temp_dir.path.to_string_lossy())) + .unwrap() + .assert_no_errors(); + let stores = Stores::parse(&mut config).await; let store_id = std::env::var("STORE") .expect("Missing store type. Try running `STORE= cargo test`");