diff --git a/.gitignore b/.gitignore index abd0eab2..22557e38 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ .vscode *.failed *_failed +stalwart.toml diff --git a/Cargo.toml b/Cargo.toml index fc328f05..e6ba5294 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ path = "crates/main/src/main.rs" store = { path = "crates/store" } jmap = { path = "crates/jmap" } jmap_proto = { path = "crates/jmap-proto" } +smtp = { path = "crates/smtp" } utils = { path = "crates/utils" } tests = { path = "tests" } @@ -26,6 +27,7 @@ tests = { path = "tests" } members = [ "crates/jmap", "crates/jmap-proto", + "crates/smtp", "crates/store", "crates/utils", "crates/maybe-async", diff --git a/crates/jmap/Cargo.toml b/crates/jmap/Cargo.toml index 8597a673..1606c33a 100644 --- a/crates/jmap/Cargo.toml +++ b/crates/jmap/Cargo.toml @@ -24,7 +24,7 @@ aes-gcm-siv = "0.11.1" bincode = "1.3.3" form-data = { version = "0.4.2", features = ["sync"], default-features = false } mime = "0.3.17" -sqlx = { git = "https://github.com/mdecimus/sqlx", features = [ "runtime-tokio-rustls", "postgres", "mysql", "sqlite" ] } +sqlx = { version = "0.7.0-alpha.3", features = [ "runtime-tokio-rustls", "postgres", "mysql", "sqlite" ] } futures-util = "0.3.28" async-stream = "0.3.5" base64 = "0.21" diff --git a/crates/jmap/src/api/config.rs b/crates/jmap/src/api/config.rs index b1c7c133..a1d704f7 100644 --- a/crates/jmap/src/api/config.rs +++ b/crates/jmap/src/api/config.rs @@ -29,9 +29,6 @@ impl crate::Config { request_max_concurrent: settings .property("jmap.protocol.request.max-concurrent")? .unwrap_or(4), - request_max_concurrent_total: settings - .property("jmap.protocol.request.max-concurrent-total")? - .unwrap_or(4), get_max_objects: settings .property("jmap.protocol.get.max-objects")? .unwrap_or(500), diff --git a/crates/jmap/src/api/http.rs b/crates/jmap/src/api/http.rs index 30296cb8..1f2a03b7 100644 --- a/crates/jmap/src/api/http.rs +++ b/crates/jmap/src/api/http.rs @@ -242,10 +242,6 @@ impl SessionManager for super::SessionManager { } }); } - - fn max_concurrent(&self) -> u64 { - self.inner.config.request_max_concurrent_total - } } async fn handle_request( diff --git a/crates/jmap/src/lib.rs b/crates/jmap/src/lib.rs index a601ac7f..b686b697 100644 --- a/crates/jmap/src/lib.rs +++ b/crates/jmap/src/lib.rs @@ -58,7 +58,6 @@ pub struct Config { pub request_max_size: usize, pub request_max_calls: usize, pub request_max_concurrent: u64, - pub request_max_concurrent_total: u64, pub get_max_objects: usize, pub set_max_objects: usize, diff --git a/crates/smtp/Cargo.toml b/crates/smtp/Cargo.toml new file mode 100644 index 00000000..2a62ef5a --- /dev/null +++ b/crates/smtp/Cargo.toml @@ -0,0 +1,62 @@ +[package] +name = "smtp" +description = "Stalwart SMTP Server" +authors = [ "Stalwart Labs Ltd. "] +repository = "https://github.com/stalwartlabs/smtp-server" +homepage = "https://stalw.art/smtp" +keywords = ["smtp", "email", "mail", "server"] +categories = ["email"] +license = "AGPL-3.0-only" +version = "0.1.1" +edition = "2021" +resolver = "2" + +[dependencies] +utils = { path = "../utils" } +mail-auth = { git = "https://github.com/stalwartlabs/mail-auth" } +mail-send = { git = "https://github.com/stalwartlabs/mail-send", default-features = false, features = ["cram-md5", "skip-ehlo"] } +mail-parser = { git = "https://github.com/stalwartlabs/mail-parser", features = ["full_encoding", "ludicrous_mode"] } +mail-builder = { git = "https://github.com/stalwartlabs/mail-builder", features = ["ludicrous_mode"] } +smtp-proto = { git = "https://github.com/stalwartlabs/smtp-proto" } +sieve-rs = { git = "https://github.com/stalwartlabs/sieve" } +ahash = { version = "0.8" } +rustls = "0.21.0" +rustls-pemfile = "1.0" +tokio = { version = "1.23", features = ["full"] } +tokio-rustls = { version = "0.24.0"} +webpki-roots = { version = "0.23.0"} +hyper = { version = "1.0.0-rc.3", features = ["server", "http1", "http2"] } +http-body-util = "0.1.0-rc.2" +form_urlencoded = "1.1.0" +sha1 = "0.10" +sha2 = "0.10.6" +rayon = "1.5" +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } +tracing-appender = "0.2" +tracing-opentelemetry = "0.18.0" +opentelemetry = { version = "0.18.0", features = ["rt-tokio"] } +opentelemetry-otlp = { version = "0.11.0", features = ["http-proto", "reqwest-client", "reqwest-rustls"] } +opentelemetry-semantic-conventions = { version = "0.10.0" } +parking_lot = "0.12" +regex = "1.7.0" +dashmap = "5.4" +blake3 = "1.3" +lru-cache = "0.1.2" +rand = "0.8.5" +x509-parser = "0.15.0" +sqlx = { version = "0.7.0-alpha.3", features = [ "runtime-tokio-rustls", "postgres", "mysql", "sqlite" ] } +reqwest = { version = "0.11", default-features = false, features = ["rustls-tls", "blocking"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +num_cpus = "1.15.0" + +[target.'cfg(unix)'.dependencies] +privdrop = "0.5.3" + +[features] +test_mode = [] + +#[[bench]] +#name = "hash" +#harness = false diff --git a/crates/smtp/src/config/auth.rs b/crates/smtp/src/config/auth.rs new file mode 100644 index 00000000..33ac64b7 --- /dev/null +++ b/crates/smtp/src/config/auth.rs @@ -0,0 +1,404 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{sync::Arc, time::Duration}; + +use mail_auth::{ + common::crypto::{Algorithm, Ed25519Key, HashAlgorithm, RsaKey, Sha256, SigningKey}, + dkim::{Canonicalization, Done}, +}; +use mail_parser::decoders::base64::base64_decode; +use utils::config::{ + utils::{AsKey, ParseValue}, + Config, +}; + +use super::{ + if_block::ConfigIf, ArcAuthConfig, ArcSealer, ConfigContext, DkimAuthConfig, + DkimCanonicalization, DkimSigner, DmarcAuthConfig, DnsBlConfig, EnvelopeKey, IfBlock, IfThen, + IpRevAuthConfig, MailAuthConfig, SpfAuthConfig, VerifyStrategy, DNSBL_EHLO, DNSBL_FROM, + DNSBL_IP, DNSBL_IPREV, DNSBL_RETURN_PATH, +}; + +pub trait ConfigAuth { + fn parse_mail_auth(&self, ctx: &ConfigContext) -> super::Result; + fn parse_dnsbl(&self, ctx: &ConfigContext) -> super::Result; + fn parse_signatures(&self, ctx: &mut ConfigContext) -> super::Result<()>; +} + +impl ConfigAuth for Config { + fn parse_mail_auth(&self, ctx: &ConfigContext) -> super::Result { + let envelope_sender_keys = [ + EnvelopeKey::Sender, + EnvelopeKey::SenderDomain, + EnvelopeKey::Priority, + EnvelopeKey::AuthenticatedAs, + EnvelopeKey::Listener, + EnvelopeKey::RemoteIp, + EnvelopeKey::LocalIp, + ]; + let envelope_conn_keys = [ + EnvelopeKey::Listener, + EnvelopeKey::RemoteIp, + EnvelopeKey::LocalIp, + ]; + + Ok(MailAuthConfig { + dkim: DkimAuthConfig { + verify: self + .parse_if_block("auth.dkim.verify", ctx, &envelope_sender_keys)? + .unwrap_or_else(|| IfBlock::new(VerifyStrategy::Relaxed)), + sign: self + .parse_if_block::>("auth.dkim.sign", ctx, &envelope_sender_keys)? + .unwrap_or_default() + .map_if_block(&ctx.signers, "auth.dkim.sign", "signature")?, + }, + arc: ArcAuthConfig { + verify: self + .parse_if_block("auth.arc.verify", ctx, &envelope_sender_keys)? + .unwrap_or_else(|| IfBlock::new(VerifyStrategy::Relaxed)), + seal: self + .parse_if_block::>("auth.arc.seal", ctx, &envelope_sender_keys)? + .unwrap_or_default() + .map_if_block(&ctx.sealers, "auth.arc.seal", "signature")?, + }, + spf: SpfAuthConfig { + verify_ehlo: self + .parse_if_block("auth.spf.verify.ehlo", ctx, &envelope_conn_keys)? + .unwrap_or_else(|| IfBlock::new(VerifyStrategy::Relaxed)), + verify_mail_from: self + .parse_if_block("auth.spf.verify.mail-from", ctx, &envelope_conn_keys)? + .unwrap_or_else(|| IfBlock::new(VerifyStrategy::Relaxed)), + }, + dmarc: DmarcAuthConfig { + verify: self + .parse_if_block("auth.dmarc.verify", ctx, &envelope_sender_keys)? + .unwrap_or_else(|| IfBlock::new(VerifyStrategy::Relaxed)), + }, + iprev: IpRevAuthConfig { + verify: self + .parse_if_block("auth.iprev.verify", ctx, &envelope_conn_keys)? + .unwrap_or_else(|| IfBlock::new(VerifyStrategy::Relaxed)), + }, + dnsbl: self.parse_dnsbl(ctx)?, + }) + } + + fn parse_dnsbl(&self, ctx: &ConfigContext) -> super::Result { + let verify = self + .parse_if_block::>( + "auth.dnsbl.verify", + ctx, + &[EnvelopeKey::RemoteIp, EnvelopeKey::Listener], + )? + .unwrap_or_default(); + + Ok(DnsBlConfig { + verify: IfBlock { + if_then: { + let mut if_then = Vec::with_capacity(verify.if_then.len()); + for cond in verify.if_then { + if_then.push(IfThen { + conditions: cond.conditions, + then: cond.then.into_dnsbl("auth.dnsbl.verify.if")?, + }); + } + if_then + }, + default: verify.default.into_dnsbl("auth.dnsbl.verify.else")?, + }, + ip_lookup: self + .values("auth.dnsbl.lookup.ip") + .filter_map(|(_, v)| { + if !v.is_empty() { + if !v.ends_with('.') { + format!("{v}.") + } else { + v.to_string() + } + .into() + } else { + None + } + }) + .collect(), + domain_lookup: self + .values("auth.dnsbl.lookup.domain") + .filter_map(|(_, v)| { + if !v.is_empty() { + if !v.ends_with('.') { + format!("{v}.") + } else { + v.to_string() + } + .into() + } else { + None + } + }) + .collect(), + }) + } + + #[allow(clippy::type_complexity)] + fn parse_signatures(&self, ctx: &mut ConfigContext) -> super::Result<()> { + for id in self.sub_keys("signature") { + let (signer, sealer) = + match self.property_require::(("signature", id, "algorithm"))? { + Algorithm::RsaSha256 => { + let key = RsaKey::::from_rsa_pem( + &String::from_utf8(self.file_contents(( + "signature", + id, + "private-key", + ))?) + .unwrap_or_default(), + ) + .map_err(|err| { + format!( + "Failed to build RSA key for {}: {}", + ("signature", id, "private-key",).as_key(), + err + ) + })?; + let key_clone = RsaKey::::from_rsa_pem( + &String::from_utf8(self.file_contents(( + "signature", + id, + "private-key", + ))?) + .unwrap_or_default(), + ) + .map_err(|err| { + format!( + "Failed to build RSA key for {}: {}", + ("signature", id, "private-key",).as_key(), + err + ) + })?; + let (signer, sealer) = parse_signature(self, id, key_clone, key)?; + (DkimSigner::RsaSha256(signer), ArcSealer::RsaSha256(sealer)) + } + Algorithm::Ed25519Sha256 => { + let public_key = + base64_decode(&self.file_contents(("signature", id, "public-key"))?) + .ok_or_else(|| { + format!( + "Failed to base64 decode public key for {}.", + ("signature", id, "public-key",).as_key(), + ) + })?; + let private_key = + base64_decode(&self.file_contents(("signature", id, "private-key"))?) + .ok_or_else(|| { + format!( + "Failed to base64 decode private key for {}.", + ("signature", id, "private-key",).as_key(), + ) + })?; + let key = Ed25519Key::from_seed_and_public_key(&private_key, &public_key) + .map_err(|err| { + format!("Failed to build ED25519 key for signature {id:?}: {err}") + })?; + let key_clone = + Ed25519Key::from_seed_and_public_key(&private_key, &public_key) + .map_err(|err| { + format!( + "Failed to build ED25519 key for signature {id:?}: {err}" + ) + })?; + + let (signer, sealer) = parse_signature(self, id, key_clone, key)?; + ( + DkimSigner::Ed25519Sha256(signer), + ArcSealer::Ed25519Sha256(sealer), + ) + } + Algorithm::RsaSha1 => { + return Err(format!( + "Could not build signature {id:?}: SHA1 signatures are deprecated.", + )) + } + }; + ctx.signers.insert(id.to_string(), Arc::new(signer)); + ctx.sealers.insert(id.to_string(), Arc::new(sealer)); + } + + Ok(()) + } +} + +fn parse_signature>( + config: &Config, + id: &str, + key_dkim: T, + key_arc: U, +) -> super::Result<( + mail_auth::dkim::DkimSigner, + mail_auth::arc::ArcSealer, +)> { + let domain = config.value_require(("signature", id, "domain"))?; + let selector = config.value_require(("signature", id, "selector"))?; + let mut headers = config + .values(("signature", id, "headers")) + .filter_map(|(_, v)| { + if !v.is_empty() { + v.to_string().into() + } else { + None + } + }) + .collect::>(); + if headers.is_empty() { + headers = vec![ + "From".to_string(), + "To".to_string(), + "Date".to_string(), + "Subject".to_string(), + "Message-ID".to_string(), + ]; + } + + let mut signer = mail_auth::dkim::DkimSigner::from_key(key_dkim) + .domain(domain) + .selector(selector) + .headers(headers.clone()); + if !headers + .iter() + .any(|h| h.eq_ignore_ascii_case("DKIM-Signature")) + { + headers.push("DKIM-Signature".to_string()); + } + let mut sealer = mail_auth::arc::ArcSealer::from_key(key_arc) + .domain(domain) + .selector(selector) + .headers(headers); + + if let Some(c) = + config.property::(("signature", id, "canonicalization"))? + { + signer = signer + .body_canonicalization(c.body) + .header_canonicalization(c.headers); + sealer = sealer + .body_canonicalization(c.body) + .header_canonicalization(c.headers); + } + + if let Some(c) = config.property::(("signature", id, "expire"))? { + signer = signer.expiration(c.as_secs()); + sealer = sealer.expiration(c.as_secs()); + } + + if let Some(true) = config.property::(("signature", id, "set-body-length"))? { + signer = signer.body_length(true); + sealer = sealer.body_length(true); + } + + if let Some(true) = config.property::(("signature", id, "report"))? { + signer = signer.reporting(true); + } + + if let Some(auid) = config.property::(("signature", id, "auid"))? { + signer = signer.agent_user_identifier(auid); + } + + if let Some(atps) = config.property::(("signature", id, "third-party"))? { + signer = signer.atps(atps); + } + + if let Some(atpsh) = config.property::(("signature", id, "third-party-algo"))? { + signer = signer.atpsh(atpsh); + } + + Ok((signer, sealer)) +} + +impl ParseValue for VerifyStrategy { + fn parse_value(key: impl AsKey, value: &str) -> super::Result { + match value { + "relaxed" => Ok(VerifyStrategy::Relaxed), + "strict" => Ok(VerifyStrategy::Strict), + "disable" | "disabled" | "never" | "none" => Ok(VerifyStrategy::Disable), + _ => Err(format!( + "Invalid value {:?} for key {:?}.", + value, + key.as_key() + )), + } + } +} + +impl ParseValue for DkimCanonicalization { + fn parse_value(key: impl AsKey, value: &str) -> super::Result { + if let Some((headers, body)) = value.split_once('/') { + Ok(DkimCanonicalization { + headers: Canonicalization::parse_value(key.clone(), headers.trim())?, + body: Canonicalization::parse_value(key, body.trim())?, + }) + } else { + let c = Canonicalization::parse_value(key, value)?; + Ok(DkimCanonicalization { + headers: c, + body: c, + }) + } + } +} + +impl Default for DkimCanonicalization { + fn default() -> Self { + Self { + headers: Canonicalization::Relaxed, + body: Canonicalization::Relaxed, + } + } +} + +trait IntoDnsbl { + fn into_dnsbl(self, key: impl AsKey) -> super::Result; +} + +impl IntoDnsbl for Vec { + fn into_dnsbl(self, key: impl AsKey) -> super::Result { + let mut dns_bl = 0; + for value in self { + dns_bl |= match value.as_str() { + "ip" => DNSBL_IP, + "iprev" => DNSBL_IPREV, + "ehlo" | "helo" => DNSBL_EHLO, + "return-path" | "sender" | "mail-from" => DNSBL_RETURN_PATH, + "from" => DNSBL_FROM, + _ => { + return Err(format!( + "Invalid DNSBL value {:?} for key {:?}.", + value, + key.as_key() + )) + } + }; + } + + Ok(dns_bl) + } +} diff --git a/crates/smtp/src/config/condition.rs b/crates/smtp/src/config/condition.rs new file mode 100644 index 00000000..999d3729 --- /dev/null +++ b/crates/smtp/src/config/condition.rs @@ -0,0 +1,491 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::net::IpAddr; + +use regex::Regex; + +use crate::config::StringMatch; + +use super::{Condition, ConditionMatch, Conditions, ConfigContext, EnvelopeKey, IpAddrMask}; +use utils::config::{ + utils::{AsKey, ParseKey, ParseValue}, + Config, +}; + +pub trait ConfigCondition { + fn parse_condition( + &self, + key: impl AsKey, + ctx: &ConfigContext, + available_keys: &[EnvelopeKey], + ) -> super::Result; + fn parse_conditions( + &self, + ctx: &ConfigContext, + ) -> super::Result>; +} + +impl ConfigCondition for Config { + fn parse_condition( + &self, + key_: impl AsKey, + ctx: &ConfigContext, + available_keys: &[EnvelopeKey], + ) -> super::Result { + let mut conditions = Vec::new(); + let mut stack = Vec::new(); + let mut iter = None; + let mut jmp_pos = Vec::new(); + let mut prefix = key_.as_key(); + let mut is_all = false; + let mut is_not = false; + + 'outer: loop { + let mut op_str = ""; + + for key in self.sub_keys(prefix.as_str()) { + if !["if", "then"].contains(&key) { + if op_str.is_empty() { + op_str = key; + } else { + return Err(format!( + "Multiple operations found for condition {prefix:?}.", + )); + } + } + } + + if op_str.is_empty() { + return Err(format!("Missing operation for condition {prefix:?}.")); + } else if ["any-of", "all-of", "none-of"].contains(&op_str) { + stack.push(( + std::mem::replace( + &mut iter, + self.sub_keys((&prefix, op_str).as_key()).peekable().into(), + ), + (&prefix, op_str).as_key(), + std::mem::take(&mut jmp_pos), + is_all, + is_not, + )); + + match op_str { + "any-of" => { + if !is_not { + is_all = false; + is_not = false; + } else { + is_all = true; + is_not = true; + } + } + "all-of" => { + if !is_not { + is_all = true; + is_not = false; + } else { + is_all = false; + is_not = true; + } + } + _ => { + is_not = !is_not; + if !is_not { + is_all = true; + is_not = false; + } else { + is_all = false; + is_not = true; + } + } + } + } else { + let key = self.property_require::((&prefix, "if"))?; + if !available_keys.contains(&key) { + return Err(format!( + "Envelope key {key:?} is not available in this context for property {prefix:?}", + )); + } + + enum MatchType { + Equal, + Regex, + Lookup, + StartsWith, + EndsWith, + } + + let (op, op_is_not) = match op_str { + "eq" | "equal-to" | "ne" | "not-equal-to" => { + (MatchType::Equal, op_str == "ne" || op_str == "not-equal-to") + } + "in-list" | "not-in-list" => (MatchType::Lookup, op_str == "not-in-list"), + "matches" | "not-matches" => (MatchType::Regex, op_str.starts_with("not-")), + "starts-with" | "not-starts-with" => { + (MatchType::StartsWith, op_str == "not-starts-with") + } + "ends-with" | "not-ends-with" => { + (MatchType::EndsWith, op_str == "not-ends-with") + } + _ => { + return Err(format!("Invalid operation {op_str:?} for key {prefix:?}.")); + } + }; + + let value_str = self.value_require((&prefix, op_str))?; + let value = match (key, &op) { + (EnvelopeKey::Listener, MatchType::Equal) => ConditionMatch::UInt( + ctx.servers + .iter() + .find_map(|s| { + if s.id == value_str { + s.internal_id.into() + } else { + None + } + }) + .ok_or_else(|| { + format!( + "Listener {:?} does not exist for property {:?}.", + value_str, + (&prefix, op_str).as_key() + ) + })?, + ), + (EnvelopeKey::LocalIp | EnvelopeKey::RemoteIp, MatchType::Equal) => { + ConditionMatch::IpAddrMask(value_str.parse_key((&prefix, op_str))?) + } + (EnvelopeKey::Priority, MatchType::Equal) => { + ConditionMatch::Int(value_str.parse_key((&prefix, op_str))?) + } + ( + EnvelopeKey::Recipient + | EnvelopeKey::RecipientDomain + | EnvelopeKey::Sender + | EnvelopeKey::SenderDomain + | EnvelopeKey::AuthenticatedAs + | EnvelopeKey::Mx + | EnvelopeKey::LocalIp + | EnvelopeKey::RemoteIp, + _, + ) => match op { + MatchType::Equal => { + ConditionMatch::String(StringMatch::Equal(value_str.to_string())) + } + MatchType::StartsWith => { + ConditionMatch::String(StringMatch::StartsWith(value_str.to_string())) + } + MatchType::EndsWith => { + ConditionMatch::String(StringMatch::EndsWith(value_str.to_string())) + } + MatchType::Regex => { + ConditionMatch::Regex(Regex::new(value_str).map_err(|err| { + format!( + "Failed to compile regular expression {:?} for key {:?}: {}.", + value_str, + (&prefix, value_str).as_key(), + err + ) + })?) + } + MatchType::Lookup => { + if let Some(list) = ctx.lookup.get(value_str) { + ConditionMatch::Lookup(list.clone()) + } else { + return Err(format!( + "Lookup {:?} not found for property {:?}.", + value_str, + (&prefix, value_str).as_key() + )); + } + } + }, + _ => { + return Err(format!( + "Invalid 'op'/'value' combination for key {:?}.", + key_.as_key() + )); + } + }; + conditions.push(Condition::Match { + key, + value, + not: is_not ^ op_is_not, + }); + if iter.as_mut().map_or(false, |it| it.peek().is_some()) { + jmp_pos.push(conditions.len()); + conditions.push(if is_all { + Condition::JumpIfFalse { + positions: usize::MAX, + } + } else { + Condition::JumpIfTrue { + positions: usize::MAX, + } + }); + } + } + + loop { + if let Some(array_pos) = iter.as_mut().and_then(|it| it.next()) { + prefix = (stack.last().unwrap().1.as_str(), array_pos).as_key(); + break; + } else if let Some((prev_iter, _, prev_jmp_pos, prev_is_all, prev_is_not)) = + stack.pop() + { + let cur_pos = conditions.len() - 1; + for pos in jmp_pos { + if let Condition::JumpIfFalse { positions } + | Condition::JumpIfTrue { positions } = &mut conditions[pos] + { + *positions = cur_pos - pos; + } + } + + iter = prev_iter; + jmp_pos = prev_jmp_pos; + is_all = prev_is_all; + is_not = prev_is_not; + } else { + break 'outer; + } + } + } + + Ok(Conditions { conditions }) + } + + #[cfg(feature = "test_mode")] + fn parse_conditions( + &self, + ctx: &ConfigContext, + ) -> super::Result> { + use ahash::AHashMap; + let mut conditions = AHashMap::new(); + let available_keys = vec![ + EnvelopeKey::Recipient, + EnvelopeKey::RecipientDomain, + EnvelopeKey::Sender, + EnvelopeKey::SenderDomain, + EnvelopeKey::AuthenticatedAs, + EnvelopeKey::Listener, + EnvelopeKey::RemoteIp, + EnvelopeKey::LocalIp, + EnvelopeKey::Priority, + EnvelopeKey::Mx, + ]; + + for rule_name in self.sub_keys("rule") { + conditions.insert( + rule_name.to_string(), + self.parse_condition(("rule", rule_name), ctx, &available_keys)?, + ); + } + + Ok(conditions) + } +} + +impl ParseValue for IpAddrMask { + fn parse_value(key: impl AsKey, value: &str) -> super::Result { + if let Some((addr, mask)) = value.rsplit_once('/') { + if let (Ok(addr), Ok(mask)) = + (addr.trim().parse::(), mask.trim().parse::()) + { + match addr { + IpAddr::V4(addr) if (8..=32).contains(&mask) => { + return Ok(IpAddrMask::V4 { + addr, + mask: u32::MAX << (32 - mask), + }) + } + IpAddr::V6(addr) if (8..=128).contains(&mask) => { + return Ok(IpAddrMask::V6 { + addr, + mask: u128::MAX << (128 - mask), + }) + } + _ => (), + } + } + } else { + match value.trim().parse::() { + Ok(IpAddr::V4(addr)) => { + return Ok(IpAddrMask::V4 { + addr, + mask: u32::MAX, + }) + } + Ok(IpAddr::V6(addr)) => { + return Ok(IpAddrMask::V6 { + addr, + mask: u128::MAX, + }) + } + _ => (), + } + } + + Err(format!( + "Invalid IP address {:?} for property {:?}.", + value, + key.as_key() + )) + } +} + +#[cfg(test)] +mod tests { + use std::{fs, path::PathBuf, sync::Arc}; + + use ahash::AHashMap; + use utils::config::{Config, Server}; + + use crate::{ + config::{ + Condition, ConditionMatch, Conditions, ConfigContext, EnvelopeKey, IpAddrMask, + StringMatch, + }, + lookup::Lookup, + }; + + use super::ConfigCondition; + + #[test] + fn parse_conditions() { + let mut file = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + file.push("resources"); + file.push("smtp"); + file.push("config"); + file.push("rules.toml"); + + let config = Config::parse(&fs::read_to_string(file).unwrap()).unwrap(); + let mut context = ConfigContext::default(); + let list = Arc::new(Lookup::default()); + context.lookup.insert("test-list".to_string(), list.clone()); + context.servers.push(Server { + id: "smtp".to_string(), + internal_id: 123, + ..Default::default() + }); + let mut conditions = config.parse_conditions(&context).unwrap(); + let expected_rules = AHashMap::from_iter([ + ( + "simple".to_string(), + Conditions { + conditions: vec![Condition::Match { + key: EnvelopeKey::Listener, + value: ConditionMatch::UInt(123), + not: false, + }], + }, + ), + ( + "is-authenticated".to_string(), + Conditions { + conditions: vec![Condition::Match { + key: EnvelopeKey::AuthenticatedAs, + value: ConditionMatch::String(StringMatch::Equal("".to_string())), + not: true, + }], + }, + ), + ( + "expanded".to_string(), + Conditions { + conditions: vec![ + Condition::Match { + key: EnvelopeKey::SenderDomain, + value: ConditionMatch::String(StringMatch::StartsWith( + "example".to_string(), + )), + not: false, + }, + Condition::JumpIfFalse { positions: 1 }, + Condition::Match { + key: EnvelopeKey::Sender, + value: ConditionMatch::Lookup(list), + not: false, + }, + ], + }, + ), + ( + "my-nested-rule".to_string(), + Conditions { + conditions: vec![ + Condition::Match { + key: EnvelopeKey::RecipientDomain, + value: ConditionMatch::String(StringMatch::Equal( + "example.org".to_string(), + )), + not: false, + }, + Condition::JumpIfTrue { positions: 9 }, + Condition::Match { + key: EnvelopeKey::RemoteIp, + value: ConditionMatch::IpAddrMask(IpAddrMask::V4 { + addr: "192.168.0.0".parse().unwrap(), + mask: u32::MAX << (32 - 24), + }), + not: false, + }, + Condition::JumpIfTrue { positions: 7 }, + Condition::Match { + key: EnvelopeKey::Recipient, + value: ConditionMatch::String(StringMatch::StartsWith( + "no-reply@".to_string(), + )), + not: false, + }, + Condition::JumpIfFalse { positions: 5 }, + Condition::Match { + key: EnvelopeKey::Sender, + value: ConditionMatch::String(StringMatch::EndsWith( + "@domain.org".to_string(), + )), + not: false, + }, + Condition::JumpIfFalse { positions: 3 }, + Condition::Match { + key: EnvelopeKey::Priority, + value: ConditionMatch::Int(1), + not: true, + }, + Condition::JumpIfTrue { positions: 1 }, + Condition::Match { + key: EnvelopeKey::Priority, + value: ConditionMatch::Int(-2), + not: false, + }, + ], + }, + ), + ]); + + for (key, rule) in expected_rules { + assert_eq!(Some(rule), conditions.remove(&key), "failed for {key}"); + } + } +} diff --git a/crates/smtp/src/config/database.rs b/crates/smtp/src/config/database.rs new file mode 100644 index 00000000..6142bede --- /dev/null +++ b/crates/smtp/src/config/database.rs @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{sync::Arc, time::Duration}; + +use parking_lot::Mutex; +use sqlx::{mysql::MySqlPoolOptions, postgres::PgPoolOptions, sqlite::SqlitePoolOptions}; + +use crate::lookup::{cache::LookupCache, Lookup, SqlDatabase, SqlQuery}; +use utils::config::{utils::AsKey, Config}; + +use super::ConfigContext; + +pub trait ConfigDatabase { + fn parse_databases(&self, ctx: &mut ConfigContext) -> super::Result<()>; + fn parse_database(&self, id: &str, ctx: &mut ConfigContext) -> super::Result<()>; +} + +impl ConfigDatabase for Config { + fn parse_databases(&self, ctx: &mut ConfigContext) -> super::Result<()> { + for id in self.sub_keys("database") { + self.parse_database(id, ctx)?; + } + + Ok(()) + } + + fn parse_database(&self, id: &str, ctx: &mut ConfigContext) -> super::Result<()> { + let address = self.value_require(("database", id, "address"))?; + let pool = if address.starts_with("postgres:") { + SqlDatabase::Postgres( + PgPoolOptions::new() + .max_connections( + self.property(("database", id, "max-connections"))? + .unwrap_or(10), + ) + .min_connections( + self.property(("database", id, "min-connections"))? + .unwrap_or(0), + ) + .idle_timeout(self.property(("database", id, "idle-timeout"))?) + .connect_lazy(address) + .map_err(|err| { + format!("Failed to create connection pool for {address:?}: {err}") + })?, + ) + } else if address.starts_with("mysql:") { + SqlDatabase::MySql( + MySqlPoolOptions::new() + .max_connections( + self.property(("database", id, "max-connections"))? + .unwrap_or(10), + ) + .min_connections( + self.property(("database", id, "min-connections"))? + .unwrap_or(0), + ) + .idle_timeout(self.property(("database", id, "idle-timeout"))?) + .connect_lazy(address) + .map_err(|err| { + format!("Failed to create connection pool for {address:?}: {err}") + })?, + ) + } else if address.starts_with("mssql:") { + unimplemented!("MSSQL support is not yet implemented") + /*SqlDatabase::MsSql( + MssqlPoolOptions::new() + .max_connections( + self.property(("database", id, "max-connections"))? + .unwrap_or(10), + ) + .min_connections( + self.property(("database", id, "min-connections"))? + .unwrap_or(0), + ) + .idle_timeout(self.property(("database", id, "idle-timeout"))?) + .connect_lazy(address) + .map_err(|err| { + format!("Failed to create connection pool for {address:?}: {err}") + })?, + )*/ + } else if address.starts_with("sqlite:") { + SqlDatabase::SqlLite( + SqlitePoolOptions::new() + .max_connections( + self.property(("database", id, "max-connections"))? + .unwrap_or(10), + ) + .min_connections( + self.property(("database", id, "min-connections"))? + .unwrap_or(0), + ) + .idle_timeout(self.property(("database", id, "idle-timeout"))?) + .connect_lazy(address) + .map_err(|err| { + format!("Failed to create connection pool for {address:?}: {err}") + })?, + ) + } else { + return Err(format!( + "Invalid database address {:?} for key {:?}", + address, + ("database", id, "address").as_key() + )); + }; + + // Add database + ctx.databases.insert(id.to_string(), pool.clone()); + + // Parse cache + let cache_entries = self + .property(("database", id, "cache.entries"))? + .unwrap_or(1024); + let cache_ttl_positive = self + .property(("database", id, "cache.ttl.positive"))? + .unwrap_or(Duration::from_secs(86400)); + let cache_ttl_negative = self + .property(("database", id, "cache.ttl.positive"))? + .unwrap_or(Duration::from_secs(3600)); + let cache_enable = self + .values(("database", id, "cache.enable")) + .map(|(_, v)| v) + .collect::>(); + + // Parse lookups + for lookup_id in self.sub_keys(("database", id, "lookup")) { + ctx.lookup.insert( + format!("db/{id}/{lookup_id}"), + Arc::new(Lookup::Sql(SqlQuery { + query: self + .value_require(("database", id, "lookup", lookup_id))? + .to_string(), + db: pool.clone(), + cache: if cache_enable.contains(&lookup_id) { + Mutex::new(LookupCache::new( + cache_entries, + cache_ttl_positive, + cache_ttl_negative, + )) + .into() + } else { + None + }, + })), + ); + } + + Ok(()) + } +} diff --git a/crates/smtp/src/config/if_block.rs b/crates/smtp/src/config/if_block.rs new file mode 100644 index 00000000..650fcc92 --- /dev/null +++ b/crates/smtp/src/config/if_block.rs @@ -0,0 +1,468 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::sync::Arc; + +use ahash::AHashMap; + +use super::{condition::ConfigCondition, ConfigContext, EnvelopeKey, IfBlock, IfThen}; +use utils::config::{ + utils::{AsKey, ParseValues}, + Config, +}; + +pub trait ConfigIf { + fn parse_if_block( + &self, + prefix: impl AsKey, + ctx: &ConfigContext, + available_keys: &[EnvelopeKey], + ) -> super::Result>>; +} + +impl ConfigIf for Config { + fn parse_if_block( + &self, + prefix: impl AsKey, + ctx: &ConfigContext, + available_keys: &[EnvelopeKey], + ) -> super::Result>> { + let key = prefix.as_key(); + let prefix = prefix.as_prefix(); + + let mut found_if = false; + let mut found_else = ""; + let mut found_then = false; + + // Parse conditions + let mut if_block = IfBlock::new(T::default()); + let mut last_array_pos = ""; + + for item in self.keys.keys() { + if let Some(suffix_) = item.strip_prefix(&prefix) { + if let Some((array_pos, suffix)) = suffix_.split_once('.') { + let if_key = suffix.split_once('.').map(|(v, _)| v).unwrap_or(suffix); + if ["if", "any-of", "all-of", "none-of"].contains(&if_key) { + if array_pos != last_array_pos { + if !last_array_pos.is_empty() && !found_then && !T::is_multivalue() { + return Err(format!( + "Missing 'then' in 'if' condition {} for property {:?}.", + last_array_pos.parse().unwrap_or(0) + 1, + key + )); + } + + if_block.if_then.push(IfThen { + conditions: self.parse_condition( + (key.as_str(), array_pos), + ctx, + available_keys, + )?, + then: T::default(), + }); + + found_then = false; + last_array_pos = array_pos; + } + + found_if = true; + } else if if_key == "else" { + if found_else.is_empty() { + if found_if { + if_block.default = T::parse_values( + (key.as_str(), suffix_.split_once(".else").unwrap().0, "else"), + self, + )?; + found_else = array_pos; + } else { + return Err(format!( + "Found 'else' before 'if' for property {key:?}.", + )); + } + } else if array_pos != found_else { + return Err(format!("Multiple 'else' found for property {key:?}.")); + } + } else if if_key == "then" { + if found_else.is_empty() { + if array_pos == last_array_pos { + if !found_then { + if_block.if_then.last_mut().unwrap().then = T::parse_values( + ( + key.as_str(), + suffix_.split_once(".then").unwrap().0, + "then", + ), + self, + )?; + found_then = true; + } + } else { + return Err(format!( + "Found 'then' without 'if' for property {key:?}.", + )); + } + } else { + return Err(format!( + "Found 'then' in 'else' block for property {key:?}.", + )); + } + } + } else if !found_if { + // Found probably a multi-value, parse and return + if_block.default = T::parse_values(key.as_str(), self)?; + return Ok(Some(if_block)); + } else { + return Err(format!("Invalid property {item:?} found in 'if' block.")); + } + } else if item == &key { + // There is a single value, parse and return + if_block.default = T::parse_values(key.as_str(), self)?; + return Ok(Some(if_block)); + } + } + + if !found_if { + Ok(None) + } else if !found_then && !T::is_multivalue() { + Err(format!( + "Missing 'then' in 'if' condition {} for property {:?}.", + last_array_pos.parse().unwrap_or(0) + 1, + key + )) + } else if found_else.is_empty() && !T::is_multivalue() { + Err(format!("Missing 'else' for property {key:?}.")) + } else { + Ok(Some(if_block)) + } + } +} + +impl IfBlock { + pub fn new(value: T) -> Self { + Self { + if_then: Vec::with_capacity(0), + default: value, + } + } +} + +impl IfBlock> { + pub fn try_unwrap(self, key: &str) -> super::Result> { + let mut if_then = Vec::with_capacity(self.if_then.len()); + for if_clause in self.if_then { + if_then.push(IfThen { + conditions: if_clause.conditions, + then: if_clause + .then + .ok_or_else(|| format!("Property {key:?} cannot contain null values."))?, + }); + } + + Ok(IfBlock { + if_then, + default: self + .default + .ok_or_else(|| format!("Property {key:?} cannot contain null values."))?, + }) + } +} + +impl IfBlock> { + pub fn map_if_block( + self, + map: &AHashMap>, + key_name: impl AsKey, + object_name: &str, + ) -> super::Result>>> { + let key_name = key_name.as_key(); + let mut if_then = Vec::with_capacity(self.if_then.len()); + for if_clause in self.if_then.into_iter() { + if_then.push(IfThen { + conditions: if_clause.conditions, + then: Self::map_value(map, if_clause.then, object_name, &key_name)?, + }); + } + + Ok(IfBlock { + if_then, + default: Self::map_value(map, self.default, object_name, &key_name)?, + }) + } + + fn map_value( + map: &AHashMap>, + value: Option, + object_name: &str, + key_name: &str, + ) -> super::Result>> { + if let Some(value) = value { + if let Some(value) = map.get(&value) { + Ok(Some(value.clone())) + } else { + Err(format!( + "Unable to find {object_name} {value:?} declared for {key_name:?}", + )) + } + } else { + Ok(None) + } + } +} + +impl IfBlock> { + pub fn map_if_block( + self, + map: &AHashMap>, + key_name: &str, + object_name: &str, + ) -> super::Result>>> { + let mut if_then = Vec::with_capacity(self.if_then.len()); + for if_clause in self.if_then.into_iter() { + if_then.push(IfThen { + conditions: if_clause.conditions, + then: Self::map_value(map, if_clause.then, object_name, key_name)?, + }); + } + + Ok(IfBlock { + if_then, + default: Self::map_value(map, self.default, object_name, key_name)?, + }) + } + + fn map_value( + map: &AHashMap>, + values: Vec, + object_name: &str, + key_name: &str, + ) -> super::Result>> { + let mut result = Vec::with_capacity(values.len()); + for value in values { + if let Some(value) = map.get(&value) { + result.push(value.clone()); + } else { + return Err(format!( + "Unable to find {object_name} {value:?} declared for {key_name:?}", + )); + } + } + Ok(result) + } +} + +impl IfBlock> { + pub fn has_empty_list(&self) -> bool { + self.default.is_empty() || self.if_then.iter().any(|v| v.then.is_empty()) + } +} + +#[cfg(test)] +mod tests { + use std::{fs, path::PathBuf, time::Duration}; + + use utils::config::Config; + + use crate::config::{ + if_block::ConfigIf, Condition, ConditionMatch, Conditions, ConfigContext, EnvelopeKey, + IfBlock, IfThen, StringMatch, + }; + + #[test] + fn parse_if_blocks() { + let mut file = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + file.push("resources"); + file.push("smtp"); + file.push("config"); + file.push("if-blocks.toml"); + + let config = Config::parse(&fs::read_to_string(file).unwrap()).unwrap(); + + // Create context and add some conditions + let context = ConfigContext::default(); + let available_keys = vec![ + EnvelopeKey::Recipient, + EnvelopeKey::RecipientDomain, + EnvelopeKey::Sender, + EnvelopeKey::SenderDomain, + EnvelopeKey::AuthenticatedAs, + EnvelopeKey::Listener, + EnvelopeKey::RemoteIp, + EnvelopeKey::LocalIp, + EnvelopeKey::Priority, + ]; + + assert_eq!( + config + .parse_if_block::>("durations", &context, &available_keys) + .unwrap() + .unwrap(), + IfBlock { + if_then: vec![ + IfThen { + conditions: Conditions { + conditions: vec![Condition::Match { + key: EnvelopeKey::Sender, + value: ConditionMatch::String(StringMatch::Equal( + "jdoe".to_string() + )), + not: false + }] + }, + then: Duration::from_secs(5 * 86400).into() + }, + IfThen { + conditions: Conditions { + conditions: vec![ + Condition::Match { + key: EnvelopeKey::Priority, + value: ConditionMatch::Int(-1), + not: false + }, + Condition::JumpIfTrue { positions: 1 }, + Condition::Match { + key: EnvelopeKey::Recipient, + value: ConditionMatch::String(StringMatch::StartsWith( + "jane".to_string() + )), + not: false + } + ] + }, + then: Duration::from_secs(3600).into() + } + ], + default: None + } + ); + + assert_eq!( + config + .parse_if_block::>("string-list", &context, &available_keys) + .unwrap() + .unwrap(), + IfBlock { + if_then: vec![ + IfThen { + conditions: Conditions { + conditions: vec![Condition::Match { + key: EnvelopeKey::Sender, + value: ConditionMatch::String(StringMatch::Equal( + "jdoe".to_string() + )), + not: false + }] + }, + then: vec!["From".to_string(), "To".to_string(), "Date".to_string()] + }, + IfThen { + conditions: Conditions { + conditions: vec![ + Condition::Match { + key: EnvelopeKey::Priority, + value: ConditionMatch::Int(-1), + not: false + }, + Condition::JumpIfTrue { positions: 1 }, + Condition::Match { + key: EnvelopeKey::Recipient, + value: ConditionMatch::String(StringMatch::StartsWith( + "jane".to_string() + )), + not: false + } + ] + }, + then: vec!["Other-ID".to_string()] + } + ], + default: vec![] + } + ); + + assert_eq!( + config + .parse_if_block::>("string-list-bis", &context, &available_keys) + .unwrap() + .unwrap(), + IfBlock { + if_then: vec![ + IfThen { + conditions: Conditions { + conditions: vec![Condition::Match { + key: EnvelopeKey::Sender, + value: ConditionMatch::String(StringMatch::Equal( + "jdoe".to_string() + )), + not: false + }] + }, + then: vec!["From".to_string(), "To".to_string(), "Date".to_string()] + }, + IfThen { + conditions: Conditions { + conditions: vec![ + Condition::Match { + key: EnvelopeKey::Priority, + value: ConditionMatch::Int(-1), + not: false + }, + Condition::JumpIfTrue { positions: 1 }, + Condition::Match { + key: EnvelopeKey::Recipient, + value: ConditionMatch::String(StringMatch::StartsWith( + "jane".to_string() + )), + not: false + } + ] + }, + then: vec![] + } + ], + default: vec!["ID-Bis".to_string()] + } + ); + + assert_eq!( + config + .parse_if_block::("single-value", &context, &available_keys) + .unwrap() + .unwrap(), + IfBlock { + if_then: vec![], + default: "hello world".to_string() + } + ); + + for bad_rule in [ + "bad-multi-value", + "bad-if-without-then", + "bad-if-without-else", + "bad-multiple-else", + ] { + if let Ok(value) = config.parse_if_block::(bad_rule, &context, &available_keys) { + panic!("Condition {bad_rule:?} had unexpected result {value:?}"); + } + } + } +} diff --git a/crates/smtp/src/config/list.rs b/crates/smtp/src/config/list.rs new file mode 100644 index 00000000..4940dfc4 --- /dev/null +++ b/crates/smtp/src/config/list.rs @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{ + fs::File, + io::{BufRead, BufReader}, + sync::Arc, +}; + +use ahash::AHashSet; +use utils::config::Config; + +use crate::lookup::Lookup; + +use super::ConfigContext; + +pub trait ConfigList { + fn parse_lists(&self, ctx: &mut ConfigContext) -> super::Result<()>; + fn parse_list(&self, id: &str) -> super::Result; +} + +impl ConfigList for Config { + fn parse_lists(&self, ctx: &mut ConfigContext) -> super::Result<()> { + for id in self.sub_keys("list") { + ctx.lookup + .insert(format!("list/{id}"), Arc::new(self.parse_list(id)?)); + } + + Ok(()) + } + + fn parse_list(&self, id: &str) -> super::Result { + let mut entries = AHashSet::new(); + for (_, value) in self.values(("list", id)) { + if let Some(path) = value.strip_prefix("file://") { + for line in BufReader::new(File::open(path).map_err(|err| { + format!("Failed to read file {path:?} for list {id:?}: {err}") + })?) + .lines() + { + let line_ = line.map_err(|err| { + format!("Failed to read file {path:?} for list {id:?}: {err}") + })?; + let line = line_.trim(); + if !line.is_empty() { + entries.insert(line.to_string()); + } + } + } else { + entries.insert(value.to_string()); + } + } + Ok(Lookup::Local(entries)) + } +} + +#[cfg(test)] +mod tests { + use std::{fs, path::PathBuf, sync::Arc}; + + use ahash::{AHashMap, AHashSet}; + use utils::config::Config; + + use crate::{ + config::{remote::ConfigHost, ConfigContext}, + lookup::Lookup, + }; + + use super::ConfigList; + + #[test] + fn parse_lists() { + let mut file = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + file.push("resources"); + file.push("smtp"); + file.push("config"); + file.push("lists.toml"); + + let mut list_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + list_path.push("resources"); + list_path.push("smtp"); + list_path.push("lists"); + let mut list1 = list_path.clone(); + list1.push("test-list1.txt"); + let mut list2 = list_path.clone(); + list2.push("test-list2.txt"); + + let toml = fs::read_to_string(file) + .unwrap() + .replace("{LIST1}", list1.as_path().to_str().unwrap()) + .replace("{LIST2}", list2.as_path().to_str().unwrap()); + + let config = Config::parse(&toml).unwrap(); + let mut context = ConfigContext::default(); + config.parse_remote_hosts(&mut context).unwrap(); + config.parse_lists(&mut context).unwrap(); + + let mut expected_lists = AHashMap::from_iter([ + ( + "list/local-domains".to_string(), + Arc::new(Lookup::Local(AHashSet::from_iter([ + "example.org".to_string(), + "example.net".to_string(), + ]))), + ), + ( + "list/spammer-domains".to_string(), + Arc::new(Lookup::Local(AHashSet::from_iter([ + "thatdomain.net".to_string() + ]))), + ), + ( + "list/local-users".to_string(), + Arc::new(Lookup::Local(AHashSet::from_iter([ + "user1@domain.org".to_string(), + "user2@domain.org".to_string(), + ]))), + ), + ( + "list/power-users".to_string(), + Arc::new(Lookup::Local(AHashSet::from_iter([ + "user1@domain.org".to_string(), + "user2@domain.org".to_string(), + "user3@example.net".to_string(), + "user4@example.net".to_string(), + "user5@example.net".to_string(), + ]))), + ), + ( + "remote/lmtp".to_string(), + context.lookup.get("remote/lmtp").unwrap().clone(), + ), + ]); + + for (key, list) in context.lookup { + assert_eq!(Some(list), expected_lists.remove(&key), "failed for {key}"); + } + } +} diff --git a/crates/smtp/src/config/mod.rs b/crates/smtp/src/config/mod.rs new file mode 100644 index 00000000..0c77d9f5 --- /dev/null +++ b/crates/smtp/src/config/mod.rs @@ -0,0 +1,540 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +pub mod auth; +pub mod condition; +pub mod database; +pub mod if_block; +pub mod list; +pub mod queue; +pub mod remote; +pub mod report; +pub mod resolver; +pub mod scripts; +pub mod session; +pub mod throttle; + +use std::{ + net::{Ipv4Addr, Ipv6Addr}, + path::PathBuf, + sync::{atomic::AtomicU64, Arc}, + time::Duration, +}; + +use ahash::AHashMap; +use mail_auth::{ + common::crypto::{Ed25519Key, RsaKey, Sha256}, + dkim::{Canonicalization, Done}, + IpLookupStrategy, +}; +use mail_send::Credentials; +use regex::Regex; +use sieve::Sieve; +use smtp_proto::MtPriority; +use tokio::sync::mpsc; +use utils::config::{Server, ServerProtocol}; + +use crate::lookup::{self, Lookup, SqlDatabase}; + +#[derive(Debug)] +pub struct Host { + pub address: String, + pub port: u16, + pub protocol: ServerProtocol, + pub concurrency: usize, + pub timeout: Duration, + pub tls_implicit: bool, + pub tls_allow_invalid_certs: bool, + pub username: Option, + pub secret: Option, + pub max_errors: usize, + pub max_requests: usize, + pub cache_entries: usize, + pub cache_ttl_positive: Duration, + pub cache_ttl_negative: Duration, + pub channel_tx: mpsc::Sender, + pub channel_rx: mpsc::Receiver, + pub lookup: bool, +} + +#[derive(Debug, Clone)] +#[cfg_attr(feature = "test_mode", derive(PartialEq, Eq))] +pub enum Condition { + Match { + key: EnvelopeKey, + value: ConditionMatch, + not: bool, + }, + JumpIfTrue { + positions: usize, + }, + JumpIfFalse { + positions: usize, + }, +} + +#[derive(Debug, PartialEq, Eq, Clone)] +pub enum StringMatch { + Equal(String), + StartsWith(String), + EndsWith(String), +} + +#[derive(Debug, Clone)] +pub enum ConditionMatch { + String(StringMatch), + UInt(u16), + Int(i16), + IpAddrMask(IpAddrMask), + Lookup(Arc), + Regex(Regex), +} + +#[cfg(feature = "test_mode")] +impl PartialEq for ConditionMatch { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Self::String(l0), Self::String(r0)) => l0 == r0, + (Self::UInt(l0), Self::UInt(r0)) => l0 == r0, + (Self::Int(l0), Self::Int(r0)) => l0 == r0, + (Self::IpAddrMask(l0), Self::IpAddrMask(r0)) => l0 == r0, + (Self::Lookup(l0), Self::Lookup(r0)) => l0 == r0, + (Self::Regex(_), Self::Regex(_)) => false, + _ => false, + } + } +} + +#[cfg(feature = "test_mode")] +impl Eq for ConditionMatch {} + +#[cfg(feature = "test_mode")] +impl PartialEq for Lookup { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (Self::Local(l0), Self::Local(r0)) => l0 == r0, + (Self::Remote(_), Self::Remote(_)) => true, + _ => false, + } + } +} + +impl Default for Condition { + fn default() -> Self { + Condition::JumpIfFalse { positions: 0 } + } +} + +#[derive(Debug, PartialEq, Eq, Clone, Copy)] +pub enum EnvelopeKey { + Recipient, + RecipientDomain, + Sender, + SenderDomain, + Mx, + HeloDomain, + AuthenticatedAs, + Listener, + RemoteIp, + LocalIp, + Priority, +} + +#[derive(Debug, Clone, Default)] +#[cfg_attr(feature = "test_mode", derive(PartialEq, Eq))] +pub struct IfThen { + pub conditions: Conditions, + pub then: T, +} + +#[derive(Debug, Clone, Default)] +#[cfg_attr(feature = "test_mode", derive(PartialEq, Eq))] +pub struct Conditions { + pub conditions: Vec, +} + +#[derive(Debug, Clone, Default)] +#[cfg_attr(feature = "test_mode", derive(PartialEq, Eq))] +pub struct IfBlock { + pub if_then: Vec>, + pub default: T, +} + +#[derive(Debug, Default)] +#[cfg_attr(feature = "test_mode", derive(PartialEq, Eq))] +pub struct Throttle { + pub conditions: Conditions, + pub keys: u16, + pub concurrency: Option, + pub rate: Option, +} + +pub const THROTTLE_RCPT: u16 = 1 << 0; +pub const THROTTLE_RCPT_DOMAIN: u16 = 1 << 1; +pub const THROTTLE_SENDER: u16 = 1 << 2; +pub const THROTTLE_SENDER_DOMAIN: u16 = 1 << 3; +pub const THROTTLE_AUTH_AS: u16 = 1 << 4; +pub const THROTTLE_LISTENER: u16 = 1 << 5; +pub const THROTTLE_MX: u16 = 1 << 6; +pub const THROTTLE_REMOTE_IP: u16 = 1 << 7; +pub const THROTTLE_LOCAL_IP: u16 = 1 << 8; +pub const THROTTLE_HELO_DOMAIN: u16 = 1 << 9; + +#[derive(Debug, Default, PartialEq, Eq, Clone)] +pub struct Rate { + pub requests: u64, + pub period: Duration, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum IpAddrMask { + V4 { addr: Ipv4Addr, mask: u32 }, + V6 { addr: Ipv6Addr, mask: u128 }, +} + +pub struct Connect { + pub script: IfBlock>>, +} + +pub struct Ehlo { + pub script: IfBlock>>, + pub require: IfBlock, + pub reject_non_fqdn: IfBlock, +} + +pub struct Extensions { + pub pipelining: IfBlock, + pub chunking: IfBlock, + pub requiretls: IfBlock, + pub dsn: IfBlock, + pub no_soliciting: IfBlock>, + pub future_release: IfBlock>, + pub deliver_by: IfBlock>, + pub mt_priority: IfBlock>, +} + +pub struct Auth { + pub lookup: IfBlock>>, + pub mechanisms: IfBlock, + pub require: IfBlock, + pub errors_max: IfBlock, + pub errors_wait: IfBlock, +} + +pub struct Mail { + pub script: IfBlock>>, +} + +pub struct Rcpt { + pub script: IfBlock>>, + pub relay: IfBlock, + pub lookup_domains: IfBlock>>, + pub lookup_addresses: IfBlock>>, + pub lookup_expn: IfBlock>>, + pub lookup_vrfy: IfBlock>>, + + // Errors + pub errors_max: IfBlock, + pub errors_wait: IfBlock, + + // Limits + pub max_recipients: IfBlock, +} + +pub struct Data { + pub script: IfBlock>>, + pub pipe_commands: Vec, + + // Limits + pub max_messages: IfBlock, + pub max_message_size: IfBlock, + pub max_received_headers: IfBlock, + + // Headers + pub add_received: IfBlock, + pub add_received_spf: IfBlock, + pub add_return_path: IfBlock, + pub add_auth_results: IfBlock, + pub add_message_id: IfBlock, + pub add_date: IfBlock, +} + +pub struct Pipe { + pub command: IfBlock>, + pub arguments: IfBlock>, + pub timeout: IfBlock, +} + +pub struct SessionConfig { + pub timeout: IfBlock, + pub duration: IfBlock, + pub transfer_limit: IfBlock, + pub throttle: SessionThrottle, + + pub connect: Connect, + pub ehlo: Ehlo, + pub auth: Auth, + pub mail: Mail, + pub rcpt: Rcpt, + pub data: Data, + pub extensions: Extensions, +} + +pub struct SessionThrottle { + pub connect: Vec, + pub mail_from: Vec, + pub rcpt_to: Vec, +} + +pub struct RelayHost { + pub address: String, + pub port: u16, + pub protocol: ServerProtocol, + pub auth: Option>, + pub tls_implicit: bool, + pub tls_allow_invalid_certs: bool, +} + +pub struct QueueConfig { + pub path: IfBlock, + pub hash: IfBlock, + + // Schedule + pub retry: IfBlock>, + pub notify: IfBlock>, + pub expire: IfBlock, + + // Outbound + pub hostname: IfBlock, + pub next_hop: IfBlock>, + pub max_mx: IfBlock, + pub max_multihomed: IfBlock, + pub ip_strategy: IfBlock, + pub source_ip: QueueOutboundSourceIp, + pub tls: QueueOutboundTls, + pub dsn: Dsn, + + // Timeouts + pub timeout: QueueOutboundTimeout, + + // Throttle and Quotas + pub throttle: QueueThrottle, + pub quota: QueueQuotas, + pub management_lookup: Arc, +} + +pub struct QueueOutboundSourceIp { + pub ipv4: IfBlock>, + pub ipv6: IfBlock>, +} + +pub struct ReportConfig { + pub path: IfBlock, + pub hash: IfBlock, + pub submitter: IfBlock, + pub analysis: ReportAnalysis, + + pub dkim: Report, + pub spf: Report, + pub dmarc: Report, + pub dmarc_aggregate: AggregateReport, + pub tls: AggregateReport, +} + +pub struct ReportAnalysis { + pub addresses: Vec, + pub forward: bool, + pub store: Option, + pub report_id: AtomicU64, +} + +pub enum AddressMatch { + StartsWith(String), + EndsWith(String), + Equals(String), +} + +pub struct Dsn { + pub name: IfBlock, + pub address: IfBlock, + pub sign: IfBlock>>, +} + +pub struct AggregateReport { + pub name: IfBlock, + pub address: IfBlock, + pub org_name: IfBlock>, + pub contact_info: IfBlock>, + pub send: IfBlock, + pub sign: IfBlock>>, + pub max_size: IfBlock, +} + +pub struct Report { + pub name: IfBlock, + pub address: IfBlock, + pub subject: IfBlock, + pub sign: IfBlock>>, + pub send: IfBlock>, +} + +pub struct QueueOutboundTls { + pub dane: IfBlock, + pub mta_sts: IfBlock, + pub start: IfBlock, +} + +pub struct QueueOutboundTimeout { + pub connect: IfBlock, + pub greeting: IfBlock, + pub tls: IfBlock, + pub ehlo: IfBlock, + pub mail: IfBlock, + pub rcpt: IfBlock, + pub data: IfBlock, + pub mta_sts: IfBlock, +} + +#[derive(Debug)] +pub struct QueueThrottle { + pub sender: Vec, + pub rcpt: Vec, + pub host: Vec, +} + +pub struct QueueQuotas { + pub sender: Vec, + pub rcpt: Vec, + pub rcpt_domain: Vec, +} + +pub struct QueueQuota { + pub conditions: Conditions, + pub keys: u16, + pub size: Option, + pub messages: Option, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum AggregateFrequency { + Hourly, + Daily, + Weekly, + #[default] + Never, +} + +#[derive(Debug, Clone, Copy, Default)] +pub struct TlsStrategy { + pub dane: RequireOptional, + pub mta_sts: RequireOptional, + pub tls: RequireOptional, +} + +#[derive(Debug, Clone, Copy, Default)] +pub enum RequireOptional { + #[default] + Optional, + Require, + Disable, +} + +pub struct MailAuthConfig { + pub dkim: DkimAuthConfig, + pub arc: ArcAuthConfig, + pub spf: SpfAuthConfig, + pub dmarc: DmarcAuthConfig, + pub iprev: IpRevAuthConfig, + pub dnsbl: DnsBlConfig, +} + +pub enum DkimSigner { + RsaSha256(mail_auth::dkim::DkimSigner, Done>), + Ed25519Sha256(mail_auth::dkim::DkimSigner), +} + +pub enum ArcSealer { + RsaSha256(mail_auth::arc::ArcSealer, Done>), + Ed25519Sha256(mail_auth::arc::ArcSealer), +} + +pub struct DkimAuthConfig { + pub verify: IfBlock, + pub sign: IfBlock>>, +} + +pub struct ArcAuthConfig { + pub verify: IfBlock, + pub seal: IfBlock>>, +} + +pub struct SpfAuthConfig { + pub verify_ehlo: IfBlock, + pub verify_mail_from: IfBlock, +} +pub struct DmarcAuthConfig { + pub verify: IfBlock, +} + +pub struct IpRevAuthConfig { + pub verify: IfBlock, +} + +pub struct DnsBlConfig { + pub verify: IfBlock, + pub ip_lookup: Vec, + pub domain_lookup: Vec, +} + +pub const DNSBL_IP: u32 = 1; +pub const DNSBL_IPREV: u32 = 1 << 1; +pub const DNSBL_EHLO: u32 = 1 << 2; +pub const DNSBL_RETURN_PATH: u32 = 1 << 3; +pub const DNSBL_FROM: u32 = 1 << 4; + +#[derive(Debug, Clone)] +pub struct DkimCanonicalization { + pub headers: Canonicalization, + pub body: Canonicalization, +} + +#[derive(Debug, Clone, Copy, Default)] +pub enum VerifyStrategy { + #[default] + Relaxed, + Strict, + Disable, +} + +#[derive(Default)] +pub struct ConfigContext { + pub servers: Vec, + pub hosts: AHashMap, + pub scripts: AHashMap>, + pub lookup: AHashMap>, + pub databases: AHashMap, + pub signers: AHashMap>, + pub sealers: AHashMap>, +} + +pub type Result = std::result::Result; diff --git a/crates/smtp/src/config/queue.rs b/crates/smtp/src/config/queue.rs new file mode 100644 index 00000000..483c5c93 --- /dev/null +++ b/crates/smtp/src/config/queue.rs @@ -0,0 +1,457 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::time::Duration; + +use mail_send::Credentials; + +use super::{ + condition::ConfigCondition, + if_block::ConfigIf, + throttle::{ConfigThrottle, ParseTrottleKey}, + *, +}; +use utils::config::{ + utils::{AsKey, ParseValue}, + Config, +}; + +pub trait ConfigQueue { + fn parse_queue(&self, ctx: &ConfigContext) -> super::Result; + fn parse_queue_throttle(&self, ctx: &ConfigContext) -> super::Result; + fn parse_queue_quota(&self, ctx: &ConfigContext) -> super::Result; + fn parse_queue_quota_item( + &self, + prefix: impl AsKey, + ctx: &ConfigContext, + ) -> super::Result; +} + +impl ConfigQueue for Config { + fn parse_queue(&self, ctx: &ConfigContext) -> super::Result { + let rcpt_envelope_keys = [ + EnvelopeKey::RecipientDomain, + EnvelopeKey::Sender, + EnvelopeKey::SenderDomain, + EnvelopeKey::Priority, + ]; + let sender_envelope_keys = [ + EnvelopeKey::Sender, + EnvelopeKey::SenderDomain, + EnvelopeKey::Priority, + ]; + let mx_envelope_keys = [ + EnvelopeKey::RecipientDomain, + EnvelopeKey::Sender, + EnvelopeKey::SenderDomain, + EnvelopeKey::Priority, + EnvelopeKey::Mx, + ]; + let host_envelope_keys = [ + EnvelopeKey::RecipientDomain, + EnvelopeKey::Sender, + EnvelopeKey::SenderDomain, + EnvelopeKey::Priority, + EnvelopeKey::LocalIp, + EnvelopeKey::RemoteIp, + EnvelopeKey::Mx, + ]; + + let next_hop = self + .parse_if_block::>("queue.outbound.next-hop", ctx, &rcpt_envelope_keys)? + .unwrap_or_else(|| IfBlock::new(None)); + + let default_hostname = self.value_require("server.hostname")?; + + let config = QueueConfig { + path: self + .parse_if_block("queue.path", ctx, &sender_envelope_keys)? + .ok_or("Missing \"queue.path\" property.")?, + hash: self + .parse_if_block("queue.hash", ctx, &sender_envelope_keys)? + .unwrap_or_else(|| IfBlock::new(32)), + + retry: self + .parse_if_block("queue.schedule.retry", ctx, &host_envelope_keys)? + .unwrap_or_else(|| { + IfBlock::new(vec![ + Duration::from_secs(60), + Duration::from_secs(2 * 60), + Duration::from_secs(5 * 60), + Duration::from_secs(10 * 60), + Duration::from_secs(15 * 60), + Duration::from_secs(30 * 60), + Duration::from_secs(3600), + Duration::from_secs(2 * 3600), + ]) + }), + notify: self + .parse_if_block("queue.schedule.notify", ctx, &rcpt_envelope_keys)? + .unwrap_or_else(|| { + IfBlock::new(vec![ + Duration::from_secs(86400), + Duration::from_secs(3 * 86400), + ]) + }), + expire: self + .parse_if_block("queue.schedule.expire", ctx, &rcpt_envelope_keys)? + .unwrap_or_else(|| IfBlock::new(Duration::from_secs(5 * 86400))), + hostname: self + .parse_if_block("queue.outbound.hostname", ctx, &sender_envelope_keys)? + .unwrap_or_else(|| IfBlock::new(default_hostname.to_string())), + max_mx: self + .parse_if_block("queue.outbound.limits.mx", ctx, &rcpt_envelope_keys)? + .unwrap_or_else(|| IfBlock::new(5)), + max_multihomed: self + .parse_if_block("queue.outbound.limits.multihomed", ctx, &rcpt_envelope_keys)? + .unwrap_or_else(|| IfBlock::new(2)), + ip_strategy: self + .parse_if_block("queue.outbound.ip-strategy", ctx, &sender_envelope_keys)? + .unwrap_or_else(|| IfBlock::new(IpLookupStrategy::Ipv4thenIpv6)), + source_ip: QueueOutboundSourceIp { + ipv4: self + .parse_if_block("queue.outbound.source-ip.v4", ctx, &mx_envelope_keys)? + .unwrap_or_else(|| IfBlock::new(Vec::new())), + ipv6: self + .parse_if_block("queue.outbound.source-ip.v6", ctx, &mx_envelope_keys)? + .unwrap_or_else(|| IfBlock::new(Vec::new())), + }, + next_hop: next_hop.into_relay_host(ctx)?, + tls: QueueOutboundTls { + dane: self + .parse_if_block("queue.outbound.tls.dane", ctx, &mx_envelope_keys)? + .unwrap_or_else(|| IfBlock::new(RequireOptional::Optional)), + mta_sts: self + .parse_if_block("queue.outbound.tls.mta-sts", ctx, &rcpt_envelope_keys)? + .unwrap_or_else(|| IfBlock::new(RequireOptional::Optional)), + start: self + .parse_if_block("queue.outbound.tls.starttls", ctx, &mx_envelope_keys)? + .unwrap_or_else(|| IfBlock::new(RequireOptional::Optional)), + }, + throttle: self.parse_queue_throttle(ctx)?, + quota: self.parse_queue_quota(ctx)?, + timeout: QueueOutboundTimeout { + connect: self + .parse_if_block("queue.outbound.timeouts.connect", ctx, &host_envelope_keys)? + .unwrap_or_else(|| IfBlock::new(Duration::from_secs(5 * 60))), + greeting: self + .parse_if_block("queue.outbound.timeouts.greeting", ctx, &host_envelope_keys)? + .unwrap_or_else(|| IfBlock::new(Duration::from_secs(5 * 60))), + tls: self + .parse_if_block("queue.outbound.timeouts.tls", ctx, &host_envelope_keys)? + .unwrap_or_else(|| IfBlock::new(Duration::from_secs(3 * 60))), + ehlo: self + .parse_if_block("queue.outbound.timeouts.ehlo", ctx, &host_envelope_keys)? + .unwrap_or_else(|| IfBlock::new(Duration::from_secs(5 * 60))), + mail: self + .parse_if_block( + "queue.outbound.timeouts.mail-from", + ctx, + &host_envelope_keys, + )? + .unwrap_or_else(|| IfBlock::new(Duration::from_secs(5 * 60))), + rcpt: self + .parse_if_block("queue.outbound.timeouts.rcpt-to", ctx, &host_envelope_keys)? + .unwrap_or_else(|| IfBlock::new(Duration::from_secs(5 * 60))), + data: self + .parse_if_block("queue.outbound.timeouts.data", ctx, &host_envelope_keys)? + .unwrap_or_else(|| IfBlock::new(Duration::from_secs(10 * 60))), + mta_sts: self + .parse_if_block("queue.outbound.timeouts.mta-sts", ctx, &rcpt_envelope_keys)? + .unwrap_or_else(|| IfBlock::new(Duration::from_secs(10 * 60))), + }, + dsn: Dsn { + name: self + .parse_if_block("report.dsn.from-name", ctx, &sender_envelope_keys)? + .unwrap_or_else(|| IfBlock::new("Mail Delivery Subsystem".to_string())), + address: self + .parse_if_block("report.dsn.from-address", ctx, &sender_envelope_keys)? + .unwrap_or_else(|| IfBlock::new(format!("MAILER-DAEMON@{default_hostname}"))), + sign: self + .parse_if_block::>("report.dsn.sign", ctx, &sender_envelope_keys)? + .unwrap_or_default() + .map_if_block(&ctx.signers, "report.dsn.sign", "signature")?, + }, + management_lookup: if let Some(lookup) = self.value("management.auth.lookup") { + ctx.lookup + .get(lookup) + .ok_or_else(|| { + format!("Lookup {lookup:?} not found for key \"management.auth.lookup\".") + })? + .clone() + } else { + Arc::new(Lookup::default()) + }, + }; + + if config.retry.has_empty_list() { + Err("Property \"queue.schedule.retry\" cannot contain empty lists.".to_string()) + } else if config.notify.has_empty_list() { + Err("Property \"queue.schedule.notify\" cannot contain empty lists.".to_string()) + } else { + Ok(config) + } + } + + fn parse_queue_throttle(&self, ctx: &ConfigContext) -> super::Result { + // Parse throttle + let mut throttle = QueueThrottle { + sender: Vec::new(), + rcpt: Vec::new(), + host: Vec::new(), + }; + let envelope_keys = [ + EnvelopeKey::RecipientDomain, + EnvelopeKey::Sender, + EnvelopeKey::SenderDomain, + EnvelopeKey::Priority, + EnvelopeKey::Mx, + EnvelopeKey::RemoteIp, + EnvelopeKey::LocalIp, + ]; + let all_throttles = self.parse_throttle( + "queue.throttle", + ctx, + &envelope_keys, + THROTTLE_RCPT_DOMAIN + | THROTTLE_SENDER + | THROTTLE_SENDER_DOMAIN + | THROTTLE_MX + | THROTTLE_REMOTE_IP + | THROTTLE_LOCAL_IP, + )?; + for t in all_throttles { + if (t.keys & (THROTTLE_MX | THROTTLE_REMOTE_IP | THROTTLE_LOCAL_IP)) != 0 + || t.conditions.conditions.iter().any(|c| { + matches!( + c, + Condition::Match { + key: EnvelopeKey::Mx | EnvelopeKey::RemoteIp | EnvelopeKey::LocalIp, + .. + } + ) + }) + { + throttle.host.push(t); + } else if (t.keys & (THROTTLE_RCPT_DOMAIN)) != 0 + || t.conditions.conditions.iter().any(|c| { + matches!( + c, + Condition::Match { + key: EnvelopeKey::RecipientDomain, + .. + } + ) + }) + { + throttle.rcpt.push(t); + } else { + throttle.sender.push(t); + } + } + + Ok(throttle) + } + + fn parse_queue_quota(&self, ctx: &ConfigContext) -> super::Result { + let mut capacities = QueueQuotas { + sender: Vec::new(), + rcpt: Vec::new(), + rcpt_domain: Vec::new(), + }; + + for array_pos in self.sub_keys("queue.quota") { + let quota = self.parse_queue_quota_item(("queue.quota", array_pos), ctx)?; + + if (quota.keys & THROTTLE_RCPT) != 0 + || quota.conditions.conditions.iter().any(|c| { + matches!( + c, + Condition::Match { + key: EnvelopeKey::Recipient, + .. + } + ) + }) + { + capacities.rcpt.push(quota); + } else if (quota.keys & THROTTLE_RCPT_DOMAIN) != 0 + || quota.conditions.conditions.iter().any(|c| { + matches!( + c, + Condition::Match { + key: EnvelopeKey::RecipientDomain, + .. + } + ) + }) + { + capacities.rcpt_domain.push(quota); + } else { + capacities.sender.push(quota); + } + } + + Ok(capacities) + } + + fn parse_queue_quota_item( + &self, + prefix: impl AsKey, + ctx: &ConfigContext, + ) -> super::Result { + let prefix = prefix.as_key(); + let mut keys = 0; + for (key_, value) in self.values((&prefix, "key")) { + let key = value.parse_throttle_key(key_)?; + if (key + & (THROTTLE_RCPT_DOMAIN | THROTTLE_RCPT | THROTTLE_SENDER | THROTTLE_SENDER_DOMAIN)) + != 0 + { + keys |= key; + } else { + return Err(format!( + "Key {value:?} is not available in this context for property {key_:?}" + )); + } + } + + let quota = QueueQuota { + conditions: if self.values((&prefix, "match")).next().is_some() { + self.parse_condition( + (&prefix, "match"), + ctx, + &[ + EnvelopeKey::Recipient, + EnvelopeKey::RecipientDomain, + EnvelopeKey::Sender, + EnvelopeKey::SenderDomain, + EnvelopeKey::Priority, + ], + )? + } else { + Conditions { + conditions: Vec::with_capacity(0), + } + }, + keys, + size: self + .property::((prefix.as_str(), "size"))? + .filter(|&v| v > 0), + messages: self + .property::((prefix.as_str(), "messages"))? + .filter(|&v| v > 0), + }; + + // Validate + if quota.size.is_none() && quota.messages.is_none() { + Err(format!( + concat!( + "Queue quota {:?} needs to define a ", + "valid 'size' and/or 'messages' property." + ), + prefix + )) + } else { + Ok(quota) + } + } +} + +impl IfBlock> { + pub fn into_relay_host(self, ctx: &ConfigContext) -> super::Result>> { + Ok(IfBlock { + if_then: { + let mut if_then = Vec::with_capacity(self.if_then.len()); + + for i in self.if_then { + if_then.push(IfThen { + conditions: i.conditions, + then: if let Some(then) = i.then { + Some( + ctx.hosts + .get(&then) + .ok_or_else(|| { + format!( + "Host {then:?} not found for property \"queue.next-hop\".", + ) + })? + .into(), + ) + } else { + None + }, + }); + } + + if_then + }, + default: if let Some(default) = self.default { + Some( + ctx.hosts + .get(&default) + .ok_or_else(|| { + format!( + "Relay host {default:?} not found for property \"queue.next-hop\".", + ) + })? + .into(), + ) + } else { + None + }, + }) + } +} + +impl From<&Host> for RelayHost { + fn from(host: &Host) -> Self { + RelayHost { + address: host.address.to_string(), + port: host.port, + protocol: host.protocol, + auth: if let (Some(username), Some(secret)) = (&host.username, &host.secret) { + Credentials::new(username.to_string(), secret.to_string()).into() + } else { + None + }, + tls_implicit: host.tls_implicit, + tls_allow_invalid_certs: host.tls_allow_invalid_certs, + } + } +} + +impl ParseValue for RequireOptional { + fn parse_value(key: impl AsKey, value: &str) -> super::Result { + match value { + "optional" => Ok(RequireOptional::Optional), + "require" | "required" => Ok(RequireOptional::Require), + "disable" | "disabled" | "none" | "false" => Ok(RequireOptional::Disable), + _ => Err(format!( + "Invalid TLS option value {:?} for key {:?}.", + value, + key.as_key() + )), + } + } +} diff --git a/crates/smtp/src/config/remote.rs b/crates/smtp/src/config/remote.rs new file mode 100644 index 00000000..039fa247 --- /dev/null +++ b/crates/smtp/src/config/remote.rs @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{sync::Arc, time::Duration}; + +use tokio::sync::mpsc; +use utils::config::Config; + +use crate::lookup::Lookup; + +use super::{ConfigContext, Host}; + +pub trait ConfigHost { + fn parse_remote_hosts(&self, ctx: &mut ConfigContext) -> super::Result<()>; + fn parse_host(&self, id: &str) -> super::Result; +} + +impl ConfigHost for Config { + fn parse_remote_hosts(&self, ctx: &mut ConfigContext) -> super::Result<()> { + for id in self.sub_keys("remote") { + let host = self.parse_host(id)?; + if host.lookup { + ctx.lookup.insert( + format!("remote/{id}"), + Arc::new(Lookup::Remote(host.channel_tx.clone().into())), + ); + } + ctx.hosts.insert(id.to_string(), host); + } + + Ok(()) + } + + fn parse_host(&self, id: &str) -> super::Result { + let (channel_tx, channel_rx) = mpsc::channel(1024); + + Ok(Host { + address: self.property_require(("remote", id, "address"))?, + port: self.property_require(("remote", id, "port"))?, + protocol: self.property_require(("remote", id, "protocol"))?, + concurrency: self.property(("remote", id, "concurrency"))?.unwrap_or(10), + tls_implicit: self + .property(("remote", id, "tls.implicit"))? + .unwrap_or(true), + tls_allow_invalid_certs: self + .property(("remote", id, "tls.allow-invalid-certs"))? + .unwrap_or(false), + username: self.property(("remote", id, "auth.username"))?, + secret: self.property(("remote", id, "auth.secret"))?, + cache_entries: self + .property(("remote", id, "cache.entries"))? + .unwrap_or(1024), + cache_ttl_positive: self + .property(("remote", id, "cache.ttl.positive"))? + .unwrap_or(Duration::from_secs(86400)), + cache_ttl_negative: self + .property(("remote", id, "cache.ttl.positive"))? + .unwrap_or(Duration::from_secs(3600)), + timeout: self + .property(("remote", id, "timeout"))? + .unwrap_or(Duration::from_secs(60)), + max_errors: self.property(("remote", id, "limits.errors"))?.unwrap_or(3), + max_requests: self + .property(("remote", id, "limits.requests"))? + .unwrap_or(50), + channel_tx, + channel_rx, + lookup: self.property(("remote", id, "lookup"))?.unwrap_or(false), + }) + } +} diff --git a/crates/smtp/src/config/report.rs b/crates/smtp/src/config/report.rs new file mode 100644 index 00000000..218b46da --- /dev/null +++ b/crates/smtp/src/config/report.rs @@ -0,0 +1,233 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use super::{ + if_block::ConfigIf, AddressMatch, AggregateFrequency, AggregateReport, ConfigContext, + EnvelopeKey, IfBlock, Report, ReportAnalysis, ReportConfig, +}; +use utils::config::{ + utils::{AsKey, ParseValue}, + Config, +}; + +pub trait ConfigReport { + fn parse_reports(&self, ctx: &ConfigContext) -> super::Result; + fn parse_report( + &self, + ctx: &ConfigContext, + id: &str, + default_hostname: &str, + available_keys: &[EnvelopeKey], + ) -> super::Result; + fn parse_aggregate_report( + &self, + ctx: &ConfigContext, + id: &str, + default_hostname: &str, + available_keys: &[EnvelopeKey], + ) -> super::Result; +} + +impl ConfigReport for Config { + fn parse_reports(&self, ctx: &ConfigContext) -> super::Result { + let sender_envelope_keys = [ + EnvelopeKey::Sender, + EnvelopeKey::SenderDomain, + EnvelopeKey::Priority, + EnvelopeKey::AuthenticatedAs, + EnvelopeKey::Listener, + EnvelopeKey::RemoteIp, + EnvelopeKey::LocalIp, + ]; + let rcpt_envelope_keys = [ + EnvelopeKey::Sender, + EnvelopeKey::SenderDomain, + EnvelopeKey::Priority, + EnvelopeKey::RemoteIp, + EnvelopeKey::LocalIp, + EnvelopeKey::RecipientDomain, + ]; + let mut addresses = Vec::new(); + for address in self.properties::("report.analysis.addresses") { + addresses.push(address?.1); + } + + let default_hostname = self.value_require("server.hostname")?; + Ok(ReportConfig { + dkim: self.parse_report(ctx, "dkim", default_hostname, &sender_envelope_keys)?, + spf: self.parse_report(ctx, "spf", default_hostname, &sender_envelope_keys)?, + dmarc: self.parse_report(ctx, "dmarc", default_hostname, &sender_envelope_keys)?, + dmarc_aggregate: self.parse_aggregate_report( + ctx, + "dmarc", + default_hostname, + &sender_envelope_keys, + )?, + tls: self.parse_aggregate_report(ctx, "tls", default_hostname, &rcpt_envelope_keys)?, + path: self + .parse_if_block("report.path", ctx, &sender_envelope_keys)? + .ok_or("Missing \"report.path\" property.")?, + submitter: self + .parse_if_block("report.submitter", ctx, &[EnvelopeKey::RecipientDomain])? + .unwrap_or_else(|| IfBlock::new(default_hostname.to_string())), + hash: self + .parse_if_block("report.hash", ctx, &sender_envelope_keys)? + .unwrap_or_else(|| IfBlock::new(32)), + analysis: ReportAnalysis { + addresses, + forward: self.property("report.analysis.forward")?.unwrap_or(false), + store: self.property("report.analysis.store")?, + report_id: 0.into(), + }, + }) + } + + fn parse_report( + &self, + ctx: &ConfigContext, + id: &str, + default_hostname: &str, + available_keys: &[EnvelopeKey], + ) -> super::Result { + Ok(Report { + name: self + .parse_if_block(("report", id, "from-name"), ctx, available_keys)? + .unwrap_or_else(|| IfBlock::new("Mail Delivery Subsystem".to_string())), + address: self + .parse_if_block(("report", id, "from-address"), ctx, available_keys)? + .unwrap_or_else(|| IfBlock::new(format!("MAILER-DAEMON@{default_hostname}"))), + subject: self + .parse_if_block(("report", id, "subject"), ctx, available_keys)? + .unwrap_or_else(|| IfBlock::new(format!("{} Report", id.to_ascii_uppercase()))), + sign: self + .parse_if_block::>(("report", id, "sign"), ctx, available_keys)? + .unwrap_or_default() + .map_if_block(&ctx.signers, &("report", id, "sign").as_key(), "signature")?, + send: self + .parse_if_block(("report", id, "send"), ctx, available_keys)? + .unwrap_or_default(), + }) + } + + fn parse_aggregate_report( + &self, + ctx: &ConfigContext, + id: &str, + default_hostname: &str, + available_keys: &[EnvelopeKey], + ) -> super::Result { + let rcpt_envelope_keys = [EnvelopeKey::RecipientDomain]; + + Ok(AggregateReport { + name: self + .parse_if_block( + ("report", id, "aggregate.from-name"), + ctx, + &rcpt_envelope_keys, + )? + .unwrap_or_else(|| { + IfBlock::new(format!("{} Aggregate Report", id.to_ascii_uppercase())) + }), + address: self + .parse_if_block( + ("report", id, "aggregate.from-address"), + ctx, + &rcpt_envelope_keys, + )? + .unwrap_or_else(|| IfBlock::new(format!("noreply-{id}@{default_hostname}"))), + org_name: self + .parse_if_block( + ("report", id, "aggregate.org-name"), + ctx, + &rcpt_envelope_keys, + )? + .unwrap_or_default(), + contact_info: self + .parse_if_block( + ("report", id, "aggregate.contact-info"), + ctx, + &rcpt_envelope_keys, + )? + .unwrap_or_default(), + send: self + .parse_if_block(("report", id, "aggregate.send"), ctx, available_keys)? + .unwrap_or_default(), + sign: self + .parse_if_block::>( + ("report", id, "aggregate.sign"), + ctx, + &rcpt_envelope_keys, + )? + .unwrap_or_default() + .map_if_block( + &ctx.signers, + &("report", id, "aggregate.sign").as_key(), + "signature", + )?, + max_size: self + .parse_if_block( + ("report", id, "aggregate.max-size"), + ctx, + &rcpt_envelope_keys, + )? + .unwrap_or_else(|| IfBlock::new(25 * 1024 * 1024)), + }) + } +} + +impl ParseValue for AggregateFrequency { + fn parse_value(key: impl AsKey, value: &str) -> super::Result { + match value { + "daily" | "day" => Ok(AggregateFrequency::Daily), + "hourly" | "hour" => Ok(AggregateFrequency::Hourly), + "weekly" | "week" => Ok(AggregateFrequency::Weekly), + "never" | "disable" | "false" => Ok(AggregateFrequency::Never), + _ => Err(format!( + "Invalid aggregate frequency value {:?} for key {:?}.", + value, + key.as_key() + )), + } + } +} + +impl ParseValue for AddressMatch { + fn parse_value(key: impl AsKey, value: &str) -> super::Result { + if let Some(value) = value.strip_prefix('*').map(|v| v.trim()) { + if !value.is_empty() { + return Ok(AddressMatch::EndsWith(value.to_lowercase())); + } + } else if let Some(value) = value.strip_suffix('*').map(|v| v.trim()) { + if !value.is_empty() { + return Ok(AddressMatch::StartsWith(value.to_lowercase())); + } + } else if value.contains('@') { + return Ok(AddressMatch::Equals(value.trim().to_lowercase())); + } + Err(format!( + "Invalid address match value {:?} for key {:?}.", + value, + key.as_key() + )) + } +} diff --git a/crates/smtp/src/config/resolver.rs b/crates/smtp/src/config/resolver.rs new file mode 100644 index 00000000..701089eb --- /dev/null +++ b/crates/smtp/src/config/resolver.rs @@ -0,0 +1,103 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use mail_auth::{ + common::lru::{DnsCache, LruCache}, + trust_dns_resolver::{ + config::{ResolverConfig, ResolverOpts}, + system_conf::read_system_conf, + }, + Resolver, +}; + +use crate::{core::Resolvers, outbound::dane::DnssecResolver}; +use utils::config::Config; + +pub trait ConfigResolver { + fn build_resolvers(&self) -> super::Result; +} + +impl ConfigResolver for Config { + fn build_resolvers(&self) -> super::Result { + let (config, mut opts) = match self.value_require("resolver.type")? { + "cloudflare" => (ResolverConfig::cloudflare(), ResolverOpts::default()), + "cloudflare-tls" => (ResolverConfig::cloudflare_tls(), ResolverOpts::default()), + "quad9" => (ResolverConfig::quad9(), ResolverOpts::default()), + "quad9-tls" => (ResolverConfig::quad9_tls(), ResolverOpts::default()), + "google" => (ResolverConfig::google(), ResolverOpts::default()), + "system" => read_system_conf() + .map_err(|err| format!("Failed to read system DNS config: {err}"))?, + other => return Err(format!("Unknown resolver type {other:?}.")), + }; + if let Some(concurrency) = self.property("resolver.concurrency")? { + opts.num_concurrent_reqs = concurrency; + } + if let Some(timeout) = self.property("resolver.timeout")? { + opts.timeout = timeout; + } + if let Some(preserve) = self.property("resolver.preserve-intermediates")? { + opts.preserve_intermediates = preserve; + } + if let Some(try_tcp_on_error) = self.property("resolver.try-tcp-on-error")? { + opts.try_tcp_on_error = try_tcp_on_error; + } + if let Some(attempts) = self.property("resolver.attempts")? { + opts.attempts = attempts; + } + + // Prepare DNSSEC resolver options + let config_dnssec = config.clone(); + let mut opts_dnssec = opts; + opts_dnssec.validate = true; + + let mut capacities = [1024usize; 5]; + for (pos, key) in ["txt", "mx", "ipv4", "ipv6", "ptr"].into_iter().enumerate() { + if let Some(capacity) = self.property(("resolver.cache", key))? { + capacities[pos] = capacity; + } + } + + Ok(Resolvers { + dns: Resolver::with_capacities( + config, + opts, + capacities[0], + capacities[1], + capacities[2], + capacities[3], + capacities[4], + ) + .map_err(|err| format!("Failed to build DNS resolver: {err}"))?, + dnssec: DnssecResolver::with_capacity(config_dnssec, opts_dnssec) + .map_err(|err| format!("Failed to build DNSSEC resolver: {err}"))?, + cache: crate::core::DnsCache { + tlsa: LruCache::with_capacity( + self.property("resolver.cache.tlsa")?.unwrap_or(1024), + ), + mta_sts: LruCache::with_capacity( + self.property("resolver.cache.mta-sts")?.unwrap_or(1024), + ), + }, + }) + } +} diff --git a/crates/smtp/src/config/scripts.rs b/crates/smtp/src/config/scripts.rs new file mode 100644 index 00000000..504d9769 --- /dev/null +++ b/crates/smtp/src/config/scripts.rs @@ -0,0 +1,152 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::time::Duration; + +use sieve::{compiler::grammar::Capability, Compiler, Runtime}; + +use crate::core::{SieveConfig, SieveCore}; +use utils::config::{utils::AsKey, Config}; + +use super::ConfigContext; + +pub trait ConfigSieve { + fn parse_sieve(&self, ctx: &mut ConfigContext) -> super::Result; +} + +impl ConfigSieve for Config { + fn parse_sieve(&self, ctx: &mut ConfigContext) -> super::Result { + // Allocate compiler and runtime + let compiler = Compiler::new() + .with_max_string_size(52428800) + .with_max_string_size(10240) + .with_max_variable_name_size(100) + .with_max_nested_blocks(50) + .with_max_nested_tests(50) + .with_max_nested_foreverypart(10) + .with_max_local_variables(128) + .with_max_header_size(10240) + .with_max_includes(10); + let mut runtime = Runtime::new() + .without_capabilities([ + Capability::FileInto, + Capability::Vacation, + Capability::VacationSeconds, + Capability::Fcc, + Capability::Mailbox, + Capability::MailboxId, + Capability::MboxMetadata, + Capability::ServerMetadata, + Capability::ImapSieve, + Capability::Duplicate, + ]) + .with_capability(Capability::Execute) + .with_max_variable_size(102400) + .with_max_header_size(10240) + .with_valid_notification_uri("mailto") + .with_valid_ext_lists(ctx.lookup.keys().map(|k| k.to_string())); + + if let Some(value) = self.property("sieve.limits.redirects")? { + runtime.set_max_redirects(value); + } + if let Some(value) = self.property("sieve.limits.out-messages")? { + runtime.set_max_out_messages(value); + } + if let Some(value) = self.property("sieve.limits.cpu")? { + runtime.set_cpu_limit(value); + } + if let Some(value) = self.property("sieve.limits.nested-includes")? { + runtime.set_max_nested_includes(value); + } + if let Some(value) = self.property("sieve.limits.received-headers")? { + runtime.set_max_received_headers(value); + } + if let Some(value) = self.property::("sieve.limits.duplicate-expiry")? { + runtime.set_default_duplicate_expiry(value.as_secs()); + } + let hostname = if let Some(hostname) = self.value("sieve.hostname") { + hostname + } else { + self.value_require("server.hostname")? + }; + runtime.set_local_hostname(hostname.to_string()); + + // Parse scripts + for id in self.sub_keys("sieve.scripts") { + let script = self.file_contents(("sieve.scripts", id))?; + ctx.scripts.insert( + id.to_string(), + compiler + .compile(&script) + .map_err(|err| format!("Failed to compile Sieve script {id:?}: {err}"))? + .into(), + ); + } + + // Parse DKIM signatures + let mut sign = Vec::new(); + for (pos, id) in self.values("sieve.sign") { + if let Some(dkim) = ctx.signers.get(id) { + sign.push(dkim.clone()); + } else { + return Err(format!( + "No DKIM signer found with id {:?} for key {:?}.", + id, + ("sieve.sign", pos).as_key() + )); + } + } + + Ok(SieveCore { + runtime, + scripts: ctx.scripts.clone(), + lookup: ctx.lookup.clone(), + config: SieveConfig { + from_addr: self + .value("sieve.from-addr") + .map(|a| a.to_string()) + .unwrap_or(format!("MAILER-DAEMON@{hostname}")), + from_name: self + .value("sieve.from-name") + .unwrap_or("Mailer Daemon") + .to_string(), + return_path: self + .value("sieve.return-path") + .unwrap_or_default() + .to_string(), + sign, + db: if let Some(db) = self.value("sieve.use-database") { + if let Some(db) = ctx.databases.get(db) { + Some(db.clone()) + } else { + return Err(format!( + "Database {db:?} not found for key \"sieve.use-database\"." + )); + } + } else { + None + }, + }, + }) + } +} diff --git a/crates/smtp/src/config/session.rs b/crates/smtp/src/config/session.rs new file mode 100644 index 00000000..25d0629a --- /dev/null +++ b/crates/smtp/src/config/session.rs @@ -0,0 +1,480 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::time::Duration; + +use smtp_proto::*; + +use super::{if_block::ConfigIf, throttle::ConfigThrottle, *}; +use utils::config::{ + utils::{AsKey, ParseValue}, + Config, +}; + +pub trait ConfigSession { + fn parse_session_config(&self, ctx: &ConfigContext) -> super::Result; + fn parse_session_throttle(&self, ctx: &ConfigContext) -> super::Result; + fn parse_session_connect(&self, ctx: &ConfigContext) -> super::Result; + fn parse_extensions(&self, ctx: &ConfigContext) -> super::Result; + fn parse_session_ehlo(&self, ctx: &ConfigContext) -> super::Result; + fn parse_session_auth(&self, ctx: &ConfigContext) -> super::Result; + fn parse_session_mail(&self, ctx: &ConfigContext) -> super::Result; + fn parse_session_rcpt(&self, ctx: &ConfigContext) -> super::Result; + fn parse_session_data(&self, ctx: &ConfigContext) -> super::Result; + fn parse_pipes( + &self, + ctx: &ConfigContext, + available_keys: &[EnvelopeKey], + ) -> super::Result>; +} + +impl ConfigSession for Config { + fn parse_session_config(&self, ctx: &ConfigContext) -> super::Result { + let available_keys = [ + EnvelopeKey::Listener, + EnvelopeKey::RemoteIp, + EnvelopeKey::LocalIp, + ]; + + Ok(SessionConfig { + duration: self + .parse_if_block("session.duration", ctx, &available_keys)? + .unwrap_or_else(|| IfBlock::new(Duration::from_secs(15 * 60))), + transfer_limit: self + .parse_if_block("session.transfer-limit", ctx, &available_keys)? + .unwrap_or_else(|| IfBlock::new(250 * 1024 * 1024)), + timeout: self + .parse_if_block::>("session.timeout", ctx, &available_keys)? + .unwrap_or_else(|| IfBlock::new(Some(Duration::from_secs(5 * 60)))) + .try_unwrap("session.timeout") + .unwrap_or_else(|_| IfBlock::new(Duration::from_secs(5 * 60))), + throttle: self.parse_session_throttle(ctx)?, + connect: self.parse_session_connect(ctx)?, + ehlo: self.parse_session_ehlo(ctx)?, + auth: self.parse_session_auth(ctx)?, + mail: self.parse_session_mail(ctx)?, + rcpt: self.parse_session_rcpt(ctx)?, + data: self.parse_session_data(ctx)?, + extensions: self.parse_extensions(ctx)?, + }) + } + + fn parse_session_throttle(&self, ctx: &ConfigContext) -> super::Result { + // Parse throttle + let mut throttle = SessionThrottle { + connect: Vec::new(), + mail_from: Vec::new(), + rcpt_to: Vec::new(), + }; + let all_throttles = self.parse_throttle( + "session.throttle", + ctx, + &[ + EnvelopeKey::Sender, + EnvelopeKey::SenderDomain, + EnvelopeKey::Recipient, + EnvelopeKey::RecipientDomain, + EnvelopeKey::AuthenticatedAs, + EnvelopeKey::Listener, + EnvelopeKey::RemoteIp, + EnvelopeKey::LocalIp, + EnvelopeKey::Priority, + EnvelopeKey::HeloDomain, + ], + THROTTLE_LISTENER + | THROTTLE_REMOTE_IP + | THROTTLE_LOCAL_IP + | THROTTLE_AUTH_AS + | THROTTLE_HELO_DOMAIN + | THROTTLE_RCPT + | THROTTLE_RCPT_DOMAIN + | THROTTLE_SENDER + | THROTTLE_SENDER_DOMAIN, + )?; + for t in all_throttles { + if (t.keys & (THROTTLE_RCPT | THROTTLE_RCPT_DOMAIN)) != 0 + || t.conditions.conditions.iter().any(|c| { + matches!( + c, + Condition::Match { + key: EnvelopeKey::Recipient | EnvelopeKey::RecipientDomain, + .. + } + ) + }) + { + throttle.rcpt_to.push(t); + } else if (t.keys + & (THROTTLE_SENDER + | THROTTLE_SENDER_DOMAIN + | THROTTLE_HELO_DOMAIN + | THROTTLE_AUTH_AS)) + != 0 + || t.conditions.conditions.iter().any(|c| { + matches!( + c, + Condition::Match { + key: EnvelopeKey::Sender + | EnvelopeKey::SenderDomain + | EnvelopeKey::HeloDomain + | EnvelopeKey::AuthenticatedAs, + .. + } + ) + }) + { + throttle.mail_from.push(t); + } else { + throttle.connect.push(t); + } + } + + Ok(throttle) + } + + fn parse_session_connect(&self, ctx: &ConfigContext) -> super::Result { + let available_keys = [ + EnvelopeKey::Listener, + EnvelopeKey::RemoteIp, + EnvelopeKey::LocalIp, + ]; + Ok(Connect { + script: self + .parse_if_block::>("session.connect.script", ctx, &available_keys)? + .unwrap_or_default() + .map_if_block(&ctx.scripts, "session.connect.script", "script")?, + }) + } + + fn parse_extensions(&self, ctx: &ConfigContext) -> super::Result { + let available_keys = [ + EnvelopeKey::Listener, + EnvelopeKey::RemoteIp, + EnvelopeKey::LocalIp, + EnvelopeKey::Sender, + EnvelopeKey::SenderDomain, + EnvelopeKey::AuthenticatedAs, + ]; + + Ok(Extensions { + pipelining: self + .parse_if_block("session.extensions.pipelining", ctx, &available_keys)? + .unwrap_or_else(|| IfBlock::new(true)), + dsn: self + .parse_if_block("session.extensions.dsn", ctx, &available_keys)? + .unwrap_or_else(|| IfBlock::new(true)), + chunking: self + .parse_if_block("session.extensions.chunking", ctx, &available_keys)? + .unwrap_or_else(|| IfBlock::new(true)), + requiretls: self + .parse_if_block("session.extensions.requiretls", ctx, &available_keys)? + .unwrap_or_default(), + no_soliciting: self + .parse_if_block("session.extensions.no-soliciting", ctx, &available_keys)? + .unwrap_or_default(), + future_release: self + .parse_if_block("session.extensions.future-release", ctx, &available_keys)? + .unwrap_or_default(), + deliver_by: self + .parse_if_block("session.extensions.deliver-by", ctx, &available_keys)? + .unwrap_or_default(), + mt_priority: self + .parse_if_block("session.extensions.mt-priority", ctx, &available_keys)? + .unwrap_or_default(), + }) + } + + fn parse_session_ehlo(&self, ctx: &ConfigContext) -> super::Result { + let available_keys = [ + EnvelopeKey::Listener, + EnvelopeKey::RemoteIp, + EnvelopeKey::LocalIp, + ]; + + Ok(Ehlo { + script: self + .parse_if_block::>("session.ehlo.script", ctx, &available_keys)? + .unwrap_or_default() + .map_if_block(&ctx.scripts, "session.ehlo.script", "script")?, + require: self + .parse_if_block("session.ehlo.require", ctx, &available_keys)? + .unwrap_or_else(|| IfBlock::new(true)), + reject_non_fqdn: self + .parse_if_block("session.ehlo.reject-non-fqdn", ctx, &available_keys)? + .unwrap_or_else(|| IfBlock::new(true)), + }) + } + + fn parse_session_auth(&self, ctx: &ConfigContext) -> super::Result { + let available_keys = [ + EnvelopeKey::Listener, + EnvelopeKey::RemoteIp, + EnvelopeKey::LocalIp, + EnvelopeKey::HeloDomain, + ]; + + let mechanisms = self + .parse_if_block::>("session.auth.mechanisms", ctx, &available_keys)? + .unwrap_or_default(); + + Ok(Auth { + lookup: self + .parse_if_block::>("session.auth.lookup", ctx, &available_keys)? + .unwrap_or_default() + .map_if_block(&ctx.lookup, "session.auth.lookup", "lookup list")?, + mechanisms: IfBlock { + if_then: mechanisms + .if_then + .into_iter() + .map(|i| IfThen { + conditions: i.conditions, + then: i.then.into_iter().fold(0, |acc, m| acc | m.mechanism), + }) + .collect(), + default: mechanisms + .default + .into_iter() + .fold(0, |acc, m| acc | m.mechanism), + }, + require: self + .parse_if_block("session.auth.require", ctx, &available_keys)? + .unwrap_or_else(|| IfBlock::new(false)), + errors_max: self + .parse_if_block("session.auth.errors.max", ctx, &available_keys)? + .unwrap_or_else(|| IfBlock::new(3)), + errors_wait: self + .parse_if_block("session.auth.errors.wait", ctx, &available_keys)? + .unwrap_or_else(|| IfBlock::new(Duration::from_secs(30))), + }) + } + + fn parse_session_mail(&self, ctx: &ConfigContext) -> super::Result { + let available_keys = [ + EnvelopeKey::AuthenticatedAs, + EnvelopeKey::Listener, + EnvelopeKey::RemoteIp, + EnvelopeKey::LocalIp, + EnvelopeKey::HeloDomain, + ]; + Ok(Mail { + script: self + .parse_if_block::>("session.mail.script", ctx, &available_keys)? + .unwrap_or_default() + .map_if_block(&ctx.scripts, "session.mail.script", "script")?, + }) + } + + fn parse_session_rcpt(&self, ctx: &ConfigContext) -> super::Result { + let available_keys = [ + EnvelopeKey::Sender, + EnvelopeKey::SenderDomain, + EnvelopeKey::AuthenticatedAs, + EnvelopeKey::Listener, + EnvelopeKey::RemoteIp, + EnvelopeKey::LocalIp, + EnvelopeKey::HeloDomain, + ]; + Ok(Rcpt { + script: self + .parse_if_block::>("session.rcpt.script", ctx, &available_keys)? + .unwrap_or_default() + .map_if_block(&ctx.scripts, "session.rcpt.script", "script")?, + relay: self + .parse_if_block("session.rcpt.relay", ctx, &available_keys)? + .unwrap_or_else(|| IfBlock::new(false)), + + lookup_domains: self + .parse_if_block::>( + "session.rcpt.lookup.domains", + ctx, + &available_keys, + )? + .unwrap_or_default() + .map_if_block(&ctx.lookup, "session.rcpt.lookup.domains", "lookup list")?, + lookup_addresses: self + .parse_if_block::>( + "session.rcpt.lookup.addresses", + ctx, + &available_keys, + )? + .unwrap_or_default() + .map_if_block(&ctx.lookup, "session.rcpt.lookup.addresses", "lookup list")?, + lookup_expn: self + .parse_if_block::>("session.rcpt.lookup.expn", ctx, &available_keys)? + .unwrap_or_default() + .map_if_block(&ctx.lookup, "session.rcpt.lookup.expn", "lookup list")?, + lookup_vrfy: self + .parse_if_block::>("session.rcpt.lookup.vrfy", ctx, &available_keys)? + .unwrap_or_default() + .map_if_block(&ctx.lookup, "session.rcpt.lookup.vrfy", "lookup list")?, + errors_max: self + .parse_if_block("session.rcpt.errors.max", ctx, &available_keys)? + .unwrap_or_else(|| IfBlock::new(10)), + errors_wait: self + .parse_if_block("session.rcpt.errors.wait", ctx, &available_keys)? + .unwrap_or_else(|| IfBlock::new(Duration::from_secs(30))), + max_recipients: self + .parse_if_block("session.rcpt.max-recipients", ctx, &available_keys)? + .unwrap_or_else(|| IfBlock::new(100)), + }) + } + + fn parse_session_data(&self, ctx: &ConfigContext) -> super::Result { + let available_keys = [ + EnvelopeKey::Sender, + EnvelopeKey::SenderDomain, + EnvelopeKey::AuthenticatedAs, + EnvelopeKey::Listener, + EnvelopeKey::RemoteIp, + EnvelopeKey::LocalIp, + EnvelopeKey::Priority, + EnvelopeKey::HeloDomain, + ]; + Ok(Data { + script: self + .parse_if_block::>("session.data.script", ctx, &available_keys)? + .unwrap_or_default() + .map_if_block(&ctx.scripts, "session.data.script", "script")?, + max_messages: self + .parse_if_block("session.data.limits.messages", ctx, &available_keys)? + .unwrap_or_else(|| IfBlock::new(10)), + max_message_size: self + .parse_if_block("session.data.limits.size", ctx, &available_keys)? + .unwrap_or_else(|| IfBlock::new(25 * 1024 * 1024)), + max_received_headers: self + .parse_if_block("session.data.limits.received-headers", ctx, &available_keys)? + .unwrap_or_else(|| IfBlock::new(50)), + add_received: self + .parse_if_block("session.data.add-headers.received", ctx, &available_keys)? + .unwrap_or_else(|| IfBlock::new(true)), + add_received_spf: self + .parse_if_block( + "session.data.add-headers.received-spf", + ctx, + &available_keys, + )? + .unwrap_or_else(|| IfBlock::new(true)), + add_return_path: self + .parse_if_block("session.data.add-headers.return-path", ctx, &available_keys)? + .unwrap_or_else(|| IfBlock::new(true)), + add_auth_results: self + .parse_if_block( + "session.data.add-headers.auth-results", + ctx, + &available_keys, + )? + .unwrap_or_else(|| IfBlock::new(true)), + add_message_id: self + .parse_if_block("session.data.add-headers.message-id", ctx, &available_keys)? + .unwrap_or_else(|| IfBlock::new(true)), + add_date: self + .parse_if_block("session.data.add-headers.date", ctx, &available_keys)? + .unwrap_or_else(|| IfBlock::new(true)), + pipe_commands: self.parse_pipes(ctx, &available_keys)?, + }) + } + + fn parse_pipes( + &self, + ctx: &ConfigContext, + available_keys: &[EnvelopeKey], + ) -> super::Result> { + let mut pipes = Vec::new(); + for id in self.sub_keys("session.data.pipe") { + pipes.push(Pipe { + command: self + .parse_if_block(("session.data.pipe", id, "command"), ctx, available_keys)? + .unwrap_or_default(), + arguments: self + .parse_if_block(("session.data.pipe", id, "arguments"), ctx, available_keys)? + .unwrap_or_default(), + timeout: self + .parse_if_block(("session.data.pipe", id, "timeout"), ctx, available_keys)? + .unwrap_or_else(|| IfBlock::new(Duration::from_secs(30))), + }) + } + Ok(pipes) + } +} + +struct Mechanism { + mechanism: u64, +} + +impl ParseValue for Mechanism { + fn parse_value(key: impl AsKey, value: &str) -> super::Result { + Ok(Mechanism { + mechanism: match value.to_ascii_uppercase().as_str() { + "LOGIN" => AUTH_LOGIN, + "PLAIN" => AUTH_PLAIN, + "XOAUTH2" => AUTH_XOAUTH2, + "OAUTHBEARER" => AUTH_OAUTHBEARER, + /*"SCRAM-SHA-256-PLUS" => AUTH_SCRAM_SHA_256_PLUS, + "SCRAM-SHA-256" => AUTH_SCRAM_SHA_256, + "SCRAM-SHA-1-PLUS" => AUTH_SCRAM_SHA_1_PLUS, + "SCRAM-SHA-1" => AUTH_SCRAM_SHA_1, + "XOAUTH" => AUTH_XOAUTH, + "9798-M-DSA-SHA1" => AUTH_9798_M_DSA_SHA1, + "9798-M-ECDSA-SHA1" => AUTH_9798_M_ECDSA_SHA1, + "9798-M-RSA-SHA1-ENC" => AUTH_9798_M_RSA_SHA1_ENC, + "9798-U-DSA-SHA1" => AUTH_9798_U_DSA_SHA1, + "9798-U-ECDSA-SHA1" => AUTH_9798_U_ECDSA_SHA1, + "9798-U-RSA-SHA1-ENC" => AUTH_9798_U_RSA_SHA1_ENC, + "EAP-AES128" => AUTH_EAP_AES128, + "EAP-AES128-PLUS" => AUTH_EAP_AES128_PLUS, + "ECDH-X25519-CHALLENGE" => AUTH_ECDH_X25519_CHALLENGE, + "ECDSA-NIST256P-CHALLENGE" => AUTH_ECDSA_NIST256P_CHALLENGE, + "EXTERNAL" => AUTH_EXTERNAL, + "GS2-KRB5" => AUTH_GS2_KRB5, + "GS2-KRB5-PLUS" => AUTH_GS2_KRB5_PLUS, + "GSS-SPNEGO" => AUTH_GSS_SPNEGO, + "GSSAPI" => AUTH_GSSAPI, + "KERBEROS_V4" => AUTH_KERBEROS_V4, + "KERBEROS_V5" => AUTH_KERBEROS_V5, + "NMAS-SAMBA-AUTH" => AUTH_NMAS_SAMBA_AUTH, + "NMAS_AUTHEN" => AUTH_NMAS_AUTHEN, + "NMAS_LOGIN" => AUTH_NMAS_LOGIN, + "NTLM" => AUTH_NTLM, + "OAUTH10A" => AUTH_OAUTH10A, + "OPENID20" => AUTH_OPENID20, + "OTP" => AUTH_OTP, + "SAML20" => AUTH_SAML20, + "SECURID" => AUTH_SECURID, + "SKEY" => AUTH_SKEY, + "SPNEGO" => AUTH_SPNEGO, + "SPNEGO-PLUS" => AUTH_SPNEGO_PLUS, + "SXOVER-PLUS" => AUTH_SXOVER_PLUS, + "CRAM-MD5" => AUTH_CRAM_MD5, + "DIGEST-MD5" => AUTH_DIGEST_MD5, + "ANONYMOUS" => AUTH_ANONYMOUS,*/ + _ => { + return Err(format!( + "Unsupported mechanism {:?} for property {:?}.", + value, + key.as_key() + )) + } + }, + }) + } +} diff --git a/crates/smtp/src/config/throttle.rs b/crates/smtp/src/config/throttle.rs new file mode 100644 index 00000000..453c8491 --- /dev/null +++ b/crates/smtp/src/config/throttle.rs @@ -0,0 +1,267 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use super::{condition::ConfigCondition, *}; +use utils::config::{ + utils::{AsKey, ParseKey, ParseValue}, + Config, +}; + +pub trait ConfigThrottle { + fn parse_throttle( + &self, + prefix: impl AsKey, + ctx: &ConfigContext, + available_envelope_keys: &[EnvelopeKey], + available_throttle_keys: u16, + ) -> super::Result>; + + fn parse_throttle_item( + &self, + prefix: impl AsKey, + ctx: &ConfigContext, + available_envelope_keys: &[EnvelopeKey], + available_throttle_keys: u16, + ) -> super::Result; +} + +impl ConfigThrottle for Config { + fn parse_throttle( + &self, + prefix: impl AsKey, + ctx: &ConfigContext, + available_envelope_keys: &[EnvelopeKey], + available_throttle_keys: u16, + ) -> super::Result> { + let prefix_ = prefix.as_key(); + let mut throttles = Vec::new(); + for array_pos in self.sub_keys(prefix) { + throttles.push(self.parse_throttle_item( + (&prefix_, array_pos), + ctx, + available_envelope_keys, + available_throttle_keys, + )?); + } + + Ok(throttles) + } + + fn parse_throttle_item( + &self, + prefix: impl AsKey, + ctx: &ConfigContext, + available_envelope_keys: &[EnvelopeKey], + available_throttle_keys: u16, + ) -> super::Result { + let prefix = prefix.as_key(); + let mut keys = 0; + for (key_, value) in self.values((&prefix, "key")) { + let key = value.parse_throttle_key(key_)?; + if (key & available_throttle_keys) != 0 { + keys |= key; + } else { + return Err(format!( + "Throttle key {value:?} is not available in this context for property {key_:?}" + )); + } + } + + let throttle = Throttle { + conditions: if self.values((&prefix, "match")).next().is_some() { + self.parse_condition((&prefix, "match"), ctx, available_envelope_keys)? + } else { + Conditions { + conditions: Vec::with_capacity(0), + } + }, + keys, + concurrency: self + .property::((prefix.as_str(), "concurrency"))? + .filter(|&v| v > 0), + rate: self + .property::((prefix.as_str(), "rate"))? + .filter(|v| v.requests > 0), + }; + + // Validate + if throttle.rate.is_none() && throttle.concurrency.is_none() { + Err(format!( + concat!( + "Throttle {:?} needs to define a ", + "valid 'rate' and/or 'concurrency' property." + ), + prefix + )) + } else { + Ok(throttle) + } + } +} + +impl ParseValue for Rate { + fn parse_value(key: impl AsKey, value: &str) -> super::Result { + if let Some((requests, period)) = value.split_once('/') { + Ok(Rate { + requests: requests + .trim() + .parse::() + .ok() + .and_then(|r| if r > 0 { Some(r) } else { None }) + .ok_or_else(|| { + format!( + "Invalid rate value {:?} for property {:?}.", + value, + key.as_key() + ) + })?, + period: period.parse_key(key)?, + }) + } else if ["false", "none", "unlimited"].contains(&value) { + Ok(Rate::default()) + } else { + Err(format!( + "Invalid rate value {:?} for property {:?}.", + value, + key.as_key() + )) + } + } +} + +impl ParseValue for EnvelopeKey { + fn parse_value(key: impl AsKey, value: &str) -> super::Result { + Ok(match value { + "rcpt" => EnvelopeKey::Recipient, + "rcpt-domain" => EnvelopeKey::RecipientDomain, + "sender" => EnvelopeKey::Sender, + "sender-domain" => EnvelopeKey::SenderDomain, + "listener" => EnvelopeKey::Listener, + "remote-ip" => EnvelopeKey::RemoteIp, + "local-ip" => EnvelopeKey::LocalIp, + "priority" => EnvelopeKey::Priority, + "authenticated-as" => EnvelopeKey::AuthenticatedAs, + "mx" => EnvelopeKey::Mx, + _ => { + return Err(format!( + "Invalid context key {:?} for property {:?}.", + value, + key.as_key() + )) + } + }) + } +} + +pub trait ParseTrottleKey { + fn parse_throttle_key(&self, key: &str) -> super::Result; +} + +impl ParseTrottleKey for &str { + fn parse_throttle_key(&self, key: &str) -> super::Result { + match *self { + "rcpt" => Ok(THROTTLE_RCPT), + "rcpt-domain" => Ok(THROTTLE_RCPT_DOMAIN), + "sender" => Ok(THROTTLE_SENDER), + "sender-domain" => Ok(THROTTLE_SENDER_DOMAIN), + "authenticated-as" => Ok(THROTTLE_AUTH_AS), + "listener" => Ok(THROTTLE_LISTENER), + "mx" => Ok(THROTTLE_MX), + "remote-ip" => Ok(THROTTLE_REMOTE_IP), + "local-ip" => Ok(THROTTLE_LOCAL_IP), + "helo-domain" => Ok(THROTTLE_HELO_DOMAIN), + _ => Err(format!("Invalid throttle key {self:?} found in {key:?}")), + } + } +} + +#[cfg(test)] +mod tests { + use std::{fs, path::PathBuf, time::Duration}; + + use utils::config::Config; + + use crate::config::{ + throttle::ConfigThrottle, Condition, ConditionMatch, Conditions, ConfigContext, + EnvelopeKey, IpAddrMask, Rate, Throttle, THROTTLE_AUTH_AS, THROTTLE_REMOTE_IP, + THROTTLE_SENDER_DOMAIN, + }; + + #[test] + fn parse_throttle() { + let mut file = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + file.push("resources"); + file.push("smtp"); + file.push("config"); + file.push("throttle.toml"); + + let available_keys = vec![ + EnvelopeKey::Recipient, + EnvelopeKey::RecipientDomain, + EnvelopeKey::Sender, + EnvelopeKey::SenderDomain, + EnvelopeKey::AuthenticatedAs, + EnvelopeKey::Listener, + EnvelopeKey::RemoteIp, + EnvelopeKey::LocalIp, + EnvelopeKey::Priority, + ]; + + let config = Config::parse(&fs::read_to_string(file).unwrap()).unwrap(); + let context = ConfigContext::default(); + let throttle = config + .parse_throttle("throttle", &context, &available_keys, u16::MAX) + .unwrap(); + + assert_eq!( + throttle, + vec![ + Throttle { + conditions: Conditions { + conditions: vec![Condition::Match { + key: EnvelopeKey::RemoteIp, + value: ConditionMatch::IpAddrMask(IpAddrMask::V4 { + addr: "127.0.0.1".parse().unwrap(), + mask: u32::MAX + }), + not: false + }] + }, + keys: THROTTLE_REMOTE_IP | THROTTLE_AUTH_AS, + concurrency: 100.into(), + rate: Rate { + requests: 50, + period: Duration::from_secs(30) + } + .into() + }, + Throttle { + conditions: Conditions { conditions: vec![] }, + keys: THROTTLE_SENDER_DOMAIN, + concurrency: 10000.into(), + rate: None + } + ] + ); + } +} diff --git a/crates/smtp/src/core/if_block.rs b/crates/smtp/src/core/if_block.rs new file mode 100644 index 00000000..ba95b0c6 --- /dev/null +++ b/crates/smtp/src/core/if_block.rs @@ -0,0 +1,296 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::net::{IpAddr, Ipv4Addr}; + +use crate::config::{ + Condition, ConditionMatch, Conditions, EnvelopeKey, IfBlock, IpAddrMask, StringMatch, +}; + +use super::Envelope; + +impl IfBlock { + pub async fn eval(&self, envelope: &impl Envelope) -> &T { + for if_then in &self.if_then { + if if_then.conditions.eval(envelope).await { + return &if_then.then; + } + } + + &self.default + } +} + +impl Conditions { + pub async fn eval(&self, envelope: &impl Envelope) -> bool { + let mut conditions = self.conditions.iter(); + let mut matched = false; + + while let Some(rule) = conditions.next() { + match rule { + Condition::Match { key, value, not } => { + matched = match value { + ConditionMatch::String(value) => { + let ctx_value = envelope.key_to_string(key); + match value { + StringMatch::Equal(value) => value.eq(ctx_value.as_ref()), + StringMatch::StartsWith(value) => ctx_value.starts_with(value), + StringMatch::EndsWith(value) => ctx_value.ends_with(value), + } + } + ConditionMatch::IpAddrMask(value) => value.matches(&match key { + EnvelopeKey::RemoteIp => envelope.remote_ip(), + EnvelopeKey::LocalIp => envelope.local_ip(), + _ => IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), + }), + ConditionMatch::UInt(value) => { + *value + == if key == &EnvelopeKey::Listener { + envelope.listener_id() + } else { + debug_assert!(false, "Invalid value for UInt context key."); + u16::MAX + } + } + ConditionMatch::Int(value) => { + *value + == if key == &EnvelopeKey::Listener { + envelope.priority() + } else { + debug_assert!(false, "Invalid value for UInt context key."); + i16::MAX + } + } + ConditionMatch::Lookup(lookup) => { + if let Some(result) = + lookup.contains(envelope.key_to_string(key).as_ref()).await + { + result + } else { + return false; + } + } + ConditionMatch::Regex(value) => { + value.is_match(envelope.key_to_string(key).as_ref()) + } + } ^ not; + } + Condition::JumpIfTrue { positions } => { + if matched { + //TODO use advance_by when stabilized + for _ in 0..*positions { + conditions.next(); + } + } + } + Condition::JumpIfFalse { positions } => { + if !matched { + //TODO use advance_by when stabilized + for _ in 0..*positions { + conditions.next(); + } + } + } + } + } + + matched + } +} + +impl IpAddrMask { + pub fn matches(&self, remote: &IpAddr) -> bool { + match self { + IpAddrMask::V4 { addr, mask } => { + if *mask == u32::MAX { + match remote { + IpAddr::V4(remote) => addr == remote, + IpAddr::V6(remote) => { + if let Some(remote) = remote.to_ipv4_mapped() { + addr == &remote + } else { + false + } + } + } + } else { + u32::from_be_bytes(match remote { + IpAddr::V4(ip) => ip.octets(), + IpAddr::V6(ip) => { + if let Some(ip) = ip.to_ipv4() { + ip.octets() + } else { + return false; + } + } + }) & mask + == u32::from_be_bytes(addr.octets()) & mask + } + } + IpAddrMask::V6 { addr, mask } => { + if mask == &u128::MAX { + match remote { + IpAddr::V6(remote) => remote == addr, + IpAddr::V4(remote) => &remote.to_ipv6_mapped() == addr, + } + } else { + u128::from_be_bytes(match remote { + IpAddr::V6(ip) => ip.octets(), + IpAddr::V4(ip) => ip.to_ipv6_mapped().octets(), + }) & mask + == u128::from_be_bytes(addr.octets()) & mask + } + } + } + } +} + +#[cfg(test)] +mod tests { + use std::{fs, net::IpAddr, path::PathBuf}; + + use utils::config::{Config, Server}; + + use crate::{ + config::{condition::ConfigCondition, list::ConfigList, ConfigContext, IfBlock, IfThen}, + core::Envelope, + }; + + struct TestEnvelope { + pub local_ip: IpAddr, + pub remote_ip: IpAddr, + pub sender_domain: String, + pub sender: String, + pub rcpt_domain: String, + pub rcpt: String, + pub helo_domain: String, + pub authenticated_as: String, + pub mx: String, + pub listener_id: u16, + pub priority: i16, + } + + impl Envelope for TestEnvelope { + fn local_ip(&self) -> IpAddr { + self.local_ip + } + + fn remote_ip(&self) -> IpAddr { + self.remote_ip + } + + fn sender_domain(&self) -> &str { + self.sender_domain.as_str() + } + + fn sender(&self) -> &str { + self.sender.as_str() + } + + fn rcpt_domain(&self) -> &str { + self.rcpt_domain.as_str() + } + + fn rcpt(&self) -> &str { + self.rcpt.as_str() + } + + fn helo_domain(&self) -> &str { + self.helo_domain.as_str() + } + + fn authenticated_as(&self) -> &str { + self.authenticated_as.as_str() + } + + fn mx(&self) -> &str { + self.mx.as_str() + } + + fn listener_id(&self) -> u16 { + self.listener_id + } + + fn priority(&self) -> i16 { + self.priority + } + } + + #[tokio::test] + async fn eval_if() { + let mut file = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + file.push("resources"); + file.push("smtp"); + file.push("config"); + file.push("rules-eval.toml"); + + let config = Config::parse(&fs::read_to_string(file).unwrap()).unwrap(); + let mut context = ConfigContext::default(); + context.servers.push(Server { + id: "smtp".to_string(), + internal_id: 123, + ..Default::default() + }); + context.servers.push(Server { + id: "smtps".to_string(), + internal_id: 456, + ..Default::default() + }); + config.parse_lists(&mut context).unwrap(); + let conditions = config.parse_conditions(&context).unwrap(); + + let envelope = TestEnvelope { + local_ip: config.property_require("envelope.local-ip").unwrap(), + remote_ip: config.property_require("envelope.remote-ip").unwrap(), + sender_domain: config.property_require("envelope.sender-domain").unwrap(), + sender: config.property_require("envelope.sender").unwrap(), + rcpt_domain: config.property_require("envelope.rcpt-domain").unwrap(), + rcpt: config.property_require("envelope.rcpt").unwrap(), + authenticated_as: config + .property_require("envelope.authenticated-as") + .unwrap(), + mx: config.property_require("envelope.mx").unwrap(), + listener_id: config.property_require("envelope.listener").unwrap(), + priority: config.property_require("envelope.priority").unwrap(), + helo_domain: config.property_require("envelope.helo-domain").unwrap(), + }; + + for (key, conditions) in conditions { + //println!("============= Testing {:?} ==================", key); + let (_, expected_result) = key.rsplit_once('-').unwrap(); + assert_eq!( + IfBlock { + if_then: vec![IfThen { + conditions, + then: true + }], + default: false, + } + .eval(&envelope) + .await, + &expected_result.parse::().unwrap(), + "failed for {key:?}" + ); + } + } +} diff --git a/crates/smtp/src/core/management.rs b/crates/smtp/src/core/management.rs new file mode 100644 index 00000000..e7837fa6 --- /dev/null +++ b/crates/smtp/src/core/management.rs @@ -0,0 +1,962 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{borrow::Cow, fmt::Display, net::IpAddr, sync::Arc, time::Instant}; + +use http_body_util::{combinators::BoxBody, BodyExt, Empty, Full}; +use hyper::{ + body::{self, Bytes}, + header::{self, AUTHORIZATION}, + server::conn::http1, + service::service_fn, + Method, StatusCode, +}; +use mail_parser::{decoders::base64::base64_decode, DateTime}; +use mail_send::Credentials; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use tokio::{ + io::{AsyncRead, AsyncWrite}, + sync::oneshot, +}; + +use utils::listener::{limiter::InFlight, SessionManager}; + +use crate::{ + lookup::{Item, LookupResult}, + queue::{self, instant_to_timestamp, InstantFromTimestamp, QueueId, Status}, + reporting::{ + self, + scheduler::{ReportKey, ReportPolicy, ReportType, ReportValue}, + }, +}; + +use super::{Core, HttpAdminSessionManager}; + +#[derive(Debug)] +pub enum QueueRequest { + List { + from: Option, + to: Option, + before: Option, + after: Option, + result_tx: oneshot::Sender>, + }, + Status { + queue_ids: Vec, + result_tx: oneshot::Sender>>, + }, + Cancel { + queue_ids: Vec, + item: Option, + result_tx: oneshot::Sender>, + }, + Retry { + queue_ids: Vec, + item: Option, + time: Instant, + result_tx: oneshot::Sender>, + }, +} + +#[derive(Debug)] +pub enum ReportRequest { + List { + type_: Option>, + domain: Option, + result_tx: oneshot::Sender>, + }, + Status { + report_ids: Vec, + result_tx: oneshot::Sender>>, + }, + Cancel { + report_ids: Vec, + result_tx: oneshot::Sender>, + }, +} + +#[derive(Debug, Serialize)] +pub struct Response { + data: T, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct Message { + pub return_path: String, + pub domains: Vec, + #[serde(deserialize_with = "deserialize_datetime")] + #[serde(serialize_with = "serialize_datetime")] + pub created: DateTime, + pub size: usize, + #[serde(skip_serializing_if = "is_zero")] + #[serde(default)] + pub priority: i16, + #[serde(skip_serializing_if = "Option::is_none")] + pub env_id: Option, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct Domain { + pub name: String, + pub status: Status, + pub recipients: Vec, + + pub retry_num: u32, + #[serde(deserialize_with = "deserialize_maybe_datetime")] + #[serde(serialize_with = "serialize_maybe_datetime")] + pub next_retry: Option, + #[serde(deserialize_with = "deserialize_maybe_datetime")] + #[serde(serialize_with = "serialize_maybe_datetime")] + pub next_notify: Option, + #[serde(deserialize_with = "deserialize_datetime")] + #[serde(serialize_with = "serialize_datetime")] + pub expires: DateTime, +} + +#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)] +pub struct Recipient { + pub address: String, + pub status: Status, + #[serde(skip_serializing_if = "Option::is_none")] + pub orcpt: Option, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Report { + pub domain: String, + #[serde(rename = "type")] + pub type_: String, + #[serde(deserialize_with = "deserialize_datetime")] + #[serde(serialize_with = "serialize_datetime")] + pub range_from: DateTime, + #[serde(deserialize_with = "deserialize_datetime")] + #[serde(serialize_with = "serialize_datetime")] + pub range_to: DateTime, + pub size: usize, +} + +impl SessionManager for HttpAdminSessionManager { + fn spawn(&self, session: utils::listener::SessionData) { + let core = self.inner.clone(); + tokio::spawn(async move { + if let Some(tls_acceptor) = &session.instance.tls_acceptor { + match tls_acceptor.accept(session.stream).await { + Ok(stream) => { + handle_request(stream, core, session.remote_ip, session.in_flight).await; + } + Err(err) => { + tracing::debug!( + context = "tls", + event = "error", + remote.ip = session.remote_ip.to_string(), + "Failed to accept TLS management connection: {}", + err + ); + } + } + } else { + handle_request(session.stream, core, session.remote_ip, session.in_flight).await; + } + }); + } +} + +async fn handle_request( + stream: impl AsyncRead + AsyncWrite + Unpin + 'static, + core: Arc, + remote_addr: IpAddr, + _in_flight: InFlight, +) { + if let Err(http_err) = http1::Builder::new() + .keep_alive(true) + .serve_connection( + stream, + service_fn(|req: hyper::Request| { + let core = core.clone(); + + async move { + let response = core.parse_request(&req).await; + + tracing::debug!( + context = "management", + event = "request", + remote.ip = remote_addr.to_string(), + uri = req.uri().to_string(), + status = match &response { + Ok(response) => response.status().to_string(), + Err(error) => error.to_string(), + } + ); + + response + } + }), + ) + .await + { + tracing::debug!( + context = "management", + event = "http-error", + remote.ip = remote_addr.to_string(), + reason = %http_err, + ); + } +} + +impl Core { + async fn parse_request( + &self, + req: &hyper::Request, + ) -> Result>, hyper::Error> { + // Authenticate request + let mut is_authenticated = false; + if let Some((mechanism, payload)) = req + .headers() + .get(AUTHORIZATION) + .and_then(|h| h.to_str().ok()) + .and_then(|h| h.trim().split_once(' ')) + { + if mechanism.eq_ignore_ascii_case("basic") { + // Decode the base64 encoded credentials + if let Some((username, secret)) = base64_decode(payload.as_bytes()) + .and_then(|token| String::from_utf8(token).ok()) + .and_then(|token| { + token.split_once(':').map(|(login, secret)| { + (login.trim().to_lowercase(), secret.to_string()) + }) + }) + { + match self + .queue + .config + .management_lookup + .lookup(Item::Authenticate(Credentials::Plain { username, secret })) + .await + { + Some(LookupResult::True) => { + is_authenticated = true; + } + Some(LookupResult::False) => { + tracing::debug!( + context = "management", + event = "auth-error", + "Invalid username or password." + ); + } + _ => { + tracing::debug!( + context = "management", + event = "auth-error", + "Temporary authentication failure." + ); + } + } + } else { + tracing::debug!( + context = "management", + event = "auth-error", + "Failed to decode base64 Authorization header." + ); + } + } else { + tracing::debug!( + context = "management", + event = "auth-error", + mechanism = mechanism, + "Unsupported authentication mechanism." + ); + } + } + if !is_authenticated { + return Ok(hyper::Response::builder() + .status(StatusCode::UNAUTHORIZED) + .header(header::WWW_AUTHENTICATE, "Basic realm=\"Stalwart SMTP\"") + .body( + Empty::::new() + .map_err(|never| match never {}) + .boxed(), + ) + .unwrap()); + } + + let mut path = req.uri().path().split('/'); + path.next(); + let (status, response) = match (req.method(), path.next(), path.next()) { + (&Method::GET, Some("queue"), Some("list")) => { + let mut from = None; + let mut to = None; + let mut before = None; + let mut after = None; + let mut error = None; + + if let Some(query) = req.uri().query() { + for (key, value) in form_urlencoded::parse(query.as_bytes()) { + match key.as_ref() { + "from" => { + from = value.into_owned().into(); + } + "to" => { + to = value.into_owned().into(); + } + "after" => match value.parse_timestamp() { + Ok(dt) => { + after = dt.into(); + } + Err(reason) => { + error = reason.into(); + break; + } + }, + "before" => match value.parse_timestamp() { + Ok(dt) => { + before = dt.into(); + } + Err(reason) => { + error = reason.into(); + break; + } + }, + _ => { + error = format!("Invalid parameter {key:?}.").into(); + break; + } + } + } + } + + match error { + None => { + let (result_tx, result_rx) = oneshot::channel(); + self.send_queue_event( + QueueRequest::List { + from, + to, + before, + after, + result_tx, + }, + result_rx, + ) + .await + } + Some(error) => error.into_bad_request(), + } + } + (&Method::GET, Some("queue"), Some("status")) => { + let mut queue_ids = Vec::new(); + let mut error = None; + + if let Some(query) = req.uri().query() { + for (key, value) in form_urlencoded::parse(query.as_bytes()) { + match key.as_ref() { + "id" | "ids" => match value.parse_queue_ids() { + Ok(ids) => { + queue_ids = ids; + } + Err(reason) => { + error = reason.into(); + break; + } + }, + _ => { + error = format!("Invalid parameter {key:?}.").into(); + break; + } + } + } + } + + match error { + None => { + let (result_tx, result_rx) = oneshot::channel(); + self.send_queue_event( + QueueRequest::Status { + queue_ids, + result_tx, + }, + result_rx, + ) + .await + } + Some(error) => error.into_bad_request(), + } + } + (&Method::GET, Some("queue"), Some("retry")) => { + let mut queue_ids = Vec::new(); + let mut time = Instant::now(); + let mut item = None; + let mut error = None; + + if let Some(query) = req.uri().query() { + for (key, value) in form_urlencoded::parse(query.as_bytes()) { + match key.as_ref() { + "id" | "ids" => match value.parse_queue_ids() { + Ok(ids) => { + queue_ids = ids; + } + Err(reason) => { + error = reason.into(); + break; + } + }, + "at" => match value.parse_timestamp() { + Ok(dt) => { + time = dt; + } + Err(reason) => { + error = reason.into(); + break; + } + }, + "filter" => { + item = value.into_owned().into(); + } + _ => { + error = format!("Invalid parameter {key:?}.").into(); + break; + } + } + } + } + + match error { + None => { + let (result_tx, result_rx) = oneshot::channel(); + self.send_queue_event( + QueueRequest::Retry { + queue_ids, + item, + time, + result_tx, + }, + result_rx, + ) + .await + } + Some(error) => error.into_bad_request(), + } + } + (&Method::GET, Some("queue"), Some("cancel")) => { + let mut queue_ids = Vec::new(); + let mut item = None; + let mut error = None; + + if let Some(query) = req.uri().query() { + for (key, value) in form_urlencoded::parse(query.as_bytes()) { + match key.as_ref() { + "id" | "ids" => match value.parse_queue_ids() { + Ok(ids) => { + queue_ids = ids; + } + Err(reason) => { + error = reason.into(); + break; + } + }, + "filter" => { + item = value.into_owned().into(); + } + _ => { + error = format!("Invalid parameter {key:?}.").into(); + break; + } + } + } + } + + match error { + None => { + let (result_tx, result_rx) = oneshot::channel(); + self.send_queue_event( + QueueRequest::Cancel { + queue_ids, + item, + result_tx, + }, + result_rx, + ) + .await + } + Some(error) => error.into_bad_request(), + } + } + (&Method::GET, Some("report"), Some("list")) => { + let mut domain = None; + let mut type_ = None; + let mut error = None; + + if let Some(query) = req.uri().query() { + for (key, value) in form_urlencoded::parse(query.as_bytes()) { + match key.as_ref() { + "type" => match value.as_ref() { + "dmarc" => { + type_ = ReportType::Dmarc(()).into(); + } + "tls" => { + type_ = ReportType::Tls(()).into(); + } + _ => { + error = format!("Invalid report type {value:?}.").into(); + break; + } + }, + "domain" => { + domain = value.into_owned().into(); + } + _ => { + error = format!("Invalid parameter {key:?}.").into(); + break; + } + } + } + } + + match error { + None => { + let (result_tx, result_rx) = oneshot::channel(); + self.send_report_event( + ReportRequest::List { + type_, + domain, + result_tx, + }, + result_rx, + ) + .await + } + Some(error) => error.into_bad_request(), + } + } + (&Method::GET, Some("report"), Some("status")) => { + let mut report_ids = Vec::new(); + let mut error = None; + + if let Some(query) = req.uri().query() { + for (key, value) in form_urlencoded::parse(query.as_bytes()) { + match key.as_ref() { + "id" | "ids" => match value.parse_report_ids() { + Ok(ids) => { + report_ids = ids; + } + Err(reason) => { + error = reason.into(); + break; + } + }, + _ => { + error = format!("Invalid parameter {key:?}.").into(); + break; + } + } + } + } + + match error { + None => { + let (result_tx, result_rx) = oneshot::channel(); + self.send_report_event( + ReportRequest::Status { + report_ids, + result_tx, + }, + result_rx, + ) + .await + } + Some(error) => error.into_bad_request(), + } + } + (&Method::GET, Some("report"), Some("cancel")) => { + let mut report_ids = Vec::new(); + let mut error = None; + + if let Some(query) = req.uri().query() { + for (key, value) in form_urlencoded::parse(query.as_bytes()) { + match key.as_ref() { + "id" | "ids" => match value.parse_report_ids() { + Ok(ids) => { + report_ids = ids; + } + Err(reason) => { + error = reason.into(); + break; + } + }, + _ => { + error = format!("Invalid parameter {key:?}.").into(); + break; + } + } + } + } + + match error { + None => { + let (result_tx, result_rx) = oneshot::channel(); + self.send_report_event( + ReportRequest::Cancel { + report_ids, + result_tx, + }, + result_rx, + ) + .await + } + Some(error) => error.into_bad_request(), + } + } + _ => ( + StatusCode::NOT_FOUND, + format!( + "{{\"error\": \"not-found\", \"details\": \"URL {} does not exist.\"}}", + req.uri().path() + ), + ), + }; + + Ok(hyper::Response::builder() + .status(status) + .header(header::CONTENT_TYPE, "application/json; charset=utf-8") + .body( + Full::new(Bytes::from(response)) + .map_err(|never| match never {}) + .boxed(), + ) + .unwrap()) + } + + async fn send_queue_event( + &self, + request: QueueRequest, + rx: oneshot::Receiver, + ) -> (StatusCode, String) { + match self.queue.tx.send(queue::Event::Manage(request)).await { + Ok(_) => match rx.await { + Ok(result) => { + return ( + StatusCode::OK, + serde_json::to_string(&Response { data: result }).unwrap_or_default(), + ) + } + Err(_) => { + tracing::debug!( + context = "queue", + event = "recv-error", + reason = "Failed to receive manage request response." + ); + } + }, + Err(_) => { + tracing::debug!( + context = "queue", + event = "send-error", + reason = "Failed to send manage request event." + ); + } + } + + ( + StatusCode::INTERNAL_SERVER_ERROR, + "{\"error\": \"internal-error\", \"details\": \"Resource unavailable, try again later.\"}" + .to_string(), + ) + } + + async fn send_report_event( + &self, + request: ReportRequest, + rx: oneshot::Receiver, + ) -> (StatusCode, String) { + match self.report.tx.send(reporting::Event::Manage(request)).await { + Ok(_) => match rx.await { + Ok(result) => { + return ( + StatusCode::OK, + serde_json::to_string(&Response { data: result }).unwrap_or_default(), + ) + } + Err(_) => { + tracing::debug!( + context = "queue", + event = "recv-error", + reason = "Failed to receive manage request response." + ); + } + }, + Err(_) => { + tracing::debug!( + context = "queue", + event = "send-error", + reason = "Failed to send manage request event." + ); + } + } + + ( + StatusCode::INTERNAL_SERVER_ERROR, + "{\"error\": \"internal-error\", \"details\": \"Resource unavailable, try again later.\"}" + .to_string(), + ) + } +} + +impl From<&queue::Message> for Message { + fn from(message: &queue::Message) -> Self { + let now = Instant::now(); + + Message { + return_path: message.return_path.clone(), + created: DateTime::from_timestamp(message.created as i64), + size: message.size, + priority: message.priority, + env_id: message.env_id.clone(), + domains: message + .domains + .iter() + .enumerate() + .map(|(idx, domain)| Domain { + name: domain.domain.clone(), + status: match &domain.status { + Status::Scheduled => Status::Scheduled, + Status::Completed(_) => Status::Completed(String::new()), + Status::TemporaryFailure(status) => { + Status::TemporaryFailure(status.to_string()) + } + Status::PermanentFailure(status) => { + Status::PermanentFailure(status.to_string()) + } + }, + retry_num: domain.retry.inner, + next_retry: if domain.retry.due > now { + DateTime::from_timestamp(instant_to_timestamp(now, domain.retry.due) as i64) + .into() + } else { + None + }, + next_notify: if domain.notify.due > now { + DateTime::from_timestamp( + instant_to_timestamp( + now, + domain.notify.due, + ) + as i64, + ) + .into() + } else { + None + }, + recipients: message + .recipients + .iter() + .filter(|rcpt| rcpt.domain_idx == idx) + .map(|rcpt| Recipient { + address: rcpt.address.clone(), + status: match &rcpt.status { + Status::Scheduled => Status::Scheduled, + Status::Completed(status) => { + Status::Completed(status.response.to_string()) + } + Status::TemporaryFailure(status) => { + Status::TemporaryFailure(status.response.to_string()) + } + Status::PermanentFailure(status) => { + Status::PermanentFailure(status.response.to_string()) + } + }, + orcpt: rcpt.orcpt.clone(), + }) + .collect(), + expires: DateTime::from_timestamp( + instant_to_timestamp(now, domain.expires) as i64 + ), + }) + .collect(), + } + } +} + +impl From<(&ReportKey, &ReportValue)> for Report { + fn from((key, value): (&ReportKey, &ReportValue)) -> Self { + match (key, value) { + (ReportType::Dmarc(domain), ReportType::Dmarc(value)) => Report { + domain: domain.inner.clone(), + range_from: DateTime::from_timestamp(value.created as i64), + range_to: DateTime::from_timestamp( + (value.created + value.deliver_at.as_secs()) as i64, + ), + size: value.size, + type_: "dmarc".to_string(), + }, + (ReportType::Tls(domain), ReportType::Tls(value)) => Report { + domain: domain.clone(), + range_from: DateTime::from_timestamp(value.created as i64), + range_to: DateTime::from_timestamp( + (value.created + value.deliver_at.as_secs()) as i64, + ), + size: value.size, + type_: "tls".to_string(), + }, + _ => unreachable!(), + } + } +} + +impl Display for ReportKey { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ReportType::Dmarc(policy) => write!(f, "d!{}!{}", policy.inner, policy.policy), + ReportType::Tls(domain) => write!(f, "t!{domain}"), + } + } +} + +trait ParseValues { + fn parse_timestamp(&self) -> Result; + fn parse_queue_ids(&self) -> Result, String>; + fn parse_report_ids(&self) -> Result, String>; +} + +impl ParseValues for Cow<'_, str> { + fn parse_timestamp(&self) -> Result { + if let Some(dt) = DateTime::parse_rfc3339(self.as_ref()) { + let instant = (dt.to_timestamp() as u64).to_instant(); + if instant >= Instant::now() { + return Ok(instant); + } + } + + Err(format!("Invalid timestamp {self:?}.")) + } + + fn parse_queue_ids(&self) -> Result, String> { + let mut ids = Vec::new(); + for id in self.split(',') { + if !id.is_empty() { + match id.parse() { + Ok(id) => { + ids.push(id); + } + Err(_) => { + return Err(format!("Failed to parse id {id:?}.")); + } + } + } + } + Ok(ids) + } + + fn parse_report_ids(&self) -> Result, String> { + let mut ids = Vec::new(); + for id in self.split(',') { + if !id.is_empty() { + let mut parts = id.split('!'); + match (parts.next(), parts.next()) { + (Some("d"), Some(domain)) if !domain.is_empty() => { + if let Some(policy) = parts.next().and_then(|policy| policy.parse().ok()) { + ids.push(ReportType::Dmarc(ReportPolicy { + inner: domain.to_string(), + policy, + })); + continue; + } + } + (Some("t"), Some(domain)) if !domain.is_empty() => { + ids.push(ReportType::Tls(domain.to_string())); + continue; + } + _ => (), + } + + return Err(format!("Failed to parse id {id:?}.")); + } + } + Ok(ids) + } +} + +trait BadRequest { + fn into_bad_request(self) -> (StatusCode, String); +} + +impl BadRequest for String { + fn into_bad_request(self) -> (StatusCode, String) { + ( + StatusCode::BAD_REQUEST, + format!( + "{{\"error\": \"bad-parameters\", \"details\": {}}}", + serde_json::to_string(&self).unwrap() + ), + ) + } +} + +fn is_zero(num: &i16) -> bool { + *num == 0 +} + +fn serialize_maybe_datetime(value: &Option, serializer: S) -> Result +where + S: Serializer, +{ + match value { + Some(value) => serializer.serialize_some(&value.to_rfc3339()), + None => serializer.serialize_none(), + } +} + +fn deserialize_maybe_datetime<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + if let Some(value) = Option::<&str>::deserialize(deserializer)? { + if let Some(value) = DateTime::parse_rfc3339(value) { + Ok(Some(value)) + } else { + Err(serde::de::Error::custom( + "Failed to parse RFC3339 timestamp", + )) + } + } else { + Ok(None) + } +} + +fn serialize_datetime(value: &DateTime, serializer: S) -> Result +where + S: Serializer, +{ + serializer.serialize_str(&value.to_rfc3339()) +} + +fn deserialize_datetime<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + if let Some(value) = DateTime::parse_rfc3339(<&str>::deserialize(deserializer)?) { + Ok(value) + } else { + Err(serde::de::Error::custom( + "Failed to parse RFC3339 timestamp", + )) + } +} diff --git a/crates/smtp/src/core/mod.rs b/crates/smtp/src/core/mod.rs new file mode 100644 index 00000000..b85465fb --- /dev/null +++ b/crates/smtp/src/core/mod.rs @@ -0,0 +1,350 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{ + borrow::Cow, + hash::Hash, + net::IpAddr, + sync::{atomic::AtomicU32, Arc}, + time::{Duration, Instant}, +}; + +use ahash::AHashMap; +use dashmap::DashMap; +use mail_auth::{common::lru::LruCache, IprevOutput, Resolver, SpfOutput}; +use sieve::{Runtime, Sieve}; +use smtp_proto::request::receiver::{ + BdatReceiver, DataReceiver, DummyDataReceiver, DummyLineReceiver, LineReceiver, RequestReceiver, +}; +use tokio::{ + io::{AsyncRead, AsyncWrite}, + sync::mpsc, +}; +use tokio_rustls::TlsConnector; +use tracing::Span; +use utils::listener::{limiter::InFlight, ServerInstance}; + +use crate::{ + config::{ + DkimSigner, EnvelopeKey, MailAuthConfig, QueueConfig, ReportConfig, SessionConfig, + VerifyStrategy, + }, + inbound::auth::SaslToken, + lookup::{Lookup, SqlDatabase}, + outbound::{ + dane::{DnssecResolver, Tlsa}, + mta_sts, + }, + queue::{self, QuotaLimiter}, + reporting, +}; + +use self::throttle::{Limiter, ThrottleKey, ThrottleKeyHasherBuilder}; + +pub mod if_block; +pub mod management; +pub mod params; +pub mod scripts; +pub mod throttle; +pub mod worker; + +#[derive(Clone)] +pub struct SmtpSessionManager { + pub inner: Arc, +} + +#[derive(Clone)] +pub struct HttpAdminSessionManager { + pub inner: Arc, +} + +impl SmtpSessionManager { + pub fn new(inner: Arc) -> Self { + Self { inner } + } +} + +impl HttpAdminSessionManager { + pub fn new(inner: Arc) -> Self { + Self { inner } + } +} + +pub struct Core { + pub worker_pool: rayon::ThreadPool, + pub session: SessionCore, + pub queue: QueueCore, + pub resolvers: Resolvers, + pub mail_auth: MailAuthConfig, + pub report: ReportCore, + pub sieve: SieveCore, +} + +pub struct SieveCore { + pub runtime: Runtime, + pub scripts: AHashMap>, + pub lookup: AHashMap>, + pub config: SieveConfig, +} + +pub struct SieveConfig { + pub from_addr: String, + pub from_name: String, + pub return_path: String, + pub sign: Vec>, + pub db: Option, +} + +pub struct Resolvers { + pub dns: Resolver, + pub dnssec: DnssecResolver, + pub cache: DnsCache, +} + +pub struct DnsCache { + pub tlsa: LruCache>, + pub mta_sts: LruCache>, +} + +pub struct SessionCore { + pub config: SessionConfig, + pub throttle: DashMap, +} + +pub struct QueueCore { + pub config: QueueConfig, + pub throttle: DashMap, + pub quota: DashMap, ThrottleKeyHasherBuilder>, + pub tx: mpsc::Sender, + pub id_seq: AtomicU32, + pub connectors: TlsConnectors, +} + +pub struct ReportCore { + pub config: ReportConfig, + pub tx: mpsc::Sender, +} + +pub struct TlsConnectors { + pub pki_verify: TlsConnector, + pub dummy_verify: TlsConnector, +} + +pub enum State { + Request(RequestReceiver), + Bdat(BdatReceiver), + Data(DataReceiver), + Sasl(LineReceiver), + DataTooLarge(DummyDataReceiver), + RequestTooLarge(DummyLineReceiver), + None, +} + +pub struct Session { + pub state: State, + pub instance: Arc, + pub core: Arc, + pub span: Span, + pub stream: T, + pub data: SessionData, + pub params: SessionParameters, + pub in_flight: Vec, +} + +pub struct SessionData { + pub local_ip: IpAddr, + pub remote_ip: IpAddr, + pub helo_domain: String, + + pub mail_from: Option, + pub rcpt_to: Vec, + pub rcpt_errors: usize, + pub message: Vec, + + pub authenticated_as: String, + pub auth_errors: usize, + + pub priority: i16, + pub delivery_by: i64, + pub future_release: u64, + + pub valid_until: Instant, + pub bytes_left: usize, + pub messages_sent: usize, + + pub iprev: Option, + pub spf_ehlo: Option, + pub spf_mail_from: Option, + pub dnsbl_error: Option>, +} + +#[derive(Clone)] +pub struct SessionAddress { + pub address: String, + pub address_lcase: String, + pub domain: String, + pub flags: u64, + pub dsn_info: Option, +} + +#[derive(Debug, Default)] +pub struct SessionParameters { + // Global parameters + pub timeout: Duration, + + // Ehlo parameters + pub ehlo_require: bool, + pub ehlo_reject_non_fqdn: bool, + + // Auth parameters + pub auth_lookup: Option>, + pub auth_require: bool, + pub auth_errors_max: usize, + pub auth_errors_wait: Duration, + + // Rcpt parameters + pub rcpt_script: Option>, + pub rcpt_relay: bool, + pub rcpt_errors_max: usize, + pub rcpt_errors_wait: Duration, + pub rcpt_max: usize, + pub rcpt_dsn: bool, + pub rcpt_lookup_domain: Option>, + pub rcpt_lookup_addresses: Option>, + pub rcpt_lookup_expn: Option>, + pub rcpt_lookup_vrfy: Option>, + pub max_message_size: usize, + + // Mail authentication parameters + pub iprev: VerifyStrategy, + pub spf_ehlo: VerifyStrategy, + pub spf_mail_from: VerifyStrategy, + pub dnsbl_policy: u32, +} + +impl SessionData { + pub fn new(local_ip: IpAddr, remote_ip: IpAddr) -> Self { + SessionData { + local_ip, + remote_ip, + helo_domain: String::new(), + mail_from: None, + rcpt_to: Vec::new(), + authenticated_as: String::new(), + priority: 0, + valid_until: Instant::now(), + rcpt_errors: 0, + message: Vec::with_capacity(0), + auth_errors: 0, + messages_sent: 0, + bytes_left: 0, + delivery_by: 0, + future_release: 0, + iprev: None, + spf_ehlo: None, + spf_mail_from: None, + dnsbl_error: None, + } + } +} + +impl Default for State { + fn default() -> Self { + State::Request(RequestReceiver::default()) + } +} + +pub trait Envelope { + fn local_ip(&self) -> IpAddr; + fn remote_ip(&self) -> IpAddr; + fn sender_domain(&self) -> &str; + fn sender(&self) -> &str; + fn rcpt_domain(&self) -> &str; + fn rcpt(&self) -> &str; + fn helo_domain(&self) -> &str; + fn authenticated_as(&self) -> &str; + fn mx(&self) -> &str; + fn listener_id(&self) -> u16; + fn priority(&self) -> i16; + + #[inline(always)] + fn key_to_string(&self, key: &EnvelopeKey) -> Cow<'_, str> { + match key { + EnvelopeKey::Recipient => self.rcpt().into(), + EnvelopeKey::RecipientDomain => self.rcpt_domain().into(), + EnvelopeKey::Sender => self.sender().into(), + EnvelopeKey::SenderDomain => self.sender_domain().into(), + EnvelopeKey::Mx => self.mx().into(), + EnvelopeKey::AuthenticatedAs => self.authenticated_as().into(), + EnvelopeKey::HeloDomain => self.helo_domain().into(), + EnvelopeKey::Listener => self.listener_id().to_string().into(), + EnvelopeKey::RemoteIp => self.remote_ip().to_string().into(), + EnvelopeKey::LocalIp => self.local_ip().to_string().into(), + EnvelopeKey::Priority => self.priority().to_string().into(), + } + } +} + +impl VerifyStrategy { + #[inline(always)] + pub fn verify(&self) -> bool { + matches!(self, VerifyStrategy::Strict | VerifyStrategy::Relaxed) + } + + #[inline(always)] + pub fn is_strict(&self) -> bool { + matches!(self, VerifyStrategy::Strict) + } +} + +impl PartialEq for SessionAddress { + fn eq(&self, other: &Self) -> bool { + self.address_lcase == other.address_lcase + } +} + +impl Eq for SessionAddress {} + +impl Hash for SessionAddress { + fn hash(&self, state: &mut H) { + self.address_lcase.hash(state); + } +} + +impl Ord for SessionAddress { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + match self.domain.cmp(&other.domain) { + std::cmp::Ordering::Equal => self.address_lcase.cmp(&other.address_lcase), + order => order, + } + } +} + +impl PartialOrd for SessionAddress { + fn partial_cmp(&self, other: &Self) -> Option { + match self.domain.partial_cmp(&other.domain) { + Some(std::cmp::Ordering::Equal) => self.address_lcase.partial_cmp(&other.address_lcase), + order => order, + } + } +} diff --git a/crates/smtp/src/core/params.rs b/crates/smtp/src/core/params.rs new file mode 100644 index 00000000..16191391 --- /dev/null +++ b/crates/smtp/src/core/params.rs @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use tokio::io::{AsyncRead, AsyncWrite}; + +use super::Session; + +impl Session { + pub async fn eval_session_params(&mut self) { + let c = &self.core.session.config; + self.data.bytes_left = *c.transfer_limit.eval(self).await; + self.data.valid_until += *c.duration.eval(self).await; + + self.params.timeout = *c.timeout.eval(self).await; + self.params.spf_ehlo = *self.core.mail_auth.spf.verify_ehlo.eval(self).await; + self.params.spf_mail_from = *self.core.mail_auth.spf.verify_mail_from.eval(self).await; + self.params.iprev = *self.core.mail_auth.iprev.verify.eval(self).await; + self.params.dnsbl_policy = *self.core.mail_auth.dnsbl.verify.eval(self).await; + + // Ehlo parameters + let ec = &self.core.session.config.ehlo; + self.params.ehlo_require = *ec.require.eval(self).await; + self.params.ehlo_reject_non_fqdn = *ec.reject_non_fqdn.eval(self).await; + + // Auth parameters + let ac = &self.core.session.config.auth; + self.params.auth_lookup = ac.lookup.eval(self).await.clone(); + self.params.auth_require = *ac.require.eval(self).await; + self.params.auth_errors_max = *ac.errors_max.eval(self).await; + self.params.auth_errors_wait = *ac.errors_wait.eval(self).await; + + // VRFY/EXPN parameters + let rc = &self.core.session.config.rcpt; + self.params.rcpt_lookup_expn = rc.lookup_expn.eval(self).await.clone(); + self.params.rcpt_lookup_vrfy = rc.lookup_vrfy.eval(self).await.clone(); + } + + pub async fn eval_post_auth_params(&mut self) { + // Refresh VRFY/EXPN parameters + let rc = &self.core.session.config.rcpt; + self.params.rcpt_lookup_expn = rc.lookup_expn.eval(self).await.clone(); + self.params.rcpt_lookup_vrfy = rc.lookup_vrfy.eval(self).await.clone(); + } + + pub async fn eval_rcpt_params(&mut self) { + let rc = &self.core.session.config.rcpt; + self.params.rcpt_script = rc.script.eval(self).await.clone(); + self.params.rcpt_relay = *rc.relay.eval(self).await; + self.params.rcpt_errors_max = *rc.errors_max.eval(self).await; + self.params.rcpt_errors_wait = *rc.errors_wait.eval(self).await; + self.params.rcpt_max = *rc.max_recipients.eval(self).await; + self.params.rcpt_lookup_domain = rc.lookup_domains.eval(self).await.clone(); + self.params.rcpt_lookup_addresses = rc.lookup_addresses.eval(self).await.clone(); + self.params.rcpt_dsn = *self.core.session.config.extensions.dsn.eval(self).await; + + self.params.max_message_size = *self + .core + .session + .config + .data + .max_message_size + .eval(self) + .await; + } +} diff --git a/crates/smtp/src/core/scripts.rs b/crates/smtp/src/core/scripts.rs new file mode 100644 index 00000000..dfaba78b --- /dev/null +++ b/crates/smtp/src/core/scripts.rs @@ -0,0 +1,476 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{borrow::Cow, process::Command, sync::Arc, time::Duration}; + +use ahash::AHashMap; +use mail_auth::common::headers::HeaderWriter; +use sieve::{ + compiler::grammar::actions::action_redirect::{ByMode, ByTime, Notify, NotifyItem, Ret}, + CommandType, Envelope, Event, Input, MatchAs, Recipient, Sieve, +}; +use smtp_proto::{ + MAIL_BY_NOTIFY, MAIL_BY_RETURN, MAIL_BY_TRACE, MAIL_RET_FULL, MAIL_RET_HDRS, RCPT_NOTIFY_DELAY, + RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_NEVER, RCPT_NOTIFY_SUCCESS, +}; +use tokio::{ + io::{AsyncRead, AsyncWrite}, + runtime::Handle, +}; + +use crate::{ + lookup::Lookup, + queue::{DomainPart, InstantFromTimestamp, Message}, +}; + +use super::{Core, Session}; + +pub enum ScriptResult { + Accept, + Replace(Vec), + Reject(String), +} + +impl Session { + pub async fn run_script( + &self, + script: Arc, + message: Option>>, + ) -> ScriptResult { + let core = self.core.clone(); + let span = self.span.clone(); + + // Set environment variables + let mut vars_env: AHashMap> = AHashMap::with_capacity(6); + vars_env.insert( + "remote_ip".to_string(), + self.data.remote_ip.to_string().into(), + ); + vars_env.insert( + "helo_domain".to_string(), + self.data.helo_domain.clone().into(), + ); + vars_env.insert( + "authenticated_as".to_string(), + self.data.authenticated_as.clone().into(), + ); + + // Set envelope + let envelope = if let Some(mail_from) = &self.data.mail_from { + let mut envelope: Vec<(Envelope, Cow)> = Vec::with_capacity(6); + envelope.push((Envelope::From, mail_from.address.clone().into())); + if let Some(env_id) = &mail_from.dsn_info { + envelope.push((Envelope::Envid, env_id.clone().into())); + } + if let Some(rcpt) = self.data.rcpt_to.last() { + envelope.push((Envelope::To, rcpt.address.clone().into())); + if let Some(orcpt) = &rcpt.dsn_info { + envelope.push((Envelope::Orcpt, orcpt.clone().into())); + } + } + if (mail_from.flags & MAIL_RET_FULL) != 0 { + envelope.push((Envelope::Ret, "FULL".into())); + } else if (mail_from.flags & MAIL_RET_HDRS) != 0 { + envelope.push((Envelope::Ret, "HDRS".into())); + } + if (mail_from.flags & MAIL_BY_NOTIFY) != 0 { + envelope.push((Envelope::ByMode, "N".into())); + } else if (mail_from.flags & MAIL_BY_RETURN) != 0 { + envelope.push((Envelope::ByMode, "R".into())); + } + envelope + } else { + Vec::with_capacity(0) + }; + + let handle = Handle::current(); + self.core + .spawn_worker(move || { + core.run_script_blocking(script, vars_env, envelope, message, handle, span) + }) + .await + .unwrap_or(ScriptResult::Accept) + } +} + +impl Core { + fn run_script_blocking( + &self, + script: Arc, + vars_env: AHashMap>, + envelope: Vec<(Envelope, Cow<'static, str>)>, + message: Option>>, + handle: Handle, + span: tracing::Span, + ) -> ScriptResult { + // Create filter instance + let mut instance = self + .sieve + .runtime + .filter(message.as_deref().map_or(b"", |m| &m[..])) + .with_vars_env(vars_env) + .with_envelope_list(envelope) + .with_user_address(&self.sieve.config.from_addr) + .with_user_full_name(&self.sieve.config.from_name); + let mut input = Input::script("__script", script); + let mut messages: Vec> = Vec::new(); + + let mut reject_reason = None; + let mut keep_id = usize::MAX; + + // Start event loop + while let Some(result) = instance.run(input) { + match result { + Ok(event) => match event { + Event::IncludeScript { name, optional } => { + if let Some(script) = self.sieve.scripts.get(name.as_str()) { + input = Input::script(name, script.clone()); + } else if optional { + input = false.into(); + } else { + tracing::warn!( + parent: &span, + context = "sieve", + event = "script-not-found", + script = name.as_str() + ); + break; + } + } + Event::ListContains { + lists, + values, + match_as, + } => { + input = false.into(); + 'outer: for list in lists { + if let Some(list) = self.sieve.lookup.get(&list) { + for value in &values { + let result = if !matches!(match_as, MatchAs::Lowercase) { + handle.block_on(list.contains(value)) + } else { + handle.block_on(list.contains(&value.to_lowercase())) + }; + if let Some(true) = result { + input = true.into(); + break 'outer; + } + } + } else { + tracing::debug!( + parent: &span, + context = "sieve", + event = "list-not-found", + list = list, + ); + } + } + } + Event::Execute { + command_type, + command, + arguments, + } => match command_type { + CommandType::Query => { + if let Some(db) = &self.sieve.config.db { + if command + .as_bytes() + .get(..6) + .map_or(false, |q| q.eq_ignore_ascii_case(b"SELECT")) + { + input = handle + .block_on(db.exists(&command, arguments.into_iter())) + .unwrap_or(false) + .into(); + } else { + input = handle + .block_on(db.execute(&command, arguments.into_iter())) + .into(); + } + } else { + tracing::warn!( + parent: &span, + context = "sieve", + event = "config-error", + reason = "No database configured", + ); + input = false.into(); + } + } + CommandType::Binary => { + match Command::new(command).args(arguments).output() { + Ok(result) => { + input = result.status.success().into(); + } + Err(err) => { + tracing::warn!( + parent: &span, + context = "sieve", + event = "execute-failed", + reason = %err, + ); + input = false.into(); + } + } + } + }, + Event::Keep { message_id, .. } => { + keep_id = message_id; + input = true.into(); + } + Event::Discard => { + reject_reason = "503 5.5.3 Message rejected.\r\n".to_string().into(); + input = true.into(); + } + Event::Reject { reason, .. } => { + reject_reason = reason.into(); + input = true.into(); + } + Event::SendMessage { + recipient, + notify, + return_of_content, + by_time, + message_id, + } => { + // Build message + let return_path_lcase = self.sieve.config.return_path.to_lowercase(); + let return_path_domain = return_path_lcase.domain_part().to_string(); + let mut message = Message::new_boxed( + self.sieve.config.return_path.clone(), + return_path_lcase, + return_path_domain, + ); + match recipient { + Recipient::Address(rcpt) => { + handle.block_on(message.add_recipient(rcpt, &self.queue.config)); + } + Recipient::Group(rcpt_list) => { + for rcpt in rcpt_list { + handle + .block_on(message.add_recipient(rcpt, &self.queue.config)); + } + } + Recipient::List(list) => { + if let Some(list) = self.sieve.lookup.get(&list) { + match list.as_ref() { + Lookup::Local(items) => { + for rcpt in items { + handle.block_on( + message.add_recipient(rcpt, &self.queue.config), + ); + } + } + Lookup::Sql(sql) => { + if let Some(items) = handle.block_on(sql.fetch_many("")) + { + for rcpt in items { + handle.block_on( + message.add_recipient( + rcpt, + &self.queue.config, + ), + ); + } + } + } + _ => (), + } + } else { + tracing::warn!( + parent: &span, + context = "sieve", + event = "send-failed", + reason = format!("Lookup {list:?} not found.") + ); + } + } + } + + // Set notify flags + let mut flags = 0; + match notify { + Notify::Never => { + flags = RCPT_NOTIFY_NEVER; + } + Notify::Items(items) => { + for item in items { + flags |= match item { + NotifyItem::Success => RCPT_NOTIFY_SUCCESS, + NotifyItem::Failure => RCPT_NOTIFY_FAILURE, + NotifyItem::Delay => RCPT_NOTIFY_DELAY, + }; + } + } + Notify::Default => (), + } + if flags > 0 { + for rcpt in &mut message.recipients { + rcpt.flags |= flags; + } + } + + // Set ByTime flags + match by_time { + ByTime::Relative { + rlimit, + mode, + trace, + } => { + if trace { + message.flags |= MAIL_BY_TRACE; + } + let rlimit = Duration::from_secs(rlimit); + match mode { + ByMode::Notify => { + for domain in &mut message.domains { + domain.notify.due += rlimit; + } + } + ByMode::Return => { + for domain in &mut message.domains { + domain.notify.due += rlimit; + } + } + ByMode::Default => (), + } + } + ByTime::Absolute { + alimit, + mode, + trace, + } => { + if trace { + message.flags |= MAIL_BY_TRACE; + } + let alimit = (alimit as u64).to_instant(); + match mode { + ByMode::Notify => { + for domain in &mut message.domains { + domain.notify.due = alimit; + } + } + ByMode::Return => { + for domain in &mut message.domains { + domain.expires = alimit; + } + } + ByMode::Default => (), + } + } + ByTime::None => (), + }; + + // Set ret + match return_of_content { + Ret::Full => { + message.flags |= MAIL_RET_FULL; + } + Ret::Hdrs => { + message.flags |= MAIL_RET_HDRS; + } + Ret::Default => (), + } + + // Queue message + if let Some(raw_message) = messages.get(message_id - 1) { + let headers = if !self.sieve.config.sign.is_empty() { + let mut headers = Vec::new(); + for dkim in &self.sieve.config.sign { + match dkim.sign(raw_message) { + Ok(signature) => { + signature.write_header(&mut headers); + } + Err(err) => { + tracing::warn!(parent: &span, + context = "dkim", + event = "sign-failed", + reason = %err); + } + } + } + Some(headers) + } else { + None + }; + + handle.block_on(self.queue.queue_message( + message, + headers.as_deref(), + raw_message, + &span, + )); + } + + input = true.into(); + } + Event::CreatedMessage { message, .. } => { + messages.push(message); + input = true.into(); + } + unsupported => { + tracing::warn!( + parent: &span, + context = "sieve", + event = "runtime-error", + reason = format!("Unsupported event: {unsupported:?}") + ); + break; + } + }, + Err(err) => { + tracing::warn!(parent: &span, + context = "sieve", + event = "runtime-error", + reason = %err + ); + break; + } + } + } + + if keep_id == 0 { + ScriptResult::Accept + } else if let Some(mut reject_reason) = reject_reason { + if !reject_reason.ends_with('\n') { + reject_reason.push_str("\r\n"); + } + let mut reject_bytes = reject_reason.as_bytes().iter(); + if matches!(reject_bytes.next(), Some(ch) if ch.is_ascii_digit()) + && matches!(reject_bytes.next(), Some(ch) if ch.is_ascii_digit()) + && matches!(reject_bytes.next(), Some(ch) if ch.is_ascii_digit()) + && matches!(reject_bytes.next(), Some(ch) if ch == &b' ' ) + { + ScriptResult::Reject(reject_reason) + } else { + ScriptResult::Reject(format!("503 5.5.3 {reject_reason}")) + } + } else { + messages + .into_iter() + .nth(keep_id - 1) + .map(ScriptResult::Replace) + .unwrap_or(ScriptResult::Accept) + } + } +} diff --git a/crates/smtp/src/core/throttle.rs b/crates/smtp/src/core/throttle.rs new file mode 100644 index 00000000..98e5cc0e --- /dev/null +++ b/crates/smtp/src/core/throttle.rs @@ -0,0 +1,309 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use ::utils::listener::limiter::{ConcurrencyLimiter, RateLimiter}; +use dashmap::mapref::entry::Entry; +use tokio::io::{AsyncRead, AsyncWrite}; + +use std::{ + hash::{BuildHasher, Hash, Hasher}, + net::IpAddr, +}; + +use crate::config::*; + +use super::{Envelope, Session}; + +#[derive(Debug)] +pub struct Limiter { + pub rate: Option, + pub concurrency: Option, +} + +#[derive(Debug, Clone, Eq)] +pub struct ThrottleKey { + hash: [u8; 32], +} + +impl PartialEq for ThrottleKey { + fn eq(&self, other: &Self) -> bool { + self.hash == other.hash + } +} + +impl Hash for ThrottleKey { + fn hash(&self, state: &mut H) { + self.hash.hash(state); + } +} + +#[derive(Default)] +pub struct ThrottleKeyHasher { + hash: u64, +} + +impl Hasher for ThrottleKeyHasher { + fn finish(&self) -> u64 { + self.hash + } + + fn write(&mut self, bytes: &[u8]) { + self.hash = u64::from_ne_bytes((&bytes[..std::mem::size_of::()]).try_into().unwrap()); + } +} + +#[derive(Clone, Default)] +pub struct ThrottleKeyHasherBuilder {} + +impl BuildHasher for ThrottleKeyHasherBuilder { + type Hasher = ThrottleKeyHasher; + + fn build_hasher(&self) -> Self::Hasher { + ThrottleKeyHasher::default() + } +} + +impl QueueQuota { + pub fn new_key(&self, e: &impl Envelope) -> ThrottleKey { + let mut hasher = blake3::Hasher::new(); + + if (self.keys & THROTTLE_RCPT) != 0 { + hasher.update(e.rcpt().as_bytes()); + } + if (self.keys & THROTTLE_RCPT_DOMAIN) != 0 { + hasher.update(e.rcpt_domain().as_bytes()); + } + if (self.keys & THROTTLE_SENDER) != 0 { + let sender = e.sender(); + hasher.update(if !sender.is_empty() { sender } else { "<>" }.as_bytes()); + } + if (self.keys & THROTTLE_SENDER_DOMAIN) != 0 { + let sender_domain = e.sender_domain(); + hasher.update( + if !sender_domain.is_empty() { + sender_domain + } else { + "<>" + } + .as_bytes(), + ); + } + + if let Some(messages) = &self.messages { + hasher.update(&messages.to_ne_bytes()[..]); + } + + if let Some(size) = &self.size { + hasher.update(&size.to_ne_bytes()[..]); + } + + ThrottleKey { + hash: hasher.finalize().into(), + } + } +} + +impl Throttle { + pub fn new_key(&self, e: &impl Envelope) -> ThrottleKey { + let mut hasher = blake3::Hasher::new(); + + if (self.keys & THROTTLE_RCPT) != 0 { + hasher.update(e.rcpt().as_bytes()); + } + if (self.keys & THROTTLE_RCPT_DOMAIN) != 0 { + hasher.update(e.rcpt_domain().as_bytes()); + } + if (self.keys & THROTTLE_SENDER) != 0 { + let sender = e.sender(); + hasher.update(if !sender.is_empty() { sender } else { "<>" }.as_bytes()); + } + if (self.keys & THROTTLE_SENDER_DOMAIN) != 0 { + let sender_domain = e.sender_domain(); + hasher.update( + if !sender_domain.is_empty() { + sender_domain + } else { + "<>" + } + .as_bytes(), + ); + } + if (self.keys & THROTTLE_HELO_DOMAIN) != 0 { + hasher.update(e.helo_domain().as_bytes()); + } + if (self.keys & THROTTLE_AUTH_AS) != 0 { + hasher.update(e.authenticated_as().as_bytes()); + } + if (self.keys & THROTTLE_LISTENER) != 0 { + hasher.update(&e.listener_id().to_ne_bytes()[..]); + } + if (self.keys & THROTTLE_MX) != 0 { + hasher.update(e.mx().as_bytes()); + } + if (self.keys & THROTTLE_REMOTE_IP) != 0 { + match &e.remote_ip() { + IpAddr::V4(ip) => { + hasher.update(&ip.octets()[..]); + } + IpAddr::V6(ip) => { + hasher.update(&ip.octets()[..]); + } + } + } + if (self.keys & THROTTLE_LOCAL_IP) != 0 { + match &e.local_ip() { + IpAddr::V4(ip) => { + hasher.update(&ip.octets()[..]); + } + IpAddr::V6(ip) => { + hasher.update(&ip.octets()[..]); + } + } + } + if let Some(rate_limit) = &self.rate { + hasher.update(&rate_limit.period.as_secs().to_ne_bytes()[..]); + hasher.update(&rate_limit.requests.to_ne_bytes()[..]); + } + if let Some(concurrency) = &self.concurrency { + hasher.update(&concurrency.to_ne_bytes()[..]); + } + + ThrottleKey { + hash: hasher.finalize().into(), + } + } +} + +impl Session { + pub async fn is_allowed(&mut self) -> bool { + let throttles = if !self.data.rcpt_to.is_empty() { + &self.core.session.config.throttle.rcpt_to + } else if self.data.mail_from.is_some() { + &self.core.session.config.throttle.mail_from + } else { + &self.core.session.config.throttle.connect + }; + + for t in throttles { + if t.conditions.conditions.is_empty() || t.conditions.eval(self).await { + if (t.keys & THROTTLE_RCPT_DOMAIN) != 0 { + let d = self + .data + .rcpt_to + .last() + .map(|r| r.domain.as_str()) + .unwrap_or_default(); + + if self.data.rcpt_to.iter().filter(|p| p.domain == d).count() > 1 { + continue; + } + } + + // Build throttle key + match self.core.session.throttle.entry(t.new_key(self)) { + Entry::Occupied(mut e) => { + let limiter = e.get_mut(); + if let Some(limiter) = &limiter.concurrency { + if let Some(inflight) = limiter.is_allowed() { + self.in_flight.push(inflight); + } else { + tracing::debug!( + parent: &self.span, + context = "throttle", + event = "too-many-requests", + max_concurrent = limiter.max_concurrent, + "Too many concurrent requests." + ); + return false; + } + } + if let Some(limiter) = &mut limiter.rate { + if !limiter.is_allowed() { + tracing::debug!( + parent: &self.span, + context = "throttle", + event = "rate-limit-exceeded", + max_requests = limiter.max_requests as u64, + max_interval = limiter.max_interval as u64, + "Rate limit exceeded." + ); + return false; + } + } + } + Entry::Vacant(e) => { + let concurrency = t.concurrency.map(|concurrency| { + let limiter = ConcurrencyLimiter::new(concurrency); + if let Some(inflight) = limiter.is_allowed() { + self.in_flight.push(inflight); + } + limiter + }); + let rate = t.rate.as_ref().map(|rate| { + let mut r = RateLimiter::new( + rate.requests, + std::cmp::min(rate.period.as_secs(), 1), + ); + r.is_allowed(); + r + }); + + e.insert(Limiter { rate, concurrency }); + } + } + } + } + + true + } + + pub fn throttle_rcpt(&self, rcpt: &str, rate: &Rate, ctx: &str) -> bool { + let mut hasher = blake3::Hasher::new(); + hasher.update(rcpt.as_bytes()); + hasher.update(ctx.as_bytes()); + hasher.update(&rate.period.as_secs().to_ne_bytes()[..]); + hasher.update(&rate.requests.to_ne_bytes()[..]); + let key = ThrottleKey { + hash: hasher.finalize().into(), + }; + + match self.core.session.throttle.entry(key) { + Entry::Occupied(mut e) => { + if let Some(limiter) = &mut e.get_mut().rate { + limiter.is_allowed() + } else { + false + } + } + Entry::Vacant(e) => { + let mut limiter = RateLimiter::new(rate.requests, rate.period.as_secs()); + limiter.is_allowed(); + e.insert(Limiter { + rate: limiter.into(), + concurrency: None, + }); + true + } + } + } +} diff --git a/crates/smtp/src/core/worker.rs b/crates/smtp/src/core/worker.rs new file mode 100644 index 00000000..95463f1b --- /dev/null +++ b/crates/smtp/src/core/worker.rs @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::sync::{atomic::Ordering, Arc}; + +use tokio::sync::oneshot; + +use super::Core; + +impl Core { + pub async fn spawn_worker(&self, f: U) -> Option + where + U: FnOnce() -> V + Send + 'static, + V: Sync + Send + 'static, + { + let (tx, rx) = oneshot::channel(); + + self.worker_pool.spawn(move || { + tx.send(f()).ok(); + }); + + match rx.await { + Ok(result) => Some(result), + Err(err) => { + tracing::warn!( + context = "worker-pool", + event = "error", + reason = %err, + ); + None + } + } + } + + fn cleanup(&self) { + for throttle in [&self.session.throttle, &self.queue.throttle] { + throttle.retain(|_, v| { + v.concurrency + .as_ref() + .map_or(false, |c| c.concurrent.load(Ordering::Relaxed) > 0) + || v.rate + .as_ref() + .map_or(false, |r| r.elapsed().as_secs_f64() < r.max_interval) + }); + } + self.queue.quota.retain(|_, v| { + v.messages.load(Ordering::Relaxed) > 0 || v.size.load(Ordering::Relaxed) > 0 + }); + } +} + +pub trait SpawnCleanup { + fn spawn_cleanup(&self); +} + +impl SpawnCleanup for Arc { + fn spawn_cleanup(&self) { + let core = self.clone(); + self.worker_pool.spawn(move || { + core.cleanup(); + }); + } +} diff --git a/crates/smtp/src/inbound/auth.rs b/crates/smtp/src/inbound/auth.rs new file mode 100644 index 00000000..007adb7a --- /dev/null +++ b/crates/smtp/src/inbound/auth.rs @@ -0,0 +1,237 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use mail_parser::decoders::base64::base64_decode; +use mail_send::Credentials; +use smtp_proto::{IntoString, AUTH_LOGIN, AUTH_OAUTHBEARER, AUTH_PLAIN, AUTH_XOAUTH2}; +use tokio::io::{AsyncRead, AsyncWrite}; + +use crate::{core::Session, lookup::Item}; + +pub struct SaslToken { + mechanism: u64, + credentials: Credentials, +} + +impl SaslToken { + pub fn from_mechanism(mechanism: u64) -> Option { + match mechanism { + AUTH_PLAIN | AUTH_LOGIN => SaslToken { + mechanism, + credentials: Credentials::Plain { + username: String::new(), + secret: String::new(), + }, + } + .into(), + AUTH_OAUTHBEARER => SaslToken { + mechanism, + credentials: Credentials::OAuthBearer { + token: String::new(), + }, + } + .into(), + AUTH_XOAUTH2 => SaslToken { + mechanism, + credentials: Credentials::XOauth2 { + username: String::new(), + secret: String::new(), + }, + } + .into(), + _ => None, + } + } +} + +impl Session { + pub async fn handle_sasl_response( + &mut self, + token: &mut SaslToken, + response: &[u8], + ) -> Result { + if response.is_empty() { + match (token.mechanism, &token.credentials) { + (AUTH_PLAIN | AUTH_XOAUTH2 | AUTH_OAUTHBEARER, _) => { + self.write(b"334 Go ahead.\r\n").await?; + return Ok(true); + } + (AUTH_LOGIN, Credentials::Plain { username, secret }) => { + if username.is_empty() && secret.is_empty() { + self.write(b"334 VXNlciBOYW1lAA==\r\n").await?; + return Ok(true); + } + } + _ => (), + } + } else if let Some(response) = base64_decode(response) { + match (token.mechanism, &mut token.credentials) { + (AUTH_PLAIN, Credentials::Plain { username, secret }) => { + let mut b_username = Vec::new(); + let mut b_secret = Vec::new(); + let mut arg_num = 0; + for ch in response { + if ch != 0 { + if arg_num == 1 { + b_username.push(ch); + } else if arg_num == 2 { + b_secret.push(ch); + } + } else { + arg_num += 1; + } + } + match (String::from_utf8(b_username), String::from_utf8(b_secret)) { + (Ok(s_username), Ok(s_secret)) if !s_username.is_empty() => { + *username = s_username; + *secret = s_secret; + return self + .authenticate(std::mem::take(&mut token.credentials)) + .await; + } + _ => (), + } + } + (AUTH_LOGIN, Credentials::Plain { username, secret }) => { + return if username.is_empty() { + *username = response.into_string(); + self.write(b"334 UGFzc3dvcmQA\r\n").await?; + Ok(true) + } else { + *secret = response.into_string(); + self.authenticate(std::mem::take(&mut token.credentials)) + .await + }; + } + (AUTH_OAUTHBEARER, Credentials::OAuthBearer { token: token_ }) => { + let response = response.into_string(); + if response.contains("auth=") { + *token_ = response; + return self + .authenticate(std::mem::take(&mut token.credentials)) + .await; + } + } + (AUTH_XOAUTH2, Credentials::XOauth2 { username, secret }) => { + let mut b_username = Vec::new(); + let mut b_secret = Vec::new(); + let mut arg_num = 0; + let mut in_arg = false; + + for ch in response { + if in_arg { + if ch != 1 { + if arg_num == 1 { + b_username.push(ch); + } else if arg_num == 2 { + b_secret.push(ch); + } + } else { + in_arg = false; + } + } else if ch == b'=' { + arg_num += 1; + in_arg = true; + } + } + match (String::from_utf8(b_username), String::from_utf8(b_secret)) { + (Ok(s_username), Ok(s_secret)) if !s_username.is_empty() => { + *username = s_username; + *secret = s_secret; + return self + .authenticate(std::mem::take(&mut token.credentials)) + .await; + } + _ => (), + } + } + + _ => (), + } + } + + self.auth_error(b"500 5.5.6 Invalid challenge.\r\n").await + } + + pub async fn authenticate(&mut self, credentials: Credentials) -> Result { + if let Some(lookup) = &self.params.auth_lookup { + let authenticated_as = match &credentials { + Credentials::Plain { username, .. } + | Credentials::XOauth2 { username, .. } + | Credentials::OAuthBearer { token: username } => username.to_string(), + }; + if let Some(is_authenticated) = lookup + .lookup(Item::Authenticate(credentials)) + .await + .map(bool::from) + { + tracing::debug!( + parent: &self.span, + context = "auth", + event = "authenticate", + result = if is_authenticated {"success"} else {"failed"} + ); + return if is_authenticated { + self.data.authenticated_as = authenticated_as; + self.eval_post_auth_params().await; + self.write(b"235 2.7.0 Authentication succeeded.\r\n") + .await?; + Ok(false) + } else { + self.auth_error(b"535 5.7.8 Authentication credentials invalid.\r\n") + .await + }; + } + } else { + tracing::warn!( + parent: &self.span, + context = "auth", + event = "error", + "No lookup list configured for authentication." + ); + } + self.write(b"454 4.7.0 Temporary authentication failure\r\n") + .await?; + + Ok(false) + } + + pub async fn auth_error(&mut self, response: &[u8]) -> Result { + tokio::time::sleep(self.params.auth_errors_wait).await; + self.data.auth_errors += 1; + self.write(response).await?; + if self.data.auth_errors < self.params.auth_errors_max { + Ok(false) + } else { + self.write(b"421 4.3.0 Too many authentication errors, disconnecting.\r\n") + .await?; + tracing::debug!( + parent: &self.span, + event = "disconnect", + reason = "auth-errors", + "Too many authentication errors." + ); + Err(()) + } + } +} diff --git a/crates/smtp/src/inbound/data.rs b/crates/smtp/src/inbound/data.rs new file mode 100644 index 00000000..f065128c --- /dev/null +++ b/crates/smtp/src/inbound/data.rs @@ -0,0 +1,680 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{ + borrow::Cow, + path::PathBuf, + process::Stdio, + sync::Arc, + time::{Duration, Instant, SystemTime}, +}; + +use mail_auth::{ + common::headers::HeaderWriter, dmarc, AuthenticatedMessage, AuthenticationResults, DkimResult, + DmarcResult, ReceivedSpf, +}; +use mail_builder::headers::{date::Date, message_id::generate_message_id_header}; +use smtp_proto::{ + MAIL_BY_RETURN, RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_NEVER, RCPT_NOTIFY_SUCCESS, +}; +use tokio::{ + io::{AsyncRead, AsyncWrite, AsyncWriteExt}, + process::Command, +}; + +use crate::{ + config::DNSBL_FROM, + core::{scripts::ScriptResult, Session, SessionAddress}, + queue::{self, DomainPart, Message, SimpleEnvelope}, + reporting::analysis::AnalyzeReport, +}; + +use super::IsTls; + +impl Session { + pub async fn queue_message(&mut self) -> Cow<'static, [u8]> { + // Authenticate message + let raw_message = Arc::new(std::mem::take(&mut self.data.message)); + let auth_message = if let Some(auth_message) = AuthenticatedMessage::parse(&raw_message) { + auth_message + } else { + tracing::info!(parent: &self.span, + context = "data", + event = "parse-failed", + size = raw_message.len()); + + return (&b"550 5.7.7 Failed to parse message.\r\n"[..]).into(); + }; + + // Validate DNSBL + let from = auth_message.from(); + let from_domain = from.domain_part(); + if !from_domain.is_empty() + && !self + .is_domain_dnsbl_allowed(from_domain, "from", DNSBL_FROM) + .await + { + return self.reset_dnsbl_error().unwrap().into(); + } + + // Loop detection + let dc = &self.core.session.config.data; + let ac = &self.core.mail_auth; + let rc = &self.core.report.config; + if auth_message.received_headers_count() > *dc.max_received_headers.eval(self).await { + tracing::info!(parent: &self.span, + context = "data", + event = "loop-detected", + return_path = self.data.mail_from.as_ref().unwrap().address, + from = auth_message.from(), + received_headers = auth_message.received_headers_count()); + return (&b"450 4.4.6 Too many Received headers. Possible loop detected.\r\n"[..]) + .into(); + } + + // Verify DKIM + let dkim = *ac.dkim.verify.eval(self).await; + let dmarc = *ac.dmarc.verify.eval(self).await; + let dkim_output = if dkim.verify() || dmarc.verify() { + let dkim_output = self.core.resolvers.dns.verify_dkim(&auth_message).await; + let rejected = dkim.is_strict() + && !dkim_output + .iter() + .any(|d| matches!(d.result(), DkimResult::Pass)); + + // Send reports for failed signatures + if let Some(rate) = rc.dkim.send.eval(self).await { + for output in &dkim_output { + if let Some(rcpt) = output.failure_report_addr() { + self.send_dkim_report(rcpt, &auth_message, rate, rejected, output) + .await; + } + } + } + + if rejected { + tracing::info!(parent: &self.span, + context = "dkim", + event = "failed", + return_path = self.data.mail_from.as_ref().unwrap().address, + from = auth_message.from(), + result = ?dkim_output.iter().map(|d| d.result().to_string()).collect::>(), + "No passing DKIM signatures found."); + + // 'Strict' mode violates the advice of Section 6.1 of RFC6376 + return if dkim_output + .iter() + .any(|d| matches!(d.result(), DkimResult::TempError(_))) + { + (&b"451 4.7.20 No passing DKIM signatures found.\r\n"[..]).into() + } else { + (&b"550 5.7.20 No passing DKIM signatures found.\r\n"[..]).into() + }; + } else { + tracing::debug!(parent: &self.span, + context = "dkim", + event = "verify", + return_path = self.data.mail_from.as_ref().unwrap().address, + from = auth_message.from(), + result = ?dkim_output.iter().map(|d| d.result().to_string()).collect::>()); + } + dkim_output + } else { + vec![] + }; + + // Verify ARC + let arc = *ac.arc.verify.eval(self).await; + let arc_sealer = ac.arc.seal.eval(self).await; + let arc_output = if arc.verify() || arc_sealer.is_some() { + let arc_output = self.core.resolvers.dns.verify_arc(&auth_message).await; + + if arc.is_strict() + && !matches!(arc_output.result(), DkimResult::Pass | DkimResult::None) + { + tracing::info!(parent: &self.span, + context = "arc", + event = "auth-failed", + return_path = self.data.mail_from.as_ref().unwrap().address, + from = auth_message.from(), + result = %arc_output.result(), + "ARC validation failed."); + + return if matches!(arc_output.result(), DkimResult::TempError(_)) { + (&b"451 4.7.29 ARC validation failed.\r\n"[..]).into() + } else { + (&b"550 5.7.29 ARC validation failed.\r\n"[..]).into() + }; + } else { + tracing::debug!(parent: &self.span, + context = "arc", + event = "verify", + return_path = self.data.mail_from.as_ref().unwrap().address, + from = auth_message.from(), + result = %arc_output.result()); + } + arc_output.into() + } else { + None + }; + + // Build authentication results header + let mail_from = self.data.mail_from.as_ref().unwrap(); + let mut auth_results = AuthenticationResults::new(&self.instance.hostname); + if !dkim_output.is_empty() { + auth_results = auth_results.with_dkim_results(&dkim_output, auth_message.from()) + } + if let Some(spf_ehlo) = &self.data.spf_ehlo { + auth_results = auth_results.with_spf_ehlo_result( + spf_ehlo, + self.data.remote_ip, + &self.data.helo_domain, + ); + } + if let Some(spf_mail_from) = &self.data.spf_mail_from { + auth_results = auth_results.with_spf_mailfrom_result( + spf_mail_from, + self.data.remote_ip, + &mail_from.address, + &self.data.helo_domain, + ); + } + if let Some(iprev) = &self.data.iprev { + auth_results = auth_results.with_iprev_result(iprev, self.data.remote_ip); + } + + // Verify DMARC + match &self.data.spf_mail_from { + Some(spf_output) if dmarc.verify() => { + let dmarc_output = self + .core + .resolvers + .dns + .verify_dmarc( + &auth_message, + &dkim_output, + if !mail_from.domain.is_empty() { + &mail_from.domain + } else { + &self.data.helo_domain + }, + spf_output, + ) + .await; + + let rejected = dmarc.is_strict() + && dmarc_output.policy() == dmarc::Policy::Reject + && !(matches!(dmarc_output.spf_result(), DmarcResult::Pass) + || matches!(dmarc_output.dkim_result(), DmarcResult::Pass)); + let is_temp_fail = rejected + && matches!(dmarc_output.spf_result(), DmarcResult::TempError(_)) + || matches!(dmarc_output.dkim_result(), DmarcResult::TempError(_)); + + // Add to DMARC output to the Authentication-Results header + auth_results = auth_results.with_dmarc_result(&dmarc_output); + + if !rejected { + tracing::debug!(parent: &self.span, + context = "dmarc", + event = "verify", + return_path = mail_from.address, + from = auth_message.from(), + dkim_result = %dmarc_output.dkim_result(), + spf_result = %dmarc_output.spf_result()); + } else { + tracing::info!(parent: &self.span, + context = "dmarc", + event = "auth-failed", + return_path = mail_from.address, + from = auth_message.from(), + dkim_result = %dmarc_output.dkim_result(), + spf_result = %dmarc_output.spf_result()); + } + + // Send DMARC report + if dmarc_output.requested_reports() { + self.send_dmarc_report( + &auth_message, + &auth_results, + rejected, + dmarc_output, + &dkim_output, + &arc_output, + ) + .await; + } + + if rejected { + return if is_temp_fail { + (&b"451 4.7.1 Email temporarily rejected per DMARC policy.\r\n"[..]).into() + } else { + (&b"550 5.7.1 Email rejected per DMARC policy.\r\n"[..]).into() + }; + } + } + _ => (), + } + + // Analyze reports + if self.is_report() { + self.core.analyze_report(raw_message.clone()); + if !rc.analysis.forward { + self.data.messages_sent += 1; + return (b"250 2.0.0 Message queued for delivery.\r\n"[..]).into(); + } + } + + // Pipe message + let mut edited_message = None; + for pipe in &dc.pipe_commands { + if let Some(command_) = pipe.command.eval(self).await { + let piped_message = edited_message.as_ref().unwrap_or(&raw_message).clone(); + let timeout = *pipe.timeout.eval(self).await; + + let mut command = Command::new(command_); + for argument in pipe.arguments.eval(self).await { + command.arg(argument); + } + match command + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .kill_on_drop(true) + .spawn() + { + Ok(mut child) => { + if let Some(mut stdin) = child.stdin.take() { + match tokio::time::timeout(timeout, stdin.write_all(&piped_message)) + .await + { + Ok(Ok(_)) => { + drop(stdin); + match tokio::time::timeout(timeout, child.wait_with_output()) + .await + { + Ok(Ok(output)) => { + if output.status.success() + && !output.stdout.is_empty() + && output.stdout[..] != piped_message[..] + { + edited_message = Arc::new(output.stdout).into(); + } + + tracing::debug!(parent: &self.span, + context = "pipe", + event = "success", + command = command_, + status = output.status.to_string()); + } + Ok(Err(err)) => { + tracing::warn!(parent: &self.span, + context = "pipe", + event = "exec-error", + command = command_, + reason = %err); + } + Err(_) => { + tracing::warn!(parent: &self.span, + context = "pipe", + event = "timeout", + command = command_); + } + } + } + Ok(Err(err)) => { + tracing::warn!(parent: &self.span, + context = "pipe", + event = "write-error", + command = command_, + reason = %err); + } + Err(_) => { + tracing::warn!(parent: &self.span, + context = "pipe", + event = "stdin-timeout", + command = command_); + } + } + } else { + tracing::warn!(parent: &self.span, + context = "pipe", + event = "stdin-failed", + command = command_); + } + } + Err(err) => { + tracing::warn!(parent: &self.span, + context = "pipe", + event = "spawn-error", + command = command_, + reason = %err); + } + } + } + } + + // Sieve filtering + if let Some(script) = dc.script.eval(self).await { + match self + .run_script( + script.clone(), + Some(edited_message.as_ref().unwrap_or(&raw_message).clone()), + ) + .await + { + ScriptResult::Accept => (), + ScriptResult::Replace(new_message) => { + edited_message = Arc::new(new_message).into(); + } + ScriptResult::Reject(message) => { + tracing::debug!(parent: &self.span, + context = "data", + event = "sieve-reject", + reason = message); + + return message.into_bytes().into(); + } + } + } + + // Build message + let mail_from = self.data.mail_from.clone().unwrap(); + let rcpt_to = std::mem::take(&mut self.data.rcpt_to); + let mut message = self.build_message(mail_from, rcpt_to).await; + + // Add Received header + let mut headers = Vec::with_capacity(64); + if *dc.add_received.eval(self).await { + self.write_received(&mut headers, message.id) + } + + // Add authentication results header + if *dc.add_auth_results.eval(self).await { + auth_results.write_header(&mut headers); + } + + // Add Received-SPF header + if let Some(spf_output) = &self.data.spf_mail_from { + if *dc.add_received_spf.eval(self).await { + ReceivedSpf::new( + spf_output, + self.data.remote_ip, + &self.data.helo_domain, + &message.return_path, + &self.instance.hostname, + ) + .write_header(&mut headers); + } + } + + // ARC Seal + if let (Some(arc_sealer), Some(arc_output)) = (arc_sealer, &arc_output) { + if !dkim_output.is_empty() && arc_output.can_be_sealed() { + match arc_sealer.seal(&auth_message, &auth_results, arc_output) { + Ok(set) => { + set.write_header(&mut headers); + } + Err(err) => { + tracing::info!(parent: &self.span, + context = "arc", + event = "seal-failed", + return_path = message.return_path, + from = auth_message.from(), + "Failed to seal message: {}", err); + } + } + } + } + + // Add any missing headers + if !auth_message.has_date_header() && *dc.add_date.eval(self).await { + headers.extend_from_slice(b"Date: "); + headers.extend_from_slice(Date::now().to_rfc822().as_bytes()); + headers.extend_from_slice(b"\r\n"); + } + if !auth_message.has_message_id_header() && *dc.add_message_id.eval(self).await { + headers.extend_from_slice(b"Message-ID: "); + let _ = generate_message_id_header(&mut headers, &self.instance.hostname); + headers.extend_from_slice(b"\r\n"); + } + + // Add Return-Path + if *dc.add_return_path.eval(self).await { + headers.extend_from_slice(b"Return-Path: <"); + headers.extend_from_slice(message.return_path.as_bytes()); + headers.extend_from_slice(b">\r\n"); + } + + // DKIM sign + let raw_message = edited_message.unwrap_or(raw_message); + for signer in ac.dkim.sign.eval(self).await.iter() { + match signer.sign_chained(&[headers.as_ref(), &raw_message]) { + Ok(signature) => { + signature.write_header(&mut headers); + } + Err(err) => { + tracing::info!(parent: &self.span, + context = "dkim", + event = "sign-failed", + return_path = message.return_path, + "Failed to sign message: {}", err); + } + } + } + + // Update size + message.size = raw_message.len() + headers.len(); + + // Verify queue quota + if self.core.queue.has_quota(&mut message).await { + if self + .core + .queue + .queue_message(message, Some(&headers), &raw_message, &self.span) + .await + { + self.data.messages_sent += 1; + (b"250 2.0.0 Message queued for delivery.\r\n"[..]).into() + } else { + (b"451 4.3.5 Unable to accept message at this time.\r\n"[..]).into() + } + } else { + tracing::warn!( + parent: &self.span, + context = "queue", + event = "quota-exceeded", + from = message.return_path, + "Queue quota exceeded, rejecting message." + ); + (b"452 4.3.1 Mail system full, try again later.\r\n"[..]).into() + } + } + + pub async fn build_message( + &self, + mail_from: SessionAddress, + mut rcpt_to: Vec, + ) -> Box { + // Build message + let mut message = Box::new(Message { + id: self.core.queue.queue_id(), + path: PathBuf::new(), + created: SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .map_or(0, |d| d.as_secs()), + return_path: mail_from.address, + return_path_lcase: mail_from.address_lcase, + return_path_domain: mail_from.domain, + recipients: Vec::with_capacity(rcpt_to.len()), + domains: Vec::with_capacity(3), + flags: mail_from.flags, + priority: self.data.priority, + size: 0, + env_id: mail_from.dsn_info, + queue_refs: Vec::with_capacity(0), + }); + + // Add recipients + let future_release = Duration::from_secs(self.data.future_release); + rcpt_to.sort_unstable(); + for rcpt in rcpt_to { + if message + .domains + .last() + .map_or(true, |d| d.domain != rcpt.domain) + { + let envelope = SimpleEnvelope::new(message.as_ref(), &rcpt.domain); + + // Set next retry time + let retry = if self.data.future_release == 0 { + queue::Schedule::now() + } else { + queue::Schedule::later(future_release) + }; + + // Set expiration and notification times + let config = &self.core.queue.config; + let notify_intervals = config.notify.eval(&envelope).await; + let (notify, expires) = if self.data.delivery_by == 0 { + ( + queue::Schedule::later(future_release + *notify_intervals.first().unwrap()), + Instant::now() + future_release + *config.expire.eval(&envelope).await, + ) + } else if (message.flags & MAIL_BY_RETURN) != 0 { + ( + queue::Schedule::later(future_release + *notify_intervals.first().unwrap()), + Instant::now() + Duration::from_secs(self.data.delivery_by as u64), + ) + } else { + let expire = *config.expire.eval(&envelope).await; + let expire_secs = expire.as_secs(); + let notify = if self.data.delivery_by.is_positive() { + let notify_at = self.data.delivery_by as u64; + if expire_secs > notify_at { + Duration::from_secs(notify_at) + } else { + *notify_intervals.first().unwrap() + } + } else { + let notify_at = -self.data.delivery_by as u64; + if expire_secs > notify_at { + Duration::from_secs(expire_secs - notify_at) + } else { + *notify_intervals.first().unwrap() + } + }; + let mut notify = queue::Schedule::later(future_release + notify); + notify.inner = (notify_intervals.len() - 1) as u32; // Disable further notification attempts + + (notify, Instant::now() + expire) + }; + + message.domains.push(queue::Domain { + retry, + notify, + expires, + status: queue::Status::Scheduled, + domain: rcpt.domain, + changed: false, + }); + } + + message.recipients.push(queue::Recipient { + address: rcpt.address, + address_lcase: rcpt.address_lcase, + status: queue::Status::Scheduled, + flags: if rcpt.flags + & (RCPT_NOTIFY_DELAY + | RCPT_NOTIFY_FAILURE + | RCPT_NOTIFY_SUCCESS + | RCPT_NOTIFY_NEVER) + != 0 + { + rcpt.flags + } else { + rcpt.flags | RCPT_NOTIFY_DELAY | RCPT_NOTIFY_FAILURE + }, + domain_idx: message.domains.len() - 1, + orcpt: rcpt.dsn_info, + }); + } + message + } + + pub async fn can_send_data(&mut self) -> Result { + if !self.data.rcpt_to.is_empty() { + if self.data.messages_sent + < *self.core.session.config.data.max_messages.eval(self).await + { + Ok(true) + } else { + tracing::debug!( + parent: &self.span, + context = "data", + event = "too-many-messages", + "Maximum number of messages per session exceeded." + ); + self.write(b"451 4.4.5 Maximum number of messages per session exceeded.\r\n") + .await?; + Ok(false) + } + } else { + self.write(b"503 5.5.1 RCPT is required first.\r\n").await?; + Ok(false) + } + } + + fn write_received(&self, headers: &mut Vec, id: u64) { + headers.extend_from_slice(b"Received: from "); + headers.extend_from_slice(self.data.helo_domain.as_bytes()); + headers.extend_from_slice(b" ("); + headers.extend_from_slice( + self.data + .iprev + .as_ref() + .and_then(|ir| ir.ptr.as_ref()) + .and_then(|ptr| ptr.first().map(|s| s.as_str())) + .unwrap_or("unknown") + .as_bytes(), + ); + headers.extend_from_slice(b" ["); + headers.extend_from_slice(self.data.remote_ip.to_string().as_bytes()); + headers.extend_from_slice(b"])\r\n\t"); + self.stream.write_tls_header(headers); + headers.extend_from_slice(b"by "); + headers.extend_from_slice(self.instance.hostname.as_bytes()); + headers.extend_from_slice(b" (Stalwart SMTP) with "); + headers.extend_from_slice( + if self.stream.is_tls() { + "ESMTPS" + } else { + "ESMTP" + } + .as_bytes(), + ); + headers.extend_from_slice(b" id "); + headers.extend_from_slice(format!("{id:X}").as_bytes()); + headers.extend_from_slice(b";\r\n\t"); + headers.extend_from_slice(Date::now().to_rfc822().as_bytes()); + headers.extend_from_slice(b"\r\n"); + } +} diff --git a/crates/smtp/src/inbound/ehlo.rs b/crates/smtp/src/inbound/ehlo.rs new file mode 100644 index 00000000..0b776e75 --- /dev/null +++ b/crates/smtp/src/inbound/ehlo.rs @@ -0,0 +1,392 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{net::IpAddr, time::SystemTime}; + +use crate::{ + config::{DNSBL_EHLO, DNSBL_IP}, + core::{scripts::ScriptResult, Session}, +}; +use mail_auth::spf::verify::HasLabels; +use smtp_proto::*; +use tokio::io::{AsyncRead, AsyncWrite}; + +use super::IsTls; + +impl Session { + pub async fn handle_ehlo(&mut self, domain: String) -> Result<(), ()> { + // Set EHLO domain + + if domain != self.data.helo_domain { + // Reject non-FQDN EHLO domains - simply checks that the hostname has at least one dot + if self.params.ehlo_reject_non_fqdn && !domain.as_str().has_labels() { + tracing::debug!(parent: &self.span, + context = "ehlo", + event = "reject", + reason = "invalid", + domain = domain, + ); + + return self.write(b"550 5.5.0 Invalid EHLO domain.\r\n").await; + } + + // Check DNSBL + if !self + .is_domain_dnsbl_allowed(&domain, "ehlo", DNSBL_EHLO) + .await + { + self.write_dnsbl_error().await?; + self.reset_dnsbl_error(); // Reset error in case a new EHLO is issued + return Ok(()); + } + + // SPF check + let prev_helo_domain = std::mem::replace(&mut self.data.helo_domain, domain); + if self.params.spf_ehlo.verify() { + let spf_output = self + .core + .resolvers + .dns + .verify_spf_helo( + self.data.remote_ip, + &self.data.helo_domain, + &self.instance.hostname, + ) + .await; + + tracing::debug!(parent: &self.span, + context = "spf", + event = "lookup", + identity = "ehlo", + domain = self.data.helo_domain, + result = %spf_output.result(), + ); + + if self + .handle_spf(&spf_output, self.params.spf_ehlo.is_strict()) + .await? + { + self.data.spf_ehlo = spf_output.into(); + } else { + self.data.mail_from = None; + self.data.helo_domain = prev_helo_domain; + return Ok(()); + } + } + + // Sieve filtering + if let Some(script) = self.core.session.config.ehlo.script.eval(self).await { + match self.run_script(script.clone(), None).await { + ScriptResult::Accept | ScriptResult::Replace(_) => (), + ScriptResult::Reject(message) => { + tracing::debug!(parent: &self.span, + context = "ehlo", + event = "sieve-reject", + domain = &self.data.helo_domain, + reason = message); + + self.data.mail_from = None; + self.data.helo_domain = prev_helo_domain; + self.data.spf_ehlo = None; + return self.write(message.as_bytes()).await; + } + } + } + + tracing::debug!(parent: &self.span, + context = "ehlo", + event = "ehlo", + domain = self.data.helo_domain, + ); + } + + // Reset + if self.data.mail_from.is_some() { + self.reset(); + } + + let mut response = EhloResponse::new(self.instance.hostname.as_str()); + response.capabilities = + EXT_ENHANCED_STATUS_CODES | EXT_8BIT_MIME | EXT_BINARY_MIME | EXT_SMTP_UTF8; + if !self.stream.is_tls() { + response.capabilities |= EXT_START_TLS; + } + let ec = &self.core.session.config.extensions; + let rc = &self.core.session.config.rcpt; + let ac = &self.core.session.config.auth; + let dc = &self.core.session.config.data; + + // Pipelining + if *ec.pipelining.eval(self).await { + response.capabilities |= EXT_PIPELINING; + } + + // Chunking + if *ec.chunking.eval(self).await { + response.capabilities |= EXT_CHUNKING; + } + + // Address Expansion + if rc.lookup_expn.eval(self).await.is_some() { + response.capabilities |= EXT_EXPN; + } + + // Recipient Verification + if rc.lookup_vrfy.eval(self).await.is_some() { + response.capabilities |= EXT_VRFY; + } + + // Require TLS + if *ec.requiretls.eval(self).await { + response.capabilities |= EXT_REQUIRE_TLS; + } + + // DSN + if *ec.dsn.eval(self).await { + response.capabilities |= EXT_DSN; + } + + // Authentication + if self.data.authenticated_as.is_empty() { + response.auth_mechanisms = *ac.mechanisms.eval(self).await; + if response.auth_mechanisms != 0 { + if !self.stream.is_tls() { + response.auth_mechanisms &= !(AUTH_PLAIN | AUTH_LOGIN); + } + if response.auth_mechanisms != 0 { + response.capabilities |= EXT_AUTH; + } + } + } + + // Future release + if let Some(value) = ec.future_release.eval(self).await { + response.capabilities |= EXT_FUTURE_RELEASE; + response.future_release_interval = value.as_secs(); + response.future_release_datetime = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) + + value.as_secs(); + } + + // Deliver By + if let Some(value) = ec.deliver_by.eval(self).await { + response.capabilities |= EXT_DELIVER_BY; + response.deliver_by = value.as_secs(); + } + + // Priority + if let Some(value) = ec.mt_priority.eval(self).await { + response.capabilities |= EXT_MT_PRIORITY; + response.mt_priority = *value; + } + + // Size + response.size = *dc.max_message_size.eval(self).await; + if response.size > 0 { + response.capabilities |= EXT_SIZE; + } + + // No soliciting + if let Some(value) = ec.no_soliciting.eval(self).await { + response.capabilities |= EXT_NO_SOLICITING; + response.no_soliciting = if !value.is_empty() { + value.to_string().into() + } else { + None + }; + } + + // Generate response + let mut buf = Vec::with_capacity(64); + response.write(&mut buf).ok(); + self.write(&buf).await + } + + pub async fn is_domain_dnsbl_allowed( + &mut self, + domain: &str, + context: &str, + policy_type: u32, + ) -> bool { + let domain_ = domain.to_lowercase(); + let is_fqdn = domain.ends_with('.'); + if (self.params.dnsbl_policy & policy_type) != 0 { + for dnsbl in &self.core.mail_auth.dnsbl.domain_lookup { + if self + .is_dns_blocked(if is_fqdn { + format!("{domain_}{dnsbl}") + } else { + format!("{domain_}.{dnsbl}") + }) + .await + { + tracing::debug!(parent: &self.span, + context = context, + event = "reject", + reason = "dnsbl", + list = dnsbl, + domain = domain, + ); + self.data.dnsbl_error = format!( + "554 5.7.1 Service unavailable; Domain '{domain}' blocked using {dnsbl}\r\n" + ) + .into_bytes() + .into(); + return false; + } + } + } + true + } + + pub async fn verify_ip_dnsbl(&mut self) -> bool { + if (self.params.dnsbl_policy & DNSBL_IP) != 0 { + for dnsbl in &self.core.mail_auth.dnsbl.ip_lookup { + if self + .is_dns_blocked(self.data.remote_ip.to_dnsbl(dnsbl)) + .await + { + tracing::debug!(parent: &self.span, + context = "connect", + event = "reject", + reason = "dnsbl", + list = dnsbl, + ip = self.data.remote_ip.to_string(), + ); + self.data.dnsbl_error = format!( + "554 5.7.1 Service unavailable; IP address {} blocked using {}\r\n", + self.data.remote_ip, dnsbl + ) + .into_bytes() + .into(); + return false; + } + } + } + true + } + + async fn is_dns_blocked(&self, domain: String) -> bool { + match self.core.resolvers.dns.ipv4_lookup(&domain).await { + Ok(ips) => { + for ip in ips.iter() { + if ip.octets()[0..2] == [127, 0] { + return true; + } + } + tracing::debug!(parent: &self.span, + context = "dnsbl", + event = "invalid-reply", + query = domain, + reply = ?ips, + ); + } + Err(mail_auth::Error::DnsRecordNotFound(_)) => (), + Err(err) => { + tracing::debug!(parent: &self.span, + context = "dnsbl", + event = "dnserror", + query = domain, + reson = %err, + ); + } + } + false + } + + pub async fn write_dnsbl_error(&mut self) -> Result<(), ()> { + if let Some(error) = &self.data.dnsbl_error { + self.write(&error.to_vec()).await + } else { + Ok(()) + } + } + + pub fn has_dnsbl_error(&mut self) -> bool { + self.data.dnsbl_error.is_some() + } + + pub fn reset_dnsbl_error(&mut self) -> Option> { + self.data.dnsbl_error.take() + } +} + +trait ToDnsbl { + fn to_dnsbl(&self, host: &str) -> String; +} + +impl ToDnsbl for IpAddr { + fn to_dnsbl(&self, dnsbl: &str) -> String { + use std::fmt::Write; + + match self { + IpAddr::V4(ip) => { + let mut host = String::with_capacity(dnsbl.len() + 16); + for octet in ip.octets().iter().rev() { + let _ = write!(host, "{octet}."); + } + host.push_str(dnsbl); + host + } + IpAddr::V6(ip) => { + let mut host = Vec::with_capacity(dnsbl.len() + 64); + for segment in ip.segments().iter().rev() { + for &p in format!("{segment:04x}").as_bytes().iter().rev() { + host.push(p); + host.push(b'.'); + } + } + host.extend_from_slice(dnsbl.as_bytes()); + String::from_utf8(host).unwrap_or_default() + } + } + } +} + +#[cfg(test)] +mod test { + use std::net::IpAddr; + + use crate::inbound::ehlo::ToDnsbl; + + #[test] + fn ip_to_dnsbl() { + assert_eq!( + "2001:DB8:abc:123::42" + .parse::() + .unwrap() + .to_dnsbl("zen.spamhaus.org"), + "2.4.0.0.0.0.0.0.0.0.0.0.0.0.0.0.3.2.1.0.c.b.a.0.8.b.d.0.1.0.0.2.zen.spamhaus.org" + ); + + assert_eq!( + "1.2.3.4" + .parse::() + .unwrap() + .to_dnsbl("zen.spamhaus.org"), + "4.3.2.1.zen.spamhaus.org" + ); + } +} diff --git a/crates/smtp/src/inbound/mail.rs b/crates/smtp/src/inbound/mail.rs new file mode 100644 index 00000000..1e981e07 --- /dev/null +++ b/crates/smtp/src/inbound/mail.rs @@ -0,0 +1,350 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::time::SystemTime; + +use mail_auth::{IprevOutput, IprevResult, SpfOutput, SpfResult}; +use smtp_proto::{MailFrom, MAIL_BY_NOTIFY, MAIL_BY_RETURN, MAIL_REQUIRETLS}; +use tokio::io::{AsyncRead, AsyncWrite}; + +use crate::{ + config::{DNSBL_IPREV, DNSBL_RETURN_PATH}, + core::{scripts::ScriptResult, Session, SessionAddress}, + queue::DomainPart, +}; + +use super::IsTls; + +impl Session { + pub async fn handle_mail_from(&mut self, from: MailFrom) -> Result<(), ()> { + if self.data.helo_domain.is_empty() + && (self.params.ehlo_require + || self.params.spf_ehlo.verify() + || self.params.spf_mail_from.verify()) + { + return self + .write(b"503 5.5.1 Polite people say EHLO first.\r\n") + .await; + } else if self.data.mail_from.is_some() { + return self + .write(b"503 5.5.1 Multiple MAIL commands not allowed.\r\n") + .await; + } else if self.params.auth_require && self.data.authenticated_as.is_empty() { + return self + .write(b"503 5.5.1 You must authenticate first.\r\n") + .await; + } else if self.has_dnsbl_error() { + // There was a previous DNSBL error + return self.write_dnsbl_error().await; + } else if self.data.iprev.is_none() + && (self.params.iprev.verify() || (self.params.dnsbl_policy & DNSBL_IPREV) != 0) + { + let iprev = self + .core + .resolvers + .dns + .verify_iprev(self.data.remote_ip) + .await; + + tracing::debug!(parent: &self.span, + context = "iprev", + event = "lookup", + result = %iprev.result, + ptr = iprev.ptr.as_ref().and_then(|p| p.first()).map(|p| p.as_str()).unwrap_or_default() + ); + + // Validate reverse hostname against DNSBL + if let Some(ptr) = iprev.ptr.as_ref().and_then(|l| l.first()) { + if !self.is_domain_dnsbl_allowed(ptr, "ptr", DNSBL_IPREV).await { + return self.write_dnsbl_error().await; + } + } + + self.data.iprev = iprev.into(); + } + + // In strict mode reject messages from hosts that fail the reverse DNS lookup check + if self.params.iprev.is_strict() + && !matches!( + &self.data.iprev, + Some(IprevOutput { + result: IprevResult::Pass, + .. + }) + ) + { + let message = if matches!( + &self.data.iprev, + Some(IprevOutput { + result: IprevResult::TempError(_), + .. + }) + ) { + &b"451 4.7.25 Temporary error validating reverse DNS.\r\n"[..] + } else { + &b"550 5.7.25 Reverse DNS validation failed.\r\n"[..] + }; + + return self.write(message).await; + } + + let (address, address_lcase, domain) = if !from.address.is_empty() { + let address_lcase = from.address.to_lowercase(); + let domain = address_lcase.domain_part().to_string(); + (from.address, address_lcase, domain) + } else { + (String::new(), String::new(), String::new()) + }; + + // Validate domain against DNSBL + if !domain.is_empty() + && !self + .is_domain_dnsbl_allowed(&domain, "mail-from", DNSBL_RETURN_PATH) + .await + { + self.write_dnsbl_error().await?; + self.reset_dnsbl_error(); // Reset error in case a new MAIL-FROM is issued later + return Ok(()); + } + + let has_dsn = from.env_id.is_some(); + self.data.mail_from = SessionAddress { + address, + address_lcase, + domain, + flags: from.flags, + dsn_info: from.env_id, + } + .into(); + + // Sieve filtering + if let Some(script) = self.core.session.config.mail.script.eval(self).await { + match self.run_script(script.clone(), None).await { + ScriptResult::Accept | ScriptResult::Replace(_) => (), + ScriptResult::Reject(message) => { + tracing::debug!(parent: &self.span, + context = "mail-from", + event = "sieve-reject", + address = &self.data.mail_from.as_ref().unwrap().address, + reason = message); + self.data.mail_from = None; + return self.write(message.as_bytes()).await; + } + } + } + + // Validate parameters + let config = &self.core.session.config.extensions; + let config_data = &self.core.session.config.data; + if (from.flags & MAIL_REQUIRETLS) != 0 && !*config.requiretls.eval(self).await { + self.data.mail_from = None; + return self + .write(b"501 5.5.4 REQUIRETLS has been disabled.\r\n") + .await; + } + if (from.flags & (MAIL_BY_NOTIFY | MAIL_BY_RETURN)) != 0 { + if let Some(duration) = config.deliver_by.eval(self).await { + if from.by.checked_abs().unwrap_or(0) as u64 <= duration.as_secs() + && (from.by.is_positive() || (from.flags & MAIL_BY_NOTIFY) != 0) + { + self.data.delivery_by = from.by; + } else { + self.data.mail_from = None; + return self + .write( + format!( + "501 5.5.4 BY parameter exceeds maximum of {} seconds.\r\n", + duration.as_secs() + ) + .as_bytes(), + ) + .await; + } + } else { + self.data.mail_from = None; + return self + .write(b"501 5.5.4 DELIVERBY extension has been disabled.\r\n") + .await; + } + } + if from.mt_priority != 0 { + if config.mt_priority.eval(self).await.is_some() { + if (-6..6).contains(&from.mt_priority) { + self.data.priority = from.mt_priority as i16; + } else { + self.data.mail_from = None; + return self.write(b"501 5.5.4 Invalid priority value.\r\n").await; + } + } else { + self.data.mail_from = None; + return self + .write(b"501 5.5.4 MT-PRIORITY extension has been disabled.\r\n") + .await; + } + } + if from.size > 0 && from.size > *config_data.max_message_size.eval(self).await { + self.data.mail_from = None; + return self + .write(b"552 5.3.4 Message too big for system.\r\n") + .await; + } + if from.hold_for != 0 || from.hold_until != 0 { + if let Some(max_hold) = config.future_release.eval(self).await { + let max_hold = max_hold.as_secs(); + let hold_for = if from.hold_for != 0 { + from.hold_for + } else { + let now = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .map_or(0, |d| d.as_secs()); + if from.hold_until > now { + from.hold_until - now + } else { + 0 + } + }; + if hold_for <= max_hold { + self.data.future_release = hold_for; + } else { + self.data.mail_from = None; + return self + .write( + format!( + "501 5.5.4 Requested hold time exceeds maximum of {max_hold} seconds.\r\n" + ) + .as_bytes(), + ) + .await; + } + } else { + self.data.mail_from = None; + return self + .write(b"501 5.5.4 FUTURERELEASE extension has been disabled.\r\n") + .await; + } + } + if has_dsn && !*config.dsn.eval(self).await { + self.data.mail_from = None; + return self + .write(b"501 5.5.4 DSN extension has been disabled.\r\n") + .await; + } + + if self.is_allowed().await { + // Verify SPF + if self.params.spf_mail_from.verify() { + let mail_from = self.data.mail_from.as_ref().unwrap(); + let spf_output = if !mail_from.address.is_empty() { + self.core + .resolvers + .dns + .check_host( + self.data.remote_ip, + &mail_from.domain, + &self.data.helo_domain, + &self.instance.hostname, + &mail_from.address_lcase, + ) + .await + } else { + self.core + .resolvers + .dns + .check_host( + self.data.remote_ip, + &self.data.helo_domain, + &self.data.helo_domain, + &self.instance.hostname, + &format!("postmaster@{}", self.data.helo_domain), + ) + .await + }; + + tracing::debug!(parent: &self.span, + context = "spf", + event = "lookup", + identity = "mail-from", + domain = self.data.helo_domain, + sender = if !mail_from.address.is_empty() {mail_from.address.as_str()} else {"<>"}, + result = %spf_output.result(), + ); + + if self + .handle_spf(&spf_output, self.params.spf_mail_from.is_strict()) + .await? + { + self.data.spf_mail_from = spf_output.into(); + } else { + self.data.mail_from = None; + return Ok(()); + } + } + + tracing::debug!(parent: &self.span, + context = "mail-from", + event = "success", + address = &self.data.mail_from.as_ref().unwrap().address); + + self.eval_rcpt_params().await; + self.write(b"250 2.1.0 OK\r\n").await + } else { + self.data.mail_from = None; + self.write(b"451 4.4.5 Rate limit exceeded, try again later.\r\n") + .await + } + } + + pub async fn handle_spf(&mut self, spf_output: &SpfOutput, strict: bool) -> Result { + let result = match spf_output.result() { + SpfResult::Pass => true, + SpfResult::TempError if strict => { + self.write(b"451 4.7.24 Temporary SPF validation error.\r\n") + .await?; + false + } + result => { + if strict { + self.write( + format!("550 5.7.23 SPF validation failed, status: {result}.\r\n") + .as_bytes(), + ) + .await?; + false + } else { + true + } + } + }; + + // Send report + if let (Some(recipient), Some(rate)) = ( + spf_output.report_address(), + self.core.report.config.spf.send.eval(self).await, + ) { + self.send_spf_report(recipient, rate, !result, spf_output) + .await; + } + + Ok(result) + } +} diff --git a/crates/smtp/src/inbound/mod.rs b/crates/smtp/src/inbound/mod.rs new file mode 100644 index 00000000..be01186e --- /dev/null +++ b/crates/smtp/src/inbound/mod.rs @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use mail_auth::{ + arc::ArcSet, dkim::Signature, ArcOutput, AuthenticatedMessage, AuthenticationResults, +}; +use tokio::net::TcpStream; +use tokio_rustls::server::TlsStream; + +use crate::config::{ArcSealer, DkimSigner}; + +pub mod auth; +pub mod data; +pub mod ehlo; +pub mod mail; +pub mod rcpt; +pub mod session; +pub mod spawn; +pub mod vrfy; + +pub trait IsTls { + fn is_tls(&self) -> bool; + fn write_tls_header(&self, headers: &mut Vec); +} + +impl IsTls for TcpStream { + fn is_tls(&self) -> bool { + false + } + fn write_tls_header(&self, _headers: &mut Vec) {} +} + +impl IsTls for TlsStream { + fn is_tls(&self) -> bool { + true + } + + fn write_tls_header(&self, headers: &mut Vec) { + let (_, conn) = self.get_ref(); + headers.extend_from_slice(b"(using "); + headers.extend_from_slice( + match conn + .protocol_version() + .unwrap_or(rustls::ProtocolVersion::Unknown(0)) + { + rustls::ProtocolVersion::SSLv2 => "SSLv2", + rustls::ProtocolVersion::SSLv3 => "SSLv3", + rustls::ProtocolVersion::TLSv1_0 => "TLSv1.0", + rustls::ProtocolVersion::TLSv1_1 => "TLSv1.1", + rustls::ProtocolVersion::TLSv1_2 => "TLSv1.2", + rustls::ProtocolVersion::TLSv1_3 => "TLSv1.3", + rustls::ProtocolVersion::DTLSv1_0 => "DTLSv1.0", + rustls::ProtocolVersion::DTLSv1_2 => "DTLSv1.2", + rustls::ProtocolVersion::DTLSv1_3 => "DTLSv1.3", + _ => "unknown", + } + .as_bytes(), + ); + headers.extend_from_slice(b" with cipher "); + headers.extend_from_slice( + match conn.negotiated_cipher_suite() { + Some(rustls::SupportedCipherSuite::Tls13(cs)) => { + cs.common.suite.as_str().unwrap_or("unknown") + } + Some(rustls::SupportedCipherSuite::Tls12(cs)) => { + cs.common.suite.as_str().unwrap_or("unknown") + } + None => "unknown", + } + .as_bytes(), + ); + headers.extend_from_slice(b")\r\n\t"); + } +} + +impl ArcSealer { + pub fn seal<'x>( + &self, + message: &'x AuthenticatedMessage, + results: &'x AuthenticationResults, + arc_output: &'x ArcOutput, + ) -> mail_auth::Result> { + match self { + ArcSealer::RsaSha256(sealer) => sealer.seal(message, results, arc_output), + ArcSealer::Ed25519Sha256(sealer) => sealer.seal(message, results, arc_output), + } + } +} + +impl DkimSigner { + pub fn sign(&self, message: &[u8]) -> mail_auth::Result { + match self { + DkimSigner::RsaSha256(signer) => signer.sign(message), + DkimSigner::Ed25519Sha256(signer) => signer.sign(message), + } + } + pub fn sign_chained(&self, message: &[&[u8]]) -> mail_auth::Result { + match self { + DkimSigner::RsaSha256(signer) => signer.sign_chained(message.iter().copied()), + DkimSigner::Ed25519Sha256(signer) => signer.sign_chained(message.iter().copied()), + } + } +} diff --git a/crates/smtp/src/inbound/rcpt.rs b/crates/smtp/src/inbound/rcpt.rs new file mode 100644 index 00000000..be1785f3 --- /dev/null +++ b/crates/smtp/src/inbound/rcpt.rs @@ -0,0 +1,185 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use smtp_proto::{ + RcptTo, RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_NEVER, RCPT_NOTIFY_SUCCESS, +}; +use tokio::io::{AsyncRead, AsyncWrite}; + +use crate::{ + core::{scripts::ScriptResult, Session, SessionAddress}, + queue::DomainPart, +}; + +impl Session { + pub async fn handle_rcpt_to(&mut self, to: RcptTo) -> Result<(), ()> { + #[cfg(feature = "test_mode")] + if self.instance.id.ends_with("-debug") { + if to.address.contains("fail@") { + return self.write(b"503 5.5.1 Invalid recipient.\r\n").await; + } else if to.address.contains("delay@") { + return self.write(b"451 4.5.3 Try again later.\r\n").await; + } + } + + if self.data.mail_from.is_none() { + return self.write(b"503 5.5.1 MAIL is required first.\r\n").await; + } else if self.data.rcpt_to.len() >= self.params.rcpt_max { + return self.write(b"451 4.5.3 Too many recipients.\r\n").await; + } + + // Verify parameters + if ((to.flags + & (RCPT_NOTIFY_DELAY | RCPT_NOTIFY_NEVER | RCPT_NOTIFY_SUCCESS | RCPT_NOTIFY_FAILURE) + != 0) + || to.orcpt.is_some()) + && !self.params.rcpt_dsn + { + return self + .write(b"501 5.5.4 DSN extension has been disabled.\r\n") + .await; + } + + // Build RCPT + let address_lcase = to.address.to_lowercase(); + let rcpt = SessionAddress { + domain: address_lcase.domain_part().to_string(), + address_lcase, + address: to.address, + flags: to.flags, + dsn_info: to.orcpt, + }; + + // Verify address + if let (Some(domain_lookup), Some(address_lookup)) = ( + &self.params.rcpt_lookup_domain, + &self.params.rcpt_lookup_addresses, + ) { + if let Some(is_local_domain) = domain_lookup.contains(&rcpt.domain).await { + if is_local_domain { + if let Some(is_local_address) = + address_lookup.contains(&rcpt.address_lcase).await + { + if !is_local_address { + tracing::debug!(parent: &self.span, + context = "rcpt", + event = "error", + address = &rcpt.address_lcase, + "Mailbox does not exist."); + return self + .rcpt_error(b"550 5.1.2 Mailbox does not exist.\r\n") + .await; + } + } else { + tracing::debug!(parent: &self.span, + context = "rcpt", + event = "error", + address = &rcpt.address_lcase, + "Temporary address verification failure."); + return self + .write(b"451 4.4.3 Unable to verify address at this time.\r\n") + .await; + } + } else if !self.params.rcpt_relay { + tracing::debug!(parent: &self.span, + context = "rcpt", + event = "error", + address = &rcpt.address_lcase, + "Relay not allowed."); + return self.rcpt_error(b"550 5.1.2 Relay not allowed.\r\n").await; + } + } else { + tracing::debug!(parent: &self.span, + context = "rcpt", + event = "error", + address = &rcpt.address_lcase, + "Temporary address verification failure."); + + return self + .write(b"451 4.4.3 Unable to verify address at this time.\r\n") + .await; + } + } else if !self.params.rcpt_relay { + tracing::debug!(parent: &self.span, + context = "rcpt", + event = "error", + address = &rcpt.address_lcase, + "Relay not allowed."); + return self.rcpt_error(b"550 5.1.2 Relay not allowed.\r\n").await; + } + + if !self.data.rcpt_to.contains(&rcpt) { + self.data.rcpt_to.push(rcpt); + + // Sieve filtering + if let Some(script) = &self.params.rcpt_script { + match self.run_script(script.clone(), None).await { + ScriptResult::Accept | ScriptResult::Replace(_) => (), + ScriptResult::Reject(message) => { + tracing::debug!(parent: &self.span, + context = "rcpt", + event = "sieve-reject", + address = &self.data.rcpt_to.last().unwrap().address, + reason = message); + self.data.rcpt_to.pop(); + return self.write(message.as_bytes()).await; + } + } + } + + if self.is_allowed().await { + tracing::debug!(parent: &self.span, + context = "rcpt", + event = "success", + address = &self.data.rcpt_to.last().unwrap().address); + } else { + self.data.rcpt_to.pop(); + return self + .write(b"451 4.4.5 Rate limit exceeded, try again later.\r\n") + .await; + } + } + + self.write(b"250 2.1.5 OK\r\n").await + } + + async fn rcpt_error(&mut self, response: &[u8]) -> Result<(), ()> { + tokio::time::sleep(self.params.rcpt_errors_wait).await; + self.data.rcpt_errors += 1; + self.write(response).await?; + if self.data.rcpt_errors < self.params.rcpt_errors_max { + Ok(()) + } else { + self.write(b"421 4.3.0 Too many errors, disconnecting.\r\n") + .await?; + tracing::debug!( + parent: &self.span, + context = "rcpt", + event = "disconnect", + reason = "too-many-errors", + "Too many invalid RCPT commands." + ); + Err(()) + } + } +} diff --git a/crates/smtp/src/inbound/session.rs b/crates/smtp/src/inbound/session.rs new file mode 100644 index 00000000..c14fea18 --- /dev/null +++ b/crates/smtp/src/inbound/session.rs @@ -0,0 +1,456 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::net::IpAddr; + +use smtp_proto::{ + request::receiver::{ + BdatReceiver, DataReceiver, DummyDataReceiver, DummyLineReceiver, LineReceiver, + MAX_LINE_LENGTH, + }, + *, +}; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; +use utils::config::ServerProtocol; + +use crate::core::{Envelope, Session, State}; + +use super::{auth::SaslToken, IsTls}; + +impl Session { + pub async fn ingest(&mut self, bytes: &[u8]) -> Result { + let mut iter = bytes.iter(); + let mut state = std::mem::replace(&mut self.state, State::None); + + 'outer: loop { + match &mut state { + State::Request(receiver) => loop { + match receiver.ingest(&mut iter, bytes) { + Ok(request) => match request { + Request::Rcpt { to } => { + self.handle_rcpt_to(to).await?; + } + Request::Mail { from } => { + self.handle_mail_from(from).await?; + } + Request::Ehlo { host } => { + if self.instance.protocol == ServerProtocol::Smtp { + self.handle_ehlo(host).await?; + } else { + self.write(b"500 5.5.1 Invalid command.\r\n").await?; + } + } + Request::Data => { + if self.can_send_data().await? { + self.write(b"354 Start mail input; end with .\r\n") + .await?; + self.data.message = Vec::with_capacity(1024); + state = State::Data(DataReceiver::new()); + continue 'outer; + } + } + Request::Bdat { + chunk_size, + is_last, + } => { + state = if chunk_size + self.data.message.len() + < self.params.max_message_size + { + if self.data.message.is_empty() { + self.data.message = Vec::with_capacity(chunk_size); + } else { + self.data.message.reserve(chunk_size); + } + State::Bdat(BdatReceiver::new(chunk_size, is_last)) + } else { + // Chunk is too large, ignore. + State::DataTooLarge(DummyDataReceiver::new_bdat(chunk_size)) + }; + continue 'outer; + } + Request::Auth { + mechanism, + initial_response, + } => { + let auth = + *self.core.session.config.auth.mechanisms.eval(self).await; + if auth == 0 || self.params.auth_lookup.is_none() { + self.write(b"503 5.5.1 AUTH not allowed.\r\n").await?; + } else if !self.data.authenticated_as.is_empty() { + self.write(b"503 5.5.1 Already authenticated.\r\n").await?; + } else if mechanism & (AUTH_LOGIN | AUTH_PLAIN) != 0 + && !self.stream.is_tls() + { + self.write(b"503 5.5.1 Clear text authentication without TLS is forbidden.\r\n").await?; + } else if let Some(mut token) = + SaslToken::from_mechanism(mechanism & auth) + { + if self + .handle_sasl_response( + &mut token, + initial_response.as_bytes(), + ) + .await? + { + state = State::Sasl(LineReceiver::new(token)); + continue 'outer; + } + } else { + self.write( + b"554 5.7.8 Authentication mechanism not supported.\r\n", + ) + .await?; + } + } + Request::Noop { .. } => { + self.write(b"250 2.0.0 OK\r\n").await?; + } + Request::Vrfy { value } => { + self.handle_vrfy(value).await?; + } + Request::Expn { value } => { + self.handle_expn(value).await?; + } + Request::StartTls => { + if !self.stream.is_tls() { + self.write(b"220 2.0.0 Ready to start TLS.\r\n").await?; + self.state = State::default(); + return Ok(false); + } else { + self.write(b"504 5.7.4 Already in TLS mode.\r\n").await?; + } + } + Request::Rset => { + self.reset(); + self.write(b"250 2.0.0 OK\r\n").await?; + } + Request::Quit => { + self.write(b"221 2.0.0 Bye.\r\n").await?; + return Err(()); + } + Request::Help { .. } => { + self.write( + b"250 2.0.0 Help can be found at https://stalw.art/smtp/\r\n", + ) + .await?; + } + Request::Helo { host } => { + if self.instance.protocol == ServerProtocol::Smtp + && self.data.helo_domain.is_empty() + { + self.data.helo_domain = host; + self.write( + format!("250 {} says hello\r\n", self.instance.hostname) + .as_bytes(), + ) + .await?; + } else { + self.write(b"503 5.5.1 Invalid command.\r\n").await?; + } + } + Request::Lhlo { host } => { + if self.instance.protocol == ServerProtocol::Lmtp { + self.handle_ehlo(host).await?; + } else { + self.write(b"502 5.5.1 Invalid command.\r\n").await?; + } + } + Request::Etrn { .. } | Request::Atrn { .. } | Request::Burl { .. } => { + self.write(b"502 5.5.1 Command not implemented.\r\n") + .await?; + } + }, + Err(err) => match err { + Error::NeedsMoreData { .. } => break 'outer, + Error::UnknownCommand | Error::InvalidResponse { .. } => { + self.write(b"500 5.5.1 Invalid command.\r\n").await?; + } + Error::InvalidSenderAddress => { + self.write(b"501 5.1.8 Bad sender's system address.\r\n") + .await?; + } + Error::InvalidRecipientAddress => { + self.write( + b"501 5.1.3 Bad destination mailbox address syntax.\r\n", + ) + .await?; + } + Error::SyntaxError { syntax } => { + self.write( + format!("501 5.5.2 Syntax error, expected: {syntax}\r\n") + .as_bytes(), + ) + .await?; + } + Error::InvalidParameter { param } => { + self.write( + format!("501 5.5.4 Invalid parameter {param:?}.\r\n") + .as_bytes(), + ) + .await?; + } + Error::UnsupportedParameter { param } => { + self.write( + format!("504 5.5.4 Unsupported parameter {param:?}.\r\n") + .as_bytes(), + ) + .await?; + } + Error::ResponseTooLong => { + state = State::RequestTooLarge(DummyLineReceiver::default()); + continue 'outer; + } + }, + } + }, + State::Data(receiver) => { + if self.data.message.len() + bytes.len() < self.params.max_message_size { + if receiver.ingest(&mut iter, &mut self.data.message) { + let num_rcpts = self.data.rcpt_to.len(); + let message = self.queue_message().await; + if self.instance.protocol == ServerProtocol::Smtp { + self.write(message.as_ref()).await?; + } else { + for _ in 0..num_rcpts { + self.write(message.as_ref()).await?; + } + } + self.reset(); + state = State::default(); + } else { + break 'outer; + } + } else { + state = State::DataTooLarge(DummyDataReceiver::new_data(receiver)); + } + } + State::Bdat(receiver) => { + if receiver.ingest(&mut iter, &mut self.data.message) { + if self.can_send_data().await? { + if receiver.is_last { + let num_rcpts = self.data.rcpt_to.len(); + let message = self.queue_message().await; + if self.instance.protocol == ServerProtocol::Smtp { + self.write(message.as_ref()).await?; + } else { + for _ in 0..num_rcpts { + self.write(message.as_ref()).await?; + } + } + self.reset(); + } else { + self.write(b"250 2.6.0 Chunk accepted.\r\n").await?; + } + } else { + self.data.message = Vec::with_capacity(0); + } + state = State::default(); + } else { + break 'outer; + } + } + State::Sasl(receiver) => { + if receiver.ingest(&mut iter) { + if receiver.buf.len() < MAX_LINE_LENGTH { + if self + .handle_sasl_response(&mut receiver.state, &receiver.buf) + .await? + { + receiver.buf.clear(); + continue 'outer; + } + } else { + self.auth_error( + b"500 5.5.6 Authentication Exchange line is too long.\r\n", + ) + .await?; + } + state = State::default(); + } else { + break 'outer; + } + } + State::DataTooLarge(receiver) => { + if receiver.ingest(&mut iter) { + tracing::debug!( + parent: &self.span, + context = "data", + event = "too-large", + "Message is too large." + ); + + self.data.message = Vec::with_capacity(0); + self.write(b"552 5.3.4 Message too big for system.\r\n") + .await?; + state = State::default(); + } else { + break 'outer; + } + } + State::RequestTooLarge(receiver) => { + if receiver.ingest(&mut iter) { + self.write(b"554 5.3.4 Line is too long.\r\n").await?; + state = State::default(); + } else { + break 'outer; + } + } + State::None => unreachable!(), + } + } + self.state = state; + + Ok(true) + } +} + +impl Session { + pub fn reset(&mut self) { + self.data.mail_from = None; + self.data.spf_mail_from = None; + self.data.rcpt_to.clear(); + self.data.message = Vec::with_capacity(0); + self.data.priority = 0; + self.data.delivery_by = 0; + self.data.future_release = 0; + } + + #[inline(always)] + pub async fn write(&mut self, bytes: &[u8]) -> Result<(), ()> { + let err = match self.stream.write_all(bytes).await { + Ok(_) => match self.stream.flush().await { + Ok(_) => { + tracing::trace!(parent: &self.span, + event = "write", + data = std::str::from_utf8(bytes).unwrap_or_default() , + size = bytes.len()); + return Ok(()); + } + Err(err) => err, + }, + Err(err) => err, + }; + + tracing::debug!(parent: &self.span, + event = "error", + "Failed to write to stream: {:?}", err); + Err(()) + } + + #[inline(always)] + pub async fn read(&mut self, bytes: &mut [u8]) -> Result { + match self.stream.read(bytes).await { + Ok(len) => { + tracing::trace!(parent: &self.span, + event = "read", + data = if matches!(self.state, State::Request(_)) {bytes + .get(0..len) + .and_then(|bytes| std::str::from_utf8(bytes).ok()) + .unwrap_or("[invalid UTF8]")} else {"[DATA]"}, + size = len); + Ok(len) + } + Err(err) => { + tracing::debug!( + parent: &self.span, + event = "error", + "Failed to read from stream: {:?}", err + ); + Err(()) + } + } + } +} + +impl Envelope for Session { + #[inline(always)] + fn local_ip(&self) -> IpAddr { + self.data.local_ip + } + + #[inline(always)] + fn remote_ip(&self) -> IpAddr { + self.data.remote_ip + } + + #[inline(always)] + fn sender_domain(&self) -> &str { + self.data + .mail_from + .as_ref() + .map(|a| a.domain.as_str()) + .unwrap_or_default() + } + + #[inline(always)] + fn sender(&self) -> &str { + self.data + .mail_from + .as_ref() + .map(|a| a.address_lcase.as_str()) + .unwrap_or_default() + } + + #[inline(always)] + fn rcpt_domain(&self) -> &str { + self.data + .rcpt_to + .last() + .map(|r| r.domain.as_str()) + .unwrap_or_default() + } + + #[inline(always)] + fn rcpt(&self) -> &str { + self.data + .rcpt_to + .last() + .map(|r| r.address_lcase.as_str()) + .unwrap_or_default() + } + + #[inline(always)] + fn helo_domain(&self) -> &str { + self.data.helo_domain.as_str() + } + + #[inline(always)] + fn authenticated_as(&self) -> &str { + self.data.authenticated_as.as_str() + } + + #[inline(always)] + fn mx(&self) -> &str { + "" + } + + #[inline(always)] + fn listener_id(&self) -> u16 { + self.instance.listener_id + } + + #[inline(always)] + fn priority(&self) -> i16 { + self.data.priority + } +} diff --git a/crates/smtp/src/inbound/spawn.rs b/crates/smtp/src/inbound/spawn.rs new file mode 100644 index 00000000..2c71f760 --- /dev/null +++ b/crates/smtp/src/inbound/spawn.rs @@ -0,0 +1,248 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::time::Instant; + +use tokio::{ + io::{AsyncRead, AsyncWrite}, + net::TcpStream, +}; +use tokio_rustls::server::TlsStream; +use utils::listener::SessionManager; + +use crate::core::{ + scripts::ScriptResult, Session, SessionData, SessionParameters, SmtpSessionManager, State, +}; + +use super::IsTls; + +impl SessionManager for SmtpSessionManager { + fn spawn(&self, session: utils::listener::SessionData) { + // Create session + let mut session = Session { + core: self.inner.clone(), + instance: session.instance, + state: State::default(), + span: session.span, + stream: session.stream, + in_flight: vec![session.in_flight], + data: SessionData::new(session.local_ip, session.remote_ip), + params: SessionParameters::default(), + }; + + tokio::spawn(async move { + // Enforce throttle + if session.is_allowed().await { + if session.instance.is_tls_implicit { + if let Ok(mut session) = session.into_tls().await { + if session.init_conn().await { + session.handle_conn().await; + } + } + } else if session.init_conn().await { + session.handle_conn().await; + } + } + }); + } +} + +impl Session { + pub async fn into_tls(self) -> Result>, ()> { + let span = self.span; + Ok(Session { + stream: match self + .instance + .tls_acceptor + .as_ref() + .unwrap() + .accept(self.stream) + .await + { + Ok(stream) => { + tracing::info!( + parent: &span, + context = "tls", + event = "handshake", + version = ?stream.get_ref().1.protocol_version().unwrap_or(rustls::ProtocolVersion::TLSv1_3), + cipher = ?stream.get_ref().1.negotiated_cipher_suite().unwrap_or(rustls::cipher_suite::TLS13_AES_128_GCM_SHA256), + ); + stream + } + Err(err) => { + tracing::debug!( + parent: &span, + context = "tls", + event = "error", + "Failed to accept TLS connection: {}", + err + ); + return Err(()); + } + }, + state: self.state, + data: self.data, + instance: self.instance, + core: self.core, + in_flight: self.in_flight, + params: self.params, + span, + }) + } + + pub async fn handle_conn(mut self) { + if self.handle_conn_().await && self.instance.tls_acceptor.is_some() { + if let Ok(session) = self.into_tls().await { + session.handle_conn().await; + } + } + } +} + +impl Session> { + pub async fn handle_conn(mut self) { + self.handle_conn_().await; + } +} + +impl Session { + pub async fn init_conn(&mut self) -> bool { + self.eval_session_params().await; + self.verify_ip_dnsbl().await; + + // Sieve filtering + if let Some(script) = self.core.session.config.connect.script.eval(self).await { + match self.run_script(script.clone(), None).await { + ScriptResult::Accept | ScriptResult::Replace(_) => (), + ScriptResult::Reject(message) => { + tracing::debug!(parent: &self.span, + context = "connect", + event = "sieve-reject", + reason = message); + + let _ = self.write(message.as_bytes()).await; + return false; + } + } + } + + let instance = self.instance.clone(); + if self.write(instance.data.as_bytes()).await.is_err() { + return false; + } + + true + } + + pub async fn handle_conn_(&mut self) -> bool { + let mut buf = vec![0; 8192]; + let mut shutdown_rx = self.instance.shutdown_rx.clone(); + + loop { + tokio::select! { + result = tokio::time::timeout( + self.params.timeout, + self.read(&mut buf)) => { + match result { + Ok(Ok(bytes_read)) => { + if bytes_read > 0 { + if Instant::now() < self.data.valid_until && bytes_read <= self.data.bytes_left { + self.data.bytes_left -= bytes_read; + match self.ingest(&buf[..bytes_read]).await { + Ok(true) => (), + Ok(false) => { + return true; + } + Err(_) => { + break; + } + } + } else if bytes_read > self.data.bytes_left { + self + .write(format!("451 4.7.28 {} Session exceeded transfer quota.\r\n", self.instance.hostname).as_bytes()) + .await + .ok(); + tracing::debug!( + parent: &self.span, + event = "disconnect", + reason = "transfer-limit", + "Client exceeded incoming transfer limit." + ); + break; + } else { + self + .write(format!("453 4.3.2 {} Session open for too long.\r\n", self.instance.hostname).as_bytes()) + .await + .ok(); + tracing::debug!( + parent: &self.span, + event = "disconnect", + reason = "loiter", + "Session open for too long." + ); + break; + } + } else { + tracing::debug!( + parent: &self.span, + event = "disconnect", + reason = "peer", + "Connection closed by peer." + ); + break; + } + } + Ok(Err(_)) => { + break; + } + Err(_) => { + tracing::debug!( + parent: &self.span, + event = "disconnect", + reason = "timeout", + "Connection timed out." + ); + self + .write(format!("221 2.0.0 {} Disconnecting inactive client.\r\n", self.instance.hostname).as_bytes()) + .await + .ok(); + break; + } + } + }, + _ = shutdown_rx.changed() => { + tracing::debug!( + parent: &self.span, + event = "disconnect", + reason = "shutdown", + "Server shutting down." + ); + self.write(b"421 4.3.0 Server shutting down.\r\n").await.ok(); + break; + } + }; + } + + false + } +} diff --git a/crates/smtp/src/inbound/vrfy.rs b/crates/smtp/src/inbound/vrfy.rs new file mode 100644 index 00000000..2dacd6e9 --- /dev/null +++ b/crates/smtp/src/inbound/vrfy.rs @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use tokio::io::{AsyncRead, AsyncWrite}; + +use crate::{ + core::Session, + lookup::{Item, LookupResult}, +}; +use std::fmt::Write; + +impl Session { + pub async fn handle_vrfy(&mut self, address: String) -> Result<(), ()> { + if let Some(address_lookup) = &self.params.rcpt_lookup_vrfy { + if let Some(result) = address_lookup + .lookup(Item::Verify(address.to_lowercase())) + .await + { + if let LookupResult::Values(values) = result { + let mut result = String::with_capacity(32); + for (pos, value) in values.iter().enumerate() { + let _ = write!( + result, + "250{}{}\r\n", + if pos == values.len() - 1 { " " } else { "-" }, + value + ); + } + + tracing::debug!(parent: &self.span, + context = "vrfy", + event = "success", + address = &address); + + self.write(result.as_bytes()).await + } else { + tracing::debug!(parent: &self.span, + context = "vrfy", + event = "not-found", + address = &address); + + self.write(b"550 5.1.2 Address not found.\r\n").await + } + } else { + tracing::debug!(parent: &self.span, + context = "vrfy", + event = "temp-fail", + address = &address); + + self.write(b"252 2.4.3 Unable to verify address at this time.\r\n") + .await + } + } else { + tracing::debug!(parent: &self.span, + context = "vrfy", + event = "forbidden", + address = &address); + + self.write(b"252 2.5.1 VRFY is disabled.\r\n").await + } + } + + pub async fn handle_expn(&mut self, address: String) -> Result<(), ()> { + if let Some(address_lookup) = &self.params.rcpt_lookup_expn { + if let Some(result) = address_lookup + .lookup(Item::Expand(address.to_lowercase())) + .await + { + if let LookupResult::Values(values) = result { + let mut result = String::with_capacity(32); + for (pos, value) in values.iter().enumerate() { + let _ = write!( + result, + "250{}{}\r\n", + if pos == values.len() - 1 { " " } else { "-" }, + value + ); + } + tracing::debug!(parent: &self.span, + context = "expn", + event = "success", + address = &address); + self.write(result.as_bytes()).await + } else { + tracing::debug!(parent: &self.span, + context = "expn", + event = "not-found", + address = &address); + + self.write(b"550 5.1.2 Mailing list not found.\r\n").await + } + } else { + tracing::debug!(parent: &self.span, + context = "expn", + event = "temp-fail", + address = &address); + + self.write(b"252 2.4.3 Unable to expand mailing list at this time.\r\n") + .await + } + } else { + tracing::debug!(parent: &self.span, + context = "expn", + event = "forbidden", + address = &address); + + self.write(b"252 2.5.1 EXPN is disabled.\r\n").await + } + } +} diff --git a/crates/smtp/src/lib.rs b/crates/smtp/src/lib.rs new file mode 100644 index 00000000..acbd2cb1 --- /dev/null +++ b/crates/smtp/src/lib.rs @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +pub mod config; +pub mod core; +pub mod inbound; +pub mod lookup; +pub mod outbound; +pub mod queue; +pub mod reporting; + +pub static USER_AGENT: &str = concat!("StalwartSMTP/", env!("CARGO_PKG_VERSION"),); + +pub trait UnwrapFailure { + fn failed(self, action: &str) -> T; +} + +impl UnwrapFailure for Option { + fn failed(self, message: &str) -> T { + match self { + Some(result) => result, + None => { + eprintln!("{message}"); + std::process::exit(1); + } + } + } +} + +impl UnwrapFailure for Result { + fn failed(self, message: &str) -> T { + match self { + Ok(result) => result, + Err(err) => { + eprintln!("{message}: {err}"); + std::process::exit(1); + } + } + } +} + +pub fn failed(message: &str) -> ! { + eprintln!("{message}"); + std::process::exit(1); +} diff --git a/crates/smtp/src/lookup/cache.rs b/crates/smtp/src/lookup/cache.rs new file mode 100644 index 00000000..5f85d303 --- /dev/null +++ b/crates/smtp/src/lookup/cache.rs @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{ + borrow::Borrow, + hash::Hash, + time::{Duration, Instant}, +}; + +#[allow(clippy::type_complexity)] +#[derive(Debug)] +pub struct LookupCache { + cache_pos: lru_cache::LruCache, + cache_neg: lru_cache::LruCache, + ttl_pos: Duration, + ttl_neg: Duration, +} + +impl LookupCache { + pub fn new(capacity: usize, ttl_pos: Duration, ttl_neg: Duration) -> Self { + Self { + cache_pos: lru_cache::LruCache::with_hasher(capacity, ahash::RandomState::new()), + cache_neg: lru_cache::LruCache::with_hasher(capacity, ahash::RandomState::new()), + ttl_pos, + ttl_neg, + } + } + + pub fn get(&mut self, name: &Q) -> Option + where + T: Borrow, + Q: Hash + Eq, + { + // Check positive cache + if let Some(valid_until) = self.cache_pos.get_mut(name) { + if *valid_until >= Instant::now() { + return Some(true); + } else { + self.cache_pos.remove(name); + } + } + + // Check negative cache + let valid_until = self.cache_neg.get_mut(name)?; + if *valid_until >= Instant::now() { + Some(false) + } else { + self.cache_pos.remove(name); + None + } + } + + pub fn insert_pos(&mut self, item: T) { + self.cache_pos.insert(item, Instant::now() + self.ttl_pos); + } + + pub fn insert_neg(&mut self, item: T) { + self.cache_neg.insert(item, Instant::now() + self.ttl_neg); + } + + pub fn clear(&mut self) { + self.cache_pos.clear(); + self.cache_neg.clear(); + } +} diff --git a/crates/smtp/src/lookup/dispatch.rs b/crates/smtp/src/lookup/dispatch.rs new file mode 100644 index 00000000..9452fe91 --- /dev/null +++ b/crates/smtp/src/lookup/dispatch.rs @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use mail_send::Credentials; + +use super::{Item, Lookup, LookupResult}; + +impl Lookup { + pub async fn contains(&self, entry: &str) -> Option { + match self { + Lookup::Remote(tx) => tx + .lookup(Item::IsAccount(entry.to_string())) + .await + .map(|r| r.into()), + Lookup::Sql(sql) => sql.exists(entry).await, + Lookup::Local(entries) => Some(entries.contains(entry)), + } + } + + pub async fn lookup(&self, item: Item) -> Option { + match self { + Lookup::Remote(tx) => tx.lookup(item).await, + + Lookup::Sql(sql) => match item { + Item::IsAccount(account) => sql.exists(&account).await.map(LookupResult::from), + Item::Authenticate(credentials) => match credentials { + Credentials::Plain { username, secret } + | Credentials::XOauth2 { username, secret } => sql + .fetch_one(&username) + .await + .map(|pwd| LookupResult::from(pwd.map_or(false, |pwd| pwd == secret))), + Credentials::OAuthBearer { token } => { + sql.exists(&token).await.map(LookupResult::from) + } + }, + Item::Verify(account) => sql.fetch_many(&account).await.map(LookupResult::from), + Item::Expand(list) => sql.fetch_many(&list).await.map(LookupResult::from), + }, + + Lookup::Local(list) => match item { + Item::IsAccount(item) => Some(list.contains(&item).into()), + Item::Verify(_item) | Item::Expand(_item) => { + #[cfg(feature = "test_mode")] + for list_item in list { + if let Some((prefix, suffix)) = list_item.split_once(':') { + if prefix == _item { + return Some(LookupResult::Values( + suffix.split(',').map(|i| i.to_string()).collect::>(), + )); + } + } + } + Some(LookupResult::False) + } + Item::Authenticate(credentials) => { + let entry = match credentials { + Credentials::Plain { username, secret } + | Credentials::XOauth2 { username, secret } => { + format!("{username}:{secret}") + } + Credentials::OAuthBearer { token } => token, + }; + + if !list.is_empty() { + Some(list.contains(&entry).into()) + } else { + None + } + } + }, + } + } +} diff --git a/crates/smtp/src/lookup/imap.rs b/crates/smtp/src/lookup/imap.rs new file mode 100644 index 00000000..27072100 --- /dev/null +++ b/crates/smtp/src/lookup/imap.rs @@ -0,0 +1,485 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{fmt::Display, sync::Arc, time::Duration}; + +use mail_send::Credentials; +use rustls::ServerName; +use smtp_proto::{ + request::{parser::Rfc5321Parser, AUTH}, + response::generate::BitToString, + IntoString, AUTH_CRAM_MD5, AUTH_LOGIN, AUTH_OAUTHBEARER, AUTH_PLAIN, AUTH_XOAUTH2, +}; +use tokio::{ + io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}, + net::{TcpStream, ToSocketAddrs}, + sync::mpsc, +}; +use tokio_rustls::{client::TlsStream, TlsConnector}; + +use crate::lookup::spawn::LoggedUnwrap; + +use super::{Event, Item, LookupItem, RemoteLookup}; + +pub struct ImapAuthClient { + stream: T, + timeout: Duration, +} + +pub struct ImapAuthClientBuilder { + pub addr: String, + timeout: Duration, + tls_connector: TlsConnector, + tls_hostname: String, + tls_implicit: bool, + mechanisms: u64, +} + +impl ImapAuthClientBuilder { + pub fn new( + addr: String, + timeout: Duration, + tls_connector: TlsConnector, + tls_hostname: String, + tls_implicit: bool, + ) -> Self { + Self { + addr, + timeout, + tls_connector, + tls_hostname, + tls_implicit, + mechanisms: AUTH_PLAIN, + } + } + + pub async fn init(mut self) -> Self { + let err = match self.connect().await { + Ok(mut client) => match client.authentication_mechanisms().await { + Ok(mechanisms) => { + client.logout().await.ok(); + self.mechanisms = mechanisms; + return self; + } + Err(err) => err, + }, + Err(err) => err, + }; + tracing::warn!( + context = "remote", + event = "error", + remote.addr = &self.addr, + remote.protocol = "imap", + "Could not obtain auth mechanisms: {}", + err + ); + + self + } + + pub async fn connect(&self) -> Result>, Error> { + ImapAuthClient::connect( + &self.addr, + self.timeout, + &self.tls_connector, + &self.tls_hostname, + self.tls_implicit, + ) + .await + } +} + +#[derive(Debug)] +pub enum Error { + Io(std::io::Error), + Timeout, + InvalidResponse(String), + InvalidChallenge(String), + AuthenticationFailed, + TLSInvalidName, + Disconnected, +} + +impl RemoteLookup for Arc { + fn spawn_lookup(&self, lookup: LookupItem, tx: mpsc::Sender) { + let builder = self.clone(); + tokio::spawn(async move { + if let Err(err) = builder.lookup(lookup, &tx).await { + tracing::warn!( + context = "remote", + event = "error", + remote.addr = &builder.addr, + remote.protocol = "imap", + "Remote lookup failed: {}", + err + ); + tx.send(Event::WorkerFailed).await.logged_unwrap(); + } + }); + } +} + +impl ImapAuthClientBuilder { + pub async fn lookup(&self, lookup: LookupItem, tx: &mpsc::Sender) -> Result<(), Error> { + match &lookup.item { + Item::Authenticate(credentials) => { + let mut client = self.connect().await?; + let mechanism = match credentials { + Credentials::Plain { .. } + if (self.mechanisms & (AUTH_PLAIN | AUTH_LOGIN | AUTH_CRAM_MD5)) != 0 => + { + if self.mechanisms & AUTH_CRAM_MD5 != 0 { + AUTH_CRAM_MD5 + } else if self.mechanisms & AUTH_PLAIN != 0 { + AUTH_PLAIN + } else { + AUTH_LOGIN + } + } + Credentials::OAuthBearer { .. } if self.mechanisms & AUTH_OAUTHBEARER != 0 => { + AUTH_OAUTHBEARER + } + Credentials::XOauth2 { .. } if self.mechanisms & AUTH_XOAUTH2 != 0 => { + AUTH_XOAUTH2 + } + _ => { + tracing::warn!( + context = "remote", + event = "error", + remote.addr = &self.addr, + remote.protocol = "imap", + "IMAP server does not offer any supported auth mechanisms.", + ); + tx.send(Event::WorkerFailed).await.logged_unwrap(); + return Ok(()); + } + }; + + let result = match client.authenticate(mechanism, credentials).await { + Ok(_) => true, + Err(err) => match &err { + Error::AuthenticationFailed => false, + _ => return Err(err), + }, + }; + tx.send(Event::WorkerReady { + item: lookup.item, + result: Some(result), + next_lookup: None, + }) + .await + .logged_unwrap(); + lookup.result.send(result.into()).logged_unwrap(); + } + _ => { + tracing::warn!( + context = "remote", + event = "error", + remote.addr = &self.addr, + remote.protocol = "imap", + "IMAP does not support validating recipients.", + ); + tx.send(Event::WorkerFailed).await.logged_unwrap(); + } + } + Ok(()) + } +} + +impl ImapAuthClient { + async fn start_tls( + mut self, + tls_connector: &TlsConnector, + tls_hostname: &str, + ) -> Result>, Error> { + let line = tokio::time::timeout(self.timeout, async { + self.write(b"C7 STARTTLS\r\n").await?; + + self.read_line().await + }) + .await + .map_err(|_| Error::Timeout)??; + + if matches!(line.get(..5), Some(b"C7 OK")) { + self.into_tls(tls_connector, tls_hostname).await + } else { + Err(Error::InvalidResponse(line.into_string())) + } + } + + async fn into_tls( + self, + tls_connector: &TlsConnector, + tls_hostname: &str, + ) -> Result>, Error> { + tokio::time::timeout(self.timeout, async { + Ok(ImapAuthClient { + stream: tls_connector + .connect( + ServerName::try_from(tls_hostname).map_err(|_| Error::TLSInvalidName)?, + self.stream, + ) + .await?, + timeout: self.timeout, + }) + }) + .await + .map_err(|_| Error::Timeout)? + } +} + +impl ImapAuthClient> { + pub async fn connect( + addr: impl ToSocketAddrs, + timeout: Duration, + tls_connector: &TlsConnector, + tls_hostname: &str, + tls_implicit: bool, + ) -> Result { + let mut client: ImapAuthClient = tokio::time::timeout(timeout, async { + match TcpStream::connect(addr).await { + Ok(stream) => Ok(ImapAuthClient { stream, timeout }), + Err(err) => Err(Error::Io(err)), + } + }) + .await + .map_err(|_| Error::Timeout)??; + + if tls_implicit { + let mut client = client.into_tls(tls_connector, tls_hostname).await?; + client.expect_greeting().await?; + Ok(client) + } else { + client.expect_greeting().await?; + client.start_tls(tls_connector, tls_hostname).await + } + } +} + +impl ImapAuthClient { + pub async fn authenticate( + &mut self, + mechanism: u64, + credentials: &Credentials, + ) -> Result<(), Error> { + if (mechanism & (AUTH_PLAIN | AUTH_XOAUTH2 | AUTH_OAUTHBEARER)) != 0 { + self.write( + format!( + "C3 AUTHENTICATE {} {}\r\n", + mechanism.to_mechanism(), + credentials + .encode(mechanism, "") + .map_err(|err| Error::InvalidChallenge(err.to_string()))? + ) + .as_bytes(), + ) + .await?; + } else { + self.write(format!("C3 AUTHENTICATE {}\r\n", mechanism.to_mechanism()).as_bytes()) + .await?; + } + let mut line = self.read_line().await?; + + for _ in 0..3 { + if matches!(line.first(), Some(b'+')) { + self.write( + format!( + "{}\r\n", + credentials + .encode( + mechanism, + std::str::from_utf8(line.get(2..).unwrap_or_default()) + .unwrap_or_default() + ) + .map_err(|err| Error::InvalidChallenge(err.to_string()))? + ) + .as_bytes(), + ) + .await?; + line = self.read_line().await?; + } else if matches!(line.get(..5), Some(b"C3 OK")) { + return Ok(()); + } else if matches!(line.get(..5), Some(b"C3 NO")) + || matches!(line.get(..6), Some(b"C3 BAD")) + { + return Err(Error::AuthenticationFailed); + } else { + return Err(Error::InvalidResponse(line.into_string())); + } + } + + Err(Error::InvalidResponse(line.into_string())) + } + + pub async fn authentication_mechanisms(&mut self) -> Result { + tokio::time::timeout(self.timeout, async { + self.write(b"C0 CAPABILITY\r\n").await?; + + let line = self.read_line().await?; + if !matches!(line.get(..12), Some(b"* CAPABILITY")) { + return Err(Error::InvalidResponse(line.into_string())); + } + + let mut line_iter = line.iter(); + let mut parser = Rfc5321Parser::new(&mut line_iter); + let mut mechanisms = 0; + + 'outer: while let Ok(ch) = parser.read_char() { + if ch == b' ' { + loop { + if parser.hashed_value().unwrap_or(0) == AUTH && parser.stop_char == b'=' { + if let Ok(Some(mechanism)) = parser.mechanism() { + mechanisms |= mechanism; + } + match parser.stop_char { + b' ' => (), + b'\n' => break 'outer, + _ => break, + } + } + } + } else if ch == b'\n' { + break; + } + } + + Ok(mechanisms) + }) + .await + .map_err(|_| Error::Timeout)? + } + + pub async fn noop(&mut self) -> Result<(), Error> { + tokio::time::timeout(self.timeout, async { + self.write(b"C8 NOOP\r\n").await?; + self.read_line().await?; + Ok(()) + }) + .await + .map_err(|_| Error::Timeout)? + } + + pub async fn logout(&mut self) -> Result<(), Error> { + tokio::time::timeout(self.timeout, async { + self.write(b"C9 LOGOUT\r\n").await?; + Ok(()) + }) + .await + .map_err(|_| Error::Timeout)? + } + + pub async fn expect_greeting(&mut self) -> Result<(), Error> { + tokio::time::timeout(self.timeout, async { + let line = self.read_line().await?; + if matches!(line.get(..4), Some(b"* OK")) { + Ok(()) + } else { + Err(Error::InvalidResponse(line.into_string())) + } + }) + .await + .map_err(|_| Error::Timeout)? + } + + pub async fn read_line(&mut self) -> Result, Error> { + let mut buf = vec![0u8; 1024]; + let mut buf_extended = Vec::with_capacity(0); + + loop { + let br = self.stream.read(&mut buf).await?; + + if br > 0 { + if matches!(buf.get(br - 1), Some(b'\n')) { + //println!("{:?}", std::str::from_utf8(&buf[..br]).unwrap()); + return Ok(if buf_extended.is_empty() { + buf.truncate(br); + buf + } else { + buf_extended.extend_from_slice(&buf[..br]); + buf_extended + }); + } else if buf_extended.is_empty() { + buf_extended = buf[..br].to_vec(); + } else { + buf_extended.extend_from_slice(&buf[..br]); + } + } else { + return Err(Error::Disconnected); + } + } + } + + async fn write(&mut self, bytes: &[u8]) -> Result<(), std::io::Error> { + self.stream.write_all(bytes).await?; + self.stream.flush().await + } +} + +impl From for Error { + fn from(error: std::io::Error) -> Self { + Error::Io(error) + } +} + +impl Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Error::Io(io) => write!(f, "I/O error: {io}"), + Error::Timeout => f.write_str("Connection time-out"), + Error::InvalidResponse(response) => write!(f, "Unexpected response: {response:?}"), + Error::InvalidChallenge(response) => write!(f, "Invalid auth challenge: {response}"), + Error::TLSInvalidName => f.write_str("Invalid TLS name"), + Error::Disconnected => f.write_str("Connection disconnected by peer"), + Error::AuthenticationFailed => f.write_str("Authentication failed"), + } + } +} + +#[cfg(test)] +mod test { + use crate::lookup::imap::ImapAuthClient; + use mail_send::smtp::tls::build_tls_connector; + use smtp_proto::{AUTH_OAUTHBEARER, AUTH_PLAIN, AUTH_XOAUTH, AUTH_XOAUTH2}; + use std::time::Duration; + + #[ignore] + #[tokio::test] + async fn imap_auth() { + let connector = build_tls_connector(false); + + let mut client = ImapAuthClient::connect( + "imap.gmail.com:993", + Duration::from_secs(5), + &connector, + "imap.gmail.com", + true, + ) + .await + .unwrap(); + assert_eq!( + AUTH_PLAIN | AUTH_XOAUTH | AUTH_XOAUTH2 | AUTH_OAUTHBEARER, + client.authentication_mechanisms().await.unwrap() + ); + client.logout().await.unwrap(); + } +} diff --git a/crates/smtp/src/lookup/mod.rs b/crates/smtp/src/lookup/mod.rs new file mode 100644 index 00000000..14fc3203 --- /dev/null +++ b/crates/smtp/src/lookup/mod.rs @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use ahash::AHashSet; +use mail_send::Credentials; +use parking_lot::Mutex; +use tokio::sync::{mpsc, oneshot}; + +use self::cache::LookupCache; + +pub mod cache; +pub mod dispatch; +pub mod imap; +pub mod smtp; +pub mod spawn; +pub mod sql; + +#[derive(Debug)] +pub enum Lookup { + Local(AHashSet), + Remote(LookupChannel), + Sql(SqlQuery), +} + +#[derive(Debug, Clone)] +pub enum SqlDatabase { + Postgres(sqlx::Pool), + MySql(sqlx::Pool), + //MsSql(sqlx::Pool), + SqlLite(sqlx::Pool), +} + +#[derive(Debug)] +pub struct SqlQuery { + pub query: String, + pub db: SqlDatabase, + pub cache: Option>>, +} + +impl Default for Lookup { + fn default() -> Self { + Lookup::Local(AHashSet::default()) + } +} + +#[derive(Debug)] +pub enum Event { + Lookup(LookupItem), + WorkerReady { + item: Item, + result: Option, + next_lookup: Option>>, + }, + WorkerFailed, + Reload, + Stop, +} + +#[derive(Clone, PartialEq, Eq, Hash)] +pub enum Item { + IsAccount(String), + Authenticate(Credentials), + Verify(String), + Expand(String), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum LookupResult { + True, + False, + Values(Vec), +} + +#[derive(Debug)] +pub struct LookupItem { + pub item: Item, + pub result: oneshot::Sender, +} + +#[derive(Debug, Clone)] +pub struct LookupChannel { + pub tx: mpsc::Sender, +} + +#[derive(Clone)] +struct RemoteHost { + tx: mpsc::Sender, + host: T, +} + +pub trait RemoteLookup: Clone { + fn spawn_lookup(&self, lookup: LookupItem, tx: mpsc::Sender); +} diff --git a/crates/smtp/src/lookup/smtp.rs b/crates/smtp/src/lookup/smtp.rs new file mode 100644 index 00000000..282ef336 --- /dev/null +++ b/crates/smtp/src/lookup/smtp.rs @@ -0,0 +1,180 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::sync::Arc; + +use mail_send::smtp::AssertReply; +use smtp_proto::Severity; +use tokio::sync::{mpsc, oneshot}; + +use super::{spawn::LoggedUnwrap, Event, Item, LookupItem, LookupResult, RemoteLookup}; + +pub struct SmtpClientBuilder { + pub builder: mail_send::SmtpClientBuilder, + pub max_rcpt: usize, + pub max_auth_errors: usize, +} + +impl SmtpClientBuilder { + pub async fn lookup_smtp( + &self, + mut lookup: LookupItem, + tx: &mpsc::Sender, + ) -> Result<(), mail_send::Error> { + let mut client = self.builder.connect().await?; + let mut sent_mail_from = false; + let mut num_rcpts = 0; + let mut num_auth_failures = 0; + let capabilities = client + .capabilities(&self.builder.local_host, self.builder.is_lmtp) + .await?; + + loop { + let (result, is_reusable): (LookupResult, bool) = match &lookup.item { + Item::IsAccount(rcpt_to) => { + if !sent_mail_from { + client + .cmd(b"MAIL FROM:<>\r\n") + .await? + .assert_positive_completion()?; + sent_mail_from = true; + } + let reply = client + .cmd(format!("RCPT TO:<{rcpt_to}>\r\n").as_bytes()) + .await?; + let result = match reply.severity() { + Severity::PositiveCompletion => { + num_rcpts += 1; + LookupResult::True + } + Severity::PermanentNegativeCompletion => LookupResult::False, + _ => return Err(mail_send::Error::UnexpectedReply(reply)), + }; + + // Try to reuse the connection with any queued requests + (result, num_rcpts < self.max_rcpt) + } + Item::Authenticate(credentials) => { + let result = match client.authenticate(credentials, &capabilities).await { + Ok(_) => true, + Err(err) => match &err { + mail_send::Error::AuthenticationFailed(err) if err.code() == 535 => { + num_auth_failures += 1; + false + } + _ => { + return Err(err); + } + }, + }; + ( + result.into(), + !result && num_auth_failures < self.max_auth_errors, + ) + } + Item::Verify(address) | Item::Expand(address) => { + let reply = client + .cmd( + if matches!(&lookup.item, Item::Verify(_)) { + format!("VRFY {address}\r\n") + } else { + format!("EXPN {address}\r\n") + } + .as_bytes(), + ) + .await?; + match reply.code() { + 250 | 251 => ( + reply + .message() + .split('\n') + .map(|p| p.to_string()) + .collect::>() + .into(), + true, + ), + 550 | 551 | 553 | 500 | 502 => (LookupResult::False, true), + _ => { + return Err(mail_send::Error::UnexpectedReply(reply)); + } + } + } + }; + + // Try to reuse the connection with any queued requests + let cached_result = match &result { + LookupResult::True => Some(true), + LookupResult::False => Some(false), + LookupResult::Values(_) => None, + }; + lookup.result.send(result).logged_unwrap(); + if is_reusable { + let (next_lookup_tx, next_lookup_rx) = oneshot::channel::>(); + if tx + .send(Event::WorkerReady { + item: lookup.item, + result: cached_result, + next_lookup: next_lookup_tx.into(), + }) + .await + .logged_unwrap() + { + if let Ok(Some(next_lookup)) = next_lookup_rx.await { + lookup = next_lookup; + continue; + } + } + } else { + tx.send(Event::WorkerReady { + item: lookup.item, + result: cached_result, + next_lookup: None, + }) + .await + .logged_unwrap(); + } + break; + } + + Ok(()) + } +} + +impl RemoteLookup for Arc { + fn spawn_lookup(&self, lookup: LookupItem, tx: mpsc::Sender) { + let builder = self.clone(); + tokio::spawn(async move { + if let Err(err) = builder.lookup_smtp(lookup, &tx).await { + tracing::warn!( + context = "remote", + event = "lookup-failed", + remote.addr = &builder.builder.addr, + remote.protocol = "smtp", + "Remote lookup failed: {}", + err + ); + tx.send(Event::WorkerFailed).await.logged_unwrap(); + } + }); + } +} diff --git a/crates/smtp/src/lookup/spawn.rs b/crates/smtp/src/lookup/spawn.rs new file mode 100644 index 00000000..b62f61ff --- /dev/null +++ b/crates/smtp/src/lookup/spawn.rs @@ -0,0 +1,259 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{collections::VecDeque, fmt::Debug, sync::Arc, time::Duration}; + +use crate::config::Host; +use mail_send::smtp::tls::build_tls_connector; +use tokio::sync::{mpsc, oneshot}; +use utils::config::{Config, ServerProtocol}; + +use super::{ + cache::LookupCache, imap::ImapAuthClientBuilder, smtp::SmtpClientBuilder, Event, Item, + LookupChannel, LookupItem, LookupResult, RemoteHost, RemoteLookup, +}; + +impl Host { + pub fn spawn(self, config: &Config) -> LookupChannel { + // Create channel + let local_host = config + .value("server.hostname") + .unwrap_or("[127.0.0.1]") + .to_string(); + let tx_ = self.channel_tx.clone(); + + tokio::spawn(async move { + // Prepare builders + match self.protocol { + ServerProtocol::Smtp | ServerProtocol::Lmtp => { + RemoteHost { + tx: self.channel_tx, + host: Arc::new(SmtpClientBuilder { + builder: mail_send::SmtpClientBuilder { + addr: format!("{}:{}", self.address, self.port), + timeout: self.timeout, + tls_connector: build_tls_connector(self.tls_allow_invalid_certs), + tls_hostname: self.address, + tls_implicit: self.tls_implicit, + is_lmtp: matches!(self.protocol, ServerProtocol::Lmtp), + credentials: None, + local_host, + }, + max_rcpt: self.max_requests, + max_auth_errors: self.max_errors, + }), + } + .run( + self.channel_rx, + self.cache_entries, + self.cache_ttl_positive, + self.cache_ttl_negative, + self.concurrency, + ) + .await; + } + ServerProtocol::Imap => { + RemoteHost { + tx: self.channel_tx, + host: Arc::new( + ImapAuthClientBuilder::new( + format!("{}:{}", self.address, self.port), + self.timeout, + build_tls_connector(self.tls_allow_invalid_certs), + self.address, + self.tls_implicit, + ) + .init() + .await, + ), + } + .run( + self.channel_rx, + self.cache_entries, + self.cache_ttl_positive, + self.cache_ttl_negative, + self.concurrency, + ) + .await; + } + ServerProtocol::Http | ServerProtocol::Jmap => { + eprintln!("HTTP/JMAP lookups are not supported."); + std::process::exit(0); + } + } + }); + + LookupChannel { tx: tx_ } + } +} + +impl RemoteHost { + pub async fn run( + &self, + mut rx: mpsc::Receiver, + entries: usize, + ttl_pos: Duration, + ttl_neg: Duration, + max_concurrent: usize, + ) { + // Create caches and queue + let mut cache = LookupCache::new(entries, ttl_pos, ttl_neg); + let mut queue = VecDeque::new(); + let mut active_lookups = 0; + + while let Some(event) = rx.recv().await { + match event { + Event::Lookup(lookup) => { + if let Some(result) = cache.get(&lookup.item) { + lookup.result.send(result.into()).logged_unwrap(); + } else if active_lookups < max_concurrent { + active_lookups += 1; + self.host.spawn_lookup(lookup, self.tx.clone()); + } else { + queue.push_back(lookup); + } + } + Event::WorkerReady { + item, + result, + next_lookup, + } => { + match result { + Some(true) => cache.insert_pos(item), + Some(false) => cache.insert_neg(item), + _ => (), + } + + let mut lookup = None; + while let Some(queued_lookup) = queue.pop_front() { + if let Some(result) = cache.get(&queued_lookup.item) { + queued_lookup.result.send(result.into()).logged_unwrap(); + } else { + lookup = queued_lookup.into(); + break; + } + } + if let Some(next_lookup) = next_lookup { + if lookup.is_none() { + active_lookups -= 1; + } + next_lookup.send(lookup).logged_unwrap(); + } else if let Some(lookup) = lookup { + self.host.spawn_lookup(lookup, self.tx.clone()); + } else { + active_lookups -= 1; + } + } + Event::WorkerFailed => { + if let Some(queued_lookup) = queue.pop_front() { + self.host.spawn_lookup(queued_lookup, self.tx.clone()); + } else { + active_lookups -= 1; + } + } + Event::Stop => { + queue.clear(); + break; + } + Event::Reload => { + cache.clear(); + } + } + } + } +} + +impl LookupChannel { + pub async fn lookup(&self, item: Item) -> Option { + let (tx, rx) = oneshot::channel(); + if self + .tx + .send(Event::Lookup(LookupItem { item, result: tx })) + .await + .is_ok() + { + rx.await.ok() + } else { + None + } + } +} + +impl From> for LookupChannel { + fn from(tx: mpsc::Sender) -> Self { + LookupChannel { tx } + } +} + +impl From for bool { + fn from(value: LookupResult) -> Self { + matches!(value, LookupResult::True | LookupResult::Values(_)) + } +} + +impl From for LookupResult { + fn from(value: bool) -> Self { + if value { + LookupResult::True + } else { + LookupResult::False + } + } +} + +impl From> for LookupResult { + fn from(value: Vec) -> Self { + if !value.is_empty() { + LookupResult::Values(value) + } else { + LookupResult::False + } + } +} + +pub trait LoggedUnwrap { + fn logged_unwrap(self) -> bool; +} + +impl LoggedUnwrap for Result { + fn logged_unwrap(self) -> bool { + match self { + Ok(_) => true, + Err(err) => { + tracing::debug!("Failed to send message over channel: {:?}", err); + false + } + } + } +} + +impl Debug for Item { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::IsAccount(arg0) => f.debug_tuple("Rcpt").field(arg0).finish(), + Self::Authenticate(_) => f.debug_tuple("Auth").finish(), + Self::Expand(arg0) => f.debug_tuple("Expn").field(arg0).finish(), + Self::Verify(arg0) => f.debug_tuple("Vrfy").field(arg0).finish(), + } + } +} diff --git a/crates/smtp/src/lookup/sql.rs b/crates/smtp/src/lookup/sql.rs new file mode 100644 index 00000000..eab1a5f5 --- /dev/null +++ b/crates/smtp/src/lookup/sql.rs @@ -0,0 +1,237 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use super::{SqlDatabase, SqlQuery}; + +impl SqlQuery { + pub async fn exists(&self, param: &str) -> Option { + if let Some(result) = self + .cache + .as_ref() + .and_then(|cache| cache.lock().get(param)) + { + return Some(result); + } + let result = match &self.db { + super::SqlDatabase::Postgres(pool) => { + sqlx::query_scalar::<_, bool>(&self.query) + .bind(param) + .fetch_one(pool) + .await + } + super::SqlDatabase::MySql(pool) => { + sqlx::query_scalar::<_, bool>(&self.query) + .bind(param) + .fetch_one(pool) + .await + } + /*super::SqlDatabase::MsSql(pool) => { + sqlx::query_scalar::<_, bool>(&self.query) + .bind(param) + .fetch_one(pool) + .await + }*/ + super::SqlDatabase::SqlLite(pool) => { + sqlx::query_scalar::<_, bool>(&self.query) + .bind(param) + .fetch_one(pool) + .await + } + }; + + match result { + Ok(result) => { + if let Some(cache) = &self.cache { + if result { + cache.lock().insert_pos(param.to_string()); + } else { + cache.lock().insert_neg(param.to_string()); + } + } + Some(result) + } + Err(err) => { + tracing::warn!(context = "sql", event = "error", query = self.query, reason = ?err); + None + } + } + } + + pub async fn fetch_one(&self, param: &str) -> Option> { + let result = match &self.db { + super::SqlDatabase::Postgres(pool) => { + sqlx::query_scalar::<_, String>(&self.query) + .bind(param) + .fetch_optional(pool) + .await + } + super::SqlDatabase::MySql(pool) => { + sqlx::query_scalar::<_, String>(&self.query) + .bind(param) + .fetch_optional(pool) + .await + } + /*super::SqlDatabase::MsSql(pool) => { + sqlx::query_scalar::<_, String>(&self.query) + .bind(param) + .fetch_optional(pool) + .await + }*/ + super::SqlDatabase::SqlLite(pool) => { + sqlx::query_scalar::<_, String>(&self.query) + .bind(param) + .fetch_optional(pool) + .await + } + }; + + match result { + Ok(result) => Some(result), + Err(err) => { + tracing::warn!(context = "sql", event = "error", query = self.query, reason = ?err); + None + } + } + } + + pub async fn fetch_many(&self, param: &str) -> Option> { + let result = match &self.db { + super::SqlDatabase::Postgres(pool) => { + sqlx::query_scalar::<_, String>(&self.query) + .bind(param) + .fetch_all(pool) + .await + } + super::SqlDatabase::MySql(pool) => { + sqlx::query_scalar::<_, String>(&self.query) + .bind(param) + .fetch_all(pool) + .await + } + /*super::SqlDatabase::MsSql(pool) => { + sqlx::query_scalar::<_, String>(&self.query) + .bind(param) + .fetch_all(pool) + .await + }*/ + super::SqlDatabase::SqlLite(pool) => { + sqlx::query_scalar::<_, String>(&self.query) + .bind(param) + .fetch_all(pool) + .await + } + }; + + match result { + Ok(result) => Some(result), + Err(err) => { + tracing::warn!(context = "sql", event = "error", query = self.query, reason = ?err); + None + } + } + } +} + +impl SqlDatabase { + pub async fn exists(&self, query: &str, params: impl Iterator) -> Option { + let result = match self { + super::SqlDatabase::Postgres(pool) => { + let mut q = sqlx::query_scalar::<_, bool>(query); + for param in params { + q = q.bind(param); + } + q.fetch_one(pool).await + } + super::SqlDatabase::MySql(pool) => { + let mut q = sqlx::query_scalar::<_, bool>(query); + for param in params { + q = q.bind(param); + } + q.fetch_one(pool).await + } + /*super::SqlDatabase::MsSql(pool) => { + let mut q = sqlx::query_scalar::<_, bool>(query); + for param in params { + q = q.bind(param); + } + q.fetch_one(pool).await + }*/ + super::SqlDatabase::SqlLite(pool) => { + let mut q = sqlx::query_scalar::<_, bool>(query); + for param in params { + q = q.bind(param); + } + q.fetch_one(pool).await + } + }; + + match result { + Ok(result) => Some(result), + Err(err) => { + tracing::warn!(context = "sql", event = "error", query = query, reason = ?err); + None + } + } + } + + pub async fn execute(&self, query: &str, params: impl Iterator) -> bool { + let result = match self { + super::SqlDatabase::Postgres(pool) => { + let mut q = sqlx::query(query); + for param in params { + q = q.bind(param); + } + q.execute(pool).await.map(|_| ()) + } + super::SqlDatabase::MySql(pool) => { + let mut q = sqlx::query(query); + for param in params { + q = q.bind(param); + } + q.execute(pool).await.map(|_| ()) + } + /*super::SqlDatabase::MsSql(pool) => { + let mut q = sqlx::query(query); + for param in params { + q = q.bind(param); + } + q.execute(pool).await.map(|_| ()) + }*/ + super::SqlDatabase::SqlLite(pool) => { + let mut q = sqlx::query(query); + for param in params { + q = q.bind(param); + } + q.execute(pool).await.map(|_| ()) + } + }; + + match result { + Ok(_) => true, + Err(err) => { + tracing::warn!(context = "sql", event = "error", query = query, reason = ?err); + false + } + } + } +} diff --git a/crates/smtp/src/old_main.rs b/crates/smtp/src/old_main.rs new file mode 100644 index 00000000..97d686c1 --- /dev/null +++ b/crates/smtp/src/old_main.rs @@ -0,0 +1,385 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{collections::HashMap, fs, sync::Arc, time::Duration}; + +use dashmap::DashMap; +use mail_send::smtp::tls::build_tls_connector; +use opentelemetry::{ + sdk::{ + trace::{self, Sampler}, + Resource, + }, + KeyValue, +}; +use opentelemetry_otlp::WithExportConfig; +use opentelemetry_semantic_conventions::resource::{SERVICE_NAME, SERVICE_VERSION}; +use stalwart_smtp::{ + config::{Config, ConfigContext, ServerProtocol}, + core::{ + throttle::{ConcurrencyLimiter, ThrottleKeyHasherBuilder}, + Core, QueueCore, ReportCore, SessionCore, TlsConnectors, + }, + failed, + queue::{self, manager::SpawnQueue}, + reporting::{self, scheduler::SpawnReport}, + UnwrapFailure, +}; +use tokio::sync::{mpsc, watch}; +use tracing_appender::non_blocking::WorkerGuard; +use tracing_subscriber::{prelude::__tracing_subscriber_SubscriberExt, EnvFilter}; + +#[tokio::main] +async fn main() -> std::io::Result<()> { + // Read configuration parameters + let config = parse_config(); + let mut config_context = ConfigContext::default(); + config + .parse_servers(&mut config_context) + .failed("Configuration error"); + config + .parse_remote_hosts(&mut config_context) + .failed("Configuration error"); + config + .parse_databases(&mut config_context) + .failed("Configuration error"); + config + .parse_lists(&mut config_context) + .failed("Configuration error"); + config + .parse_signatures(&mut config_context) + .failed("Configuration error"); + let sieve_config = config + .parse_sieve(&mut config_context) + .failed("Configuration error"); + let session_config = config + .parse_session_config(&config_context) + .failed("Configuration error"); + let queue_config = config + .parse_queue(&config_context) + .failed("Configuration error"); + let mail_auth_config = config + .parse_mail_auth(&config_context) + .failed("Configuration error"); + let report_config = config + .parse_reports(&config_context) + .failed("Configuration error"); + + // Build core + let (queue_tx, queue_rx) = mpsc::channel(1024); + let (report_tx, report_rx) = mpsc::channel(1024); + let core = Arc::new(Core { + worker_pool: rayon::ThreadPoolBuilder::new() + .num_threads( + config + .property::("global.thread-pool") + .failed("Failed to parse thread pool size") + .filter(|v| *v > 0) + .unwrap_or_else(num_cpus::get), + ) + .build() + .unwrap(), + resolvers: config.build_resolvers().failed("Failed to build resolvers"), + session: SessionCore { + config: session_config, + throttle: DashMap::with_capacity_and_hasher_and_shard_amount( + config + .property("global.shared-map.capacity") + .failed("Failed to parse shared map capacity") + .unwrap_or(2), + ThrottleKeyHasherBuilder::default(), + config + .property::("global.shared-map.shard") + .failed("Failed to parse shared map shard amount") + .unwrap_or(32) + .next_power_of_two() as usize, + ), + }, + queue: QueueCore { + config: queue_config, + throttle: DashMap::with_capacity_and_hasher_and_shard_amount( + config + .property("global.shared-map.capacity") + .failed("Failed to parse shared map capacity") + .unwrap_or(2), + ThrottleKeyHasherBuilder::default(), + config + .property::("global.shared-map.shard") + .failed("Failed to parse shared map shard amount") + .unwrap_or(32) + .next_power_of_two() as usize, + ), + id_seq: 0.into(), + quota: DashMap::with_capacity_and_hasher_and_shard_amount( + config + .property("global.shared-map.capacity") + .failed("Failed to parse shared map capacity") + .unwrap_or(2), + ThrottleKeyHasherBuilder::default(), + config + .property::("global.shared-map.shard") + .failed("Failed to parse shared map shard amount") + .unwrap_or(32) + .next_power_of_two() as usize, + ), + tx: queue_tx, + connectors: TlsConnectors { + pki_verify: build_tls_connector(false), + dummy_verify: build_tls_connector(true), + }, + }, + report: ReportCore { + tx: report_tx, + config: report_config, + }, + mail_auth: mail_auth_config, + sieve: sieve_config, + }); + + // Bind ports before dropping privileges + for server in &config_context.servers { + for listener in &server.listeners { + listener + .socket + .bind(listener.addr) + .failed(&format!("Failed to bind to {}", listener.addr)); + } + } + + // Drop privileges + #[cfg(not(target_env = "msvc"))] + { + if let Some(run_as_user) = config.value("server.run-as.user") { + let mut pd = privdrop::PrivDrop::default().user(run_as_user); + if let Some(run_as_group) = config.value("server.run-as.group") { + pd = pd.group(run_as_group); + } + pd.apply().failed("Failed to drop privileges"); + } + } + + // Enable tracing + let _tracer = enable_tracing(&config).failed("Failed to enable tracing"); + tracing::info!( + "Starting Stalwart SMTP server v{}...", + env!("CARGO_PKG_VERSION") + ); + + // Spawn queue manager + queue_rx.spawn(core.clone(), core.queue.read_queue().await); + + // Spawn report manager + report_rx.spawn(core.clone(), core.report.read_reports().await); + + // Spawn remote hosts + for host in config_context.hosts.into_values() { + if host.lookup { + host.spawn(&config); + } + } + + // Spawn listeners + let (shutdown_tx, shutdown_rx) = watch::channel(false); + for server in config_context.servers { + match server.protocol { + ServerProtocol::Smtp | ServerProtocol::Lmtp => server + .spawn(core.clone(), shutdown_rx.clone()) + .failed("Failed to start listener"), + ServerProtocol::Http => server + .spawn_management(core.clone(), shutdown_rx.clone()) + .failed("Failed to start management interface"), + ServerProtocol::Imap => { + eprintln!("Invalid protocol 'imap' for listener '{}'.", server.id); + std::process::exit(0); + } + } + } + + // Wait for shutdown signal + #[cfg(not(target_env = "msvc"))] + { + use tokio::signal::unix::{signal, SignalKind}; + + let mut h_term = signal(SignalKind::terminate()).failed("start signal handler"); + let mut h_int = signal(SignalKind::interrupt()).failed("start signal handler"); + + tokio::select! { + _ = h_term.recv() => tracing::debug!("Received SIGTERM."), + _ = h_int.recv() => tracing::debug!("Received SIGINT."), + }; + } + + #[cfg(target_env = "msvc")] + { + match tokio::signal::ctrl_c().await { + Ok(()) => {} + Err(err) => { + eprintln!("Unable to listen for shutdown signal: {}", err); + } + } + } + + // Shutdown the system + tracing::info!( + "Shutting down Stalwart SMTP server v{}...", + env!("CARGO_PKG_VERSION") + ); + + // Stop services + shutdown_tx.send(true).ok(); + core.queue.tx.send(queue::Event::Stop).await.ok(); + core.report.tx.send(reporting::Event::Stop).await.ok(); + + // Wait for services to finish + tokio::time::sleep(Duration::from_secs(1)).await; + + Ok(()) +} + +fn enable_tracing(config: &Config) -> stalwart_smtp::config::Result> { + let level = config.value("global.tracing.level").unwrap_or("info"); + let env_filter = EnvFilter::builder() + .parse(format!("stalwart_smtp={}", level)) + .failed("Failed to log level"); + match config.value("global.tracing.method").unwrap_or_default() { + "log" => { + let path = config.value_require("global.tracing.path")?; + let prefix = config.value_require("global.tracing.prefix")?; + let file_appender = match config.value("global.tracing.rotate").unwrap_or("daily") { + "daily" => tracing_appender::rolling::daily(path, prefix), + "hourly" => tracing_appender::rolling::hourly(path, prefix), + "minutely" => tracing_appender::rolling::minutely(path, prefix), + "never" => tracing_appender::rolling::never(path, prefix), + rotate => { + return Err(format!("Unsupported log rotation strategy {rotate:?}")); + } + }; + + let (non_blocking, guard) = tracing_appender::non_blocking(file_appender); + tracing::subscriber::set_global_default( + tracing_subscriber::FmtSubscriber::builder() + .with_env_filter(env_filter) + .with_writer(non_blocking) + .finish(), + ) + .failed("Failed to set subscriber"); + Ok(guard.into()) + } + "stdout" => { + tracing::subscriber::set_global_default( + tracing_subscriber::FmtSubscriber::builder() + .with_env_filter(env_filter) + .finish(), + ) + .failed("Failed to set subscriber"); + + Ok(None) + } + "otel" | "open-telemetry" => { + let tracer = match config.value_require("global.tracing.transport")? { + "grpc" => { + let mut exporter = opentelemetry_otlp::new_exporter().tonic(); + if let Some(endpoint) = config.value("global.tracing.endpoint") { + exporter = exporter.with_endpoint(endpoint); + } + opentelemetry_otlp::new_pipeline() + .tracing() + .with_exporter(exporter) + } + "http" => { + let mut headers = HashMap::new(); + for (_, value) in config.values("global.tracing.headers") { + if let Some((key, value)) = value.split_once(':') { + headers.insert(key.trim().to_string(), value.trim().to_string()); + } else { + return Err(format!("Invalid open-telemetry header {value:?}")); + } + } + let mut exporter = opentelemetry_otlp::new_exporter() + .http() + .with_endpoint(config.value_require("global.tracing.endpoint")?); + if !headers.is_empty() { + exporter = exporter.with_headers(headers); + } + opentelemetry_otlp::new_pipeline() + .tracing() + .with_exporter(exporter) + } + transport => { + return Err(format!( + "Unsupported open-telemetry transport {transport:?}" + )); + } + } + .with_trace_config( + trace::config() + .with_resource(Resource::new(vec![ + KeyValue::new(SERVICE_NAME, "stalwart-smtp".to_string()), + KeyValue::new(SERVICE_VERSION, env!("CARGO_PKG_VERSION").to_string()), + ])) + .with_sampler(Sampler::AlwaysOn), + ) + .install_batch(opentelemetry::runtime::Tokio) + .failed("Failed to create tracer"); + + tracing::subscriber::set_global_default( + tracing_subscriber::Registry::default() + .with(tracing_opentelemetry::layer().with_tracer(tracer)) + .with(env_filter), + ) + .failed("Failed to set subscriber"); + + Ok(None) + } + _ => Ok(None), + } +} + +fn parse_config() -> Config { + let mut config_path = None; + let mut found_param = false; + + for arg in std::env::args().skip(1) { + if let Some((key, value)) = arg.split_once('=') { + if key.starts_with("--config") { + config_path = value.trim().to_string().into(); + break; + } else { + failed(&format!("Invalid command line argument: {key}")); + } + } else if found_param { + config_path = arg.into(); + break; + } else if arg.starts_with("--config") { + found_param = true; + } else { + failed(&format!("Invalid command line argument: {arg}")); + } + } + + Config::parse( + &fs::read_to_string(config_path.failed("Missing parameter --config=.")) + .failed("Could not read configuration file"), + ) + .failed("Invalid configuration file") +} diff --git a/crates/smtp/src/outbound/dane/dnssec.rs b/crates/smtp/src/outbound/dane/dnssec.rs new file mode 100644 index 00000000..ddd90a70 --- /dev/null +++ b/crates/smtp/src/outbound/dane/dnssec.rs @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use mail_auth::{ + common::{lru::DnsCache, resolver::IntoFqdn}, + trust_dns_resolver::{ + config::{ResolverConfig, ResolverOpts}, + error::{ResolveError, ResolveErrorKind}, + proto::{ + error::ProtoErrorKind, + rr::rdata::tlsa::{CertUsage, Matching, Selector}, + }, + AsyncResolver, + }, +}; +use std::sync::Arc; + +use crate::core::Resolvers; + +use super::{DnssecResolver, Tlsa, TlsaEntry}; + +impl DnssecResolver { + pub fn with_capacity( + config: ResolverConfig, + options: ResolverOpts, + ) -> Result { + Ok(Self { + resolver: AsyncResolver::tokio(config, options)?, + }) + } +} + +impl Resolvers { + pub async fn tlsa_lookup<'x>( + &self, + key: impl IntoFqdn<'x>, + ) -> mail_auth::Result>> { + let key = key.into_fqdn(); + if let Some(value) = self.cache.tlsa.get(key.as_ref()) { + return Ok(Some(value)); + } + + #[cfg(any(test, feature = "test_mode"))] + if true { + return mail_auth::common::resolver::mock_resolve(key.as_ref()); + } + + let mut entries = Vec::new(); + let tlsa_lookup = match self.dnssec.resolver.tlsa_lookup(key.as_ref()).await { + Ok(tlsa_lookup) => tlsa_lookup, + Err(err) => { + return match &err.kind() { + ResolveErrorKind::Proto(proto_err) + if matches!(proto_err.kind(), ProtoErrorKind::RrsigsNotPresent { .. }) => + { + Ok(None) + } + _ => Err(err.into()), + }; + } + }; + + let mut has_end_entities = false; + let mut has_intermediates = false; + + for record in tlsa_lookup.as_lookup().record_iter() { + if let Some(tlsa) = record.data().and_then(|r| r.as_tlsa()) { + let is_end_entity = match tlsa.cert_usage() { + CertUsage::DomainIssued => true, + CertUsage::TrustAnchor => false, + _ => continue, + }; + if is_end_entity { + has_end_entities = true; + } else { + has_intermediates = true; + } + entries.push(TlsaEntry { + is_end_entity, + is_sha256: match tlsa.matching() { + Matching::Sha256 => true, + Matching::Sha512 => false, + _ => continue, + }, + is_spki: match tlsa.selector() { + Selector::Spki => true, + Selector::Full => false, + _ => continue, + }, + data: tlsa.cert_data().to_vec(), + }); + } + } + + Ok(Some(self.cache.tlsa.insert( + key.into_owned(), + Arc::new(Tlsa { + entries, + has_end_entities, + has_intermediates, + }), + tlsa_lookup.valid_until(), + ))) + } + + #[cfg(feature = "test_mode")] + pub fn tlsa_add<'x>( + &self, + key: impl IntoFqdn<'x>, + value: impl Into>, + valid_until: std::time::Instant, + ) { + self.cache + .tlsa + .insert(key.into_fqdn().into_owned(), value.into(), valid_until); + } +} diff --git a/crates/smtp/src/outbound/dane/mod.rs b/crates/smtp/src/outbound/dane/mod.rs new file mode 100644 index 00000000..7d1118dd --- /dev/null +++ b/crates/smtp/src/outbound/dane/mod.rs @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use mail_auth::trust_dns_resolver::TokioAsyncResolver; + +pub mod dnssec; +pub mod verify; + +pub struct DnssecResolver { + pub resolver: TokioAsyncResolver, +} + +#[derive(Debug, Hash, PartialEq, Eq)] +pub struct TlsaEntry { + pub is_end_entity: bool, + pub is_sha256: bool, + pub is_spki: bool, + pub data: Vec, +} + +#[derive(Debug, Hash, PartialEq, Eq)] +pub struct Tlsa { + pub entries: Vec, + pub has_end_entities: bool, + pub has_intermediates: bool, +} diff --git a/crates/smtp/src/outbound/dane/verify.rs b/crates/smtp/src/outbound/dane/verify.rs new file mode 100644 index 00000000..3d599d95 --- /dev/null +++ b/crates/smtp/src/outbound/dane/verify.rs @@ -0,0 +1,307 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use rustls::Certificate; +use sha1::Digest; +use sha2::{Sha256, Sha512}; +use x509_parser::prelude::{FromDer, X509Certificate}; + +use crate::queue::{Error, ErrorDetails, Status}; + +use super::Tlsa; + +impl Tlsa { + pub fn verify( + &self, + span: &tracing::Span, + hostname: &str, + certificates: Option<&[Certificate]>, + ) -> Result<(), Status<(), Error>> { + let certificates = if let Some(certificates) = certificates { + certificates + } else { + tracing::info!( + parent: span, + context = "dane", + event = "no-server-certs-found", + mx = hostname, + "No certificates were provided." + ); + return Err(Status::TemporaryFailure(Error::DaneError(ErrorDetails { + entity: hostname.to_string(), + details: "No certificates were provided by host".to_string(), + }))); + }; + + let mut matched_end_entity = false; + let mut matched_intermediate = false; + 'outer: for (pos, der_certificate) in certificates.iter().enumerate() { + // Parse certificate + let certificate = match X509Certificate::from_der(der_certificate.as_ref()) { + Ok((_, certificate)) => certificate, + Err(err) => { + tracing::debug!( + parent: span, + context = "dane", + event = "cert-parse-error", + "Failed to parse X.509 certificate for host {}: {}", + hostname, + err + ); + return Err(Status::TemporaryFailure(Error::DaneError(ErrorDetails { + entity: hostname.to_string(), + details: "Failed to parse X.509 certificate".to_string(), + }))); + } + }; + + // Match against TLSA records + let is_end_entity = pos == 0; + let mut sha256 = [None, None]; + let mut sha512 = [None, None]; + for record in self.entries.iter() { + if record.is_end_entity == is_end_entity { + let hash: &[u8] = if record.is_sha256 { + &sha256[usize::from(record.is_spki)].get_or_insert_with(|| { + let mut hasher = Sha256::new(); + hasher.update(if record.is_spki { + certificate.public_key().raw + } else { + der_certificate.as_ref() + }); + hasher.finalize() + })[..] + } else { + &sha512[usize::from(record.is_spki)].get_or_insert_with(|| { + let mut hasher = Sha512::new(); + hasher.update(if record.is_spki { + certificate.public_key().raw + } else { + der_certificate.as_ref() + }); + hasher.finalize() + })[..] + }; + + if hash == record.data { + tracing::debug!( + parent: span, + context = "dane", + event = "info", + mx = hostname, + certificate = if is_end_entity { + "end-entity" + } else { + "intermediate" + }, + "Matched TLSA record with hash {:x?}.", + hash + ); + + if is_end_entity { + matched_end_entity = true; + if !self.has_intermediates { + break 'outer; + } + } else { + matched_intermediate = true; + break 'outer; + } + } + } + } + } + + if (self.has_end_entities == matched_end_entity) + && (self.has_intermediates == matched_intermediate) + { + tracing::info!( + parent: span, + context = "dane", + event = "authenticated", + mx = hostname, + "DANE authentication successful.", + ); + Ok(()) + } else { + tracing::warn!( + parent: span, + context = "dane", + event = "auth-failure", + mx = hostname, + "No matching certificates found in TLSA records.", + ); + Err(Status::PermanentFailure(Error::DaneError(ErrorDetails { + entity: hostname.to_string(), + details: "No matching certificates found in TLSA records".to_string(), + }))) + } + } +} + +#[cfg(test)] +mod test { + use std::{ + collections::BTreeSet, + fs::{self, File}, + io::{BufRead, BufReader}, + num::ParseIntError, + path::PathBuf, + time::{Duration, Instant}, + }; + + use mail_auth::{ + common::lru::{DnsCache, LruCache}, + trust_dns_resolver::{ + config::{ResolverConfig, ResolverOpts}, + AsyncResolver, + }, + Resolver, + }; + use rustls::Certificate; + + use crate::{ + core::Resolvers, + outbound::dane::{DnssecResolver, Tlsa, TlsaEntry}, + queue::{Error, ErrorDetails, Status}, + }; + + #[tokio::test] + async fn dane_test() { + let conf = ResolverConfig::cloudflare_tls(); + let mut opts = ResolverOpts::default(); + opts.validate = true; + opts.try_tcp_on_error = true; + + let r = Resolvers { + dns: Resolver::new_cloudflare().unwrap(), + dnssec: DnssecResolver { + resolver: AsyncResolver::tokio(conf, opts).unwrap(), + }, + cache: crate::core::DnsCache { + tlsa: LruCache::with_capacity(10), + mta_sts: LruCache::with_capacity(10), + }, + }; + + // Add dns entries + let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + path.push("resources"); + path.push("smtp"); + path.push("dane"); + let mut file = path.clone(); + file.push("dns.txt"); + + let mut hosts = BTreeSet::new(); + let mut tlsa = Tlsa { + entries: Vec::new(), + has_end_entities: false, + has_intermediates: false, + }; + let mut hostname = String::new(); + + for line in BufReader::new(File::open(file).unwrap()).lines() { + let line = line.unwrap(); + let mut is_end_entity = false; + for (pos, item) in line.split_whitespace().enumerate() { + match pos { + 0 => { + if hostname != item && !hostname.is_empty() { + r.tlsa_add(hostname, tlsa, Instant::now() + Duration::from_secs(30)); + tlsa = Tlsa { + entries: Vec::new(), + has_end_entities: false, + has_intermediates: false, + }; + } + hosts.insert(item.strip_prefix("_25._tcp.").unwrap().to_string()); + hostname = item.to_string(); + } + 1 => { + is_end_entity = item == "3"; + } + 4 => { + if is_end_entity { + tlsa.has_end_entities = true; + } else { + tlsa.has_intermediates = true; + } + tlsa.entries.push(TlsaEntry { + is_end_entity, + is_sha256: true, + is_spki: true, + data: decode_hex(item).unwrap(), + }); + } + _ => (), + } + } + } + r.tlsa_add(hostname, tlsa, Instant::now() + Duration::from_secs(30)); + + // Add certificates + assert!(!hosts.is_empty()); + for host in hosts { + // Add certificates + let mut certs = Vec::new(); + for num in 0..6 { + let mut file = path.clone(); + file.push(format!("{host}.{num}.cert")); + if file.exists() { + certs.push(Certificate(fs::read(file).unwrap())); + } else { + break; + } + } + + // Successful DANE verification + let tlsa = r + .tlsa_lookup(format!("_25._tcp.{host}.")) + .await + .unwrap() + .unwrap(); + + assert_eq!( + tlsa.verify(&tracing::info_span!("test_span"), &host, Some(&certs)), + Ok(()) + ); + + // Failed DANE verification + certs.remove(0); + assert_eq!( + tlsa.verify(&tracing::info_span!("test_span"), &host, Some(&certs)), + Err(Status::PermanentFailure(Error::DaneError(ErrorDetails { + entity: host.to_string(), + details: "No matching certificates found in TLSA records".to_string() + }))) + ); + } + } + + pub fn decode_hex(s: &str) -> Result, ParseIntError> { + (0..s.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&s[i..i + 2], 16)) + .collect() + } +} diff --git a/crates/smtp/src/outbound/delivery.rs b/crates/smtp/src/outbound/delivery.rs new file mode 100644 index 00000000..3b0ea3e8 --- /dev/null +++ b/crates/smtp/src/outbound/delivery.rs @@ -0,0 +1,1009 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{ + net::{IpAddr, Ipv4Addr, SocketAddr}, + sync::Arc, + time::{Duration, Instant}, +}; + +use mail_auth::{ + mta_sts::TlsRpt, + report::tlsrpt::{FailureDetails, ResultType}, +}; +use mail_send::SmtpClient; +use smtp_proto::MAIL_REQUIRETLS; +use utils::config::ServerProtocol; + +use crate::{ + config::{AggregateFrequency, TlsStrategy}, + core::Core, + queue::ErrorDetails, + reporting::{tls::TlsRptOptions, PolicyType, TlsEvent}, +}; + +use super::{ + lookup::ToRemoteHost, + mta_sts, + session::{read_greeting, say_helo, try_start_tls, SessionParams, StartTlsResult}, + RemoteHost, +}; +use crate::queue::{ + manager::Queue, throttle, DeliveryAttempt, Domain, Error, Event, OnHold, QueueEnvelope, + Schedule, Status, WorkerResult, +}; + +impl DeliveryAttempt { + pub async fn try_deliver(mut self, core: Arc, queue: &mut Queue) { + // Check that the message still has recipients to be delivered + let has_pending_delivery = self.has_pending_delivery(); + + // Send any due Delivery Status Notifications + core.queue.send_dsn(&mut self).await; + + if has_pending_delivery { + // Re-queue the message if its not yet due for delivery + let due = self.message.next_delivery_event(); + if due > Instant::now() { + // Save changes to disk + self.message.save_changes().await; + + queue.schedule(Schedule { + due, + inner: self.message, + }); + return; + } + } else { + // All message recipients expired, do not re-queue. (DSN has been already sent) + self.message.remove().await; + return; + } + + // Throttle sender + for throttle in &core.queue.config.throttle.sender { + if let Err(err) = core + .queue + .is_allowed( + throttle, + self.message.as_ref(), + &mut self.in_flight, + &self.span, + ) + .await + { + // Save changes to disk + self.message.save_changes().await; + + match err { + throttle::Error::Concurrency { limiter } => { + queue.on_hold(OnHold { + next_due: self.message.next_event_after(Instant::now()), + limiters: vec![limiter], + message: self.message, + }); + } + throttle::Error::Rate { retry_at } => { + queue.schedule(Schedule { + due: retry_at, + inner: self.message, + }); + } + } + return; + } + } + + tokio::spawn(async move { + let queue_config = &core.queue.config; + let mut on_hold = Vec::new(); + let no_ip = IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)); + + let mut domains = std::mem::take(&mut self.message.domains); + let mut recipients = std::mem::take(&mut self.message.recipients); + 'next_domain: for (domain_idx, domain) in domains.iter_mut().enumerate() { + // Only process domains due for delivery + if !matches!(&domain.status, Status::Scheduled | Status::TemporaryFailure(_) + if domain.retry.due <= Instant::now()) + { + continue; + } + + // Create new span for domain + let span = tracing::info_span!( + parent: &self.span, + "attempt", + domain = domain.domain, + attempt_number = domain.retry.inner, + ); + + // Build envelope + let mut envelope = QueueEnvelope { + message: self.message.as_ref(), + domain: &domain.domain, + mx: "", + remote_ip: no_ip, + local_ip: no_ip, + }; + + // Throttle recipient domain + let mut in_flight = Vec::new(); + for throttle in &queue_config.throttle.rcpt { + if let Err(err) = core + .queue + .is_allowed(throttle, &envelope, &mut in_flight, &span) + .await + { + domain.set_throttle_error(err, &mut on_hold); + continue 'next_domain; + } + } + + // Obtain next hop + let (mut remote_hosts, is_smtp) = + if let Some(next_hop) = queue_config.next_hop.eval(&envelope).await { + ( + vec![RemoteHost::Relay(next_hop)], + next_hop.protocol == ServerProtocol::Smtp, + ) + } else { + (Vec::with_capacity(0), true) + }; + + // Prepare TLS strategy + let mut tls_strategy = TlsStrategy { + mta_sts: *queue_config.tls.mta_sts.eval(&envelope).await, + ..Default::default() + }; + + // Obtain TLS reporting + let tls_report = match core.report.config.tls.send.eval(&envelope).await { + interval @ (AggregateFrequency::Hourly + | AggregateFrequency::Daily + | AggregateFrequency::Weekly) + if is_smtp => + { + match core + .resolvers + .dns + .txt_lookup::(format!("_smtp._tls.{}.", envelope.domain)) + .await + { + Ok(record) => { + tracing::debug!(parent: &span, + context = "tlsrpt", + event = "record-fetched", + record = ?record); + + TlsRptOptions { + record, + interval: *interval, + } + .into() + } + Err(err) => { + tracing::debug!( + parent: &span, + context = "tlsrpt", + "Failed to retrieve TLSRPT record: {}", + err + ); + None + } + } + } + _ => None, + }; + + // Obtain MTA-STS policy for domain + let mta_sts_policy = if tls_strategy.try_mta_sts() && is_smtp { + match core + .lookup_mta_sts_policy( + envelope.domain, + *queue_config.timeout.mta_sts.eval(&envelope).await, + ) + .await + { + Ok(mta_sts_policy) => { + tracing::debug!( + parent: &span, + context = "sts", + event = "policy-fetched", + policy = ?mta_sts_policy, + ); + + mta_sts_policy.into() + } + Err(err) => { + // Report MTA-STS error + if let Some(tls_report) = &tls_report { + match &err { + mta_sts::Error::Dns(mail_auth::Error::DnsRecordNotFound(_)) => { + if tls_strategy.is_mta_sts_required() { + core.schedule_report(TlsEvent { + policy: PolicyType::Sts(None), + domain: envelope.domain.to_string(), + failure: FailureDetails::new(ResultType::Other) + .with_failure_reason_code("MTA-STS is required and no policy was found.") + .into(), + tls_record: tls_report.record.clone(), + interval: tls_report.interval, + }) + .await; + } + } + mta_sts::Error::Dns(mail_auth::Error::DnsError(_)) => (), + _ => { + core.schedule_report(TlsEvent { + policy: PolicyType::Sts(None), + domain: envelope.domain.to_string(), + failure: FailureDetails::new(&err) + .with_failure_reason_code(err.to_string()) + .into(), + tls_record: tls_report.record.clone(), + interval: tls_report.interval, + }) + .await; + } + } + } + + if tls_strategy.is_mta_sts_required() { + tracing::info!( + parent: &span, + context = "sts", + event = "policy-fetch-failure", + "Failed to retrieve MTA-STS policy: {}", + err + ); + domain.set_status(err, queue_config.retry.eval(&envelope).await); + continue 'next_domain; + } else { + tracing::debug!( + parent: &span, + context = "sts", + event = "policy-fetch-failure", + "Failed to retrieve MTA-STS policy: {}", + err + ); + } + + None + } + } + } else { + None + }; + + // Obtain remote hosts list + let mx_list; + if is_smtp { + // Lookup MX + mx_list = match core.resolvers.dns.mx_lookup(&domain.domain).await { + Ok(mx) => mx, + Err(err) => { + tracing::info!( + parent: &span, + context = "dns", + event = "mx-lookup-failed", + reason = %err, + ); + domain.set_status(err, queue_config.retry.eval(&envelope).await); + continue 'next_domain; + } + }; + + if let Some(remote_hosts_) = mx_list + .to_remote_hosts(&domain.domain, *queue_config.max_mx.eval(&envelope).await) + { + remote_hosts = remote_hosts_; + } else { + tracing::info!( + parent: &span, + context = "dns", + event = "null-mx", + reason = "Domain does not accept messages (mull MX)", + ); + domain.set_status( + Status::PermanentFailure(Error::DnsError( + "Domain does not accept messages (null MX)".to_string(), + )), + queue_config.retry.eval(&envelope).await, + ); + continue 'next_domain; + } + } + + // Try delivering message + let max_multihomed = *queue_config.max_multihomed.eval(&envelope).await; + let mut last_status = Status::Scheduled; + 'next_host: for remote_host in &remote_hosts { + // Validate MTA-STS + envelope.mx = remote_host.hostname(); + if let Some(mta_sts_policy) = &mta_sts_policy { + if !mta_sts_policy.verify(envelope.mx) { + // Report MTA-STS failed verification + if let Some(tls_report) = &tls_report { + core.schedule_report(TlsEvent { + policy: mta_sts_policy.into(), + domain: envelope.domain.to_string(), + failure: FailureDetails::new(ResultType::ValidationFailure) + .with_receiving_mx_hostname(envelope.mx) + .with_failure_reason_code("MX not authorized by policy.") + .into(), + tls_record: tls_report.record.clone(), + interval: tls_report.interval, + }) + .await; + } + + tracing::warn!( + parent: &span, + context = "sts", + event = "policy-error", + mx = envelope.mx, + "MX not authorized by policy." + ); + + if mta_sts_policy.enforce() { + last_status = Status::PermanentFailure(Error::MtaStsError( + format!("MX {:?} not authorized by policy.", envelope.mx), + )); + continue 'next_host; + } + } + } + + // Obtain source and remote IPs + let (source_ip, remote_ips) = match core + .resolve_host(remote_host, &envelope, max_multihomed) + .await + { + Ok(result) => result, + Err(status) => { + tracing::info!( + parent: &span, + context = "dns", + event = "ip-lookup-failed", + mx = envelope.mx, + status = %status, + ); + + last_status = status; + continue 'next_host; + } + }; + + // Update TLS strategy + tls_strategy.dane = *queue_config.tls.dane.eval(&envelope).await; + tls_strategy.tls = *queue_config.tls.start.eval(&envelope).await; + + // Lookup DANE policy + let dane_policy = if tls_strategy.try_dane() && is_smtp { + match core + .resolvers + .tlsa_lookup(format!("_25._tcp.{}.", envelope.mx)) + .await + { + Ok(Some(tlsa)) => { + if tlsa.has_end_entities { + tracing::debug!( + parent: &span, + context = "dane", + event = "record-fetched", + mx = envelope.mx, + record = ?tlsa, + ); + + tlsa.into() + } else { + tracing::info!( + parent: &span, + context = "dane", + event = "no-tlsa-records", + mx = envelope.mx, + "No valid TLSA records were found.", + ); + + // Report invalid TLSA record + if let Some(tls_report) = &tls_report { + core.schedule_report(TlsEvent { + policy: tlsa.into(), + domain: envelope.domain.to_string(), + failure: FailureDetails::new(ResultType::TlsaInvalid) + .with_receiving_mx_hostname(envelope.mx) + .with_failure_reason_code("Invalid TLSA record.") + .into(), + tls_record: tls_report.record.clone(), + interval: tls_report.interval, + }) + .await; + } + + if tls_strategy.is_dane_required() { + last_status = Status::PermanentFailure(Error::DaneError( + ErrorDetails { + entity: envelope.mx.to_string(), + details: "No valid TLSA records were found" + .to_string(), + }, + )); + continue 'next_host; + } + None + } + } + Ok(None) => { + if tls_strategy.is_dane_required() { + // Report DANE required + if let Some(tls_report) = &tls_report { + core.schedule_report(TlsEvent { + policy: PolicyType::Tlsa(None), + domain: envelope.domain.to_string(), + failure: FailureDetails::new(ResultType::DaneRequired) + .with_receiving_mx_hostname(envelope.mx) + .with_failure_reason_code( + "No TLSA DNSSEC records found.", + ) + .into(), + tls_record: tls_report.record.clone(), + interval: tls_report.interval, + }) + .await; + } + + tracing::info!( + parent: &span, + context = "dane", + event = "tlsa-dnssec-missing", + mx = envelope.mx, + "No TLSA DNSSEC records found." + ); + + last_status = + Status::PermanentFailure(Error::DaneError(ErrorDetails { + entity: envelope.mx.to_string(), + details: "No TLSA DNSSEC records found".to_string(), + })); + continue 'next_host; + } + None + } + Err(err) => { + if tls_strategy.is_dane_required() { + tracing::info!( + parent: &span, + context = "dane", + event = "tlsa-missing", + mx = envelope.mx, + "No TLSA records found." + ); + + last_status = + if matches!(&err, mail_auth::Error::DnsRecordNotFound(_)) { + // Report DANE required + if let Some(tls_report) = &tls_report { + core.schedule_report(TlsEvent { + policy: PolicyType::Tlsa(None), + domain: envelope.domain.to_string(), + failure: FailureDetails::new( + ResultType::DaneRequired, + ) + .with_receiving_mx_hostname(envelope.mx) + .with_failure_reason_code( + "No TLSA records found for MX.", + ) + .into(), + tls_record: tls_report.record.clone(), + interval: tls_report.interval, + }) + .await; + } + + Status::PermanentFailure(Error::DaneError( + ErrorDetails { + entity: envelope.mx.to_string(), + details: "No TLSA records found".to_string(), + }, + )) + } else { + err.into() + }; + continue 'next_host; + } + None + } + } + } else { + None + }; + + // Try each IP address + envelope.local_ip = source_ip.unwrap_or(no_ip); + 'next_ip: for remote_ip in remote_ips { + // Throttle remote host + let mut in_flight_host = Vec::new(); + envelope.remote_ip = remote_ip; + for throttle in &queue_config.throttle.host { + if let Err(err) = core + .queue + .is_allowed(throttle, &envelope, &mut in_flight_host, &span) + .await + { + domain.set_throttle_error(err, &mut on_hold); + continue 'next_domain; + } + } + + // Connect + let mut smtp_client = match if let Some(ip_addr) = source_ip { + SmtpClient::connect_using( + ip_addr, + SocketAddr::new(remote_ip, remote_host.port()), + *queue_config.timeout.connect.eval(&envelope).await, + ) + .await + } else { + SmtpClient::connect( + SocketAddr::new(remote_ip, remote_host.port()), + *queue_config.timeout.connect.eval(&envelope).await, + ) + .await + } { + Ok(smtp_client) => { + tracing::debug!( + parent: &span, + context = "connect", + event = "success", + mx = envelope.mx, + source_ip = %source_ip.unwrap_or(no_ip), + remote_ip = %remote_ip, + remote_port = remote_host.port(), + ); + + smtp_client + } + Err(err) => { + tracing::info!( + parent: &span, + context = "connect", + event = "failed", + mx = envelope.mx, + reason = %err, + ); + last_status = Status::from_smtp_error(envelope.mx, "", err); + continue 'next_ip; + } + }; + + // Obtail session parameters + let params = SessionParams { + span: &span, + credentials: remote_host.credentials(), + is_smtp: remote_host.is_smtp(), + hostname: envelope.mx, + local_hostname: queue_config.hostname.eval(&envelope).await, + timeout_ehlo: *queue_config.timeout.ehlo.eval(&envelope).await, + timeout_mail: *queue_config.timeout.mail.eval(&envelope).await, + timeout_rcpt: *queue_config.timeout.rcpt.eval(&envelope).await, + timeout_data: *queue_config.timeout.data.eval(&envelope).await, + }; + + // Prepare TLS connector + let tls_connector = if !remote_host.allow_invalid_certs() { + &core.queue.connectors.pki_verify + } else { + &core.queue.connectors.dummy_verify + }; + + let delivery_result = if !remote_host.implicit_tls() { + // Read greeting + smtp_client.timeout = + *queue_config.timeout.greeting.eval(&envelope).await; + if let Err(status) = read_greeting(&mut smtp_client, envelope.mx).await + { + tracing::info!( + parent: &span, + context = "greeting", + event = "invalid", + mx = envelope.mx, + status = %status, + ); + + last_status = status; + continue 'next_host; + } + + // Say EHLO + let capabilties = match say_helo(&mut smtp_client, ¶ms).await { + Ok(capabilities) => capabilities, + Err(status) => { + tracing::info!( + parent: &span, + context = "ehlo", + event = "rejected", + mx = envelope.mx, + status = %status, + ); + + last_status = status; + continue 'next_host; + } + }; + + // Try starting TLS + smtp_client.timeout = *queue_config.timeout.tls.eval(&envelope).await; + match try_start_tls( + smtp_client, + tls_connector, + envelope.mx, + &capabilties, + ) + .await + { + StartTlsResult::Success { smtp_client } => { + // Verify DANE + if let Some(dane_policy) = &dane_policy { + if let Err(status) = dane_policy.verify( + &span, + envelope.mx, + smtp_client.tls_connection().peer_certificates(), + ) { + // Report DANE verification failure + if let Some(tls_report) = &tls_report { + core.schedule_report(TlsEvent { + policy: dane_policy.into(), + domain: envelope.domain.to_string(), + failure: FailureDetails::new( + ResultType::ValidationFailure, + ) + .with_receiving_mx_hostname(envelope.mx) + .with_receiving_ip(remote_ip) + .with_failure_reason_code( + "No matching certificates found.", + ) + .into(), + tls_record: tls_report.record.clone(), + interval: tls_report.interval, + }) + .await; + } + + last_status = status; + continue 'next_host; + } + } + + // Report TLS success + if let Some(tls_report) = &tls_report { + core.schedule_report(TlsEvent { + policy: (&mta_sts_policy, &dane_policy).into(), + domain: envelope.domain.to_string(), + failure: None, + tls_record: tls_report.record.clone(), + interval: tls_report.interval, + }) + .await; + } + + // Deliver message over TLS + self.message + .deliver( + smtp_client, + recipients + .iter_mut() + .filter(|r| r.domain_idx == domain_idx), + params, + ) + .await + } + StartTlsResult::Unavailable { + response, + smtp_client, + } => { + // Report unavailable STARTTLS + let reason = + response.as_ref().map(|r| r.to_string()).unwrap_or_else( + || "STARTTLS was not advertised by host".to_string(), + ); + + tracing::info!( + parent: &span, + context = "tls", + event = "unavailable", + mx = envelope.mx, + reason = reason, + ); + + if let Some(tls_report) = &tls_report { + core.schedule_report(TlsEvent { + policy: (&mta_sts_policy, &dane_policy).into(), + domain: envelope.domain.to_string(), + failure: FailureDetails::new( + ResultType::StartTlsNotSupported, + ) + .with_receiving_mx_hostname(envelope.mx) + .with_receiving_ip(remote_ip) + .with_failure_reason_code(reason) + .into(), + tls_record: tls_report.record.clone(), + interval: tls_report.interval, + }) + .await; + } + + if tls_strategy.is_tls_required() + || (self.message.flags & MAIL_REQUIRETLS) != 0 + || mta_sts_policy.is_some() + || dane_policy.is_some() + { + last_status = + Status::from_starttls_error(envelope.mx, response); + continue 'next_host; + } else { + // TLS is not required, proceed in plain-text + self.message + .deliver( + smtp_client, + recipients + .iter_mut() + .filter(|r| r.domain_idx == domain_idx), + params, + ) + .await + } + } + StartTlsResult::Error { error } => { + tracing::info!( + parent: &span, + context = "tls", + event = "failed", + mx = envelope.mx, + error = %error, + ); + + // Report TLS failure + if let (Some(tls_report), mail_send::Error::Tls(error)) = + (&tls_report, &error) + { + core.schedule_report(TlsEvent { + policy: (&mta_sts_policy, &dane_policy).into(), + domain: envelope.domain.to_string(), + failure: FailureDetails::new( + ResultType::CertificateNotTrusted, + ) + .with_receiving_mx_hostname(envelope.mx) + .with_receiving_ip(remote_ip) + .with_failure_reason_code(error.to_string()) + .into(), + tls_record: tls_report.record.clone(), + interval: tls_report.interval, + }) + .await; + } + last_status = Status::from_tls_error(envelope.mx, error); + continue 'next_host; + } + } + } else { + // Start TLS + smtp_client.timeout = *queue_config.timeout.tls.eval(&envelope).await; + let mut smtp_client = + match smtp_client.into_tls(tls_connector, envelope.mx).await { + Ok(smtp_client) => smtp_client, + Err(error) => { + tracing::info!( + parent: &span, + context = "tls", + event = "failed", + mx = envelope.mx, + error = %error, + ); + + last_status = Status::from_tls_error(envelope.mx, error); + continue 'next_host; + } + }; + + // Read greeting + smtp_client.timeout = + *queue_config.timeout.greeting.eval(&envelope).await; + if let Err(status) = read_greeting(&mut smtp_client, envelope.mx).await + { + tracing::info!( + parent: &span, + context = "greeting", + event = "invalid", + mx = envelope.mx, + status = %status, + ); + + last_status = status; + continue 'next_host; + } + + // Deliver message + self.message + .deliver( + smtp_client, + recipients.iter_mut().filter(|r| r.domain_idx == domain_idx), + params, + ) + .await + }; + + // Update status for the current domain and continue with the next one + domain + .set_status(delivery_result, queue_config.retry.eval(&envelope).await); + continue 'next_domain; + } + } + + // Update status + domain.set_status(last_status, queue_config.retry.eval(&envelope).await); + } + self.message.domains = domains; + self.message.recipients = recipients; + + // Send Delivery Status Notifications + core.queue.send_dsn(&mut self).await; + + // Notify queue manager + let span = self.span; + let result = if !on_hold.is_empty() { + // Release quota for completed deliveries + self.message.release_quota(); + + // Save changes to disk + self.message.save_changes().await; + + tracing::info!( + parent: &span, + context = "queue", + event = "requeue", + reason = "concurrency-limited", + "Too many outbound concurrenct connections, message moved to on-hold queue." + ); + + WorkerResult::OnHold(OnHold { + next_due: self.message.next_event_after(Instant::now()), + limiters: on_hold, + message: self.message, + }) + } else if let Some(due) = self.message.next_event() { + // Release quota for completed deliveries + self.message.release_quota(); + + // Save changes to disk + self.message.save_changes().await; + + tracing::info!( + parent: &span, + context = "queue", + event = "requeue", + reason = "delivery-incomplete", + "Delivery was not possible, message re-queued for delivery." + ); + + WorkerResult::Retry(Schedule { + due, + inner: self.message, + }) + } else { + // Delete message from queue + self.message.remove().await; + + tracing::info!( + parent: &span, + context = "queue", + event = "completed", + "Delivery completed." + ); + + WorkerResult::Done + }; + if core.queue.tx.send(Event::Done(result)).await.is_err() { + tracing::warn!( + parent: &span, + "Channel closed while trying to notify queue manager." + ); + } + }); + } + + /// Marks as failed all domains that reached their expiration time + pub fn has_pending_delivery(&mut self) -> bool { + let now = Instant::now(); + let mut has_pending_delivery = false; + let span = self.span.clone(); + + for (idx, domain) in self.message.domains.iter_mut().enumerate() { + match &domain.status { + Status::TemporaryFailure(err) if domain.expires <= now => { + tracing::info!( + parent: &span, + event = "delivery-expired", + domain = domain.domain, + reason = %err, + ); + + for rcpt in &mut self.message.recipients { + if rcpt.domain_idx == idx { + rcpt.status = std::mem::replace(&mut rcpt.status, Status::Scheduled) + .into_permanent(); + } + } + + domain.status = + std::mem::replace(&mut domain.status, Status::Scheduled).into_permanent(); + domain.changed = true; + } + Status::Scheduled if domain.expires <= now => { + tracing::info!( + parent: &span, + event = "delivery-expired", + domain = domain.domain, + reason = "Queue rate limit exceeded.", + ); + + for rcpt in &mut self.message.recipients { + if rcpt.domain_idx == idx { + rcpt.status = std::mem::replace(&mut rcpt.status, Status::Scheduled) + .into_permanent(); + } + } + + domain.status = Status::PermanentFailure(Error::Io( + "Queue rate limit exceeded.".to_string(), + )); + domain.changed = true; + } + Status::Completed(_) | Status::PermanentFailure(_) => (), + _ => { + has_pending_delivery = true; + } + } + } + + has_pending_delivery + } +} + +impl Domain { + pub fn set_status(&mut self, status: impl Into>, schedule: &[Duration]) { + self.status = status.into(); + self.changed = true; + if matches!( + &self.status, + Status::TemporaryFailure(_) | Status::Scheduled + ) { + self.retry(schedule); + } + } + + pub fn retry(&mut self, schedule: &[Duration]) { + self.retry.due = + Instant::now() + schedule[std::cmp::min(self.retry.inner as usize, schedule.len() - 1)]; + self.retry.inner += 1; + } +} diff --git a/crates/smtp/src/outbound/lookup.rs b/crates/smtp/src/outbound/lookup.rs new file mode 100644 index 00000000..224405bf --- /dev/null +++ b/crates/smtp/src/outbound/lookup.rs @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::net::IpAddr; + +use mail_auth::MX; +use rand::{seq::SliceRandom, Rng}; + +use crate::{ + core::{Core, Envelope}, + queue::{Error, ErrorDetails, Status}, +}; + +use super::RemoteHost; + +impl Core { + pub(super) async fn resolve_host( + &self, + remote_host: &RemoteHost<'_>, + envelope: &impl Envelope, + max_multihomed: usize, + ) -> Result<(Option, Vec), Status<(), Error>> { + let remote_ips = self + .resolvers + .dns + .ip_lookup( + remote_host.fqdn_hostname().as_ref(), + *self.queue.config.ip_strategy.eval(envelope).await, + max_multihomed, + ) + .await + .map_err(|err| { + if let mail_auth::Error::DnsRecordNotFound(_) = &err { + Status::PermanentFailure(Error::ConnectionError(ErrorDetails { + entity: remote_host.hostname().to_string(), + details: "record not found for MX".to_string(), + })) + } else { + Status::TemporaryFailure(Error::ConnectionError(ErrorDetails { + entity: remote_host.hostname().to_string(), + details: format!("lookup error: {err}"), + })) + } + })?; + + if let Some(remote_ip) = remote_ips.first() { + let mut source_ip = None; + + if remote_ip.is_ipv4() { + let source_ips = self.queue.config.source_ip.ipv4.eval(envelope).await; + match source_ips.len().cmp(&1) { + std::cmp::Ordering::Equal => { + source_ip = IpAddr::from(*source_ips.first().unwrap()).into(); + } + std::cmp::Ordering::Greater => { + source_ip = IpAddr::from( + source_ips[rand::thread_rng().gen_range(0..source_ips.len())], + ) + .into(); + } + std::cmp::Ordering::Less => (), + } + } else { + let source_ips = self.queue.config.source_ip.ipv6.eval(envelope).await; + match source_ips.len().cmp(&1) { + std::cmp::Ordering::Equal => { + source_ip = IpAddr::from(*source_ips.first().unwrap()).into(); + } + std::cmp::Ordering::Greater => { + source_ip = IpAddr::from( + source_ips[rand::thread_rng().gen_range(0..source_ips.len())], + ) + .into(); + } + std::cmp::Ordering::Less => (), + } + } + + Ok((source_ip, remote_ips)) + } else { + Err(Status::TemporaryFailure(Error::DnsError(format!( + "No IP addresses found for {:?}.", + envelope.mx() + )))) + } + } +} + +pub(super) trait ToRemoteHost { + fn to_remote_hosts<'x, 'y: 'x>( + &'x self, + domain: &'y str, + max_mx: usize, + ) -> Option>>; +} + +impl ToRemoteHost for Vec { + fn to_remote_hosts<'x, 'y: 'x>( + &'x self, + domain: &'y str, + max_mx: usize, + ) -> Option>> { + if !self.is_empty() { + // Obtain max number of MX hosts to process + let mut remote_hosts = Vec::with_capacity(max_mx); + + 'outer: for mx in self.iter() { + if mx.exchanges.len() > 1 { + let mut slice = mx.exchanges.iter().collect::>(); + slice.shuffle(&mut rand::thread_rng()); + for remote_host in slice { + remote_hosts.push(RemoteHost::MX(remote_host.as_str())); + if remote_hosts.len() == max_mx { + break 'outer; + } + } + } else if let Some(remote_host) = mx.exchanges.first() { + // Check for Null MX + if mx.preference == 0 && remote_host == "." { + return None; + } + remote_hosts.push(RemoteHost::MX(remote_host.as_str())); + if remote_hosts.len() == max_mx { + break; + } + } + } + remote_hosts.into() + } else { + // If an empty list of MXs is returned, the address is treated as if it was + // associated with an implicit MX RR with a preference of 0, pointing to that host. + vec![RemoteHost::MX(domain)].into() + } + } +} diff --git a/crates/smtp/src/outbound/mod.rs b/crates/smtp/src/outbound/mod.rs new file mode 100644 index 00000000..50ad9f19 --- /dev/null +++ b/crates/smtp/src/outbound/mod.rs @@ -0,0 +1,304 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::borrow::Cow; + +use mail_send::Credentials; +use smtp_proto::{Response, Severity}; +use utils::config::ServerProtocol; + +use crate::{ + config::RelayHost, + queue::{DeliveryAttempt, Error, ErrorDetails, HostResponse, Message, Status}, +}; + +pub mod dane; +pub mod delivery; +pub mod lookup; +pub mod mta_sts; +pub mod session; + +impl Status<(), Error> { + pub fn from_smtp_error(hostname: &str, command: &str, err: mail_send::Error) -> Self { + match err { + mail_send::Error::Io(_) + | mail_send::Error::Tls(_) + | mail_send::Error::Base64(_) + | mail_send::Error::UnparseableReply + | mail_send::Error::AuthenticationFailed(_) + | mail_send::Error::MissingCredentials + | mail_send::Error::MissingMailFrom + | mail_send::Error::MissingRcptTo + | mail_send::Error::Timeout => { + Status::TemporaryFailure(Error::ConnectionError(ErrorDetails { + entity: hostname.to_string(), + details: err.to_string(), + })) + } + + mail_send::Error::UnexpectedReply(reply) => { + let details = ErrorDetails { + entity: hostname.to_string(), + details: command.trim().to_string(), + }; + if reply.severity() == Severity::PermanentNegativeCompletion { + Status::PermanentFailure(Error::UnexpectedResponse(HostResponse { + hostname: details, + response: reply, + })) + } else { + Status::TemporaryFailure(Error::UnexpectedResponse(HostResponse { + hostname: details, + response: reply, + })) + } + } + + mail_send::Error::Auth(_) + | mail_send::Error::UnsupportedAuthMechanism + | mail_send::Error::InvalidTLSName + | mail_send::Error::MissingStartTls => { + Status::PermanentFailure(Error::ConnectionError(ErrorDetails { + entity: hostname.to_string(), + details: err.to_string(), + })) + } + } + } + + pub fn from_starttls_error(hostname: &str, response: Option>) -> Self { + let entity = hostname.to_string(); + if let Some(response) = response { + let hostname = ErrorDetails { + entity, + details: "STARTTLS".to_string(), + }; + + if response.severity() == Severity::PermanentNegativeCompletion { + Status::PermanentFailure(Error::UnexpectedResponse(HostResponse { + hostname, + response, + })) + } else { + Status::TemporaryFailure(Error::UnexpectedResponse(HostResponse { + hostname, + response, + })) + } + } else { + Status::PermanentFailure(Error::TlsError(ErrorDetails { + entity, + details: "STARTTLS not advertised by host.".to_string(), + })) + } + } + + pub fn from_tls_error(hostname: &str, err: mail_send::Error) -> Self { + match err { + mail_send::Error::InvalidTLSName => { + Status::PermanentFailure(Error::TlsError(ErrorDetails { + entity: hostname.to_string(), + details: "Invalid hostname".to_string(), + })) + } + mail_send::Error::Timeout => Status::TemporaryFailure(Error::TlsError(ErrorDetails { + entity: hostname.to_string(), + details: "TLS handshake timed out".to_string(), + })), + mail_send::Error::Tls(err) => Status::TemporaryFailure(Error::TlsError(ErrorDetails { + entity: hostname.to_string(), + details: format!("Handshake failed: {err}"), + })), + mail_send::Error::Io(err) => Status::TemporaryFailure(Error::TlsError(ErrorDetails { + entity: hostname.to_string(), + details: format!("I/O error: {err}"), + })), + _ => Status::PermanentFailure(Error::TlsError(ErrorDetails { + entity: hostname.to_string(), + details: "Other TLS error".to_string(), + })), + } + } + + pub fn timeout(hostname: &str, stage: &str) -> Self { + Status::TemporaryFailure(Error::ConnectionError(ErrorDetails { + entity: hostname.to_string(), + details: format!("Timeout while {stage}"), + })) + } +} + +impl From for Status<(), Error> { + fn from(err: mail_auth::Error) -> Self { + match &err { + mail_auth::Error::DnsRecordNotFound(code) => { + Status::PermanentFailure(Error::DnsError(format!("Domain not found: {code:?}"))) + } + _ => Status::TemporaryFailure(Error::DnsError(err.to_string())), + } + } +} + +impl From for Status<(), Error> { + fn from(err: mta_sts::Error) -> Self { + match &err { + mta_sts::Error::Dns(err) => match err { + mail_auth::Error::DnsRecordNotFound(code) => Status::PermanentFailure( + Error::MtaStsError(format!("Record not found: {code:?}")), + ), + mail_auth::Error::InvalidRecordType => Status::PermanentFailure( + Error::MtaStsError("Failed to parse MTA-STS DNS record.".to_string()), + ), + _ => { + Status::TemporaryFailure(Error::MtaStsError(format!("DNS lookup error: {err}"))) + } + }, + mta_sts::Error::Http(err) => { + if err.is_timeout() { + Status::TemporaryFailure(Error::MtaStsError( + "Timeout fetching policy.".to_string(), + )) + } else if err.is_connect() { + Status::TemporaryFailure(Error::MtaStsError( + "Could not reach policy host.".to_string(), + )) + } else if err.is_status() + & err + .status() + .map_or(false, |s| s == reqwest::StatusCode::NOT_FOUND) + { + Status::PermanentFailure(Error::MtaStsError("Policy not found.".to_string())) + } else { + Status::TemporaryFailure(Error::MtaStsError( + "Failed to fetch policy.".to_string(), + )) + } + } + mta_sts::Error::InvalidPolicy(err) => Status::PermanentFailure(Error::MtaStsError( + format!("Failed to parse policy: {err}"), + )), + } + } +} + +impl From> for DeliveryAttempt { + fn from(message: Box) -> Self { + DeliveryAttempt { + span: tracing::info_span!( + "delivery", + "id" = message.id, + "return_path" = if !message.return_path.is_empty() { + message.return_path.as_ref() + } else { + "<>" + }, + "nrcpt" = message.recipients.len(), + "size" = message.size + ), + in_flight: Vec::new(), + message, + } + } +} + +enum RemoteHost<'x> { + Relay(&'x RelayHost), + MX(&'x str), +} + +impl<'x> RemoteHost<'x> { + #[inline(always)] + fn hostname(&self) -> &str { + match self { + RemoteHost::MX(host) => { + if let Some(host) = host.strip_suffix('.') { + host + } else { + host + } + } + RemoteHost::Relay(host) => host.address.as_str(), + } + } + + #[inline(always)] + fn fqdn_hostname(&self) -> Cow<'_, str> { + let host = match self { + RemoteHost::MX(host) => host, + RemoteHost::Relay(host) => host.address.as_str(), + }; + if !host.ends_with('.') { + format!("{host}.").into() + } else { + (*host).into() + } + } + + #[inline(always)] + fn port(&self) -> u16 { + match self { + #[cfg(feature = "test_mode")] + RemoteHost::MX(_) => 9925, + #[cfg(not(feature = "test_mode"))] + RemoteHost::MX(_) => 25, + RemoteHost::Relay(host) => host.port, + } + } + + #[inline(always)] + fn credentials(&self) -> Option<&Credentials> { + match self { + RemoteHost::MX(_) => None, + RemoteHost::Relay(host) => host.auth.as_ref(), + } + } + + #[inline(always)] + fn allow_invalid_certs(&self) -> bool { + #[cfg(feature = "test_mode")] + { + true + } + #[cfg(not(feature = "test_mode"))] + match self { + RemoteHost::MX(_) => false, + RemoteHost::Relay(host) => host.tls_allow_invalid_certs, + } + } + + #[inline(always)] + fn implicit_tls(&self) -> bool { + match self { + RemoteHost::MX(_) => false, + RemoteHost::Relay(host) => host.tls_implicit, + } + } + + #[inline(always)] + fn is_smtp(&self) -> bool { + match self { + RemoteHost::MX(_) => true, + RemoteHost::Relay(host) => host.protocol == ServerProtocol::Smtp, + } + } +} diff --git a/crates/smtp/src/outbound/mta_sts/lookup.rs b/crates/smtp/src/outbound/mta_sts/lookup.rs new file mode 100644 index 00000000..f4dda030 --- /dev/null +++ b/crates/smtp/src/outbound/mta_sts/lookup.rs @@ -0,0 +1,177 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{ + fmt::Display, + sync::Arc, + time::{Duration, Instant}, +}; + +#[cfg(feature = "test_mode")] +pub static STS_TEST_POLICY: parking_lot::Mutex> = parking_lot::Mutex::new(Vec::new()); + +use mail_auth::{common::lru::DnsCache, mta_sts::MtaSts, report::tlsrpt::ResultType}; + +use crate::core::Core; + +use super::{Error, Policy}; + +#[allow(unused_variables)] +impl Core { + pub async fn lookup_mta_sts_policy<'x>( + &self, + domain: &str, + timeout: Duration, + ) -> Result, Error> { + // Lookup MTA-STS TXT record + let record = match self + .resolvers + .dns + .txt_lookup::(format!("_mta-sts.{domain}.")) + .await + { + Ok(record) => record, + Err(err) => { + // Return the cached policy in case of failure + return if let Some(value) = self.resolvers.cache.mta_sts.get(domain) { + Ok(value) + } else { + Err(err.into()) + }; + } + }; + + // Check if the policy has been cached + if let Some(value) = self.resolvers.cache.mta_sts.get(domain) { + if value.id == record.id { + return Ok(value); + } + } + + // Fetch policy + #[cfg(not(feature = "test_mode"))] + let bytes = reqwest::Client::builder() + .user_agent(crate::USER_AGENT) + .timeout(timeout) + .redirect(reqwest::redirect::Policy::none()) + .build()? + .get(&format!("https://mta-sts.{domain}/.well-known/mta-sts.txt")) + .send() + .await? + .bytes() + .await?; + #[cfg(feature = "test_mode")] + let bytes = STS_TEST_POLICY.lock().clone(); + + // Parse policy + let policy = Policy::parse( + std::str::from_utf8(&bytes).map_err(|err| Error::InvalidPolicy(err.to_string()))?, + record.id.clone(), + )?; + let valid_until = Instant::now() + + Duration::from_secs(if (3600..31557600).contains(&policy.max_age) { + policy.max_age + } else { + 86400 + }); + + Ok(self + .resolvers + .cache + .mta_sts + .insert(domain.to_string(), Arc::new(policy), valid_until)) + } + + #[cfg(feature = "test_mode")] + pub fn policy_add<'x>( + &self, + key: impl mail_auth::common::resolver::IntoFqdn<'x>, + value: Policy, + valid_until: std::time::Instant, + ) { + self.resolvers.cache.mta_sts.insert( + key.into_fqdn().into_owned(), + Arc::new(value), + valid_until, + ); + } +} + +impl From<&Error> for ResultType { + fn from(err: &Error) -> Self { + match &err { + Error::InvalidPolicy(_) => ResultType::StsPolicyInvalid, + _ => ResultType::StsPolicyFetchError, + } + } +} + +impl Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Error::Dns(err) => match err { + mail_auth::Error::DnsRecordNotFound(code) => { + write!(f, "Record not found: {code:?}") + } + mail_auth::Error::InvalidRecordType => { + f.write_str("Failed to parse MTA-STS DNS record.") + } + _ => write!(f, "DNS lookup error: {err}"), + }, + Error::Http(err) => { + if err.is_timeout() { + f.write_str("Timeout fetching policy.") + } else if err.is_connect() { + f.write_str("Could not reach policy host.") + } else if err.is_status() + & err + .status() + .map_or(false, |s| s == reqwest::StatusCode::NOT_FOUND) + { + f.write_str("Policy not found.") + } else { + f.write_str("Failed to fetch policy.") + } + } + Error::InvalidPolicy(err) => write!(f, "Failed to parse policy: {err}"), + } + } +} + +impl From for Error { + fn from(value: mail_auth::Error) -> Self { + Error::Dns(value) + } +} + +impl From for Error { + fn from(value: reqwest::Error) -> Self { + Error::Http(value) + } +} + +impl From for Error { + fn from(value: String) -> Self { + Error::InvalidPolicy(value) + } +} diff --git a/crates/smtp/src/outbound/mta_sts/mod.rs b/crates/smtp/src/outbound/mta_sts/mod.rs new file mode 100644 index 00000000..5dbd20fb --- /dev/null +++ b/crates/smtp/src/outbound/mta_sts/mod.rs @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +pub mod lookup; +pub mod parse; +pub mod verify; + +#[derive(Debug, PartialEq, Eq, Hash)] +pub enum Mode { + Enforce, + Testing, + None, +} + +#[derive(Debug, PartialEq, Eq, Hash)] +pub enum MxPattern { + Equals(String), + StartsWith(String), +} + +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct Policy { + pub id: String, + pub mode: Mode, + pub mx: Vec, + pub max_age: u64, +} + +#[derive(Debug)] +pub enum Error { + Dns(mail_auth::Error), + Http(reqwest::Error), + InvalidPolicy(String), +} diff --git a/crates/smtp/src/outbound/mta_sts/parse.rs b/crates/smtp/src/outbound/mta_sts/parse.rs new file mode 100644 index 00000000..f762a7fa --- /dev/null +++ b/crates/smtp/src/outbound/mta_sts/parse.rs @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use super::{Mode, MxPattern, Policy}; + +impl Policy { + pub fn parse(mut data: &str, id: String) -> Result { + let mut mode = Mode::None; + let mut max_age: u64 = 86400; + let mut mx = Vec::new(); + + while !data.is_empty() { + if let Some((key, next_data)) = data.split_once(':') { + let value = if let Some((value, next_data)) = next_data.split_once('\n') { + data = next_data; + value.trim() + } else { + data = ""; + next_data.trim() + }; + match key.trim() { + "mx" => { + if let Some(suffix) = value.strip_prefix("*.") { + if !suffix.is_empty() { + mx.push(MxPattern::StartsWith(suffix.to_lowercase())); + } + } else if !value.is_empty() { + mx.push(MxPattern::Equals(value.to_lowercase())); + } + } + "max_age" => { + if let Ok(value) = value.parse() { + max_age = value; + } + } + "mode" => { + mode = match value { + "enforce" => Mode::Enforce, + "testing" => Mode::Testing, + "none" => Mode::None, + _ => return Err(format!("Unsupported mode {value:?}.")), + }; + } + "version" => { + if !value.eq_ignore_ascii_case("STSv1") { + return Err(format!("Unsupported version {value:?}.")); + } + } + _ => (), + } + } else { + break; + } + } + + if !mx.is_empty() { + Ok(Policy { + id, + mode, + mx, + max_age, + }) + } else { + Err("No 'mx' entries found.".to_string()) + } + } +} + +#[cfg(test)] +mod tests { + use crate::outbound::mta_sts::{Mode, MxPattern, Policy}; + + #[test] + fn parse_policy() { + for (policy, expected_policy) in [ + ( + r"version: STSv1 +mode: enforce +mx: mail.example.com +mx: *.example.net +mx: backupmx.example.com +max_age: 604800", + Policy { + id: "abc".to_string(), + mode: Mode::Enforce, + mx: vec![ + MxPattern::Equals("mail.example.com".to_string()), + MxPattern::StartsWith("example.net".to_string()), + MxPattern::Equals("backupmx.example.com".to_string()), + ], + max_age: 604800, + }, + ), + ( + r"version: STSv1 +mode: testing +mx: gmail-smtp-in.l.google.com +mx: *.gmail-smtp-in.l.google.com +max_age: 86400 +", + Policy { + id: "abc".to_string(), + mode: Mode::Testing, + mx: vec![ + MxPattern::Equals("gmail-smtp-in.l.google.com".to_string()), + MxPattern::StartsWith("gmail-smtp-in.l.google.com".to_string()), + ], + max_age: 86400, + }, + ), + ] { + assert_eq!( + Policy::parse(policy, expected_policy.id.to_string()).unwrap(), + expected_policy + ); + } + } +} diff --git a/crates/smtp/src/outbound/mta_sts/verify.rs b/crates/smtp/src/outbound/mta_sts/verify.rs new file mode 100644 index 00000000..af10ea56 --- /dev/null +++ b/crates/smtp/src/outbound/mta_sts/verify.rs @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use super::{Mode, MxPattern, Policy}; + +impl Policy { + pub fn verify(&self, mx_host: &str) -> bool { + if self.mode != Mode::None { + for mx_pattern in &self.mx { + match mx_pattern { + MxPattern::Equals(host) => { + if host == mx_host { + return true; + } + } + MxPattern::StartsWith(domain) => { + if let Some((_, suffix)) = mx_host.split_once('.') { + if suffix == domain { + return true; + } + } + } + } + } + + false + } else { + true + } + } + + pub fn enforce(&self) -> bool { + self.mode == Mode::Enforce + } +} diff --git a/crates/smtp/src/outbound/session.rs b/crates/smtp/src/outbound/session.rs new file mode 100644 index 00000000..d910df8f --- /dev/null +++ b/crates/smtp/src/outbound/session.rs @@ -0,0 +1,637 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use mail_send::{smtp::AssertReply, Credentials, SmtpClient}; +use smtp_proto::{ + EhloResponse, Response, Severity, EXT_CHUNKING, EXT_DSN, EXT_REQUIRE_TLS, EXT_SIZE, + EXT_SMTP_UTF8, EXT_START_TLS, MAIL_REQUIRETLS, MAIL_RET_FULL, MAIL_RET_HDRS, MAIL_SMTPUTF8, + RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_NEVER, RCPT_NOTIFY_SUCCESS, +}; +use std::fmt::Write; +use std::time::Duration; +use tokio::{ + fs, + io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}, + net::TcpStream, +}; +use tokio_rustls::{client::TlsStream, TlsConnector}; + +use crate::{ + config::{RequireOptional, TlsStrategy}, + queue::{ErrorDetails, HostResponse, RCPT_STATUS_CHANGED}, +}; + +use crate::queue::{Error, Message, Recipient, Status}; + +pub struct SessionParams<'x> { + pub span: &'x tracing::Span, + pub hostname: &'x str, + pub credentials: Option<&'x Credentials>, + pub is_smtp: bool, + pub local_hostname: &'x str, + pub timeout_ehlo: Duration, + pub timeout_mail: Duration, + pub timeout_rcpt: Duration, + pub timeout_data: Duration, +} + +impl Message { + pub async fn deliver( + &self, + mut smtp_client: SmtpClient, + recipients: impl Iterator, + params: SessionParams<'_>, + ) -> Status<(), Error> { + // Obtain capabilities + let mut capabilities = match say_helo(&mut smtp_client, ¶ms).await { + Ok(capabilities) => capabilities, + Err(status) => { + tracing::info!( + parent: params.span, + context = "ehlo", + event = "rejected", + mx = ¶ms.hostname, + reason = %status, + ); + quit(smtp_client).await; + return status; + } + }; + + // Authenticate + if let Some(credentials) = params.credentials { + if let Err(err) = smtp_client.authenticate(credentials, &capabilities).await { + tracing::info!( + parent: params.span, + context = "auth", + event = "failed", + mx = ¶ms.hostname, + reason = %err, + ); + quit(smtp_client).await; + return Status::from_smtp_error(params.hostname, "AUTH ...", err); + } + + // Refresh capabilities + capabilities = match say_helo(&mut smtp_client, ¶ms).await { + Ok(capabilities) => capabilities, + Err(status) => { + tracing::info!( + parent: params.span, + context = "ehlo", + event = "rejected", + mx = ¶ms.hostname, + reason = %status, + ); + quit(smtp_client).await; + return status; + } + }; + } + + // MAIL FROM + smtp_client.timeout = params.timeout_mail; + let cmd = self.build_mail_from(&capabilities); + if let Err(err) = smtp_client + .cmd(cmd.as_bytes()) + .await + .and_then(|r| r.assert_positive_completion()) + { + tracing::info!( + parent: params.span, + context = "sender", + event = "rejected", + mx = ¶ms.hostname, + reason = %err, + ); + quit(smtp_client).await; + return Status::from_smtp_error(params.hostname, &cmd, err); + } + + // RCPT TO + let mut total_rcpt = 0; + let mut total_completed = 0; + let mut accepted_rcpts = Vec::new(); + smtp_client.timeout = params.timeout_rcpt; + for rcpt in recipients { + total_rcpt += 1; + if matches!( + &rcpt.status, + Status::Completed(_) | Status::PermanentFailure(_) + ) { + total_completed += 1; + continue; + } + + let cmd = self.build_rcpt_to(rcpt, &capabilities); + match smtp_client.cmd(cmd.as_bytes()).await { + Ok(response) => match response.severity() { + Severity::PositiveCompletion => { + accepted_rcpts.push(( + rcpt, + Status::Completed(HostResponse { + hostname: params.hostname.to_string(), + response, + }), + )); + } + severity => { + tracing::info!( + parent: params.span, + context = "rcpt", + event = "rejected", + rcpt = rcpt.address, + mx = ¶ms.hostname, + reason = %response, + ); + + let response = HostResponse { + hostname: ErrorDetails { + entity: params.hostname.to_string(), + details: cmd.trim().to_string(), + }, + response, + }; + rcpt.flags |= RCPT_STATUS_CHANGED; + rcpt.status = if severity == Severity::PermanentNegativeCompletion { + total_completed += 1; + Status::PermanentFailure(response) + } else { + Status::TemporaryFailure(response) + }; + } + }, + Err(err) => { + tracing::info!( + parent: params.span, + context = "rcpt", + event = "failed", + mx = ¶ms.hostname, + rcpt = rcpt.address, + reason = %err, + ); + + // Something went wrong, abort. + quit(smtp_client).await; + return Status::from_smtp_error(params.hostname, "", err); + } + } + } + + // Send message + if !accepted_rcpts.is_empty() { + let bdat_cmd = if capabilities.has_capability(EXT_CHUNKING) { + format!("BDAT {} LAST\r\n", self.size).into() + } else { + None + }; + + if let Err(status) = send_message(&mut smtp_client, self, &bdat_cmd, ¶ms).await { + tracing::info!( + parent: params.span, + context = "message", + event = "rejected", + mx = ¶ms.hostname, + reason = %status, + ); + + quit(smtp_client).await; + return status; + } + + if params.is_smtp { + // Handle SMTP response + match read_smtp_data_respone(&mut smtp_client, params.hostname, &bdat_cmd).await { + Ok(response) => { + // Mark recipients as delivered + if response.code() == 250 { + for (rcpt, status) in accepted_rcpts { + tracing::info!( + parent: params.span, + context = "rcpt", + event = "delivered", + rcpt = rcpt.address, + mx = ¶ms.hostname, + response = %status, + ); + + rcpt.status = status; + rcpt.flags |= RCPT_STATUS_CHANGED; + total_completed += 1; + } + } else { + tracing::info!( + parent: params.span, + context = "message", + event = "rejected", + mx = ¶ms.hostname, + reason = %response, + ); + + quit(smtp_client).await; + return Status::from_smtp_error( + params.hostname, + bdat_cmd.as_deref().unwrap_or("DATA"), + mail_send::Error::UnexpectedReply(response), + ); + } + } + Err(status) => { + tracing::info!( + parent: params.span, + context = "message", + event = "failed", + mx = ¶ms.hostname, + reason = %status, + ); + + quit(smtp_client).await; + return status; + } + } + } else { + // Handle LMTP responses + match read_lmtp_data_respone( + &mut smtp_client, + params.hostname, + accepted_rcpts.len(), + ) + .await + { + Ok(responses) => { + for ((rcpt, _), response) in accepted_rcpts.into_iter().zip(responses) { + rcpt.flags |= RCPT_STATUS_CHANGED; + rcpt.status = match response.severity() { + Severity::PositiveCompletion => { + tracing::info!( + parent: params.span, + context = "rcpt", + event = "delivered", + rcpt = rcpt.address, + mx = ¶ms.hostname, + response = %response, + ); + + total_completed += 1; + Status::Completed(HostResponse { + hostname: params.hostname.to_string(), + response, + }) + } + severity => { + tracing::info!( + parent: params.span, + context = "rcpt", + event = "rejected", + rcpt = rcpt.address, + mx = ¶ms.hostname, + reason = %response, + ); + + let response = HostResponse { + hostname: ErrorDetails { + entity: params.hostname.to_string(), + details: bdat_cmd + .as_deref() + .unwrap_or("DATA") + .to_string(), + }, + response, + }; + if severity == Severity::PermanentNegativeCompletion { + total_completed += 1; + Status::PermanentFailure(response) + } else { + Status::TemporaryFailure(response) + } + } + }; + } + } + Err(status) => { + tracing::info!( + parent: params.span, + context = "message", + event = "rejected", + mx = ¶ms.hostname, + reason = %status, + ); + + quit(smtp_client).await; + return status; + } + } + } + } + + quit(smtp_client).await; + if total_completed == total_rcpt { + Status::Completed(()) + } else { + Status::Scheduled + } + } + + fn build_mail_from(&self, capabilities: &EhloResponse) -> String { + let mut mail_from = String::with_capacity(self.return_path.len() + 60); + let _ = write!(mail_from, "MAIL FROM:<{}>", self.return_path); + if capabilities.has_capability(EXT_SIZE) { + let _ = write!(mail_from, " SIZE={}", self.size); + } + if self.has_flag(MAIL_REQUIRETLS) & capabilities.has_capability(EXT_REQUIRE_TLS) { + mail_from.push_str(" REQUIRETLS"); + } + if self.has_flag(MAIL_SMTPUTF8) & capabilities.has_capability(EXT_SMTP_UTF8) { + mail_from.push_str(" SMTPUTF8"); + } + if capabilities.has_capability(EXT_DSN) { + if self.has_flag(MAIL_RET_FULL) { + mail_from.push_str(" RET=FULL"); + } else if self.has_flag(MAIL_RET_HDRS) { + mail_from.push_str(" RET=HDRS"); + } + if let Some(env_id) = &self.env_id { + let _ = write!(mail_from, " ENVID={env_id}"); + } + } + + mail_from.push_str("\r\n"); + mail_from + } + + fn build_rcpt_to(&self, rcpt: &Recipient, capabilities: &EhloResponse) -> String { + let mut rcpt_to = String::with_capacity(rcpt.address.len() + 60); + let _ = write!(rcpt_to, "RCPT TO:<{}>", rcpt.address); + if capabilities.has_capability(EXT_DSN) { + if rcpt.has_flag(RCPT_NOTIFY_SUCCESS | RCPT_NOTIFY_FAILURE | RCPT_NOTIFY_DELAY) { + rcpt_to.push_str(" NOTIFY="); + let mut add_comma = if rcpt.has_flag(RCPT_NOTIFY_SUCCESS) { + rcpt_to.push_str("SUCCESS"); + true + } else { + false + }; + if rcpt.has_flag(RCPT_NOTIFY_DELAY) { + if add_comma { + rcpt_to.push(','); + } else { + add_comma = true; + } + rcpt_to.push_str("DELAY"); + } + if rcpt.has_flag(RCPT_NOTIFY_FAILURE) { + if add_comma { + rcpt_to.push(','); + } + rcpt_to.push_str("FAILURE"); + } + } else if rcpt.has_flag(RCPT_NOTIFY_NEVER) { + rcpt_to.push_str(" NOTIFY=NEVER"); + } + } + rcpt_to.push_str("\r\n"); + rcpt_to + } + + #[inline(always)] + pub fn has_flag(&self, flag: u64) -> bool { + (self.flags & flag) != 0 + } +} + +impl Recipient { + #[inline(always)] + pub fn has_flag(&self, flag: u64) -> bool { + (self.flags & flag) != 0 + } +} + +pub enum StartTlsResult { + Success { + smtp_client: SmtpClient>, + }, + Error { + error: mail_send::Error, + }, + Unavailable { + response: Option>, + smtp_client: SmtpClient, + }, +} + +pub async fn try_start_tls( + mut smtp_client: SmtpClient, + tls_connector: &TlsConnector, + hostname: &str, + capabilities: &EhloResponse, +) -> StartTlsResult { + if capabilities.has_capability(EXT_START_TLS) { + match smtp_client.cmd("STARTTLS\r\n").await { + Ok(response) => { + if response.code() == 220 { + match smtp_client.into_tls(tls_connector, hostname).await { + Ok(smtp_client) => StartTlsResult::Success { smtp_client }, + Err(error) => StartTlsResult::Error { error }, + } + } else { + StartTlsResult::Unavailable { + response: response.into(), + smtp_client, + } + } + } + Err(error) => StartTlsResult::Error { error }, + } + } else { + StartTlsResult::Unavailable { + smtp_client, + response: None, + } + } +} + +pub async fn read_greeting( + smtp_client: &mut SmtpClient, + hostname: &str, +) -> Result<(), Status<(), Error>> { + tokio::time::timeout(smtp_client.timeout, smtp_client.read()) + .await + .map_err(|_| Status::timeout(hostname, "reading greeting"))? + .and_then(|r| r.assert_code(220)) + .map_err(|err| Status::from_smtp_error(hostname, "", err)) +} + +pub async fn read_smtp_data_respone( + smtp_client: &mut SmtpClient, + hostname: &str, + bdat_cmd: &Option, +) -> Result, Status<(), Error>> { + tokio::time::timeout(smtp_client.timeout, smtp_client.read()) + .await + .map_err(|_| Status::timeout(hostname, "reading SMTP DATA response"))? + .map_err(|err| { + Status::from_smtp_error(hostname, bdat_cmd.as_deref().unwrap_or("DATA"), err) + }) +} + +pub async fn read_lmtp_data_respone( + smtp_client: &mut SmtpClient, + hostname: &str, + num_responses: usize, +) -> Result>, Status<(), Error>> { + tokio::time::timeout(smtp_client.timeout, async { + smtp_client.read_many(num_responses).await + }) + .await + .map_err(|_| Status::timeout(hostname, "reading LMTP DATA responses"))? + .map_err(|err| Status::from_smtp_error(hostname, "", err)) +} + +pub async fn write_chunks( + smtp_client: &mut SmtpClient, + chunks: &[&[u8]], +) -> Result<(), mail_send::Error> { + for chunk in chunks { + smtp_client + .stream + .write_all(chunk) + .await + .map_err(mail_send::Error::from)?; + } + smtp_client + .stream + .flush() + .await + .map_err(mail_send::Error::from) +} + +pub async fn send_message( + smtp_client: &mut SmtpClient, + message: &Message, + bdat_cmd: &Option, + params: &SessionParams<'_>, +) -> Result<(), Status<(), Error>> { + let mut raw_message = vec![0u8; message.size]; + let mut file = fs::File::open(&message.path).await.map_err(|err| { + tracing::error!(parent: params.span, + context = "queue", + event = "error", + "Failed to open message file {}: {}", + message.path.display(), + err); + Status::TemporaryFailure(Error::Io("Queue system error.".to_string())) + })?; + file.read_exact(&mut raw_message).await.map_err(|err| { + tracing::error!(parent: params.span, + context = "queue", + event = "error", + "Failed to read {} bytes file {} from disk: {}", + message.size, + message.path.display(), + err); + Status::TemporaryFailure(Error::Io("Queue system error.".to_string())) + })?; + tokio::time::timeout(params.timeout_data, async { + if let Some(bdat_cmd) = bdat_cmd { + write_chunks(smtp_client, &[bdat_cmd.as_bytes(), &raw_message]).await + } else { + write_chunks(smtp_client, &[b"DATA\r\n"]).await?; + smtp_client.read().await?.assert_code(354)?; + smtp_client + .write_message(&raw_message) + .await + .map_err(mail_send::Error::from) + } + }) + .await + .map_err(|_| Status::timeout(params.hostname, "sending message"))? + .map_err(|err| { + Status::from_smtp_error(params.hostname, bdat_cmd.as_deref().unwrap_or("DATA"), err) + }) +} + +pub async fn say_helo( + smtp_client: &mut SmtpClient, + params: &SessionParams<'_>, +) -> Result, Status<(), Error>> { + let cmd = if params.is_smtp { + format!("EHLO {}\r\n", params.local_hostname) + } else { + format!("LHLO {}\r\n", params.local_hostname) + }; + tokio::time::timeout(params.timeout_ehlo, async { + smtp_client.stream.write_all(cmd.as_bytes()).await?; + smtp_client.stream.flush().await?; + smtp_client.read_ehlo().await + }) + .await + .map_err(|_| Status::timeout(params.hostname, "reading EHLO response"))? + .map_err(|err| Status::from_smtp_error(params.hostname, &cmd, err)) +} + +pub async fn quit(mut smtp_client: SmtpClient) { + let _ = tokio::time::timeout(Duration::from_secs(10), async { + if smtp_client.stream.write_all(b"QUIT\r\n").await.is_ok() + && smtp_client.stream.flush().await.is_ok() + { + let mut buf = [0u8; 128]; + let _ = smtp_client.stream.read(&mut buf).await; + } + }) + .await; +} + +impl TlsStrategy { + #[inline(always)] + pub fn try_dane(&self) -> bool { + matches!( + self.dane, + RequireOptional::Require | RequireOptional::Optional + ) + } + + #[inline(always)] + pub fn is_dane_required(&self) -> bool { + matches!(self.dane, RequireOptional::Require) + } + + #[inline(always)] + pub fn try_mta_sts(&self) -> bool { + matches!( + self.mta_sts, + RequireOptional::Require | RequireOptional::Optional + ) + } + + #[inline(always)] + pub fn is_mta_sts_required(&self) -> bool { + matches!(self.mta_sts, RequireOptional::Require) + } + + #[inline(always)] + pub fn is_tls_required(&self) -> bool { + matches!(self.tls, RequireOptional::Require) + || self.is_dane_required() + || self.is_mta_sts_required() + } +} diff --git a/crates/smtp/src/queue/dsn.rs b/crates/smtp/src/queue/dsn.rs new file mode 100644 index 00000000..5bc39850 --- /dev/null +++ b/crates/smtp/src/queue/dsn.rs @@ -0,0 +1,671 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use mail_builder::headers::content_type::ContentType; +use mail_builder::headers::HeaderType; +use mail_builder::mime::{make_boundary, BodyPart, MimePart}; +use mail_builder::MessageBuilder; +use mail_parser::DateTime; +use smtp_proto::{ + Response, RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_NEVER, RCPT_NOTIFY_SUCCESS, +}; +use std::fmt::Write; +use std::time::{Duration, Instant}; +use tokio::fs::File; +use tokio::io::AsyncReadExt; + +use crate::config::QueueConfig; +use crate::core::QueueCore; + +use super::{ + instant_to_timestamp, DeliveryAttempt, Domain, Error, ErrorDetails, HostResponse, Message, + Recipient, SimpleEnvelope, Status, RCPT_DSN_SENT, RCPT_STATUS_CHANGED, +}; + +impl QueueCore { + pub async fn send_dsn(&self, attempt: &mut DeliveryAttempt) { + if !attempt.message.return_path.is_empty() { + if let Some(dsn) = attempt.build_dsn(&self.config).await { + let mut dsn_message = Message::new_boxed("", "", ""); + dsn_message + .add_recipient_parts( + &attempt.message.return_path, + &attempt.message.return_path_lcase, + &attempt.message.return_path_domain, + &self.config, + ) + .await; + + // Sign message + let signature = attempt + .message + .sign(&self.config.dsn.sign, &dsn, &attempt.span) + .await; + self.queue_message(dsn_message, signature.as_deref(), &dsn, &attempt.span) + .await; + } + } else { + attempt.handle_double_bounce(); + } + } +} + +impl DeliveryAttempt { + pub async fn build_dsn(&mut self, config: &QueueConfig) -> Option> { + let now = Instant::now(); + + let mut txt_success = String::new(); + let mut txt_delay = String::new(); + let mut txt_failed = String::new(); + let mut dsn = String::new(); + + for rcpt in &mut self.message.recipients { + if rcpt.has_flag(RCPT_DSN_SENT | RCPT_NOTIFY_NEVER) { + continue; + } + let domain = &self.message.domains[rcpt.domain_idx]; + match &rcpt.status { + Status::Completed(response) => { + rcpt.flags |= RCPT_DSN_SENT | RCPT_STATUS_CHANGED; + if !rcpt.has_flag(RCPT_NOTIFY_SUCCESS) { + continue; + } + rcpt.write_dsn(&mut dsn); + rcpt.status.write_dsn(&mut dsn); + response.write_dsn_text(&rcpt.address, &mut txt_success); + } + Status::TemporaryFailure(response) + if domain.notify.due <= now && rcpt.has_flag(RCPT_NOTIFY_DELAY) => + { + rcpt.write_dsn(&mut dsn); + rcpt.status.write_dsn(&mut dsn); + domain.write_dsn_will_retry_until(&mut dsn); + response.write_dsn_text(&rcpt.address, &mut txt_delay); + } + Status::PermanentFailure(response) => { + rcpt.flags |= RCPT_DSN_SENT | RCPT_STATUS_CHANGED; + if !rcpt.has_flag(RCPT_NOTIFY_FAILURE) { + continue; + } + rcpt.write_dsn(&mut dsn); + rcpt.status.write_dsn(&mut dsn); + response.write_dsn_text(&rcpt.address, &mut txt_failed); + } + Status::Scheduled => { + // There is no status for this address, use the domain's status. + match &domain.status { + Status::PermanentFailure(err) => { + rcpt.flags |= RCPT_DSN_SENT | RCPT_STATUS_CHANGED; + if !rcpt.has_flag(RCPT_NOTIFY_FAILURE) { + continue; + } + rcpt.write_dsn(&mut dsn); + domain.status.write_dsn(&mut dsn); + err.write_dsn_text(&rcpt.address, &domain.domain, &mut txt_failed); + } + Status::TemporaryFailure(err) + if domain.notify.due <= now && rcpt.has_flag(RCPT_NOTIFY_DELAY) => + { + rcpt.write_dsn(&mut dsn); + domain.status.write_dsn(&mut dsn); + domain.write_dsn_will_retry_until(&mut dsn); + err.write_dsn_text(&rcpt.address, &domain.domain, &mut txt_delay); + } + Status::Scheduled + if domain.notify.due <= now && rcpt.has_flag(RCPT_NOTIFY_DELAY) => + { + // This case should not happen under normal circumstances + rcpt.write_dsn(&mut dsn); + domain.status.write_dsn(&mut dsn); + domain.write_dsn_will_retry_until(&mut dsn); + Error::ConcurrencyLimited.write_dsn_text( + &rcpt.address, + &domain.domain, + &mut txt_delay, + ); + } + Status::Completed(_) => { + #[cfg(feature = "test_mode")] + panic!("This should not have happened."); + } + _ => continue, + } + } + _ => continue, + } + + dsn.push_str("\r\n"); + } + + // Build text response + let txt_len = txt_success.len() + txt_delay.len() + txt_failed.len(); + if txt_len == 0 { + return None; + } + + let has_success = !txt_success.is_empty(); + let has_delay = !txt_delay.is_empty(); + let has_failure = !txt_failed.is_empty(); + + let mut txt = String::with_capacity(txt_len + 128); + let (subject, is_mixed) = if has_success && !has_delay && !has_failure { + txt.push_str( + "Your message has been successfully delivered to the following recipients:\r\n\r\n", + ); + ("Successfully delivered message", false) + } else if has_delay && !has_success && !has_failure { + txt.push_str("There was a temporary problem delivering your message to the following recipients:\r\n\r\n"); + ("Warning: Delay in message delivery", false) + } else if has_failure && !has_success && !has_delay { + txt.push_str( + "Your message could not be delivered to the following recipients:\r\n\r\n", + ); + ("Failed to deliver message", false) + } else if has_success { + txt.push_str("Your message has been partially delivered:\r\n\r\n"); + ("Partially delivered message", true) + } else { + txt.push_str("Your message could not be delivered to some recipients:\r\n\r\n"); + ( + "Warning: Temporary and permanent failures during message delivery", + true, + ) + }; + + if has_success { + if is_mixed { + txt.push_str( + " ----- Delivery to the following addresses was succesful -----\r\n", + ); + } + + txt.push_str(&txt_success); + txt.push_str("\r\n"); + } + + if has_delay { + if is_mixed { + txt.push_str( + " ----- There was a temporary problem delivering to these addresses -----\r\n", + ); + } + txt.push_str(&txt_delay); + txt.push_str("\r\n"); + } + + if has_failure { + if is_mixed { + txt.push_str(" ----- Delivery to the following addresses failed -----\r\n"); + } + txt.push_str(&txt_failed); + txt.push_str("\r\n"); + } + + // Update next delay notification time + if has_delay { + let mut domains = std::mem::take(&mut self.message.domains); + for domain in &mut domains { + if matches!( + &domain.status, + Status::TemporaryFailure(_) | Status::Scheduled + ) && domain.notify.due <= now + { + let envelope = SimpleEnvelope::new(&self.message, &domain.domain); + + if let Some(next_notify) = config + .notify + .eval(&envelope) + .await + .get((domain.notify.inner + 1) as usize) + { + domain.notify.inner += 1; + domain.notify.due = Instant::now() + *next_notify; + } else { + domain.notify.due = domain.expires + Duration::from_secs(10); + } + domain.changed = true; + } + } + self.message.domains = domains; + } + + // Obtain hostname and sender addresses + let from_name = config.dsn.name.eval(self.message.as_ref()).await; + let from_addr = config.dsn.address.eval(self.message.as_ref()).await; + let reporting_mta = config.hostname.eval(self.message.as_ref()).await; + + // Prepare DSN + let mut dsn_header = String::with_capacity(dsn.len() + 128); + self.message + .write_dsn_headers(&mut dsn_header, reporting_mta); + let dsn = dsn_header + &dsn; + + // Fetch up to 1024 bytes of message headers + let headers = match File::open(&self.message.path).await { + Ok(mut file) => { + let mut buf = vec![0u8; std::cmp::min(self.message.size, 1024)]; + match file.read(&mut buf).await { + Ok(br) => { + let mut prev_ch = 0; + let mut last_lf = br; + for (pos, &ch) in buf.iter().enumerate() { + match ch { + b'\n' => { + last_lf = pos + 1; + if prev_ch != b'\n' { + prev_ch = ch; + } else { + break; + } + } + b'\r' => (), + 0 => break, + _ => { + prev_ch = ch; + } + } + } + if last_lf < 1024 { + buf.truncate(last_lf); + } + String::from_utf8(buf).unwrap_or_default() + } + Err(err) => { + tracing::error!( + parent: &self.span, + context = "queue", + event = "error", + "Failed to read from {}: {}", + self.message.path.display(), + err + ); + String::new() + } + } + } + Err(err) => { + tracing::error!( + parent: &self.span, + context = "queue", + event = "error", + "Failed to open file {}: {}", + self.message.path.display(), + err + ); + String::new() + } + }; + + // Build message + MessageBuilder::new() + .from((from_name.as_str(), from_addr.as_str())) + .header( + "To", + HeaderType::Text(self.message.return_path.as_str().into()), + ) + .header("Auto-Submitted", HeaderType::Text("auto-generated".into())) + .message_id(format!("<{}@{}>", make_boundary("."), reporting_mta)) + .subject(subject) + .body(MimePart::new( + ContentType::new("multipart/report").attribute("report-type", "delivery-status"), + BodyPart::Multipart(vec![ + MimePart::new(ContentType::new("text/plain"), BodyPart::Text(txt.into())), + MimePart::new( + ContentType::new("message/delivery-status"), + BodyPart::Text(dsn.into()), + ), + MimePart::new( + ContentType::new("message/rfc822"), + BodyPart::Text(headers.into()), + ), + ]), + )) + .write_to_vec() + .unwrap_or_default() + .into() + } + + fn handle_double_bounce(&mut self) { + let mut is_double_bounce = Vec::with_capacity(0); + let message = &mut self.message; + + for rcpt in &mut message.recipients { + if !rcpt.has_flag(RCPT_DSN_SENT | RCPT_NOTIFY_NEVER) { + match &rcpt.status { + Status::PermanentFailure(err) => { + rcpt.flags |= RCPT_DSN_SENT; + let mut dsn = String::new(); + err.write_dsn_text(&rcpt.address, &mut dsn); + is_double_bounce.push(dsn); + } + Status::Scheduled => { + let domain = &message.domains[rcpt.domain_idx]; + if let Status::PermanentFailure(err) = &domain.status { + rcpt.flags |= RCPT_DSN_SENT; + let mut dsn = String::new(); + err.write_dsn_text(&rcpt.address, &domain.domain, &mut dsn); + is_double_bounce.push(dsn); + } + } + _ => (), + } + } + } + + let now = Instant::now(); + for domain in &mut message.domains { + if domain.notify.due <= now { + domain.notify.due = domain.expires + Duration::from_secs(10); + } + } + + if !is_double_bounce.is_empty() { + tracing::info!( + parent: &self.span, + context = "queue", + event = "double-bounce", + id = self.message.id, + failures = ?is_double_bounce, + "Failed delivery of message with null return path.", + ); + } + } +} + +impl HostResponse { + fn write_dsn_text(&self, addr: &str, dsn: &mut String) { + let _ = write!( + dsn, + "<{}> (delivered to '{}' with code {} ({}.{}.{}) '", + addr, + self.hostname, + self.response.code, + self.response.esc[0], + self.response.esc[1], + self.response.esc[2] + ); + self.response.write_response(dsn); + dsn.push_str("')\r\n"); + } +} + +impl HostResponse { + fn write_dsn_text(&self, addr: &str, dsn: &mut String) { + let _ = write!(dsn, "<{}> (host '{}' rejected ", addr, self.hostname.entity); + + if !self.hostname.details.is_empty() { + let _ = write!(dsn, "command '{}'", self.hostname.details,); + } else { + dsn.push_str("transaction"); + } + + let _ = write!( + dsn, + " with code {} ({}.{}.{}) '", + self.response.code, self.response.esc[0], self.response.esc[1], self.response.esc[2] + ); + self.response.write_response(dsn); + dsn.push_str("')\r\n"); + } +} + +impl Error { + fn write_dsn_text(&self, addr: &str, domain: &str, dsn: &mut String) { + match self { + Error::UnexpectedResponse(response) => { + response.write_dsn_text(addr, dsn); + } + Error::DnsError(err) => { + let _ = write!(dsn, "<{addr}> (failed to lookup '{domain}': {err})\r\n",); + } + Error::ConnectionError(details) => { + let _ = write!( + dsn, + "<{}> (connection to '{}' failed: {})\r\n", + addr, details.entity, details.details + ); + } + Error::TlsError(details) => { + let _ = write!( + dsn, + "<{}> (TLS error from '{}': {})\r\n", + addr, details.entity, details.details + ); + } + Error::DaneError(details) => { + let _ = write!( + dsn, + "<{}> (DANE failed to authenticate '{}': {})\r\n", + addr, details.entity, details.details + ); + } + Error::MtaStsError(details) => { + let _ = write!( + dsn, + "<{addr}> (MTA-STS failed to authenticate '{domain}': {details})\r\n", + ); + } + Error::RateLimited => { + let _ = write!(dsn, "<{addr}> (rate limited)\r\n"); + } + Error::ConcurrencyLimited => { + let _ = write!( + dsn, + "<{addr}> (too many concurrent connections to remote server)\r\n", + ); + } + Error::Io(err) => { + let _ = write!(dsn, "<{addr}> (queue error: {err})\r\n"); + } + } + } +} + +impl Message { + fn write_dsn_headers(&self, dsn: &mut String, reporting_mta: &str) { + let _ = write!(dsn, "Reporting-MTA: dns;{reporting_mta}\r\n"); + dsn.push_str("Arrival-Date: "); + dsn.push_str(&DateTime::from_timestamp(self.created as i64).to_rfc822()); + dsn.push_str("\r\n"); + if let Some(env_id) = &self.env_id { + let _ = write!(dsn, "Original-Envelope-Id: {env_id}\r\n"); + } + dsn.push_str("\r\n"); + } +} + +impl Recipient { + fn write_dsn(&self, dsn: &mut String) { + if let Some(orcpt) = &self.orcpt { + let _ = write!(dsn, "Original-Recipient: rfc822;{orcpt}\r\n"); + } + let _ = write!(dsn, "Final-Recipient: rfc822;{}\r\n", self.address); + } +} + +impl Domain { + fn write_dsn_will_retry_until(&self, dsn: &mut String) { + let now = Instant::now(); + if self.expires > now { + dsn.push_str("Will-Retry-Until: "); + dsn.push_str( + &DateTime::from_timestamp(instant_to_timestamp(now, self.expires) as i64) + .to_rfc822(), + ); + dsn.push_str("\r\n"); + } + } +} + +impl Status { + pub fn into_permanent(self) -> Self { + match self { + Status::TemporaryFailure(v) => Status::PermanentFailure(v), + v => v, + } + } + + fn write_dsn_action(&self, dsn: &mut String) { + dsn.push_str("Action: "); + dsn.push_str(match self { + Status::Completed(_) => "delivered", + Status::PermanentFailure(_) => "failed", + Status::TemporaryFailure(_) | Status::Scheduled => "delayed", + }); + dsn.push_str("\r\n"); + } +} + +impl Status, HostResponse> { + fn write_dsn(&self, dsn: &mut String) { + self.write_dsn_action(dsn); + self.write_dsn_status(dsn); + self.write_dsn_diagnostic(dsn); + self.write_dsn_remote_mta(dsn); + } + + fn write_dsn_status(&self, dsn: &mut String) { + dsn.push_str("Status: "); + if let Status::Completed(HostResponse { response, .. }) + | Status::PermanentFailure(HostResponse { response, .. }) + | Status::TemporaryFailure(HostResponse { response, .. }) = self + { + response.write_dsn_status(dsn); + } + dsn.push_str("\r\n"); + } + + fn write_dsn_remote_mta(&self, dsn: &mut String) { + dsn.push_str("Remote-MTA: dns;"); + if let Status::Completed(HostResponse { hostname, .. }) + | Status::PermanentFailure(HostResponse { + hostname: ErrorDetails { + entity: hostname, .. + }, + .. + }) + | Status::TemporaryFailure(HostResponse { + hostname: ErrorDetails { + entity: hostname, .. + }, + .. + }) = self + { + dsn.push_str(hostname); + } + dsn.push_str("\r\n"); + } + + fn write_dsn_diagnostic(&self, dsn: &mut String) { + if let Status::PermanentFailure(details) | Status::TemporaryFailure(details) = self { + details.response.write_dsn_diagnostic(dsn); + } + } +} + +impl Status<(), Error> { + fn write_dsn(&self, dsn: &mut String) { + self.write_dsn_action(dsn); + self.write_dsn_status(dsn); + self.write_dsn_diagnostic(dsn); + self.write_dsn_remote_mta(dsn); + } + + fn write_dsn_status(&self, dsn: &mut String) { + if let Status::PermanentFailure(err) | Status::TemporaryFailure(err) = self { + dsn.push_str("Status: "); + if let Error::UnexpectedResponse(response) = err { + response.response.write_dsn_status(dsn); + } else { + dsn.push_str(if matches!(self, Status::PermanentFailure(_)) { + "5.0.0" + } else { + "4.0.0" + }); + } + dsn.push_str("\r\n"); + } + } + + fn write_dsn_remote_mta(&self, dsn: &mut String) { + if let Status::PermanentFailure(err) | Status::TemporaryFailure(err) = self { + match err { + Error::UnexpectedResponse(HostResponse { + hostname: details, .. + }) + | Error::ConnectionError(details) + | Error::TlsError(details) + | Error::DaneError(details) => { + dsn.push_str("Remote-MTA: dns;"); + dsn.push_str(&details.entity); + dsn.push_str("\r\n"); + } + _ => (), + } + } + } + + fn write_dsn_diagnostic(&self, dsn: &mut String) { + if let Status::PermanentFailure(Error::UnexpectedResponse(response)) + | Status::TemporaryFailure(Error::UnexpectedResponse(response)) = self + { + response.response.write_dsn_diagnostic(dsn); + } + } +} + +impl WriteDsn for Response { + fn write_dsn_status(&self, dsn: &mut String) { + if self.esc[0] > 0 { + let _ = write!(dsn, "{}.{}.{}", self.esc[0], self.esc[1], self.esc[2]); + } else { + let _ = write!( + dsn, + "{}.{}.{}", + self.code / 100, + (self.code / 10) % 10, + self.code % 10 + ); + } + } + + fn write_dsn_diagnostic(&self, dsn: &mut String) { + let _ = write!(dsn, "Diagnostic-Code: smtp;{} ", self.code); + self.write_response(dsn); + dsn.push_str("\r\n"); + } + + fn write_response(&self, dsn: &mut String) { + for ch in self.message.chars() { + if ch != '\n' && ch != '\r' { + dsn.push(ch); + } + } + } +} + +trait WriteDsn { + fn write_dsn_status(&self, dsn: &mut String); + fn write_dsn_diagnostic(&self, dsn: &mut String); + fn write_response(&self, dsn: &mut String); +} diff --git a/crates/smtp/src/queue/manager.rs b/crates/smtp/src/queue/manager.rs new file mode 100644 index 00000000..80c2ed46 --- /dev/null +++ b/crates/smtp/src/queue/manager.rs @@ -0,0 +1,561 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{ + collections::BinaryHeap, + sync::{atomic::Ordering, Arc}, + time::{Duration, Instant}, +}; + +use ahash::AHashMap; +use smtp_proto::Response; +use tokio::sync::mpsc; + +use crate::core::{ + management::{self}, + Core, QueueCore, +}; + +use super::{ + DeliveryAttempt, Event, HostResponse, Message, OnHold, QueueId, Schedule, Status, WorkerResult, + RCPT_STATUS_CHANGED, +}; + +#[derive(Debug)] +pub struct Queue { + short_wait: Duration, + long_wait: Duration, + pub scheduled: BinaryHeap>, + pub on_hold: Vec>, + pub messages: AHashMap>, +} + +impl SpawnQueue for mpsc::Receiver { + fn spawn(mut self, core: Arc, mut queue: Queue) { + tokio::spawn(async move { + loop { + let result = tokio::time::timeout(queue.wake_up_time(), self.recv()).await; + + // Deliver scheduled messages + while let Some(message) = queue.next_due() { + DeliveryAttempt::from(message) + .try_deliver(core.clone(), &mut queue) + .await; + } + + match result { + Ok(Some(event)) => match event { + Event::Queue(item) => { + // Deliver any concurrency limited messages + while let Some(message) = queue.next_on_hold() { + DeliveryAttempt::from(message) + .try_deliver(core.clone(), &mut queue) + .await; + } + + if item.due <= Instant::now() { + DeliveryAttempt::from(item.inner) + .try_deliver(core.clone(), &mut queue) + .await; + } else { + queue.schedule(item); + } + } + Event::Done(result) => { + // A worker is done, try delivering concurrency limited messages + while let Some(message) = queue.next_on_hold() { + DeliveryAttempt::from(message) + .try_deliver(core.clone(), &mut queue) + .await; + } + match result { + WorkerResult::Done => (), + WorkerResult::Retry(schedule) => { + queue.schedule(schedule); + } + WorkerResult::OnHold(on_hold) => { + queue.on_hold(on_hold); + } + } + } + Event::Manage(request) => match request { + management::QueueRequest::List { + from, + to, + before, + after, + result_tx, + } => { + let mut result = Vec::with_capacity(queue.messages.len()); + for message in queue.messages.values() { + if from.as_ref().map_or(false, |from| { + !message.return_path_lcase.contains(from) + }) { + continue; + } + if to.as_ref().map_or(false, |to| { + !message + .recipients + .iter() + .any(|rcpt| rcpt.address_lcase.contains(to)) + }) { + continue; + } + + if (before.is_some() || after.is_some()) + && !message.domains.iter().any(|domain| { + matches!( + &domain.status, + Status::Scheduled | Status::TemporaryFailure(_) + ) && match (&before, &after) { + (Some(before), Some(after)) => { + domain.retry.due.lt(before) + && domain.retry.due.gt(after) + } + (Some(before), None) => domain.retry.due.lt(before), + (None, Some(after)) => domain.retry.due.gt(after), + (None, None) => false, + } + }) + { + continue; + } + + result.push(message.id); + } + result.sort_unstable_by_key(|id| *id & 0xFFFFFFFF); + let _ = result_tx.send(result); + } + management::QueueRequest::Status { + queue_ids, + result_tx, + } => { + let mut result = Vec::with_capacity(queue_ids.len()); + for queue_id in queue_ids { + result.push( + queue + .messages + .get(&queue_id) + .map(|message| message.as_ref().into()), + ); + } + let _ = result_tx.send(result); + } + management::QueueRequest::Cancel { + queue_ids, + item, + result_tx, + } => { + let mut result = Vec::with_capacity(queue_ids.len()); + for queue_id in &queue_ids { + let mut found = false; + if let Some(item) = &item { + if let Some(message) = queue.messages.get_mut(queue_id) { + // Cancel delivery for all recipients that match + for rcpt in &mut message.recipients { + if rcpt.address_lcase.contains(item) { + rcpt.flags |= RCPT_STATUS_CHANGED; + rcpt.status = Status::Completed(HostResponse { + hostname: String::new(), + response: Response { + code: 0, + esc: [0, 0, 0], + message: "Delivery canceled." + .to_string(), + }, + }); + found = true; + } + } + if found { + // Mark as completed domains without any pending deliveries + for (domain_idx, domain) in + message.domains.iter_mut().enumerate() + { + if matches!( + domain.status, + Status::TemporaryFailure(_) + | Status::Scheduled + ) { + let mut total_rcpt = 0; + let mut total_completed = 0; + + for rcpt in &message.recipients { + if rcpt.domain_idx == domain_idx { + total_rcpt += 1; + if matches!( + rcpt.status, + Status::PermanentFailure(_) + | Status::Completed(_) + ) { + total_completed += 1; + } + } + } + + if total_rcpt == total_completed { + domain.status = Status::Completed(()); + domain.changed = true; + } + } + } + + // Delete message if there are no pending deliveries + if message.domains.iter().any(|domain| { + matches!( + domain.status, + Status::TemporaryFailure(_) + | Status::Scheduled + ) + }) { + message.save_changes().await; + } else { + message.remove().await; + queue.messages.remove(queue_id); + } + } + } + } else if let Some(message) = queue.messages.remove(queue_id) { + message.remove().await; + found = true; + } + result.push(found); + } + let _ = result_tx.send(result); + } + management::QueueRequest::Retry { + queue_ids, + item, + time, + result_tx, + } => { + let mut result = Vec::with_capacity(queue_ids.len()); + for queue_id in &queue_ids { + let mut found = false; + if let Some(message) = queue.messages.get_mut(queue_id) { + for domain in &mut message.domains { + if matches!( + domain.status, + Status::Scheduled | Status::TemporaryFailure(_) + ) && item + .as_ref() + .map_or(true, |item| domain.domain.contains(item)) + { + domain.retry.due = time; + if domain.expires > time { + domain.expires = time + Duration::from_secs(10); + } + domain.changed = true; + found = true; + } + } + + if found { + queue.on_hold.retain(|oh| &oh.message != queue_id); + message.save_changes().await; + if let Some(next_event) = message.next_event() { + queue.scheduled.push(Schedule { + due: next_event, + inner: *queue_id, + }); + } + } + } + result.push(found); + } + let _ = result_tx.send(result); + } + }, + Event::Stop => break, + }, + Ok(None) => break, + Err(_) => (), + } + } + }); + } +} + +impl Queue { + pub fn schedule(&mut self, message: Schedule>) { + self.scheduled.push(Schedule { + due: message.due, + inner: message.inner.id, + }); + self.messages.insert(message.inner.id, message.inner); + } + + pub fn on_hold(&mut self, message: OnHold>) { + self.on_hold.push(OnHold { + next_due: message.next_due, + limiters: message.limiters, + message: message.message.id, + }); + self.messages.insert(message.message.id, message.message); + } + + pub fn next_due(&mut self) -> Option> { + let item = self.scheduled.peek()?; + if item.due <= Instant::now() { + self.scheduled + .pop() + .and_then(|i| self.messages.remove(&i.inner)) + } else { + None + } + } + + pub fn next_on_hold(&mut self) -> Option> { + let now = Instant::now(); + self.on_hold + .iter() + .position(|o| { + o.limiters + .iter() + .any(|l| l.concurrent.load(Ordering::Relaxed) < l.max_concurrent) + || o.next_due.map_or(false, |due| due <= now) + }) + .and_then(|pos| self.messages.remove(&self.on_hold.remove(pos).message)) + } + + pub fn wake_up_time(&self) -> Duration { + self.scheduled + .peek() + .map(|item| { + item.due + .checked_duration_since(Instant::now()) + .unwrap_or(self.short_wait) + }) + .unwrap_or(self.long_wait) + } +} + +impl Message { + pub fn next_event(&self) -> Option { + let mut next_event = Instant::now(); + let mut has_events = false; + + for domain in &self.domains { + if matches!( + domain.status, + Status::Scheduled | Status::TemporaryFailure(_) + ) { + if !has_events || domain.retry.due < next_event { + next_event = domain.retry.due; + has_events = true; + } + if domain.notify.due < next_event { + next_event = domain.notify.due; + } + if domain.expires < next_event { + next_event = domain.expires; + } + } + } + + if has_events { + next_event.into() + } else { + None + } + } + + pub fn next_delivery_event(&self) -> Instant { + let mut next_delivery = Instant::now(); + + for (pos, domain) in self + .domains + .iter() + .filter(|d| matches!(d.status, Status::Scheduled | Status::TemporaryFailure(_))) + .enumerate() + { + if pos == 0 || domain.retry.due < next_delivery { + next_delivery = domain.retry.due; + } + } + + next_delivery + } + + pub fn next_event_after(&self, instant: Instant) -> Option { + let mut next_event = None; + + for domain in &self.domains { + if matches!( + domain.status, + Status::Scheduled | Status::TemporaryFailure(_) + ) { + if domain.retry.due > instant + && next_event + .as_ref() + .map_or(true, |ne| domain.retry.due.lt(ne)) + { + next_event = domain.retry.due.into(); + } + if domain.notify.due > instant + && next_event + .as_ref() + .map_or(true, |ne| domain.notify.due.lt(ne)) + { + next_event = domain.notify.due.into(); + } + if domain.expires > instant + && next_event.as_ref().map_or(true, |ne| domain.expires.lt(ne)) + { + next_event = domain.expires.into(); + } + } + } + + next_event + } +} + +impl QueueCore { + pub async fn read_queue(&self) -> Queue { + let mut queue = Queue::default(); + let mut messages = Vec::new(); + + for path in self + .config + .path + .if_then + .iter() + .map(|t| &t.then) + .chain([&self.config.path.default]) + { + let mut dir = match tokio::fs::read_dir(path).await { + Ok(dir) => dir, + Err(_) => continue, + }; + loop { + match dir.next_entry().await { + Ok(Some(file)) => { + let file = file.path(); + if file.is_dir() { + match tokio::fs::read_dir(&file).await { + Ok(mut dir) => { + let file_ = file; + loop { + match dir.next_entry().await { + Ok(Some(file)) => { + let file = file.path(); + if file.extension().map_or(false, |e| e == "msg") { + messages.push(tokio::spawn( + Message::from_path(file), + )); + } + } + Ok(None) => break, + Err(err) => { + tracing::warn!( + "Failed to read queue directory {}: {}", + file_.display(), + err + ); + break; + } + } + } + } + Err(err) => { + tracing::warn!( + "Failed to read queue directory {}: {}", + file.display(), + err + ) + } + }; + } else if file.extension().map_or(false, |e| e == "msg") { + messages.push(tokio::spawn(Message::from_path(file))); + } + } + Ok(None) => { + break; + } + Err(err) => { + tracing::warn!( + "Failed to read queue directory {}: {}", + path.display(), + err + ); + break; + } + } + } + } + + // Join all futures + for message in messages { + match message.await { + Ok(Ok(mut message)) => { + // Reserve quota + self.has_quota(&mut message).await; + + // Schedule message + queue.schedule(Schedule { + due: message.next_event().unwrap_or_else(|| { + tracing::warn!( + context = "queue", + event = "warn", + "No due events found for message {}", + message.path.display() + ); + Instant::now() + }), + inner: Box::new(message), + }); + } + Ok(Err(err)) => { + tracing::warn!( + context = "queue", + event = "error", + "Queue startup error: {}", + err + ); + } + Err(err) => { + tracing::error!("Join error while starting queue: {}", err); + } + } + } + + queue + } +} + +impl Default for Queue { + fn default() -> Self { + Queue { + short_wait: Duration::from_millis(1), + long_wait: Duration::from_secs(86400 * 365), + scheduled: BinaryHeap::with_capacity(128), + on_hold: Vec::with_capacity(128), + messages: AHashMap::with_capacity(128), + } + } +} + +pub trait SpawnQueue { + fn spawn(self, core: Arc, queue: Queue); +} diff --git a/crates/smtp/src/queue/mod.rs b/crates/smtp/src/queue/mod.rs new file mode 100644 index 00000000..ca39de07 --- /dev/null +++ b/crates/smtp/src/queue/mod.rs @@ -0,0 +1,550 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{ + fmt::Display, + net::{IpAddr, Ipv4Addr}, + path::PathBuf, + sync::{atomic::AtomicUsize, Arc}, + time::{Duration, Instant, SystemTime}, +}; + +use serde::{Deserialize, Serialize}; +use smtp_proto::Response; +use utils::listener::limiter::{ConcurrencyLimiter, InFlight}; + +use crate::core::{management, Envelope}; + +pub mod dsn; +pub mod manager; +pub mod quota; +pub mod serialize; +pub mod spool; +pub mod throttle; + +pub type QueueId = u64; + +#[derive(Debug)] +pub enum Event { + Queue(Schedule>), + Manage(management::QueueRequest), + Done(WorkerResult), + Stop, +} + +#[derive(Debug)] +pub enum WorkerResult { + Done, + Retry(Schedule>), + OnHold(OnHold>), +} + +#[derive(Debug)] +pub struct OnHold { + pub next_due: Option, + pub limiters: Vec, + pub message: T, +} + +#[derive(Debug)] +pub struct Schedule { + pub due: Instant, + pub inner: T, +} + +#[derive(Debug)] +pub struct Message { + pub id: QueueId, + pub created: u64, + pub path: PathBuf, + + pub return_path: String, + pub return_path_lcase: String, + pub return_path_domain: String, + pub recipients: Vec, + pub domains: Vec, + + pub flags: u64, + pub env_id: Option, + pub priority: i16, + + pub size: usize, + pub queue_refs: Vec, +} + +#[derive(Debug, PartialEq, Eq)] +pub struct Domain { + pub domain: String, + pub retry: Schedule, + pub notify: Schedule, + pub expires: Instant, + pub status: Status<(), Error>, + pub changed: bool, +} + +#[derive(Debug, PartialEq, Eq)] +pub struct Recipient { + pub domain_idx: usize, + pub address: String, + pub address_lcase: String, + pub status: Status, HostResponse>, + pub flags: u64, + pub orcpt: Option, +} + +pub const RCPT_DSN_SENT: u64 = 1 << 32; +pub const RCPT_STATUS_CHANGED: u64 = 2 << 32; + +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum Status { + #[serde(rename = "scheduled")] + Scheduled, + #[serde(rename = "completed")] + Completed(T), + #[serde(rename = "temp_fail")] + TemporaryFailure(E), + #[serde(rename = "perm_fail")] + PermanentFailure(E), +} + +#[derive(Debug, PartialEq, Eq)] +pub struct HostResponse { + pub hostname: T, + pub response: Response, +} + +#[derive(Debug, PartialEq, Eq)] +pub enum Error { + DnsError(String), + UnexpectedResponse(HostResponse), + ConnectionError(ErrorDetails), + TlsError(ErrorDetails), + DaneError(ErrorDetails), + MtaStsError(String), + RateLimited, + ConcurrencyLimited, + Io(String), +} + +#[derive(Debug, PartialEq, Eq)] +pub struct ErrorDetails { + pub entity: String, + pub details: String, +} + +pub struct DeliveryAttempt { + pub span: tracing::Span, + pub in_flight: Vec, + pub message: Box, +} + +#[derive(Debug)] +pub struct QuotaLimiter { + pub max_size: usize, + pub max_messages: usize, + pub size: AtomicUsize, + pub messages: AtomicUsize, +} + +#[derive(Debug)] +pub struct UsedQuota { + id: u64, + size: usize, + limiter: Arc, +} + +impl PartialEq for UsedQuota { + fn eq(&self, other: &Self) -> bool { + self.id == other.id && self.size == other.size + } +} + +impl Eq for UsedQuota {} + +impl Ord for Schedule { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + other.due.cmp(&self.due) + } +} + +impl PartialOrd for Schedule { + fn partial_cmp(&self, other: &Self) -> Option { + other.due.partial_cmp(&self.due) + } +} + +impl PartialEq for Schedule { + fn eq(&self, other: &Self) -> bool { + self.due == other.due + } +} + +impl Eq for Schedule {} + +impl Schedule { + pub fn now() -> Self { + Schedule { + due: Instant::now(), + inner: T::default(), + } + } + + pub fn later(duration: Duration) -> Self { + Schedule { + due: Instant::now() + duration, + inner: T::default(), + } + } +} + +pub struct SimpleEnvelope<'x> { + pub message: &'x Message, + pub domain: &'x str, + pub recipient: &'x str, +} + +impl<'x> SimpleEnvelope<'x> { + pub fn new(message: &'x Message, domain: &'x str) -> Self { + Self { + message, + domain, + recipient: "", + } + } + + pub fn new_rcpt(message: &'x Message, domain: &'x str, recipient: &'x str) -> Self { + Self { + message, + domain, + recipient, + } + } +} + +impl<'x> Envelope for SimpleEnvelope<'x> { + fn local_ip(&self) -> IpAddr { + IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)) + } + + fn remote_ip(&self) -> IpAddr { + IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)) + } + + fn sender_domain(&self) -> &str { + &self.message.return_path_domain + } + + fn sender(&self) -> &str { + &self.message.return_path_lcase + } + + fn rcpt_domain(&self) -> &str { + self.domain + } + + fn rcpt(&self) -> &str { + self.recipient + } + + fn helo_domain(&self) -> &str { + "" + } + + fn authenticated_as(&self) -> &str { + "" + } + + fn mx(&self) -> &str { + "" + } + + fn listener_id(&self) -> u16 { + 0 + } + + fn priority(&self) -> i16 { + self.message.priority + } +} + +pub struct QueueEnvelope<'x> { + pub message: &'x Message, + pub domain: &'x str, + pub mx: &'x str, + pub remote_ip: IpAddr, + pub local_ip: IpAddr, +} + +impl<'x> Envelope for QueueEnvelope<'x> { + fn local_ip(&self) -> IpAddr { + self.local_ip + } + + fn remote_ip(&self) -> IpAddr { + self.remote_ip + } + + fn sender_domain(&self) -> &str { + &self.message.return_path_domain + } + + fn sender(&self) -> &str { + &self.message.return_path_lcase + } + + fn rcpt_domain(&self) -> &str { + self.domain + } + + fn rcpt(&self) -> &str { + "" + } + + fn helo_domain(&self) -> &str { + "" + } + + fn authenticated_as(&self) -> &str { + "" + } + + fn mx(&self) -> &str { + self.mx + } + + fn listener_id(&self) -> u16 { + 0 + } + + fn priority(&self) -> i16 { + self.message.priority + } +} + +impl Envelope for Message { + fn local_ip(&self) -> IpAddr { + IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)) + } + + fn remote_ip(&self) -> IpAddr { + IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)) + } + + fn sender_domain(&self) -> &str { + &self.return_path_domain + } + + fn sender(&self) -> &str { + &self.return_path_lcase + } + + fn rcpt_domain(&self) -> &str { + "" + } + + fn rcpt(&self) -> &str { + "" + } + + fn helo_domain(&self) -> &str { + "" + } + + fn authenticated_as(&self) -> &str { + "" + } + + fn mx(&self) -> &str { + "" + } + + fn listener_id(&self) -> u16 { + 0 + } + + fn priority(&self) -> i16 { + self.priority + } +} + +impl Envelope for &str { + fn local_ip(&self) -> IpAddr { + IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)) + } + + fn remote_ip(&self) -> IpAddr { + IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)) + } + + fn sender_domain(&self) -> &str { + "" + } + + fn sender(&self) -> &str { + "" + } + + fn rcpt_domain(&self) -> &str { + self + } + + fn rcpt(&self) -> &str { + "" + } + + fn helo_domain(&self) -> &str { + "" + } + + fn authenticated_as(&self) -> &str { + "" + } + + fn mx(&self) -> &str { + "" + } + + fn listener_id(&self) -> u16 { + 0 + } + + fn priority(&self) -> i16 { + 0 + } +} + +#[inline(always)] +pub fn instant_to_timestamp(now: Instant, time: Instant) -> u64 { + SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .map_or(0, |d| d.as_secs()) + + time.checked_duration_since(now).map_or(0, |d| d.as_secs()) +} + +pub trait InstantFromTimestamp { + fn to_instant(&self) -> Instant; +} + +impl InstantFromTimestamp for u64 { + fn to_instant(&self) -> Instant { + let timestamp = *self; + let current_timestamp = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .map_or(0, |d| d.as_secs()); + if timestamp > current_timestamp { + Instant::now() + Duration::from_secs(timestamp - current_timestamp) + } else { + Instant::now() + } + } +} + +pub trait DomainPart { + fn domain_part(&self) -> &str; +} + +impl DomainPart for &str { + #[inline(always)] + fn domain_part(&self) -> &str { + self.rsplit_once('@').map(|(_, d)| d).unwrap_or_default() + } +} + +impl DomainPart for String { + #[inline(always)] + fn domain_part(&self) -> &str { + self.rsplit_once('@').map(|(_, d)| d).unwrap_or_default() + } +} + +impl Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Error::UnexpectedResponse(response) => { + write!( + f, + "Unexpected response from '{}': {}", + response.hostname.entity, response.response + ) + } + Error::DnsError(err) => { + write!(f, "DNS lookup failed: {err}") + } + Error::ConnectionError(details) => { + write!( + f, + "Connection to '{}' failed: {}", + details.entity, details.details + ) + } + Error::TlsError(details) => { + write!( + f, + "TLS error from '{}': {}", + details.entity, details.details + ) + } + Error::DaneError(details) => { + write!( + f, + "DANE failed to authenticate '{}': {}", + details.entity, details.details + ) + } + Error::MtaStsError(details) => { + write!(f, "MTA-STS auth failed: {details}") + } + Error::RateLimited => { + write!(f, "Rate limited") + } + Error::ConcurrencyLimited => { + write!(f, "Too many concurrent connections to remote server") + } + Error::Io(err) => { + write!(f, "Queue error: {err}") + } + } + } +} + +impl Display for Status<(), Error> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Status::Scheduled => write!(f, "Scheduled"), + Status::Completed(_) => write!(f, "Completed"), + Status::TemporaryFailure(err) => write!(f, "Temporary Failure: {err}"), + Status::PermanentFailure(err) => write!(f, "Permanent Failure: {err}"), + } + } +} + +impl Display for Status, HostResponse> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Status::Scheduled => write!(f, "Scheduled"), + Status::Completed(response) => write!(f, "Delivered: {}", response.response), + Status::TemporaryFailure(err) => write!(f, "Temporary Failure: {}", err.response), + Status::PermanentFailure(err) => write!(f, "Permanent Failure: {}", err.response), + } + } +} diff --git a/crates/smtp/src/queue/quota.rs b/crates/smtp/src/queue/quota.rs new file mode 100644 index 00000000..e7820efc --- /dev/null +++ b/crates/smtp/src/queue/quota.rs @@ -0,0 +1,195 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::sync::{atomic::Ordering, Arc}; + +use dashmap::mapref::entry::Entry; + +use crate::{ + config::QueueQuota, + core::{Envelope, QueueCore}, +}; + +use super::{Message, QuotaLimiter, SimpleEnvelope, Status, UsedQuota}; + +impl QueueCore { + pub async fn has_quota(&self, message: &mut Message) -> bool { + let mut queue_refs = Vec::new(); + + if !self.config.quota.sender.is_empty() { + for quota in &self.config.quota.sender { + if !self + .reserve_quota(quota, message, message.size, 0, &mut queue_refs) + .await + { + return false; + } + } + } + + for quota in &self.config.quota.rcpt_domain { + for (pos, domain) in message.domains.iter().enumerate() { + if !self + .reserve_quota( + quota, + &SimpleEnvelope::new(message, &domain.domain), + message.size, + ((pos + 1) << 32) as u64, + &mut queue_refs, + ) + .await + { + return false; + } + } + } + + for quota in &self.config.quota.rcpt { + for (pos, rcpt) in message.recipients.iter().enumerate() { + if !self + .reserve_quota( + quota, + &SimpleEnvelope::new_rcpt( + message, + &message.domains[rcpt.domain_idx].domain, + &rcpt.address_lcase, + ), + message.size, + (pos + 1) as u64, + &mut queue_refs, + ) + .await + { + return false; + } + } + } + + message.queue_refs = queue_refs; + + true + } + + async fn reserve_quota( + &self, + quota: &QueueQuota, + envelope: &impl Envelope, + size: usize, + id: u64, + refs: &mut Vec, + ) -> bool { + if !quota.conditions.conditions.is_empty() && quota.conditions.eval(envelope).await { + match self.quota.entry(quota.new_key(envelope)) { + Entry::Occupied(e) => { + if let Some(qref) = e.get().is_allowed(id, size) { + refs.push(qref); + } else { + return false; + } + } + Entry::Vacant(e) => { + let limiter = Arc::new(QuotaLimiter { + max_size: quota.size.unwrap_or(0), + max_messages: quota.messages.unwrap_or(0), + size: 0.into(), + messages: 0.into(), + }); + + if let Some(qref) = limiter.is_allowed(id, size) { + refs.push(qref); + e.insert(limiter); + } else { + return false; + } + } + } + } + true + } +} + +impl Message { + pub fn release_quota(&mut self) { + let mut quota_ids = Vec::with_capacity(self.domains.len() + self.recipients.len()); + for (pos, domain) in self.domains.iter().enumerate() { + if matches!( + &domain.status, + Status::Completed(_) | Status::PermanentFailure(_) + ) { + quota_ids.push(((pos + 1) << 32) as u64); + } + } + for (pos, rcpt) in self.recipients.iter().enumerate() { + if matches!( + &rcpt.status, + Status::Completed(_) | Status::PermanentFailure(_) + ) { + quota_ids.push((pos + 1) as u64); + } + } + if !quota_ids.is_empty() { + self.queue_refs.retain(|q| !quota_ids.contains(&q.id)); + } + } +} + +trait QuotaLimiterAllowed { + fn is_allowed(&self, id: u64, size: usize) -> Option; +} + +impl QuotaLimiterAllowed for Arc { + fn is_allowed(&self, id: u64, size: usize) -> Option { + if self.max_messages > 0 { + if self.messages.load(Ordering::Relaxed) < self.max_messages { + self.messages.fetch_add(1, Ordering::Relaxed); + } else { + return None; + } + } + + if self.max_size > 0 { + if self.size.load(Ordering::Relaxed) + size < self.max_size { + self.size.fetch_add(size, Ordering::Relaxed); + } else { + return None; + } + } + + Some(UsedQuota { + id, + size, + limiter: self.clone(), + }) + } +} + +impl Drop for UsedQuota { + fn drop(&mut self) { + if self.limiter.max_messages > 0 { + self.limiter.messages.fetch_sub(1, Ordering::Relaxed); + } + if self.limiter.max_size > 0 { + self.limiter.size.fetch_sub(self.size, Ordering::Relaxed); + } + } +} diff --git a/crates/smtp/src/queue/serialize.rs b/crates/smtp/src/queue/serialize.rs new file mode 100644 index 00000000..3c7f7f35 --- /dev/null +++ b/crates/smtp/src/queue/serialize.rs @@ -0,0 +1,564 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use mail_auth::common::base32::Base32Reader; +use smtp_proto::Response; +use std::io::SeekFrom; +use std::path::PathBuf; +use std::slice::Iter; +use std::{fmt::Write, time::Instant}; +use tokio::fs; +use tokio::fs::File; +use tokio::io::{AsyncReadExt, AsyncSeekExt}; + +use super::{ + instant_to_timestamp, Domain, DomainPart, Error, ErrorDetails, HostResponse, + InstantFromTimestamp, Message, Recipient, Schedule, Status, RCPT_STATUS_CHANGED, +}; + +pub trait QueueSerializer: Sized { + fn serialize(&self, buf: &mut String); + fn deserialize(bytes: &mut Iter<'_, u8>) -> Option; +} + +impl Message { + pub fn serialize(&self) -> Vec { + let mut buf = String::with_capacity( + self.return_path.len() + + self.env_id.as_ref().map_or(0, |e| e.len()) + + (self.domains.len() * 64) + + (self.recipients.len() * 64) + + 50, + ); + + // Serialize message properties + (self.created as usize).serialize(&mut buf); + self.return_path.serialize(&mut buf); + (self.env_id.as_deref().unwrap_or_default()).serialize(&mut buf); + (self.flags as usize).serialize(&mut buf); + self.priority.serialize(&mut buf); + + // Serialize domains + let now = Instant::now(); + self.domains.len().serialize(&mut buf); + for domain in &self.domains { + domain.domain.serialize(&mut buf); + (instant_to_timestamp(now, domain.expires) as usize).serialize(&mut buf); + } + + // Serialize recipients + self.recipients.len().serialize(&mut buf); + for rcpt in &self.recipients { + rcpt.domain_idx.serialize(&mut buf); + rcpt.address.serialize(&mut buf); + (rcpt.orcpt.as_deref().unwrap_or_default()).serialize(&mut buf); + } + + // Serialize domain status + for (idx, domain) in self.domains.iter().enumerate() { + domain.serialize(idx, now, &mut buf); + } + + // Serialize recipient status + for (idx, rcpt) in self.recipients.iter().enumerate() { + rcpt.serialize(idx, &mut buf); + } + + buf.into_bytes() + } + + pub fn serialize_changes(&mut self) -> Vec { + let now = Instant::now(); + let mut buf = String::with_capacity(128); + + for (idx, domain) in self.domains.iter_mut().enumerate() { + if domain.changed { + domain.changed = false; + domain.serialize(idx, now, &mut buf); + } + } + + for (idx, rcpt) in self.recipients.iter_mut().enumerate() { + if rcpt.has_flag(RCPT_STATUS_CHANGED) { + rcpt.flags &= !RCPT_STATUS_CHANGED; + rcpt.serialize(idx, &mut buf); + } + } + + buf.into_bytes() + } + + pub async fn from_path(path: PathBuf) -> Result { + let filename = path + .file_name() + .and_then(|f| f.to_str()) + .and_then(|f| f.rsplit_once('.')) + .map(|(f, _)| f) + .ok_or_else(|| format!("Invalid queue file name {}", path.display()))?; + + // Decode file name + let mut id = [0u8; std::mem::size_of::()]; + let mut size = [0u8; std::mem::size_of::()]; + + for (pos, byte) in Base32Reader::new(filename.as_bytes()).enumerate() { + match pos { + 0..=7 => { + id[pos] = byte; + } + 8..=11 => { + size[pos - 8] = byte; + } + _ => { + return Err(format!("Invalid queue file name {}", path.display())); + } + } + } + + let id = u64::from_le_bytes(id); + let size = u32::from_le_bytes(size) as u64; + + // Obtail file size + let file_size = fs::metadata(&path) + .await + .map_err(|err| { + format!( + "Failed to obtain file metadata for {}: {}", + path.display(), + err + ) + })? + .len(); + if size == 0 || size >= file_size { + return Err(format!( + "Invalid queue file name size {} for {}", + size, + path.display() + )); + } + let mut buf = Vec::with_capacity((file_size - size) as usize); + let mut file = File::open(&path) + .await + .map_err(|err| format!("Failed to open queue file {}: {}", path.display(), err))?; + file.seek(SeekFrom::Start(size)) + .await + .map_err(|err| format!("Failed to seek queue file {}: {}", path.display(), err))?; + file.read_to_end(&mut buf) + .await + .map_err(|err| format!("Failed to read queue file {}: {}", path.display(), err))?; + + let mut message = Self::deserialize(&buf) + .ok_or_else(|| format!("Failed to deserialize metadata for file {}", path.display()))?; + message.path = path; + message.size = size as usize; + message.id = id; + Ok(message) + } + + pub fn deserialize(bytes: &[u8]) -> Option { + let mut bytes = bytes.iter(); + let created = usize::deserialize(&mut bytes)? as u64; + let return_path = String::deserialize(&mut bytes)?; + let return_path_lcase = return_path.to_lowercase(); + let env_id = String::deserialize(&mut bytes)?; + + let mut message = Message { + id: 0, + path: PathBuf::new(), + created, + return_path_domain: return_path_lcase.domain_part().to_string(), + return_path_lcase, + return_path, + env_id: if !env_id.is_empty() { + env_id.into() + } else { + None + }, + flags: usize::deserialize(&mut bytes)? as u64, + priority: i16::deserialize(&mut bytes)?, + size: 0, + recipients: vec![], + domains: vec![], + queue_refs: vec![], + }; + + // Deserialize domains + let num_domains = usize::deserialize(&mut bytes)?; + message.domains = Vec::with_capacity(num_domains); + for _ in 0..num_domains { + message.domains.push(Domain { + domain: String::deserialize(&mut bytes)?, + expires: Instant::deserialize(&mut bytes)?, + retry: Schedule::now(), + notify: Schedule::now(), + status: Status::Scheduled, + changed: false, + }); + } + + // Deserialize recipients + let num_recipients = usize::deserialize(&mut bytes)?; + message.recipients = Vec::with_capacity(num_recipients); + for _ in 0..num_recipients { + let domain_idx = usize::deserialize(&mut bytes)?; + let address = String::deserialize(&mut bytes)?; + let orcpt = String::deserialize(&mut bytes)?; + message.recipients.push(Recipient { + domain_idx, + address_lcase: address.to_lowercase(), + address, + status: Status::Scheduled, + flags: 0, + orcpt: if !orcpt.is_empty() { + orcpt.into() + } else { + None + }, + }); + } + + // Deserialize status + while let Some((ch, idx)) = bytes + .next() + .and_then(|ch| (ch, usize::deserialize(&mut bytes)?).into()) + { + match ch { + b'D' => { + if let (Some(domain), Some(retry), Some(notify), Some(status)) = ( + message.domains.get_mut(idx), + Schedule::deserialize(&mut bytes), + Schedule::deserialize(&mut bytes), + Status::deserialize(&mut bytes), + ) { + domain.retry = retry; + domain.notify = notify; + domain.status = status; + } else { + break; + } + } + b'R' => { + if let (Some(rcpt), Some(flags), Some(status)) = ( + message.recipients.get_mut(idx), + usize::deserialize(&mut bytes), + Status::deserialize(&mut bytes), + ) { + rcpt.flags = flags as u64; + rcpt.status = status; + } else { + break; + } + } + _ => break, + } + } + + message.into() + } +} + +impl QueueSerializer for Status { + fn serialize(&self, buf: &mut String) { + match self { + Status::Scheduled => buf.push('S'), + Status::Completed(s) => { + buf.push('C'); + s.serialize(buf); + } + Status::TemporaryFailure(s) => { + buf.push('T'); + s.serialize(buf); + } + Status::PermanentFailure(s) => { + buf.push('F'); + s.serialize(buf); + } + } + } + + fn deserialize(bytes: &mut Iter<'_, u8>) -> Option { + match bytes.next()? { + b'S' => Self::Scheduled.into(), + b'C' => Self::Completed(T::deserialize(bytes)?).into(), + b'T' => Self::TemporaryFailure(E::deserialize(bytes)?).into(), + b'F' => Self::PermanentFailure(E::deserialize(bytes)?).into(), + _ => None, + } + } +} + +impl QueueSerializer for Response { + fn serialize(&self, buf: &mut String) { + let _ = write!( + buf, + "{} {} {} {} {} {}", + self.code, + self.esc[0], + self.esc[1], + self.esc[2], + self.message.len(), + self.message + ); + } + + fn deserialize(bytes: &mut Iter<'_, u8>) -> Option { + Response { + code: usize::deserialize(bytes)? as u16, + esc: [ + usize::deserialize(bytes)? as u8, + usize::deserialize(bytes)? as u8, + usize::deserialize(bytes)? as u8, + ], + message: String::deserialize(bytes)?, + } + .into() + } +} + +impl QueueSerializer for usize { + fn serialize(&self, buf: &mut String) { + let _ = write!(buf, "{self} "); + } + + fn deserialize(bytes: &mut Iter<'_, u8>) -> Option { + let mut num = 0; + loop { + match bytes.next()? { + ch @ (b'0'..=b'9') => { + num = (num * 10) + (*ch - b'0') as usize; + } + b' ' => { + return num.into(); + } + _ => { + return None; + } + } + } + } +} + +impl QueueSerializer for i16 { + fn serialize(&self, buf: &mut String) { + let _ = write!(buf, "{self} "); + } + + fn deserialize(bytes: &mut Iter<'_, u8>) -> Option { + let mut num = 0; + let mut mul = 1; + loop { + match bytes.next()? { + ch @ (b'0'..=b'9') => { + num = (num * 10) + (*ch - b'0') as i16; + } + b' ' => { + return (num * mul).into(); + } + b'-' => { + mul = -1; + } + _ => { + return None; + } + } + } + } +} + +impl QueueSerializer for ErrorDetails { + fn serialize(&self, buf: &mut String) { + self.entity.serialize(buf); + self.details.serialize(buf); + } + + fn deserialize(bytes: &mut Iter<'_, u8>) -> Option { + ErrorDetails { + entity: String::deserialize(bytes)?, + details: String::deserialize(bytes)?, + } + .into() + } +} + +impl QueueSerializer for HostResponse { + fn serialize(&self, buf: &mut String) { + self.hostname.serialize(buf); + self.response.serialize(buf); + } + + fn deserialize(bytes: &mut Iter<'_, u8>) -> Option { + HostResponse { + hostname: T::deserialize(bytes)?, + response: Response::deserialize(bytes)?, + } + .into() + } +} + +impl QueueSerializer for String { + fn serialize(&self, buf: &mut String) { + if !self.is_empty() { + let _ = write!(buf, "{} {}", self.len(), self); + } else { + buf.push_str("0 "); + } + } + + fn deserialize(bytes: &mut Iter<'_, u8>) -> Option { + match usize::deserialize(bytes)? { + len @ (1..=4096) => { + String::from_utf8(bytes.take(len).copied().collect::>()).ok() + } + 0 => String::new().into(), + _ => None, + } + } +} + +impl QueueSerializer for &str { + fn serialize(&self, buf: &mut String) { + if !self.is_empty() { + let _ = write!(buf, "{} {}", self.len(), self); + } else { + buf.push_str("0 "); + } + } + + fn deserialize(_bytes: &mut Iter<'_, u8>) -> Option { + unimplemented!() + } +} + +impl QueueSerializer for Instant { + fn serialize(&self, buf: &mut String) { + let _ = write!(buf, "{} ", instant_to_timestamp(Instant::now(), *self),); + } + + fn deserialize(bytes: &mut Iter<'_, u8>) -> Option { + (usize::deserialize(bytes)? as u64).to_instant().into() + } +} + +impl QueueSerializer for Schedule { + fn serialize(&self, buf: &mut String) { + let _ = write!( + buf, + "{} {} ", + self.inner, + instant_to_timestamp(Instant::now(), self.due), + ); + } + + fn deserialize(bytes: &mut Iter<'_, u8>) -> Option { + Schedule { + inner: usize::deserialize(bytes)? as u32, + due: Instant::deserialize(bytes)?, + } + .into() + } +} + +impl QueueSerializer for Error { + fn serialize(&self, buf: &mut String) { + match self { + Error::DnsError(e) => { + buf.push('0'); + e.serialize(buf); + } + Error::UnexpectedResponse(e) => { + buf.push('1'); + e.serialize(buf); + } + Error::ConnectionError(e) => { + buf.push('2'); + e.serialize(buf); + } + Error::TlsError(e) => { + buf.push('3'); + e.serialize(buf); + } + Error::DaneError(e) => { + buf.push('4'); + e.serialize(buf); + } + Error::MtaStsError(e) => { + buf.push('5'); + e.serialize(buf); + } + Error::RateLimited => { + buf.push('6'); + } + Error::ConcurrencyLimited => { + buf.push('7'); + } + Error::Io(e) => { + buf.push('8'); + e.serialize(buf); + } + } + } + + fn deserialize(bytes: &mut Iter<'_, u8>) -> Option { + match bytes.next()? { + b'0' => Error::DnsError(String::deserialize(bytes)?).into(), + b'1' => Error::UnexpectedResponse(HostResponse::deserialize(bytes)?).into(), + b'2' => Error::ConnectionError(ErrorDetails::deserialize(bytes)?).into(), + b'3' => Error::TlsError(ErrorDetails::deserialize(bytes)?).into(), + b'4' => Error::DaneError(ErrorDetails::deserialize(bytes)?).into(), + b'5' => Error::MtaStsError(String::deserialize(bytes)?).into(), + b'6' => Error::RateLimited.into(), + b'7' => Error::ConcurrencyLimited.into(), + b'8' => Error::Io(String::deserialize(bytes)?).into(), + _ => None, + } + } +} + +impl QueueSerializer for () { + fn serialize(&self, _buf: &mut String) {} + + fn deserialize(_bytes: &mut Iter<'_, u8>) -> Option { + Some(()) + } +} + +impl Domain { + fn serialize(&self, idx: usize, now: Instant, buf: &mut String) { + let _ = write!( + buf, + "D{} {} {} {} {} ", + idx, + self.retry.inner, + instant_to_timestamp(now, self.retry.due), + self.notify.inner, + instant_to_timestamp(now, self.notify.due) + ); + self.status.serialize(buf); + } +} + +impl Recipient { + fn serialize(&self, idx: usize, buf: &mut String) { + let _ = write!(buf, "R{} {} ", idx, self.flags); + self.status.serialize(buf); + } +} diff --git a/crates/smtp/src/queue/spool.rs b/crates/smtp/src/queue/spool.rs new file mode 100644 index 00000000..f92c910f --- /dev/null +++ b/crates/smtp/src/queue/spool.rs @@ -0,0 +1,271 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use crate::queue::DomainPart; +use mail_auth::common::base32::Base32Writer; +use mail_auth::common::headers::Writer; +use std::path::PathBuf; +use std::sync::atomic::Ordering; +use std::time::Instant; +use std::time::{Duration, SystemTime}; +use tokio::fs::OpenOptions; +use tokio::{fs, io::AsyncWriteExt}; + +use crate::config::QueueConfig; +use crate::core::QueueCore; + +use super::{Domain, Event, Message, Recipient, Schedule, SimpleEnvelope, Status}; + +impl QueueCore { + pub async fn queue_message( + &self, + mut message: Box, + raw_headers: Option<&[u8]>, + raw_message: &[u8], + span: &tracing::Span, + ) -> bool { + // Generate id + if message.id == 0 { + message.id = self.queue_id(); + } + if message.size == 0 { + message.size = raw_message.len() + raw_headers.as_ref().map_or(0, |h| h.len()); + } + + // Build path + message.path = self.config.path.eval(message.as_ref()).await.clone(); + let hash = *self.config.hash.eval(message.as_ref()).await; + if hash > 0 { + message.path.push((message.id % hash).to_string()); + } + let _ = fs::create_dir(&message.path).await; + + // Encode file name + let mut encoder = Base32Writer::with_capacity(20); + encoder.write(&message.id.to_le_bytes()[..]); + encoder.write(&(message.size as u32).to_le_bytes()[..]); + let mut file = encoder.finalize(); + file.push_str(".msg"); + message.path.push(file); + + // Serialize metadata + let metadata = message.serialize(); + + // Save message + let mut file = match fs::File::create(&message.path).await { + Ok(file) => file, + Err(err) => { + tracing::error!( + parent: span, + context = "queue", + event = "error", + "Failed to create file {}: {}", + message.path.display(), + err + ); + return false; + } + }; + + let iter = if let Some(raw_headers) = raw_headers { + [raw_headers, raw_message, &metadata].into_iter() + } else { + [raw_message, &metadata, b""].into_iter() + }; + + for bytes in iter { + if !bytes.is_empty() { + if let Err(err) = file.write_all(bytes).await { + tracing::error!( + parent: span, + context = "queue", + event = "error", + "Failed to write to file {}: {}", + message.path.display(), + err + ); + return false; + } + } + } + if let Err(err) = file.flush().await { + tracing::error!( + parent: span, + context = "queue", + event = "error", + "Failed to flush file {}: {}", + message.path.display(), + err + ); + return false; + } + + tracing::info!( + parent: span, + context = "queue", + event = "scheduled", + id = message.id, + from = if !message.return_path.is_empty() { + message.return_path.as_str() + } else { + "<>" + }, + nrcpts = message.recipients.len(), + size = message.size, + "Message queued for delivery." + ); + + // Queue the message + if self + .tx + .send(Event::Queue(Schedule { + due: message.next_event().unwrap(), + inner: message, + })) + .await + .is_err() + { + tracing::warn!( + parent: span, + context = "queue", + event = "error", + "Queue channel closed: Message queued but won't be sent until next restart." + ); + } + + true + } + + pub fn queue_id(&self) -> u64 { + (SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .map_or(0, |d| d.as_secs()) + .saturating_sub(946684800) + & 0xFFFFFFFF) + | (self.id_seq.fetch_add(1, Ordering::Relaxed) as u64) << 32 + } +} + +impl Message { + pub fn new_boxed( + return_path: impl Into, + return_path_lcase: impl Into, + return_path_domain: impl Into, + ) -> Box { + Box::new(Message { + id: 0, + path: PathBuf::new(), + created: SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0), + return_path: return_path.into(), + return_path_lcase: return_path_lcase.into(), + return_path_domain: return_path_domain.into(), + recipients: Vec::with_capacity(1), + domains: Vec::with_capacity(1), + flags: 0, + env_id: None, + priority: 0, + size: 0, + queue_refs: vec![], + }) + } + + pub async fn add_recipient_parts( + &mut self, + rcpt: impl Into, + rcpt_lcase: impl Into, + rcpt_domain: impl Into, + config: &QueueConfig, + ) { + let rcpt_domain = rcpt_domain.into(); + let domain_idx = + if let Some(idx) = self.domains.iter().position(|d| d.domain == rcpt_domain) { + idx + } else { + let idx = self.domains.len(); + let expires = *config + .expire + .eval(&SimpleEnvelope::new(self, &rcpt_domain)) + .await; + self.domains.push(Domain { + domain: rcpt_domain, + retry: Schedule::now(), + notify: Schedule::later(expires + Duration::from_secs(10)), + expires: Instant::now() + expires, + status: Status::Scheduled, + changed: false, + }); + idx + }; + self.recipients.push(Recipient { + domain_idx, + address: rcpt.into(), + address_lcase: rcpt_lcase.into(), + status: Status::Scheduled, + flags: 0, + orcpt: None, + }); + } + + pub async fn add_recipient(&mut self, rcpt: impl Into, config: &QueueConfig) { + let rcpt = rcpt.into(); + let rcpt_lcase = rcpt.to_lowercase(); + let rcpt_domain = rcpt_lcase.domain_part().to_string(); + self.add_recipient_parts(rcpt, rcpt_lcase, rcpt_domain, config) + .await; + } + + pub async fn save_changes(&mut self) { + let buf = self.serialize_changes(); + if !buf.is_empty() { + let err = match OpenOptions::new().append(true).open(&self.path).await { + Ok(mut file) => match file.write_all(&buf).await { + Ok(_) => return, + Err(err) => err, + }, + Err(err) => err, + }; + tracing::error!( + context = "queue", + event = "error", + "Failed to write to {}: {}", + self.path.display(), + err + ); + } + } + + pub async fn remove(&self) { + if let Err(err) = fs::remove_file(&self.path).await { + tracing::error!( + context = "queue", + event = "error", + "Failed to delete queued message {}: {}", + self.path.display(), + err + ); + } + } +} diff --git a/crates/smtp/src/queue/throttle.rs b/crates/smtp/src/queue/throttle.rs new file mode 100644 index 00000000..e0e2d44d --- /dev/null +++ b/crates/smtp/src/queue/throttle.rs @@ -0,0 +1,123 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::time::Instant; + +use dashmap::mapref::entry::Entry; +use utils::listener::limiter::{ConcurrencyLimiter, InFlight, RateLimiter}; + +use crate::{ + config::Throttle, + core::{throttle::Limiter, Envelope, QueueCore}, +}; + +use super::{Domain, Status}; + +#[derive(Debug)] +pub enum Error { + Concurrency { limiter: ConcurrencyLimiter }, + Rate { retry_at: Instant }, +} + +impl QueueCore { + pub async fn is_allowed( + &self, + throttle: &Throttle, + envelope: &impl Envelope, + in_flight: &mut Vec, + span: &tracing::Span, + ) -> Result<(), Error> { + if throttle.conditions.conditions.is_empty() || throttle.conditions.eval(envelope).await { + match self.throttle.entry(throttle.new_key(envelope)) { + Entry::Occupied(mut e) => { + let limiter = e.get_mut(); + if let Some(limiter) = &limiter.concurrency { + if let Some(inflight) = limiter.is_allowed() { + in_flight.push(inflight); + } else { + tracing::info!( + parent: span, + context = "throttle", + event = "too-many-requests", + max_concurrent = limiter.max_concurrent, + "Queue concurrency limit exceeded." + ); + return Err(Error::Concurrency { + limiter: limiter.clone(), + }); + } + } + if let Some(limiter) = &mut limiter.rate { + if !limiter.is_allowed() { + tracing::info!( + parent: span, + context = "throttle", + event = "rate-limit-exceeded", + max_requests = limiter.max_requests as u64, + max_interval = limiter.max_interval as u64, + "Queue rate limit exceeded." + ); + return Err(Error::Rate { + retry_at: limiter.retry_at(), + }); + } + } + } + Entry::Vacant(e) => { + let concurrency = throttle.concurrency.map(|concurrency| { + let limiter = ConcurrencyLimiter::new(concurrency); + if let Some(inflight) = limiter.is_allowed() { + in_flight.push(inflight); + } + limiter + }); + let rate = throttle.rate.as_ref().map(|rate| { + let mut r = RateLimiter::new(rate.requests, rate.period.as_secs()); + r.is_allowed(); + r + }); + + e.insert(Limiter { rate, concurrency }); + } + } + } + + Ok(()) + } +} + +impl Domain { + pub fn set_throttle_error(&mut self, err: Error, on_hold: &mut Vec) { + match err { + Error::Concurrency { limiter } => { + on_hold.push(limiter); + self.status = Status::TemporaryFailure(super::Error::ConcurrencyLimited); + } + Error::Rate { retry_at } => { + self.retry.due = retry_at; + self.status = Status::TemporaryFailure(super::Error::RateLimited); + } + } + self.changed = true; + } +} diff --git a/crates/smtp/src/reporting/analysis.rs b/crates/smtp/src/reporting/analysis.rs new file mode 100644 index 00000000..b1d47949 --- /dev/null +++ b/crates/smtp/src/reporting/analysis.rs @@ -0,0 +1,488 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{ + borrow::Cow, + collections::hash_map::Entry, + io::{Cursor, Read}, + sync::{atomic::Ordering, Arc}, + time::SystemTime, +}; + +use ahash::AHashMap; +use mail_auth::{ + flate2::read::GzDecoder, + report::{tlsrpt::TlsReport, ActionDisposition, DmarcResult, Feedback, Report}, + zip, +}; +use mail_parser::{DateTime, HeaderValue, Message, MimeHeaders, PartType}; + +use crate::core::Core; + +enum Compression { + None, + Gzip, + Zip, +} + +enum Format { + Dmarc, + Tls, + Arf, +} + +struct ReportData<'x> { + compression: Compression, + format: Format, + data: &'x [u8], +} + +pub trait AnalyzeReport { + fn analyze_report(&self, message: Arc>); +} + +impl AnalyzeReport for Arc { + fn analyze_report(&self, message: Arc>) { + let core = self.clone(); + self.worker_pool.spawn(move || { + let message = if let Some(message) = Message::parse(&message) { + message + } else { + tracing::debug!(context = "report", "Failed to parse message."); + return; + }; + let from = match message.from() { + HeaderValue::Address(addr) => addr.address.as_ref().map(|a| a.as_ref()), + HeaderValue::AddressList(addr_list) => addr_list + .last() + .and_then(|a| a.address.as_ref()) + .map(|a| a.as_ref()), + _ => None, + } + .unwrap_or("unknown"); + let mut reports = Vec::new(); + + for part in &message.parts { + match &part.body { + PartType::Text(report) => { + if part + .content_type() + .and_then(|ct| ct.subtype()) + .map_or(false, |t| t.eq_ignore_ascii_case("xml")) + || part + .attachment_name() + .and_then(|n| n.rsplit_once('.')) + .map_or(false, |(_, e)| e.eq_ignore_ascii_case("xml")) + { + reports.push(ReportData { + compression: Compression::None, + format: Format::Dmarc, + data: report.as_bytes(), + }); + } else if part.is_content_type("message", "feedback-report") { + reports.push(ReportData { + compression: Compression::None, + format: Format::Arf, + data: report.as_bytes(), + }); + } + } + PartType::Binary(report) | PartType::InlineBinary(report) => { + if part.is_content_type("message", "feedback-report") { + reports.push(ReportData { + compression: Compression::None, + format: Format::Arf, + data: report.as_ref(), + }); + continue; + } + + let subtype = part + .content_type() + .and_then(|ct| ct.subtype()) + .unwrap_or(""); + let attachment_name = part.attachment_name(); + let ext = attachment_name + .and_then(|f| f.rsplit_once('.')) + .map_or("", |(_, e)| e); + let tls_parts = subtype.rsplit_once('+'); + let compression = match (tls_parts.map(|(_, c)| c).unwrap_or(subtype), ext) + { + ("gzip", _) => Compression::Gzip, + ("zip", _) => Compression::Zip, + (_, "gz") => Compression::Gzip, + (_, "zip") => Compression::Zip, + _ => Compression::None, + }; + let format = match (tls_parts.map(|(c, _)| c).unwrap_or(subtype), ext) { + ("xml", _) => Format::Dmarc, + ("tlsrpt", _) | (_, "json") => Format::Tls, + _ => { + if attachment_name + .map_or(false, |n| n.contains(".xml") || n.contains('!')) + { + Format::Dmarc + } else { + continue; + } + } + }; + + reports.push(ReportData { + compression, + format, + data: report.as_ref(), + }); + } + _ => (), + } + } + + for report in reports { + let data = match report.compression { + Compression::None => Cow::Borrowed(report.data), + Compression::Gzip => { + let mut file = GzDecoder::new(report.data); + let mut buf = Vec::new(); + if let Err(err) = file.read_to_end(&mut buf) { + tracing::debug!( + context = "report", + from = from, + "Failed to decompress report: {}", + err + ); + continue; + } + Cow::Owned(buf) + } + Compression::Zip => { + let mut archive = match zip::ZipArchive::new(Cursor::new(report.data)) { + Ok(archive) => archive, + Err(err) => { + tracing::debug!( + context = "report", + from = from, + "Failed to decompress report: {}", + err + ); + continue; + } + }; + let mut buf = Vec::with_capacity(0); + for i in 0..archive.len() { + match archive.by_index(i) { + Ok(mut file) => { + buf = Vec::with_capacity(file.compressed_size() as usize); + if let Err(err) = file.read_to_end(&mut buf) { + tracing::debug!( + context = "report", + from = from, + "Failed to decompress report: {}", + err + ); + } + break; + } + Err(err) => { + tracing::debug!( + context = "report", + from = from, + "Failed to decompress report: {}", + err + ); + } + } + } + Cow::Owned(buf) + } + }; + + match report.format { + Format::Dmarc => match Report::parse_xml(&data) { + Ok(report) => { + report.log(); + } + Err(err) => { + tracing::debug!( + context = "report", + from = from, + "Failed to parse DMARC report: {}", + err + ); + continue; + } + }, + Format::Tls => match TlsReport::parse_json(&data) { + Ok(report) => { + report.log(); + } + Err(err) => { + tracing::debug!( + context = "report", + from = from, + "Failed to parse TLS report: {:?}", + err + ); + continue; + } + }, + Format::Arf => match Feedback::parse_arf(&data) { + Some(report) => { + report.log(); + } + None => { + tracing::debug!( + context = "report", + from = from, + "Failed to parse Auth Failure report" + ); + continue; + } + }, + } + + // Save report + if let Some(report_path) = &core.report.config.analysis.store { + let (report_format, extension) = match report.format { + Format::Dmarc => ("dmarc", "xml"), + Format::Tls => ("tlsrpt", "json"), + Format::Arf => ("arf", "txt"), + }; + let c_extension = match report.compression { + Compression::None => "", + Compression::Gzip => ".gz", + Compression::Zip => ".zip", + }; + let now = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .map_or(0, |d| d.as_secs()); + let id = core + .report + .config + .analysis + .report_id + .fetch_add(1, Ordering::Relaxed); + + // Build path + let mut report_path = report_path.clone(); + report_path.push(format!( + "{report_format}_{now}_{id}.{extension}{c_extension}" + )); + if let Err(err) = std::fs::write(&report_path, report.data) { + tracing::warn!( + context = "report", + event = "error", + from = from, + "Failed to write incoming report to {}: {}", + report_path.display(), + err + ); + } + } + break; + } + }); + } +} + +trait LogReport { + fn log(&self); +} + +impl LogReport for Report { + fn log(&self) { + let mut dmarc_pass = 0; + let mut dmarc_quarantine = 0; + let mut dmarc_reject = 0; + let mut dmarc_none = 0; + let mut dkim_pass = 0; + let mut dkim_fail = 0; + let mut dkim_none = 0; + let mut spf_pass = 0; + let mut spf_fail = 0; + let mut spf_none = 0; + + for record in self.records() { + let count = std::cmp::min(record.count(), 1); + + match record.action_disposition() { + ActionDisposition::Pass => { + dmarc_pass += count; + } + ActionDisposition::Quarantine => { + dmarc_quarantine += count; + } + ActionDisposition::Reject => { + dmarc_reject += count; + } + ActionDisposition::None | ActionDisposition::Unspecified => { + dmarc_none += count; + } + } + match record.dmarc_dkim_result() { + DmarcResult::Pass => { + dkim_pass += count; + } + DmarcResult::Fail => { + dkim_fail += count; + } + DmarcResult::Unspecified => { + dkim_none += count; + } + } + match record.dmarc_spf_result() { + DmarcResult::Pass => { + spf_pass += count; + } + DmarcResult::Fail => { + spf_fail += count; + } + DmarcResult::Unspecified => { + spf_none += count; + } + } + } + + let range_from = DateTime::from_timestamp(self.date_range_begin() as i64).to_rfc3339(); + let range_to = DateTime::from_timestamp(self.date_range_end() as i64).to_rfc3339(); + + if (dmarc_reject + dmarc_quarantine + dkim_fail + spf_fail) > 0 { + tracing::warn!( + context = "dmarc", + event = "analyze", + range_from = range_from, + range_to = range_to, + domain = self.domain(), + report_email = self.email(), + report_id = self.report_id(), + dmarc_pass = dmarc_pass, + dmarc_quarantine = dmarc_quarantine, + dmarc_reject = dmarc_reject, + dmarc_none = dmarc_none, + dkim_pass = dkim_pass, + dkim_fail = dkim_fail, + dkim_none = dkim_none, + spf_pass = spf_pass, + spf_fail = spf_fail, + spf_none = spf_none, + ); + } else { + tracing::info!( + context = "dmarc", + event = "analyze", + range_from = range_from, + range_to = range_to, + domain = self.domain(), + report_email = self.email(), + report_id = self.report_id(), + dmarc_pass = dmarc_pass, + dmarc_quarantine = dmarc_quarantine, + dmarc_reject = dmarc_reject, + dmarc_none = dmarc_none, + dkim_pass = dkim_pass, + dkim_fail = dkim_fail, + dkim_none = dkim_none, + spf_pass = spf_pass, + spf_fail = spf_fail, + spf_none = spf_none, + ); + } + } +} + +impl LogReport for TlsReport { + fn log(&self) { + for policy in self.policies.iter().take(5) { + let mut details = AHashMap::with_capacity(policy.failure_details.len()); + for failure in &policy.failure_details { + let num_failures = std::cmp::min(1, failure.failed_session_count); + match details.entry(failure.result_type) { + Entry::Occupied(mut e) => { + *e.get_mut() += num_failures; + } + Entry::Vacant(e) => { + e.insert(num_failures); + } + } + } + + if policy.summary.total_failure > 0 { + tracing::warn!( + context = "tlsrpt", + event = "analyze", + range_from = self.date_range.start_datetime.to_rfc3339(), + range_to = self.date_range.end_datetime.to_rfc3339(), + domain = policy.policy.policy_domain, + report_contact = self.contact_info.as_deref().unwrap_or("unknown"), + report_id = self.report_id, + policy_type = ?policy.policy.policy_type, + total_success = policy.summary.total_success, + total_failures = policy.summary.total_failure, + details = ?details, + ); + } else { + tracing::info!( + context = "tlsrpt", + event = "analyze", + range_from = self.date_range.start_datetime.to_rfc3339(), + range_to = self.date_range.end_datetime.to_rfc3339(), + domain = policy.policy.policy_domain, + report_contact = self.contact_info.as_deref().unwrap_or("unknown"), + report_id = self.report_id, + policy_type = ?policy.policy.policy_type, + total_success = policy.summary.total_success, + total_failures = policy.summary.total_failure, + details = ?details, + ); + } + } + } +} + +impl LogReport for Feedback<'_> { + fn log(&self) { + tracing::warn!( + context = "arf", + event = "analyze", + feedback_type = ?self.feedback_type(), + arrival_date = DateTime::from_timestamp(self.arrival_date().unwrap_or_else(|| { + SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .map_or(0, |d| d.as_secs()) as i64 + })).to_rfc3339(), + authentication_results = ?self.authentication_results(), + incidents = self.incidents(), + reported_domain = ?self.reported_domain(), + reported_uri = ?self.reported_uri(), + reporting_mta = self.reporting_mta().unwrap_or_default(), + source_ip = ?self.source_ip(), + user_agent = self.user_agent().unwrap_or_default(), + auth_failure = ?self.auth_failure(), + delivery_result = ?self.delivery_result(), + dkim_domain = self.dkim_domain().unwrap_or_default(), + dkim_identity = self.dkim_identity().unwrap_or_default(), + dkim_selector = self.dkim_selector().unwrap_or_default(), + identity_alignment = ?self.identity_alignment(), + ); + } +} diff --git a/crates/smtp/src/reporting/dkim.rs b/crates/smtp/src/reporting/dkim.rs new file mode 100644 index 00000000..a59fccfd --- /dev/null +++ b/crates/smtp/src/reporting/dkim.rs @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use mail_auth::{ + common::verify::VerifySignature, AuthenticatedMessage, AuthenticationResults, DkimOutput, +}; +use tokio::io::{AsyncRead, AsyncWrite}; + +use crate::{config::Rate, core::Session}; + +impl Session { + pub async fn send_dkim_report( + &self, + rcpt: &str, + message: &AuthenticatedMessage<'_>, + rate: &Rate, + rejected: bool, + output: &DkimOutput<'_>, + ) { + // Generate report + let signature = if let Some(signature) = output.signature() { + signature + } else { + return; + }; + + // Throttle recipient + if !self.throttle_rcpt(rcpt, rate, "dkim") { + tracing::debug!( + parent: &self.span, + context = "report", + report = "dkim", + event = "throttle", + rcpt = rcpt, + ); + return; + } + + let config = &self.core.report.config.dkim; + let from_addr = config.address.eval(self).await; + let mut report = Vec::with_capacity(128); + self.new_auth_failure(output.result().into(), rejected) + .with_authentication_results( + AuthenticationResults::new(&self.instance.hostname) + .with_dkim_result(output, message.from()) + .to_string(), + ) + .with_dkim_domain(signature.domain()) + .with_dkim_selector(signature.selector()) + .with_dkim_identity(signature.identity()) + .with_headers(message.raw_headers()) + .write_rfc5322( + (config.name.eval(self).await.as_str(), from_addr.as_str()), + rcpt, + config.subject.eval(self).await, + &mut report, + ) + .ok(); + + tracing::info!( + parent: &self.span, + context = "report", + report = "dkim", + event = "queue", + rcpt = rcpt, + "Queueing DKIM authentication failure report." + ); + + // Send report + self.core + .send_report( + from_addr, + [rcpt].into_iter(), + report, + &config.sign, + &self.span, + true, + ) + .await; + } +} diff --git a/crates/smtp/src/reporting/dmarc.rs b/crates/smtp/src/reporting/dmarc.rs new file mode 100644 index 00000000..2bea08cb --- /dev/null +++ b/crates/smtp/src/reporting/dmarc.rs @@ -0,0 +1,489 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{collections::hash_map::Entry, path::PathBuf, sync::Arc}; + +use ahash::AHashMap; +use mail_auth::{ + common::verify::VerifySignature, + dmarc::{self, URI}, + report::{AuthFailureType, IdentityAlignment, PolicyPublished, Record, Report, SPFDomainScope}, + ArcOutput, AuthenticatedMessage, AuthenticationResults, DkimOutput, DkimResult, DmarcOutput, + SpfResult, +}; +use serde::{Deserialize, Serialize}; +use tokio::{ + io::{AsyncRead, AsyncWrite}, + runtime::Handle, +}; + +use crate::{ + config::AggregateFrequency, + core::{Core, Session}, + queue::{DomainPart, InstantFromTimestamp, Schedule}, +}; + +use super::{ + scheduler::{ + json_append, json_read_blocking, json_write, ReportPath, ReportPolicy, ReportType, + Scheduler, ToHash, + }, + DmarcEvent, +}; + +#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct DmarcFormat { + pub rua: Vec, + pub policy: PolicyPublished, + pub records: Vec, +} + +impl Session { + #[allow(clippy::too_many_arguments)] + pub async fn send_dmarc_report( + &self, + message: &AuthenticatedMessage<'_>, + auth_results: &AuthenticationResults<'_>, + rejected: bool, + dmarc_output: DmarcOutput, + dkim_output: &[DkimOutput<'_>], + arc_output: &Option>, + ) { + let dmarc_record = dmarc_output.dmarc_record_cloned().unwrap(); + let config = &self.core.report.config.dmarc; + + // Send failure report + if let (Some(failure_rate), Some(report_options)) = + (config.send.eval(self).await, dmarc_output.failure_report()) + { + // Verify that any external reporting addresses are authorized + let rcpts = match self + .core + .resolvers + .dns + .verify_dmarc_report_address(dmarc_output.domain(), dmarc_record.ruf()) + .await + { + Some(rcpts) => { + if !rcpts.is_empty() { + rcpts + .into_iter() + .filter_map(|rcpt| { + if self.throttle_rcpt(rcpt.uri(), failure_rate, "dmarc") { + rcpt.uri().into() + } else { + None + } + }) + .collect() + } else { + if !dmarc_record.ruf().is_empty() { + tracing::debug!( + parent: &self.span, + context = "report", + report = "dkim", + event = "unauthorized-ruf", + ruf = ?dmarc_record.ruf(), + "Unauthorized external reporting addresses" + ); + } + vec![] + } + } + None => { + tracing::debug!( + parent: &self.span, + context = "report", + report = "dmarc", + event = "dns-failure", + ruf = ?dmarc_record.ruf(), + "Failed to validate external report addresses", + ); + vec![] + } + }; + + // Throttle recipient + if !rcpts.is_empty() { + let mut report = Vec::with_capacity(128); + let from_addr = config.address.eval(self).await; + let mut auth_failure = self + .new_auth_failure(AuthFailureType::Dmarc, rejected) + .with_authentication_results(auth_results.to_string()) + .with_headers(message.raw_headers()); + + // Report the first failed signature + let dkim_failed = if let ( + dmarc::Report::Dkim + | dmarc::Report::DkimSpf + | dmarc::Report::All + | dmarc::Report::Any, + Some(signature), + ) = ( + &report_options, + dkim_output.iter().find_map(|o| { + let s = o.signature()?; + if !matches!(o.result(), DkimResult::Pass) { + Some(s) + } else { + None + } + }), + ) { + auth_failure = auth_failure + .with_dkim_domain(signature.domain()) + .with_dkim_selector(signature.selector()) + .with_dkim_identity(signature.identity()); + true + } else { + false + }; + + // Report SPF failure + let spf_failed = if let ( + dmarc::Report::Spf + | dmarc::Report::DkimSpf + | dmarc::Report::All + | dmarc::Report::Any, + Some(output), + ) = ( + &report_options, + self.data + .spf_ehlo + .as_ref() + .and_then(|s| { + if s.result() != SpfResult::Pass { + s.into() + } else { + None + } + }) + .or_else(|| { + self.data.spf_mail_from.as_ref().and_then(|s| { + if s.result() != SpfResult::Pass { + s.into() + } else { + None + } + }) + }), + ) { + auth_failure = + auth_failure.with_spf_dns(format!("txt : {} : v=SPF1", output.domain())); + // TODO use DNS record + true + } else { + false + }; + + auth_failure + .with_identity_alignment(if dkim_failed && spf_failed { + IdentityAlignment::DkimSpf + } else if dkim_failed { + IdentityAlignment::Dkim + } else { + IdentityAlignment::Spf + }) + .write_rfc5322( + (config.name.eval(self).await.as_str(), from_addr.as_str()), + &rcpts.join(", "), + config.subject.eval(self).await, + &mut report, + ) + .ok(); + + tracing::info!( + parent: &self.span, + context = "report", + report = "dmarc", + event = "queue", + rcpt = ?rcpts, + "Queueing DMARC authentication failure report." + ); + + // Send report + self.core + .send_report( + from_addr, + rcpts.into_iter(), + report, + &config.sign, + &self.span, + true, + ) + .await; + } else { + tracing::debug!( + parent: &self.span, + context = "report", + report = "dmarc", + event = "throttle", + ruf = ?dmarc_record.ruf(), + ); + } + } + + // Send agregate reports + let interval = self + .core + .report + .config + .dmarc_aggregate + .send + .eval(self) + .await; + + if matches!(interval, AggregateFrequency::Never) || dmarc_record.rua().is_empty() { + return; + } + + // Create DMARC report record + let mut report_record = Record::new() + .with_dmarc_output(&dmarc_output) + .with_dkim_output(dkim_output) + .with_source_ip(self.data.remote_ip) + .with_header_from(message.from().domain_part()) + .with_envelope_from( + self.data + .mail_from + .as_ref() + .map(|mf| mf.domain.as_str()) + .unwrap_or_else(|| self.data.helo_domain.as_str()), + ); + if let Some(spf_ehlo) = &self.data.spf_ehlo { + report_record = report_record.with_spf_output(spf_ehlo, SPFDomainScope::Helo); + } + if let Some(spf_mail_from) = &self.data.spf_mail_from { + report_record = report_record.with_spf_output(spf_mail_from, SPFDomainScope::MailFrom); + } + if let Some(arc_output) = arc_output { + report_record = report_record.with_arc_output(arc_output); + } + + // Submit DMARC report event + self.core + .schedule_report(DmarcEvent { + domain: dmarc_output.into_domain(), + report_record, + dmarc_record, + interval: *interval, + }) + .await; + } +} + +pub trait GenerateDmarcReport { + fn generate_dmarc_report(&self, domain: ReportPolicy, path: ReportPath); +} + +impl GenerateDmarcReport for Arc { + fn generate_dmarc_report(&self, domain: ReportPolicy, path: ReportPath) { + let core = self.clone(); + let handle = Handle::current(); + + self.worker_pool.spawn(move || { + let deliver_at = path.created + path.deliver_at.as_secs(); + let span = tracing::info_span!( + "dmarc-report", + domain = domain.inner, + range_from = path.created, + range_to = deliver_at, + size = path.size, + ); + + // Deserialize report + let dmarc = if let Some(dmarc) = json_read_blocking::(&path.path, &span) { + dmarc + } else { + return; + }; + + // Verify external reporting addresses + let rua = match handle.block_on( + core.resolvers + .dns + .verify_dmarc_report_address(&domain.inner, &dmarc.rua), + ) { + Some(rcpts) => { + if !rcpts.is_empty() { + rcpts + .into_iter() + .map(|u| u.uri().to_string()) + .collect::>() + } else { + tracing::info!( + parent: &span, + event = "failed", + reason = "unauthorized-rua", + rua = ?dmarc.rua, + "Unauthorized external reporting addresses" + ); + let _ = std::fs::remove_file(&path.path); + return; + } + } + None => { + tracing::info!( + parent: &span, + event = "failed", + reason = "dns-failure", + rua = ?dmarc.rua, + "Failed to validate external report addresses", + ); + let _ = std::fs::remove_file(&path.path); + return; + } + }; + + let config = &core.report.config.dmarc_aggregate; + + // Group duplicates + let mut record_map = AHashMap::with_capacity(dmarc.records.len()); + for record in dmarc.records { + match record_map.entry(record) { + Entry::Occupied(mut e) => { + *e.get_mut() += 1; + } + Entry::Vacant(e) => { + e.insert(1u32); + } + } + } + + // Create report + let mut report = Report::new() + .with_policy_published(dmarc.policy) + .with_date_range_begin(path.created) + .with_date_range_end(deliver_at) + .with_report_id(format!("{}_{}", domain.policy, path.created)) + .with_email(handle.block_on(config.address.eval(&domain.inner.as_str()))); + if let Some(org_name) = handle.block_on(config.org_name.eval(&domain.inner.as_str())) { + report = report.with_org_name(org_name); + } + if let Some(contact_info) = + handle.block_on(config.contact_info.eval(&domain.inner.as_str())) + { + report = report.with_extra_contact_info(contact_info); + } + for (record, count) in record_map { + report.add_record(record.with_count(count)); + } + let from_addr = handle.block_on(config.address.eval(&domain.inner.as_str())); + let mut message = Vec::with_capacity(path.size); + let _ = report.write_rfc5322( + handle.block_on(core.report.config.submitter.eval(&domain.inner.as_str())), + ( + handle + .block_on(config.name.eval(&domain.inner.as_str())) + .as_str(), + from_addr.as_str(), + ), + rua.iter().map(|a| a.as_str()), + &mut message, + ); + + // Send report + handle.block_on(core.send_report( + from_addr, + rua.iter(), + message, + &config.sign, + &span, + false, + )); + + if let Err(err) = std::fs::remove_file(&path.path) { + tracing::warn!( + context = "report", + event = "error", + "Failed to remove report file {}: {}", + path.path.display(), + err + ); + } + }); + } +} + +impl Scheduler { + pub async fn schedule_dmarc(&mut self, event: Box, core: &Core) { + let max_size = core + .report + .config + .dmarc_aggregate + .max_size + .eval(&event.domain.as_str()) + .await; + + let policy = event.dmarc_record.to_hash(); + let (create, path) = match self.reports.entry(ReportType::Dmarc(ReportPolicy { + inner: event.domain, + policy, + })) { + Entry::Occupied(e) => (None, e.into_mut().dmarc_path()), + Entry::Vacant(e) => { + let domain = e.key().domain_name().to_string(); + let created = event.interval.to_timestamp(); + let deliver_at = created + event.interval.as_secs(); + + self.main.push(Schedule { + due: deliver_at.to_instant(), + inner: e.key().clone(), + }); + let path = core + .build_report_path(ReportType::Dmarc(&domain), policy, created, event.interval) + .await; + let v = e.insert(ReportType::Dmarc(ReportPath { + path, + deliver_at: event.interval, + created, + size: 0, + })); + (domain.into(), v.dmarc_path()) + } + }; + + if let Some(domain) = create { + // Serialize report + let entry = DmarcFormat { + rua: event.dmarc_record.rua().to_vec(), + policy: PolicyPublished::from_record(domain, &event.dmarc_record), + records: vec![event.report_record], + }; + let bytes_written = json_write(&path.path, &entry).await; + + if bytes_written > 0 { + path.size += bytes_written; + } else { + // Something went wrong, remove record + self.reports.remove(&ReportType::Dmarc(ReportPolicy { + inner: entry.policy.domain, + policy, + })); + } + } else if path.size < *max_size { + // Append to existing report + path.size += json_append(&path.path, &event.report_record, *max_size - path.size).await; + } + } +} diff --git a/crates/smtp/src/reporting/mod.rs b/crates/smtp/src/reporting/mod.rs new file mode 100644 index 00000000..75ab301a --- /dev/null +++ b/crates/smtp/src/reporting/mod.rs @@ -0,0 +1,383 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{sync::Arc, time::SystemTime}; + +use mail_auth::{ + common::headers::HeaderWriter, + dmarc::Dmarc, + mta_sts::TlsRpt, + report::{ + tlsrpt::FailureDetails, AuthFailureType, DeliveryResult, Feedback, FeedbackType, Record, + }, +}; +use mail_parser::DateTime; + +use tokio::io::{AsyncRead, AsyncWrite}; + +use crate::{ + config::{AddressMatch, AggregateFrequency, DkimSigner, IfBlock}, + core::{management, Core, Session}, + outbound::{dane::Tlsa, mta_sts::Policy}, + queue::{DomainPart, Message}, + USER_AGENT, +}; + +use self::scheduler::{ReportKey, ReportValue}; + +pub mod analysis; +pub mod dkim; +pub mod dmarc; +pub mod scheduler; +pub mod spf; +pub mod tls; + +#[derive(Debug)] +pub enum Event { + Dmarc(Box), + Tls(Box), + Manage(management::ReportRequest), + Stop, +} + +#[derive(Debug)] +pub struct DmarcEvent { + pub domain: String, + pub report_record: Record, + pub dmarc_record: Arc, + pub interval: AggregateFrequency, +} + +#[derive(Debug)] +pub struct TlsEvent { + pub domain: String, + pub policy: PolicyType, + pub failure: Option, + pub tls_record: Arc, + pub interval: AggregateFrequency, +} + +#[derive(Debug, Hash, PartialEq, Eq)] +pub enum PolicyType { + Tlsa(Option>), + Sts(Option>), + None, +} + +impl Session { + pub fn new_auth_failure(&self, ft: AuthFailureType, rejected: bool) -> Feedback<'_> { + Feedback::new(FeedbackType::AuthFailure) + .with_auth_failure(ft) + .with_arrival_date( + SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .map_or(0, |d| d.as_secs()) as i64, + ) + .with_source_ip(self.data.remote_ip) + .with_reporting_mta(&self.instance.hostname) + .with_user_agent(USER_AGENT) + .with_delivery_result(if rejected { + DeliveryResult::Reject + } else { + DeliveryResult::Unspecified + }) + } + + pub fn is_report(&self) -> bool { + for addr_match in &self.core.report.config.analysis.addresses { + for addr in &self.data.rcpt_to { + match addr_match { + AddressMatch::StartsWith(prefix) if addr.address_lcase.starts_with(prefix) => { + return true + } + AddressMatch::EndsWith(suffix) if addr.address_lcase.ends_with(suffix) => { + return true + } + AddressMatch::Equals(value) if addr.address_lcase.eq(value) => return true, + _ => (), + } + } + } + + false + } +} + +impl Core { + pub async fn send_report( + &self, + from_addr: &str, + rcpts: impl Iterator>, + report: Vec, + sign_config: &IfBlock>>, + span: &tracing::Span, + deliver_now: bool, + ) { + // Build message + let from_addr_lcase = from_addr.to_lowercase(); + let from_addr_domain = from_addr_lcase.domain_part().to_string(); + let mut message = Message::new_boxed(from_addr, from_addr_lcase, from_addr_domain); + for rcpt_ in rcpts { + message + .add_recipient(rcpt_.as_ref(), &self.queue.config) + .await; + } + + // Sign message + let signature = message.sign(sign_config, &report, span).await; + + // Schedule delivery at a random time between now and the next 3 hours + if !deliver_now { + #[cfg(not(feature = "test_mode"))] + { + use rand::Rng; + use std::time::Duration; + + let delivery_time = Duration::from_secs(rand::thread_rng().gen_range(0..10800)); + for domain in &mut message.domains { + domain.retry.due += delivery_time; + domain.expires += delivery_time; + domain.notify.due += delivery_time; + } + } + } + + // Queue message + self.queue + .queue_message(message, signature.as_deref(), &report, span) + .await; + } + + pub async fn schedule_report(&self, report: impl Into) { + if self.report.tx.send(report.into()).await.is_err() { + tracing::warn!(contex = "report", "Channel send failed."); + } + } +} + +impl Message { + pub async fn sign( + &mut self, + config: &IfBlock>>, + bytes: &[u8], + span: &tracing::Span, + ) -> Option> { + let signers = config.eval(self).await; + if !signers.is_empty() { + let mut headers = Vec::with_capacity(64); + for signer in signers.iter() { + match signer.sign(bytes) { + Ok(signature) => { + signature.write_header(&mut headers); + } + Err(err) => { + tracing::warn!(parent: span, + context = "dkim", + event = "sign-failed", + reason = %err); + } + } + } + if !headers.is_empty() { + return Some(headers); + } + } + None + } +} + +impl AggregateFrequency { + pub fn to_timestamp(&self) -> u64 { + self.to_timestamp_(DateTime::from_timestamp( + SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .map_or(0, |d| d.as_secs()) as i64, + )) + } + + pub fn to_timestamp_(&self, mut dt: DateTime) -> u64 { + (match self { + AggregateFrequency::Hourly => { + dt.minute = 0; + dt.second = 0; + dt.to_timestamp() + } + AggregateFrequency::Daily => { + dt.hour = 0; + dt.minute = 0; + dt.second = 0; + dt.to_timestamp() + } + AggregateFrequency::Weekly => { + let dow = dt.day_of_week(); + dt.hour = 0; + dt.minute = 0; + dt.second = 0; + dt.to_timestamp() - (86400 * dow as i64) + } + AggregateFrequency::Never => dt.to_timestamp(), + }) as u64 + } + + pub fn as_secs(&self) -> u64 { + match self { + AggregateFrequency::Hourly => 3600, + AggregateFrequency::Daily => 86400, + AggregateFrequency::Weekly => 7 * 86400, + AggregateFrequency::Never => 0, + } + } +} + +impl From for Event { + fn from(value: DmarcEvent) -> Self { + Event::Dmarc(Box::new(value)) + } +} + +impl From for Event { + fn from(value: TlsEvent) -> Self { + Event::Tls(Box::new(value)) + } +} + +impl From> for PolicyType { + fn from(value: Arc) -> Self { + PolicyType::Tlsa(Some(value)) + } +} + +impl From> for PolicyType { + fn from(value: Arc) -> Self { + PolicyType::Sts(Some(value)) + } +} + +impl From<&Arc> for PolicyType { + fn from(value: &Arc) -> Self { + PolicyType::Tlsa(Some(value.clone())) + } +} + +impl From<&Arc> for PolicyType { + fn from(value: &Arc) -> Self { + PolicyType::Sts(Some(value.clone())) + } +} + +impl From<(&Option>, &Option>)> for PolicyType { + fn from(value: (&Option>, &Option>)) -> Self { + match value { + (Some(value), _) => PolicyType::Sts(Some(value.clone())), + (_, Some(value)) => PolicyType::Tlsa(Some(value.clone())), + _ => PolicyType::None, + } + } +} + +impl ReportKey { + pub fn domain(&self) -> &str { + match self { + scheduler::ReportType::Dmarc(p) => &p.inner, + scheduler::ReportType::Tls(d) => d, + } + } +} + +impl ReportValue { + pub async fn delete(&self) { + match self { + scheduler::ReportType::Dmarc(path) => { + if let Err(err) = tokio::fs::remove_file(&path.path).await { + tracing::warn!( + context = "report", + event = "error", + "Failed to remove report file {}: {}", + path.path.display(), + err + ); + } + } + scheduler::ReportType::Tls(path) => { + for path in &path.path { + if let Err(err) = tokio::fs::remove_file(&path.inner).await { + tracing::warn!( + context = "report", + event = "error", + "Failed to remove report file {}: {}", + path.inner.display(), + err + ); + } + } + } + } + } +} + +#[cfg(test)] +mod tests { + use mail_parser::DateTime; + + use crate::config::AggregateFrequency; + + #[test] + fn aggregate_to_timestamp() { + for (freq, date, expected) in [ + ( + AggregateFrequency::Hourly, + "2023-01-24T09:10:40Z", + "2023-01-24T09:00:00Z", + ), + ( + AggregateFrequency::Daily, + "2023-01-24T09:10:40Z", + "2023-01-24T00:00:00Z", + ), + ( + AggregateFrequency::Weekly, + "2023-01-24T09:10:40Z", + "2023-01-22T00:00:00Z", + ), + ( + AggregateFrequency::Weekly, + "2023-01-28T23:59:59Z", + "2023-01-22T00:00:00Z", + ), + ( + AggregateFrequency::Weekly, + "2023-01-22T23:59:59Z", + "2023-01-22T00:00:00Z", + ), + ] { + assert_eq!( + DateTime::from_timestamp( + freq.to_timestamp_(DateTime::parse_rfc3339(date).unwrap()) as i64 + ) + .to_rfc3339(), + expected, + "failed for {freq:?} {date} {expected}" + ); + } + } +} diff --git a/crates/smtp/src/reporting/scheduler.rs b/crates/smtp/src/reporting/scheduler.rs new file mode 100644 index 00000000..620e1806 --- /dev/null +++ b/crates/smtp/src/reporting/scheduler.rs @@ -0,0 +1,649 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use ahash::{AHashMap, RandomState}; +use mail_auth::{ + common::{ + base32::{Base32Reader, Base32Writer}, + headers::Writer, + }, + dmarc::Dmarc, +}; + +use serde::{de::DeserializeOwned, Serialize}; +use std::{ + collections::{hash_map::Entry, BinaryHeap}, + hash::Hash, + path::PathBuf, + sync::Arc, + time::{Duration, Instant, SystemTime}, +}; +use tokio::{ + fs::{self, OpenOptions}, + io::AsyncWriteExt, + sync::mpsc, +}; + +use crate::{ + config::AggregateFrequency, + core::{management::ReportRequest, worker::SpawnCleanup, Core, ReportCore}, + queue::{InstantFromTimestamp, Schedule}, +}; + +use super::{dmarc::GenerateDmarcReport, tls::GenerateTlsReport, Event}; + +pub type ReportKey = ReportType, String>; +pub type ReportValue = ReportType, ReportPath>>>; + +pub struct Scheduler { + short_wait: Duration, + long_wait: Duration, + pub main: BinaryHeap>, + pub reports: AHashMap, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize)] +pub enum ReportType { + Dmarc(T), + Tls(U), +} + +#[derive(Debug, PartialEq, Eq)] +pub struct ReportPath { + pub path: T, + pub size: usize, + pub created: u64, + pub deliver_at: AggregateFrequency, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct ReportPolicy { + pub inner: T, + pub policy: u64, +} + +impl SpawnReport for mpsc::Receiver { + fn spawn(mut self, core: Arc, mut scheduler: Scheduler) { + tokio::spawn(async move { + let mut last_cleanup = Instant::now(); + + loop { + match tokio::time::timeout(scheduler.wake_up_time(), self.recv()).await { + Ok(Some(event)) => match event { + Event::Dmarc(event) => { + scheduler.schedule_dmarc(event, &core).await; + } + Event::Tls(event) => { + scheduler.schedule_tls(event, &core).await; + } + Event::Manage(request) => match request { + ReportRequest::List { + type_, + domain, + result_tx, + } => { + let mut result = Vec::new(); + for key in scheduler.reports.keys() { + if domain + .as_ref() + .map_or(false, |domain| domain != key.domain()) + { + continue; + } + if let Some(type_) = &type_ { + if !matches!( + (key, type_), + (ReportType::Dmarc(_), ReportType::Dmarc(_)) + | (ReportType::Tls(_), ReportType::Tls(_)) + ) { + continue; + } + } + result.push(key.to_string()); + } + let _ = result_tx.send(result); + } + ReportRequest::Status { + report_ids, + result_tx, + } => { + let mut result = Vec::with_capacity(report_ids.len()); + for report_id in &report_ids { + result.push( + scheduler + .reports + .get(report_id) + .map(|report_value| (report_id, report_value).into()), + ); + } + let _ = result_tx.send(result); + } + ReportRequest::Cancel { + report_ids, + result_tx, + } => { + let mut result = Vec::with_capacity(report_ids.len()); + for report_id in &report_ids { + result.push( + if let Some(report) = scheduler.reports.remove(report_id) { + report.delete().await; + true + } else { + false + }, + ); + } + let _ = result_tx.send(result); + } + }, + Event::Stop => break, + }, + Ok(None) => break, + Err(_) => { + while let Some(report) = scheduler.next_due() { + match report { + (ReportType::Dmarc(domain), ReportType::Dmarc(path)) => { + core.generate_dmarc_report(domain, path); + } + (ReportType::Tls(domain), ReportType::Tls(path)) => { + core.generate_tls_report(domain, path); + } + _ => unreachable!(), + } + } + + // Cleanup expired throttles + if last_cleanup.elapsed().as_secs() >= 86400 { + last_cleanup = Instant::now(); + core.spawn_cleanup(); + } + } + } + } + }); + } +} + +impl Core { + pub async fn build_report_path( + &self, + domain: ReportType<&str, &str>, + policy: u64, + created: u64, + interval: AggregateFrequency, + ) -> PathBuf { + let (ext, domain) = match domain { + ReportType::Dmarc(domain) => ("d", domain), + ReportType::Tls(domain) => ("t", domain), + }; + + // Build base path + let mut path = self.report.config.path.eval(&domain).await.clone(); + path.push((policy % *self.report.config.hash.eval(&domain).await).to_string()); + let _ = fs::create_dir(&path).await; + + // Build filename + let mut w = Base32Writer::with_capacity(domain.len() + 13); + w.write(&policy.to_le_bytes()[..]); + w.write(&(created.saturating_sub(946684800) as u32).to_le_bytes()[..]); + w.push_byte( + match interval { + AggregateFrequency::Hourly => 0, + AggregateFrequency::Daily => 1, + AggregateFrequency::Weekly => 2, + AggregateFrequency::Never => 3, + }, + false, + ); + w.write(domain.as_bytes()); + let mut file = w.finalize(); + file.push('.'); + file.push_str(ext); + path.push(file); + path + } +} + +impl ReportCore { + pub async fn read_reports(&self) -> Scheduler { + let mut scheduler = Scheduler::default(); + + for path in self + .config + .path + .if_then + .iter() + .map(|t| &t.then) + .chain([&self.config.path.default]) + { + let mut dir = match tokio::fs::read_dir(path).await { + Ok(dir) => dir, + Err(_) => continue, + }; + loop { + match dir.next_entry().await { + Ok(Some(file)) => { + let file = file.path(); + if file.is_dir() { + match tokio::fs::read_dir(&file).await { + Ok(mut dir) => { + let file_ = file; + loop { + match dir.next_entry().await { + Ok(Some(file)) => { + let file = file.path(); + if file + .extension() + .map_or(false, |e| e == "t" || e == "d") + { + if let Err(err) = scheduler.add_path(file).await + { + tracing::warn!("{}", err); + } + } + } + Ok(None) => break, + Err(err) => { + tracing::warn!( + "Failed to read report directory {}: {}", + file_.display(), + err + ); + break; + } + } + } + } + Err(err) => { + tracing::warn!( + "Failed to read report directory {}: {}", + file.display(), + err + ) + } + }; + } else if file.extension().map_or(false, |e| e == "t" || e == "d") { + if let Err(err) = scheduler.add_path(file).await { + tracing::warn!("{}", err); + } + } + } + Ok(None) => { + break; + } + Err(err) => { + tracing::warn!( + "Failed to read report directory {}: {}", + path.display(), + err + ); + break; + } + } + } + } + + scheduler + } +} + +impl Scheduler { + pub fn next_due(&mut self) -> Option<(ReportKey, ReportValue)> { + let item = self.main.peek()?; + if item.due <= Instant::now() { + let item = self.main.pop().unwrap(); + self.reports + .remove(&item.inner) + .map(|policy| (item.inner, policy)) + } else { + None + } + } + + pub fn wake_up_time(&self) -> Duration { + self.main + .peek() + .map(|item| { + item.due + .checked_duration_since(Instant::now()) + .unwrap_or(self.short_wait) + }) + .unwrap_or(self.long_wait) + } + + pub async fn add_path(&mut self, path: PathBuf) -> Result<(), String> { + let (file, ext) = path + .file_name() + .and_then(|f| f.to_str()) + .and_then(|f| f.rsplit_once('.')) + .ok_or_else(|| format!("Invalid queue file name {}", path.display()))?; + let file_size = fs::metadata(&path) + .await + .map_err(|err| { + format!( + "Failed to obtain file metadata for {}: {}", + path.display(), + err + ) + })? + .len(); + if file_size == 0 { + let _ = fs::remove_file(&path).await; + return Err(format!( + "Removed zero length report file {}", + path.display() + )); + } + + // Decode domain name + let mut policy = [0u8; std::mem::size_of::()]; + let mut created = [0u8; std::mem::size_of::()]; + let mut deliver_at = AggregateFrequency::Never; + let mut domain = Vec::new(); + for (pos, byte) in Base32Reader::new(file.as_bytes()).enumerate() { + match pos { + 0..=7 => { + policy[pos] = byte; + } + 8..=11 => { + created[pos - 8] = byte; + } + 12 => { + deliver_at = match byte { + 0 => AggregateFrequency::Hourly, + 1 => AggregateFrequency::Daily, + 2 => AggregateFrequency::Weekly, + _ => { + return Err(format!( + "Failed to base32 decode report file {}", + path.display() + )); + } + }; + } + _ => { + domain.push(byte); + } + } + } + if domain.is_empty() { + return Err(format!( + "Failed to base32 decode report file {}", + path.display() + )); + } + let domain = String::from_utf8(domain).map_err(|err| { + format!( + "Failed to base32 decode report file {}: {}", + path.display(), + err + ) + })?; + + // Rebuild parts + let policy = u64::from_le_bytes(policy); + let created = u32::from_le_bytes(created) as u64 + 946684800; + + match ext { + "d" => { + let key = ReportType::Dmarc(ReportPolicy { + inner: domain, + policy, + }); + self.reports.insert( + key.clone(), + ReportType::Dmarc(ReportPath { + path, + size: file_size as usize, + created, + deliver_at, + }), + ); + self.main.push(Schedule { + due: (created + deliver_at.as_secs()).to_instant(), + inner: key, + }); + } + "t" => match self.reports.entry(ReportType::Tls(domain)) { + Entry::Occupied(mut e) => { + if let ReportType::Tls(tls) = e.get_mut() { + tls.size += file_size as usize; + tls.path.push(ReportPolicy { + inner: path, + policy, + }); + } + } + Entry::Vacant(e) => { + self.main.push(Schedule { + due: (created + deliver_at.as_secs()).to_instant(), + inner: e.key().clone(), + }); + e.insert(ReportType::Tls(ReportPath { + path: vec![ReportPolicy { + inner: path, + policy, + }], + size: file_size as usize, + created, + deliver_at, + })); + } + }, + _ => unreachable!(), + } + + Ok(()) + } +} + +pub async fn json_write(path: &PathBuf, entry: &impl Serialize) -> usize { + if let Ok(bytes) = serde_json::to_vec(entry) { + // Save serialized report + let bytes_written = bytes.len() - 2; + match fs::File::create(&path).await { + Ok(mut file) => match file.write_all(&bytes[..bytes_written]).await { + Ok(_) => bytes_written, + Err(err) => { + tracing::error!( + context = "report", + event = "error", + "Failed to write to report file {}: {}", + path.display(), + err + ); + 0 + } + }, + Err(err) => { + tracing::error!( + context = "report", + event = "error", + "Failed to create report file {}: {}", + path.display(), + err + ); + 0 + } + } + } else { + 0 + } +} + +pub async fn json_append(path: &PathBuf, entry: &impl Serialize, bytes_left: usize) -> usize { + let mut bytes = Vec::with_capacity(128); + bytes.push(b','); + if serde_json::to_writer(&mut bytes, entry).is_ok() && bytes.len() <= bytes_left { + let err = match OpenOptions::new().append(true).open(&path).await { + Ok(mut file) => match file.write_all(&bytes).await { + Ok(_) => return bytes.len(), + Err(err) => err, + }, + Err(err) => err, + }; + tracing::error!( + context = "report", + event = "error", + "Failed to append report to {}: {}", + path.display(), + err + ); + } + 0 +} + +pub async fn json_read(path: &PathBuf, span: &tracing::Span) -> Option { + match fs::read_to_string(&path).await { + Ok(mut json) => { + json.push_str("]}"); + match serde_json::from_str(&json) { + Ok(report) => Some(report), + Err(err) => { + tracing::error!( + parent: span, + context = "deserialize", + event = "error", + "Failed to deserialize report file {}: {}", + path.display(), + err + ); + None + } + } + } + Err(err) => { + tracing::error!( + parent: span, + context = "io", + event = "error", + "Failed to read report file {}: {}", + path.display(), + err + ); + None + } + } +} + +pub fn json_read_blocking(path: &PathBuf, span: &tracing::Span) -> Option { + match std::fs::read_to_string(path) { + Ok(mut json) => { + json.push_str("]}"); + match serde_json::from_str(&json) { + Ok(report) => Some(report), + Err(err) => { + tracing::error!( + parent: span, + context = "deserialize", + event = "error", + "Failed to deserialize report file {}: {}", + path.display(), + err + ); + None + } + } + } + Err(err) => { + tracing::error!( + parent: span, + context = "io", + event = "error", + "Failed to read report file {}: {}", + path.display(), + err + ); + None + } + } +} + +impl Default for Scheduler { + fn default() -> Self { + Self { + short_wait: Duration::from_millis(1), + long_wait: Duration::from_secs(86400 * 365), + main: BinaryHeap::with_capacity(128), + reports: AHashMap::with_capacity(128), + } + } +} + +impl ReportKey { + pub fn domain_name(&self) -> &str { + match self { + ReportType::Dmarc(domain) => domain.inner.as_str(), + ReportType::Tls(domain) => domain.as_str(), + } + } +} + +impl ReportValue { + pub fn dmarc_path(&mut self) -> &mut ReportPath { + match self { + ReportType::Dmarc(path) => path, + ReportType::Tls(_) => unreachable!(), + } + } + + pub fn tls_path(&mut self) -> &mut ReportPath>> { + match self { + ReportType::Tls(path) => path, + ReportType::Dmarc(_) => unreachable!(), + } + } +} + +pub trait ToHash { + fn to_hash(&self) -> u64; +} + +impl ToHash for Dmarc { + fn to_hash(&self) -> u64 { + RandomState::with_seeds(1, 9, 7, 9).hash_one(self) + } +} + +impl ToHash for super::PolicyType { + fn to_hash(&self) -> u64 { + RandomState::with_seeds(1, 9, 7, 9).hash_one(self) + } +} + +pub trait ToTimestamp { + fn to_timestamp(&self) -> u64; +} + +impl ToTimestamp for Duration { + fn to_timestamp(&self) -> u64 { + SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .map_or(0, |d| d.as_secs()) + + self.as_secs() + } +} + +pub trait SpawnReport { + fn spawn(self, core: Arc, scheduler: Scheduler); +} diff --git a/crates/smtp/src/reporting/spf.rs b/crates/smtp/src/reporting/spf.rs new file mode 100644 index 00000000..cf062f55 --- /dev/null +++ b/crates/smtp/src/reporting/spf.rs @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use mail_auth::{report::AuthFailureType, AuthenticationResults, SpfOutput}; +use tokio::io::{AsyncRead, AsyncWrite}; + +use crate::{config::Rate, core::Session}; + +impl Session { + pub async fn send_spf_report( + &self, + rcpt: &str, + rate: &Rate, + rejected: bool, + output: &SpfOutput, + ) { + // Throttle recipient + if !self.throttle_rcpt(rcpt, rate, "spf") { + tracing::debug!( + parent: &self.span, + context = "report", + report = "spf", + event = "throttle", + rcpt = rcpt, + ); + return; + } + + // Generate report + let config = &self.core.report.config.spf; + let from_addr = config.address.eval(self).await; + let mut report = Vec::with_capacity(128); + self.new_auth_failure(AuthFailureType::Spf, rejected) + .with_authentication_results( + if let Some(mail_from) = &self.data.mail_from { + AuthenticationResults::new(&self.instance.hostname).with_spf_mailfrom_result( + output, + self.data.remote_ip, + &mail_from.address, + &self.data.helo_domain, + ) + } else { + AuthenticationResults::new(&self.instance.hostname).with_spf_ehlo_result( + output, + self.data.remote_ip, + &self.data.helo_domain, + ) + } + .to_string(), + ) + .with_spf_dns(format!("txt : {} : v=SPF1", output.domain())) // TODO use DNS record + .write_rfc5322( + (config.name.eval(self).await.as_str(), from_addr.as_str()), + rcpt, + config.subject.eval(self).await, + &mut report, + ) + .ok(); + + tracing::info!( + parent: &self.span, + context = "report", + report = "spf", + event = "queue", + rcpt = rcpt, + "Queueing SPF authentication failure report." + ); + + // Send report + self.core + .send_report( + from_addr, + [rcpt].into_iter(), + report, + &config.sign, + &self.span, + true, + ) + .await; + } +} diff --git a/crates/smtp/src/reporting/tls.rs b/crates/smtp/src/reporting/tls.rs new file mode 100644 index 00000000..248d46f2 --- /dev/null +++ b/crates/smtp/src/reporting/tls.rs @@ -0,0 +1,450 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{collections::hash_map::Entry, path::PathBuf, sync::Arc, time::Duration}; + +use ahash::AHashMap; +use mail_auth::{ + flate2::{write::GzEncoder, Compression}, + mta_sts::{ReportUri, TlsRpt}, + report::tlsrpt::{ + DateRange, FailureDetails, Policy, PolicyDetails, PolicyType, Summary, TlsReport, + }, +}; + +use mail_parser::DateTime; +use reqwest::header::CONTENT_TYPE; +use serde::{Deserialize, Serialize}; +use std::fmt::Write; +use tokio::runtime::Handle; + +use crate::{ + config::AggregateFrequency, + core::Core, + outbound::mta_sts::{Mode, MxPattern}, + queue::{InstantFromTimestamp, Schedule}, + USER_AGENT, +}; + +use super::{ + scheduler::{ + json_append, json_read_blocking, json_write, ReportPath, ReportPolicy, ReportType, + Scheduler, ToHash, + }, + TlsEvent, +}; + +#[derive(Debug, Clone)] +pub struct TlsRptOptions { + pub record: Arc, + pub interval: AggregateFrequency, +} + +#[derive(Debug, Serialize, Deserialize)] +struct TlsFormat { + rua: Vec, + policy: PolicyDetails, + records: Vec>, +} + +pub trait GenerateTlsReport { + fn generate_tls_report(&self, domain: String, paths: ReportPath>>); +} + +#[cfg(feature = "test_mode")] +pub static TLS_HTTP_REPORT: parking_lot::Mutex> = parking_lot::Mutex::new(Vec::new()); + +impl GenerateTlsReport for Arc { + fn generate_tls_report(&self, domain: String, path: ReportPath>>) { + let core = self.clone(); + let handle = Handle::current(); + + self.worker_pool.spawn(move || { + let deliver_at = path.created + path.deliver_at.as_secs(); + let span = tracing::info_span!( + "tls-report", + domain = domain, + range_from = path.created, + range_to = deliver_at, + size = path.size, + ); + + // Deserialize report + let config = &core.report.config.tls; + let mut report = TlsReport { + organization_name: handle + .block_on(config.org_name.eval(&domain.as_str())) + .clone(), + date_range: DateRange { + start_datetime: DateTime::from_timestamp(path.created as i64), + end_datetime: DateTime::from_timestamp(deliver_at as i64), + }, + contact_info: handle + .block_on(config.contact_info.eval(&domain.as_str())) + .clone(), + report_id: format!( + "{}_{}", + path.created, + path.path.first().map_or(0, |p| p.policy) + ), + policies: Vec::with_capacity(path.path.len()), + }; + let mut rua = Vec::new(); + for path in &path.path { + if let Some(tls) = json_read_blocking::(&path.inner, &span) { + // Group duplicates + let mut total_success = 0; + let mut total_failure = 0; + let mut record_map = AHashMap::with_capacity(tls.records.len()); + for record in tls.records { + if let Some(record) = record { + match record_map.entry(record) { + Entry::Occupied(mut e) => { + *e.get_mut() += 1; + } + Entry::Vacant(e) => { + e.insert(1u32); + } + } + total_failure += 1; + } else { + total_success += 1; + } + } + report.policies.push(Policy { + policy: tls.policy, + summary: Summary { + total_success, + total_failure, + }, + failure_details: record_map + .into_iter() + .map(|(mut r, count)| { + r.failed_session_count = count; + r + }) + .collect(), + }); + + rua = tls.rua; + } + } + + if report.policies.is_empty() { + // This should not happen + tracing::warn!( + parent: &span, + event = "empty-report", + "No policies found in report" + ); + path.cleanup_blocking(); + return; + } + + // Compress and serialize report + let json = report.to_json(); + let mut e = GzEncoder::new(Vec::with_capacity(json.len()), Compression::default()); + let json = + match std::io::Write::write_all(&mut e, json.as_bytes()).and_then(|_| e.finish()) { + Ok(report) => report, + Err(err) => { + tracing::error!( + parent: &span, + event = "error", + "Failed to compress report: {}", + err + ); + return; + } + }; + + // Try delivering report over HTTP + let mut rcpts = Vec::with_capacity(rua.len()); + for uri in &rua { + match uri { + ReportUri::Http(uri) => { + if let Ok(client) = reqwest::blocking::Client::builder() + .user_agent(USER_AGENT) + .timeout(Duration::from_secs(2 * 60)) + .build() + { + #[cfg(feature = "test_mode")] + if uri == "https://127.0.0.1/tls" { + TLS_HTTP_REPORT.lock().extend_from_slice(&json); + path.cleanup_blocking(); + return; + } + + match client + .post(uri) + .header(CONTENT_TYPE, "application/tlsrpt+gzip") + .body(json.to_vec()) + .send() + { + Ok(response) => { + if response.status().is_success() { + tracing::info!( + parent: &span, + context = "http", + event = "success", + url = uri, + ); + path.cleanup_blocking(); + return; + } else { + tracing::debug!( + parent: &span, + context = "http", + event = "invalid-response", + url = uri, + status = %response.status() + ); + } + } + Err(err) => { + tracing::debug!( + parent: &span, + context = "http", + event = "error", + url = uri, + reason = %err + ); + } + } + } + } + ReportUri::Mail(mailto) => { + rcpts.push(mailto.as_str()); + } + } + } + + // Deliver report over SMTP + if !rcpts.is_empty() { + let from_addr = handle.block_on(config.address.eval(&domain.as_str())); + let mut message = Vec::with_capacity(path.size); + let _ = report.write_rfc5322_from_bytes( + &domain, + handle.block_on(core.report.config.submitter.eval(&domain.as_str())), + ( + handle.block_on(config.name.eval(&domain.as_str())).as_str(), + from_addr.as_str(), + ), + rcpts.iter().copied(), + &json, + &mut message, + ); + + // Send report + handle.block_on(core.send_report( + from_addr, + rcpts.iter(), + message, + &config.sign, + &span, + false, + )); + } else { + tracing::info!( + parent: &span, + event = "delivery-failed", + "No valid recipients found to deliver report to." + ); + } + path.cleanup_blocking(); + }); + } +} + +impl Scheduler { + pub async fn schedule_tls(&mut self, event: Box, core: &Core) { + let max_size = core + .report + .config + .tls + .max_size + .eval(&event.domain.as_str()) + .await; + let policy_hash = event.policy.to_hash(); + + let (path, pos, create) = match self.reports.entry(ReportType::Tls(event.domain)) { + Entry::Occupied(e) => { + if let ReportType::Tls(path) = e.get() { + if let Some(pos) = path.path.iter().position(|p| p.policy == policy_hash) { + (e.into_mut().tls_path(), pos, None) + } else { + let pos = path.path.len(); + let domain = e.key().domain_name().to_string(); + let path = e.into_mut().tls_path(); + path.path.push(ReportPolicy { + inner: core + .build_report_path( + ReportType::Tls(&domain), + policy_hash, + path.created, + path.deliver_at, + ) + .await, + policy: policy_hash, + }); + (path, pos, domain.into()) + } + } else { + unreachable!() + } + } + Entry::Vacant(e) => { + let created = event.interval.to_timestamp(); + let deliver_at = created + event.interval.as_secs(); + + self.main.push(Schedule { + due: deliver_at.to_instant(), + inner: e.key().clone(), + }); + let domain = e.key().domain_name().to_string(); + let path = core + .build_report_path( + ReportType::Tls(&domain), + policy_hash, + created, + event.interval, + ) + .await; + let v = e.insert(ReportType::Tls(ReportPath { + path: vec![ReportPolicy { + inner: path, + policy: policy_hash, + }], + size: 0, + created, + deliver_at: event.interval, + })); + (v.tls_path(), 0, domain.into()) + } + }; + + if let Some(domain) = create { + let mut policy = PolicyDetails { + policy_type: PolicyType::NoPolicyFound, + policy_string: vec![], + policy_domain: domain, + mx_host: vec![], + }; + + match event.policy { + super::PolicyType::Tlsa(tlsa) => { + policy.policy_type = PolicyType::Tlsa; + if let Some(tlsa) = tlsa { + for entry in &tlsa.entries { + policy.policy_string.push(format!( + "{} {} {} {}", + if entry.is_end_entity { 3 } else { 2 }, + i32::from(entry.is_spki), + if entry.is_sha256 { 1 } else { 2 }, + entry + .data + .iter() + .fold(String::with_capacity(64), |mut s, b| { + write!(s, "{b:02X}").ok(); + s + }) + )); + } + } + } + super::PolicyType::Sts(sts) => { + policy.policy_type = PolicyType::Sts; + if let Some(sts) = sts { + policy.policy_string.push("version: STSv1".to_string()); + policy.policy_string.push(format!( + "mode: {}", + match sts.mode { + Mode::Enforce => "enforce", + Mode::Testing => "testing", + Mode::None => "none", + } + )); + policy + .policy_string + .push(format!("max_age: {}", sts.max_age)); + for mx in &sts.mx { + let mx = match mx { + MxPattern::Equals(mx) => mx.to_string(), + MxPattern::StartsWith(mx) => format!("*.{mx}"), + }; + policy.policy_string.push(format!("mx: {mx}")); + policy.mx_host.push(mx); + } + } + } + _ => (), + } + + // Create report entry + let entry = TlsFormat { + rua: event.tls_record.rua.clone(), + policy, + records: vec![event.failure], + }; + let bytes_written = json_write(&path.path[pos].inner, &entry).await; + + if bytes_written > 0 { + path.size += bytes_written; + } else { + // Something went wrong, remove record + if let Entry::Occupied(mut e) = self + .reports + .entry(ReportType::Tls(entry.policy.policy_domain)) + { + if let ReportType::Tls(path) = e.get_mut() { + path.path.retain(|p| p.policy != policy_hash); + if path.path.is_empty() { + e.remove_entry(); + } + } + } + } + } else if path.size < *max_size { + // Append to existing report + path.size += + json_append(&path.path[pos].inner, &event.failure, *max_size - path.size).await; + } + } +} + +impl ReportPath>> { + fn cleanup_blocking(&self) { + for path in &self.path { + if let Err(err) = std::fs::remove_file(&path.inner) { + tracing::error!( + context = "report", + report = "tls", + event = "error", + "Failed to delete file {}: {}", + path.inner.display(), + err + ); + } + } + } +} diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index 96bbdabe..84826ce0 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -11,6 +11,8 @@ tokio = { version = "1.23", features = ["net", "macros"] } tokio-rustls = { version = "0.24.0"} serde = { version = "1.0", features = ["derive"]} tracing = "0.1" +mail-auth = { git = "https://github.com/stalwartlabs/mail-auth" } +smtp-proto = { git = "https://github.com/stalwartlabs/smtp-proto" } [target.'cfg(unix)'.dependencies] privdrop = "0.5.3" diff --git a/crates/utils/src/config/listener.rs b/crates/utils/src/config/listener.rs index 1309c7dc..0ab247b4 100644 --- a/crates/utils/src/config/listener.rs +++ b/crates/utils/src/config/listener.rs @@ -286,6 +286,12 @@ impl Config { .failed(&format!("No 'url' directive found for listener {id:?}")) .to_string() }, + max_connections: self + .property_or_default( + ("server.listener", id, "max-connections"), + "server.max-connections", + )? + .unwrap_or(8192), protocol, listeners, tls, @@ -356,149 +362,3 @@ impl ParseValue for SupportedCipherSuite { }) } } - -#[cfg(test)] -mod tests { - use std::{fs, path::PathBuf}; - - use tokio::net::TcpSocket; - - use crate::config::{Config, Listener, Server, ServerProtocol}; - - fn add_test_certs(config: &str) -> String { - let mut cert_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - cert_path.push("resources"); - cert_path.push("tests"); - cert_path.push("certs"); - let mut cert = cert_path.clone(); - cert.push("tls_cert.pem"); - let mut pk = cert_path.clone(); - pk.push("tls_privatekey.pem"); - - config - .replace("{CERT}", cert.as_path().to_str().unwrap()) - .replace("{PK}", pk.as_path().to_str().unwrap()) - } - - #[test] - fn parse_servers() { - let mut file = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - file.push("resources"); - file.push("tests"); - file.push("config"); - file.push("servers.toml"); - - let toml = add_test_certs(&fs::read_to_string(file).unwrap()); - - // Parse servers - let config = Config::parse(&toml).unwrap(); - let servers = config.parse_servers().unwrap(); - let expected_servers = vec![ - Server { - id: "smtp".to_string(), - internal_id: 0, - hostname: "mx.example.org".to_string(), - data: "Stalwart SMTP - hi there!".to_string(), - protocol: ServerProtocol::Smtp, - listeners: vec![Listener { - socket: TcpSocket::new_v4().unwrap(), - addr: "127.0.0.1:9925".parse().unwrap(), - ttl: 3600.into(), - backlog: 1024.into(), - }], - tls: None, - tls_implicit: false, - }, - Server { - id: "smtps".to_string(), - internal_id: 1, - hostname: "mx.example.org".to_string(), - data: "Stalwart SMTP - hi there!".to_string(), - protocol: ServerProtocol::Smtp, - listeners: vec![ - Listener { - socket: TcpSocket::new_v4().unwrap(), - addr: "127.0.0.1:9465".parse().unwrap(), - ttl: 4096.into(), - backlog: 1024.into(), - }, - Listener { - socket: TcpSocket::new_v4().unwrap(), - addr: "127.0.0.1:9466".parse().unwrap(), - ttl: 4096.into(), - backlog: 1024.into(), - }, - ], - tls: None, - tls_implicit: true, - }, - Server { - id: "submission".to_string(), - internal_id: 2, - hostname: "submit.example.org".to_string(), - data: "Stalwart SMTP submission at your service".to_string(), - protocol: ServerProtocol::Smtp, - listeners: vec![Listener { - socket: TcpSocket::new_v4().unwrap(), - addr: "127.0.0.1:9991".parse().unwrap(), - ttl: 3600.into(), - backlog: 2048.into(), - }], - tls: None, - tls_implicit: true, - }, - ]; - - for (server, expected_server) in servers.inner.into_iter().zip(expected_servers) { - assert_eq!( - server.id, expected_server.id, - "failed for {}", - expected_server.id - ); - assert_eq!( - server.internal_id, expected_server.internal_id, - "failed for {}", - expected_server.id - ); - assert_eq!( - server.hostname, expected_server.hostname, - "failed for {}", - expected_server.id - ); - assert_eq!( - server.data, expected_server.data, - "failed for {}", - expected_server.id - ); - assert_eq!( - server.protocol, expected_server.protocol, - "failed for {}", - expected_server.id - ); - assert_eq!( - server.tls_implicit, expected_server.tls_implicit, - "failed for {}", - expected_server.id - ); - for (listener, expected_listener) in - server.listeners.into_iter().zip(expected_server.listeners) - { - assert_eq!( - listener.addr, expected_listener.addr, - "failed for {}", - expected_server.id - ); - assert_eq!( - listener.ttl, expected_listener.ttl, - "failed for {}", - expected_server.id - ); - assert_eq!( - listener.backlog, expected_listener.backlog, - "failed for {}", - expected_server.id - ); - } - } - } -} diff --git a/crates/utils/src/config/mod.rs b/crates/utils/src/config/mod.rs index cf96ac29..d30e29c2 100644 --- a/crates/utils/src/config/mod.rs +++ b/crates/utils/src/config/mod.rs @@ -33,7 +33,7 @@ use tokio::net::TcpSocket; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Config { - keys: BTreeMap, + pub keys: BTreeMap, } #[derive(Debug, Default)] @@ -46,6 +46,7 @@ pub struct Server { pub listeners: Vec, pub tls: Option, pub tls_implicit: bool, + pub max_connections: u64, } pub struct Servers { diff --git a/crates/utils/src/config/utils.rs b/crates/utils/src/config/utils.rs index be5349e9..5c9a1ae2 100644 --- a/crates/utils/src/config/utils.rs +++ b/crates/utils/src/config/utils.rs @@ -21,7 +21,18 @@ * for more details. */ -use std::{net::IpAddr, time::Duration}; +use std::{ + net::{IpAddr, Ipv4Addr, Ipv6Addr}, + path::PathBuf, + time::Duration, +}; + +use mail_auth::{ + common::crypto::{Algorithm, HashAlgorithm}, + dkim::Canonicalization, + IpLookupStrategy, +}; +use smtp_proto::MtPriority; use super::{Config, Rate}; @@ -342,6 +353,111 @@ impl ParseValue for bool { } } +impl ParseValue for Ipv4Addr { + fn parse_value(key: impl AsKey, value: &str) -> super::Result { + value + .parse() + .map_err(|_| format!("Invalid IPv4 value {:?} for key {:?}.", value, key.as_key())) + } +} + +impl ParseValue for Ipv6Addr { + fn parse_value(key: impl AsKey, value: &str) -> super::Result { + value + .parse() + .map_err(|_| format!("Invalid IPv6 value {:?} for key {:?}.", value, key.as_key())) + } +} + +impl ParseValue for PathBuf { + fn parse_value(_key: impl AsKey, value: &str) -> super::Result { + let path = PathBuf::from(value); + + if path.exists() { + Ok(path) + } else { + Err(format!("Directory {} does not exist.", path.display())) + } + } +} + +impl ParseValue for MtPriority { + fn parse_value(key: impl AsKey, value: &str) -> super::Result { + match value.to_ascii_lowercase().as_str() { + "mixer" => Ok(MtPriority::Mixer), + "stanag4406" => Ok(MtPriority::Stanag4406), + "nsep" => Ok(MtPriority::Nsep), + _ => Err(format!( + "Invalid priority value {:?} for property {:?}.", + value, + key.as_key() + )), + } + } +} + +impl ParseValue for Canonicalization { + fn parse_value(key: impl AsKey, value: &str) -> super::Result { + match value { + "relaxed" => Ok(Canonicalization::Relaxed), + "simple" => Ok(Canonicalization::Simple), + _ => Err(format!( + "Invalid canonicalization value {:?} for key {:?}.", + value, + key.as_key() + )), + } + } +} + +impl ParseValue for IpLookupStrategy { + fn parse_value(key: impl AsKey, value: &str) -> super::Result { + Ok(match value.to_lowercase().as_str() { + "ipv4-only" => IpLookupStrategy::Ipv4Only, + "ipv6-only" => IpLookupStrategy::Ipv6Only, + //"ipv4-and-ipv6" => IpLookupStrategy::Ipv4AndIpv6, + "ipv6-then-ipv4" => IpLookupStrategy::Ipv6thenIpv4, + "ipv4-then-ipv6" => IpLookupStrategy::Ipv4thenIpv6, + _ => { + return Err(format!( + "Invalid IP lookup strategy {:?} for property {:?}.", + value, + key.as_key() + )) + } + }) + } +} + +impl ParseValue for Algorithm { + fn parse_value(key: impl AsKey, value: &str) -> super::Result { + match value { + "ed25519-sha256" | "ed25519-sha-256" => Ok(Algorithm::Ed25519Sha256), + "rsa-sha-256" | "rsa-sha256" => Ok(Algorithm::RsaSha256), + "rsa-sha-1" | "rsa-sha1" => Ok(Algorithm::RsaSha1), + _ => Err(format!( + "Invalid algorithm {:?} for key {:?}.", + value, + key.as_key() + )), + } + } +} + +impl ParseValue for HashAlgorithm { + fn parse_value(key: impl AsKey, value: &str) -> super::Result { + match value { + "sha256" | "sha-256" => Ok(HashAlgorithm::Sha256), + "sha-1" | "sha1" => Ok(HashAlgorithm::Sha1), + _ => Err(format!( + "Invalid hash algorithm {:?} for key {:?}.", + value, + key.as_key() + )), + } + } +} + impl ParseValue for Duration { fn parse_value(key: impl AsKey, value: &str) -> super::Result { let duration = value.trim_end().to_ascii_lowercase(); diff --git a/crates/utils/src/listener/listen.rs b/crates/utils/src/listener/listen.rs index c0bfb376..0a63b573 100644 --- a/crates/utils/src/listener/listen.rs +++ b/crates/utils/src/listener/listen.rs @@ -27,7 +27,7 @@ impl Server { hostname: self.hostname, tls_acceptor: self.tls.map(|config| TlsAcceptor::from(Arc::new(config))), is_tls_implicit: self.tls_implicit, - limiter: ConcurrencyLimiter::new(manager.max_concurrent()), + limiter: ConcurrencyLimiter::new(self.max_connections), shutdown_rx, }); diff --git a/crates/utils/src/listener/mod.rs b/crates/utils/src/listener/mod.rs index 3b7cc75e..4b72bf3b 100644 --- a/crates/utils/src/listener/mod.rs +++ b/crates/utils/src/listener/mod.rs @@ -37,5 +37,4 @@ pub struct SessionData { pub trait SessionManager: Sync + Send + 'static + Clone { fn spawn(&self, session: SessionData); - fn max_concurrent(&self) -> u64; } diff --git a/resources/config/config.toml b/resources/config/config.toml new file mode 100644 index 00000000..83debf0e --- /dev/null +++ b/resources/config/config.toml @@ -0,0 +1,470 @@ +[server] +hostname = "__HOST__" +#greeting = "Stalwart SMTP at your service" +protocol = "smtp" + +[server.run-as] +user = "stalwart-smtp" +group = "stalwart-smtp" + +[server.listener."smtp"] +bind = ["0.0.0.0:25"] +max-connections = 8192 + +[server.listener."submission"] +bind = ["0.0.0.0:587"] +max-connections = 8192 + +[server.listener."submissions"] +bind = ["0.0.0.0:465"] +max-connections = 8192 +tls.implicit = true + + +[server.listener."management"] +bind = ["127.0.0.1:8686"] +protocol = "http" + +[server.tls] +enable = true +implicit = false +timeout = "1m" +certificate = "default" +#sni = [{subject = "", certificate = ""}] +#protocols = ["TLSv1.2", TLSv1.3"] +#ciphers = [] +ignore-client-order = true + +[server.socket] +reuse-addr = true +#reuse-port = true +backlog = 1024 +#ttl = 3600 +#send-buffer-size = 65535 +#recv-buffer-size = 65535 +#linger = 1 +#tos = 1 + +[global] +shared-map = {shard = 32, capacity = 10} +#thread-pool = 8 + +#[global.tracing] +#method = "stdout" +#level = "trace" + +#[global.tracing] +#method = "open-telemetry" +#transport = "http" +#endpoint = "https://127.0.0.1/otel" +#headers = ["Authorization: "] +#level = "debug" + +[global.tracing] +method = "log" +path = "/usr/local/stalwart-smtp/logs" +prefix = "smtp.log" +rotate = "daily" +level = "info" + +[session] +timeout = "5m" +transfer-limit = 262144000 # 250 MB +duration = "10m" + +[session.connect] +#script = "connect.sieve" + +[session.ehlo] +require = true +reject-non-fqdn = [ { if = "listener", eq = "smtp", then = true}, + { else = false } ] +#script = "ehlo" + +[session.extensions] +pipelining = true +chunking = true +requiretls = true +no-soliciting = "" +dsn = [ { if = "authenticated-as", ne = "", then = true}, + { else = false } ] +future-release = [ { if = "authenticated-as", ne = "", then = "7d"}, + { else = false } ] +deliver-by = [ { if = "authenticated-as", ne = "", then = "15d"}, + { else = false } ] +mt-priority = [ { if = "authenticated-as", ne = "", then = "mixer"}, + { else = false } ] + +[session.auth] +mechanisms = [ { if = "listener", ne = "smtp", then = ["plain", "login"]}, + { else = [] } ] +lookup = [ { if = "listener", ne = "smtp", then = "remote/imap" }, + { else = false } ] +require = [ { if = "listener", ne = "smtp", then = true}, + { else = false } ] + +[session.auth.errors] +total = 3 +wait = "5s" + +[session.mail] +#script = "mail-from" + +[session.rcpt] +#script = "rcpt-to" +relay = [ { if = "authenticated-as", ne = "", then = true }, + { else = false } ] +max-recipients = 25 + +[session.rcpt.lookup] +domains = "list/domains" +addresses = "remote/lmtp" +vrfy = [ { if = "authenticated-as", ne = "", then = "remote/lmtp" }, + { else = false } ] +expn = [ { if = "authenticated-as", ne = "", then = "remote/lmtp" }, + { else = false } ] + +[session.rcpt.errors] +total = 5 +wait = "5s" + +[session.data] +#script = "data" + +#[session.data.pipe."spam-assassin"] +#command = "spamc" +#arguments = [] +#timeout = "10s" + +[session.data.limits] +messages = 10 +size = 104857600 +received-headers = 50 + +[session.data.add-headers] +received = [ { if = "listener", eq = "smtp", then = true }, + { else = false } ] +received-spf = [ { if = "listener", eq = "smtp", then = true }, + { else = false } ] +auth-results = [ { if = "listener", eq = "smtp", then = true }, + { else = false } ] +message-id = [ { if = "listener", eq = "smtp", then = false }, + { else = true } ] +date = [ { if = "listener", eq = "smtp", then = false }, + { else = true } ] +return-path = false + +[[session.throttle]] +#match = {if = "remote-ip", eq = "10.0.0.1"} +key = ["remote-ip"] +concurrency = 5 +#rate = "5/1h" + +[[session.throttle]] +key = ["sender-domain", "rcpt"] +rate = "25/1h" + +[auth.dnsbl] +verify = [ { if = "listener", eq = "smtp", then = ["ip", "iprev", "ehlo", "return-path", "from"] }, + { else = [] } ] +[auth.dnsbl.lookup] +ip = ["zen.spamhaus.org", "bl.spamcop.net", "b.barracudacentral.org"] +domain = ["dbl.spamhaus.org"] + +[auth.iprev] +verify = [ { if = "listener", eq = "smtp", then = "relaxed" }, + { else = "disable" } ] + +[auth.dkim] +verify = "relaxed" +sign = [ { if = "listener", ne = "smtp", then = ["rsa"] }, + { else = [] } ] + +[auth.spf.verify] +ehlo = [ { if = "listener", eq = "smtp", then = "relaxed" }, + { else = "disable" } ] +mail-from = [ { if = "listener", eq = "smtp", then = "relaxed" }, + { else = "disable" } ] + +[auth.arc] +verify = "relaxed" +seal = ["rsa"] + +[auth.dmarc] +verify = [ { if = "listener", eq = "smtp", then = "relaxed" }, + { else = "disable" } ] + +[queue] +path = "/usr/local/stalwart-smtp/queue" +hash = 64 + +[queue.schedule] +retry = ["2m", "5m", "10m", "15m", "30m", "1h", "2h"] +notify = ["1d", "3d"] +expire = "5d" + +[queue.outbound] +#hostname = "__HOST__" +next-hop = [ { if = "rcpt-domain", in-list = "list/domains", then = "lmtp" }, + { else = false } ] +ip-strategy = "ipv4-then-ipv6" + +[queue.outbound.tls] +dane = "optional" +mta-sts = "optional" +starttls = "require" + +#[queue.outbound.source-ip] +#v4 = ["10.0.0.10", "10.0.0.11"] +#v6 = ["a::b", "a::c"] + +[queue.outbound.limits] +mx = 7 +multihomed = 2 + +[queue.outbound.timeouts] +connect = "3m" +greeting = "3m" +tls = "2m" +ehlo = "3m" +mail-from = "3m" +rcpt-to = "3m" +data = "10m" +mta-sts = "2m" + +[[queue.quota]] +#match = {if = "sender-domain", eq = "foobar.org"} +#key = ["rcpt"] +messages = 100000 +size = 10737418240 # 10gb + +[[queue.throttle]] +key = ["rcpt-domain"] +#rate = "100/1h" +concurrency = 5 + +[resolver] +type = "system" +#preserve-intermediates = true +concurrency = 2 +timeout = "5s" +attempts = 2 +try-tcp-on-error = true + +[resolver.cache] +txt = 2048 +mx = 1024 +ipv4 = 1024 +ipv6 = 1024 +ptr = 1024 +tlsa = 1024 +mta-sts = 1024 + +[report] +path = "/usr/local/stalwart-smtp/reports" +hash = 64 +#submitter = "mx.domain.org" + +[report.analysis] +addresses = ["dmarc@*", "abuse@*"] +forward = true +#store = "/usr/local/stalwart-smtp/incoming" + +[report.dsn] +from-name = "Mail Delivery Subsystem" +from-address = "MAILER-DAEMON@__DOMAIN__" +sign = ["rsa"] + +[report.dkim] +from-name = "Report Subsystem" +from-address = "noreply-dkim@__DOMAIN__" +subject = "DKIM Authentication Failure Report" +sign = ["rsa"] +send = "1/1d" + +[report.spf] +from-name = "Report Subsystem" +from-address = "noreply-spf@__DOMAIN__" +subject = "SPF Authentication Failure Report" +send = "1/1d" +sign = ["rsa"] + +[report.dmarc] +from-name = "Report Subsystem" +from-address = "noreply-dmarc@__DOMAIN__" +subject = "DMARC Authentication Failure Report" +send = "1/1d" +sign = ["rsa"] + +[report.dmarc.aggregate] +from-name = "DMARC Report" +from-address = "noreply-dmarc@__DOMAIN__" +org-name = "__DOMAIN__" +#contact-info = "" +send = "daily" +max-size = 26214400 # 25mb +sign = ["rsa"] + +[report.tls.aggregate] +from-name = "TLS Report" +from-address = "noreply-tls@__DOMAIN__" +org-name = "__DOMAIN__" +#contact-info = "" +send = "daily" +max-size = 26214400 # 25 mb +sign = ["rsa"] + +[signature."rsa"] +#public-key = "file:///usr/local/stalwart-smtp/etc/certs/dkim.crt" +private-key = "file:///usr/local/stalwart-smtp/etc/private/dkim.key" +domain = "__DOMAIN__" +selector = "stalwart_smtp" +headers = ["From", "To", "Date", "Subject", "Message-ID"] +algorithm = "rsa-sha256" +canonicalization = "relaxed/relaxed" +#expire = "10d" +#third-party = "" +#third-party-algo = "" +#auid = "" +set-body-length = false +report = true + +[remote."lmtp"] +address = "__LMTP_HOST__" +port = __LMTP_PORT__ +protocol = "lmtp" +concurrency = 10 +timeout = "1m" +lookup = true + +[remote."lmtp".cache] +entries = 1000 +ttl = {positive = "1d", negative = "1h"} + +[remote."lmtp".tls] +implicit = false +allow-invalid-certs = true + +#[remote."lmtp".auth] +#username = "" +#secret = "" + +[remote."lmtp".limits] +errors = 3 +requests = 50 + +[remote."imap"] +address = "localhost" +port = 143 +protocol = "imap" +concurrency = 10 +timeout = "1m" +lookup = true + +[remote."imap".cache] +entries = 1000 +ttl = {positive = "1d", negative = "1h"} + +[remote."imap".tls] +implicit = false +allow-invalid-certs = true + +[database."sql"] +#address = "sqlite:///usr/local/stalwart-smtp/etc/sqlite.db?mode=rwc" +address = "postgres://postgres:password@localhost/test" +max-connections = 10 +min-connections = 0 +idle-timeout = "5m" + +[database."sql".lookup] +auth = "SELECT secret FROM users WHERE email=?" +rcpt = "SELECT EXISTS(SELECT 1 FROM users WHERE email=? LIMIT 1)" +vrfy = "SELECT email FROM users WHERE email LIKE '%' || ? || '%' LIMIT 5" +expn = "SELECT member FROM mailing_lists WHERE id = ?" +domains = "SELECT EXISTS(SELECT 1 FROM domains WHERE name=? LIMIT 1)" + +[database."sql".cache] +enable = ["rcpt", "domains"] +entries = 1000 +ttl = {positive = "1d", negative = "1h"} + +[sieve] +from-name = "Automated Message" +from-addr = "no-reply@__DOMAIN__" +return-path = "" +#hostname = "__HOST__" +sign = ["rsa"] +use-database = "sql" + +[sieve.limits] +redirects = 3 +out-messages = 5 +received-headers = 50 +cpu = 10000 +nested-includes = 5 +duplicate-expiry = "7d" + +[sieve.scripts] +# Note: These scripts are included here for demonstration purposes. +# They should not be used in their current form. +connect = ''' + require ["variables", "extlists", "reject"]; + + if string :list "${env.remote_ip}" "list/blocked-ips" { + reject "Your IP '${env.remote_ip}' is not welcomed here."; + } +''' +ehlo = ''' + require ["variables", "extlists", "reject"]; + + if string :list "${env.helo_domain}" "list/blocked-domains" { + reject "551 5.1.1 Your domain '${env.helo_domain}' has been blacklisted."; + } +''' +mail = ''' + require ["variables", "envelope", "reject"]; + + if envelope :localpart :is "from" "known_spammer" { + reject "We do not accept SPAM."; + } +''' +rcpt = ''' + require ["variables", "vnd.stalwart.execute", "envelope", "reject"]; + + set "triplet" "${env.remote_ip}.${envelope.from}.${envelope.to}"; + + if not execute :query "SELECT EXISTS(SELECT 1 FROM greylist WHERE addr=? LIMIT 1)" ["${triplet}"] { + execute :query "INSERT INTO greylist (addr) VALUES (?)" ["${triplet}"]; + reject "422 4.2.2 Greylisted, please try again in a few moments."; + } +''' +data = ''' + require ["envelope", "variables", "replace", "mime", "foreverypart", "editheader", "extracttext"]; + + if envelope :domain :is "to" "foobar.net" { + set "counter" "a"; + foreverypart { + if header :mime :contenttype "content-type" "text/html" { + extracttext :upper "text_content"; + replace "${text_content}"; + } + set :length "part_num" "${counter}"; + addheader :last "X-Part-Number" "${part_num}"; + set "counter" "${counter}a"; + } + } +''' + +[management.auth] +lookup = "list/admin" + +[list] +domains = ["__DOMAIN__"] +admin = ["admin:__ADMIN_PASS__"] +#blocked-ips = ["10.0.0.1"] +#blocked-domains = ["mail.spammer.com"] +#users = "file:///usr/local/stalwart-smtp/etc/users.txt" + +[certificate."default"] +cert = "file:///usr/local/stalwart-smtp/etc/certs/tls.crt" +private-key = "file:///usr/local/stalwart-smtp/etc/private/tls.key" diff --git a/resources/config/stalwart-config.zip b/resources/config/stalwart-config.zip new file mode 100644 index 00000000..68603260 Binary files /dev/null and b/resources/config/stalwart-config.zip differ diff --git a/resources/systemd/stalwart-smtp.service b/resources/systemd/stalwart-smtp.service new file mode 100644 index 00000000..415a40f8 --- /dev/null +++ b/resources/systemd/stalwart-smtp.service @@ -0,0 +1,21 @@ +[Unit] +Description=Stalwart SMTP +Conflicts=postfix.service sendmail.service exim4.service +ConditionPathExists=/usr/local/stalwart-smtp/etc/config.toml +After=network-online.target + +[Service] +Type=simple +LimitNOFILE=65536 +KillMode=process +KillSignal=SIGINT +Restart=on-failure +RestartSec=5 +ExecStart=/usr/local/stalwart-smtp/bin/stalwart-smtp --config=/usr/local/stalwart-smtp/etc/config.toml +PermissionsStartOnly=true +StandardOutput=syslog +StandardError=syslog +SyslogIdentifier=stalwart-smtp + +[Install] +WantedBy=multi-user.target diff --git a/resources/systemd/stalwart.smtp.plist b/resources/systemd/stalwart.smtp.plist new file mode 100644 index 00000000..4279dfcc --- /dev/null +++ b/resources/systemd/stalwart.smtp.plist @@ -0,0 +1,20 @@ + + + + + Label + stalwart.smtp + ServiceDescription + Stalwart SMTP Server + ProgramArguments + + /usr/local/stalwart-smtp/bin/stalwart-smtp + --config=/usr/local/stalwart-smtp/etc/config.toml + + RunAtLoad + + KeepAlive + + + diff --git a/tests/Cargo.toml b/tests/Cargo.toml index a60598bb..defc0792 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -8,12 +8,19 @@ resolver = "2" store = { path = "../crates/store", features = ["test_mode"] } jmap = { path = "../crates/jmap", features = ["test_mode"] } jmap_proto = { path = "../crates/jmap-proto" } -mail-send = { git = "https://github.com/stalwartlabs/mail-send" } +smtp = { path = "../crates/smtp", features = ["test_mode"] } +smtp-proto = { git = "https://github.com/stalwartlabs/smtp-proto" } +mail-send = { git = "https://github.com/stalwartlabs/mail-send" } +mail-auth = { git = "https://github.com/stalwartlabs/mail-auth", features = ["test"] } +sieve-rs = { git = "https://github.com/stalwartlabs/sieve" } utils = { path = "../crates/utils" } #jmap-client = { git = "https://github.com/stalwartlabs/jmap-client", features = ["websockets", "debug", "async"] } jmap-client = { path = "/home/vagrant/code/jmap-client", features = ["websockets", "debug", "async"] } mail-parser = { git = "https://github.com/stalwartlabs/mail-parser", features = ["full_encoding", "serde_support", "ludicrous_mode"] } tokio = { version = "1.23", features = ["full"] } +tokio-rustls = { version = "0.24.0"} +rustls = "0.21.0" +rustls-pemfile = "1.0" csv = "1.1" rayon = { version = "1.5.1" } flate2 = { version = "1.0.17", features = ["zlib"], default-features = false } @@ -28,3 +35,9 @@ ece = "2.2" hyper = { version = "1.0.0-rc.3", features = ["server", "http1", "http2"] } http-body-util = "0.1.0-rc.2" base64 = "0.21" +dashmap = "5.4" +ahash = { version = "0.8" } +serial_test = "2.0.0" +sqlx = { version = "0.7.0-alpha.3", features = [ "runtime-tokio-rustls", "postgres", "mysql", "sqlite" ] } +num_cpus = "1.15.0" +async-trait = "0.1.68" diff --git a/tests/resources/smtp/certs/tls_cert.pem b/tests/resources/smtp/certs/tls_cert.pem new file mode 100644 index 00000000..02b9b963 --- /dev/null +++ b/tests/resources/smtp/certs/tls_cert.pem @@ -0,0 +1,29 @@ +-----BEGIN CERTIFICATE----- +MIIFCTCCAvGgAwIBAgIUCgHGQYUqtelbHGVSzCVwBL3fyEUwDQYJKoZIhvcNAQEL +BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTIyMDUxNjExNDAzNFoXDTIzMDUx +NjExNDAzNFowFDESMBAGA1UEAwwJbG9jYWxob3N0MIICIjANBgkqhkiG9w0BAQEF +AAOCAg8AMIICCgKCAgEAtwS0Fzl3SjaCuKEXgZ/fdWbDoj/qDphyNCAKNevQ0+D0 +STNkWCO04aFSH0zcL8zoD9gokNos0i7OU9//ZhZQmex4V6EFdZn8bFwUWN/scUvW +HEFXVjtHldO2isZgIxH9LuwRv7KAgkISuWahqerOVDhe7SeQUV0AJGNEh3cT9PZr +gSY931BxB7n+5k8eoSk8Z1gtBzQzL62kVGpHDKfw8yX8m65owF9eLUBrNzgxmXfC +xpuHwj7hmVhS09PPKeN/RsFS8PsYO7bo0u8jEKalteumjRT7RyUEbioqfo6ZFOGj +FHPIq/uKXS9zN1fpoyNh3ur5hMznQhrqlwBM9KlM7GdBJ0pZ3ad0YjT8IL/GnGKR +85J2WZdLqaQdUZo7nV67FhqdDlNE4MdwiykTMjfmLRXGAVhAzJHKyRKNwmkI2aqe +S7aqeNgvuDBwY80Q9a2rb5py1Aw+L8yCkUBuHboToDpxSVRDNN8DrWNmmsXnxsOG +wRDODy4GICKyxlP+RFSM8xWSQ6y9ktS2OfDBm+Eqcw+3pZKhdz2wgxLkUBJ8X1eh +kJrCA/6LTuhy6m6mMjAfoSOFU7fu88jxaWPgvP7GKyH+LM/t9eucobz2ks5rtSjz +V4Dc5DCS94/OpVRHwHdaFSPbJKBN9Ev8gnNrAyx/aBPGoHBPG/QUiU7dcUNIPt0C +AwEAAaNTMFEwHQYDVR0OBBYEFI167IxBmErB11EqiPPqFLa31ZaMMB8GA1UdIwQY +MBaAFI167IxBmErB11EqiPPqFLa31ZaMMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI +hvcNAQELBQADggIBALU00IOiH5ubEauVCmakms5ermNTZfculnhnDfWTLMeh2+a7 +G4cqADErfMhm/mmLbrw33t9s6tCAhQltvewKR40ST9uMPSyiQbYaCXd5DXnuI6Ox +JtNW+UOWIaMf8abnkdLvREOvb8dVQS1i3xq14tAjY5XgpGwCPP8m54b7N3Q7soLn +e5PDhPNTnhRIn2RLuYoZmQmMA5fcqEUDYff4epUww7PhrM1QckZligI3566NlGOf +j1G9JrivBtY0eaJtamIFnGMBT0ThDudxVja2Nv0C2Elry0p4T/o4nc4M67BJ/y1R +vjNLAgFhbxssemU3lZqSd+pykpJBwDBjFSPrZZmQcbk7H6Uz8V1xr/xuzfw6fA13 +NWZ5vLgP/DQ13sM+XFlxThKfbPMPVe/UCTvfGtNW+3XyBgPntEkR+fNEawQmzbYl +R+X1ymT9MZnEZqRMf7/UD/SYek1aUJefoew3upjMgxYVvh4F8dqJ+39F+xoFzIA2 +1dDAEMzXtjA3zKhZ2cycZbEzpJvYA3eGLuR16Suqfi4kPvfwK0mOhCxQmpayt7/X +vuEzW6dPCH8Hgbb0WvsSppGOvhdbDaZFNfFc5eNSxhyKzu3H3ACNImZRtZE+yixx +0fR8+xz9kDLf8xupV+X9heyFGHSyYU2Lveaevtr2Ij3weLRgJ6LbNALoeKXk +-----END CERTIFICATE----- diff --git a/tests/resources/smtp/certs/tls_privatekey.pem b/tests/resources/smtp/certs/tls_privatekey.pem new file mode 100644 index 00000000..3f9ff47f --- /dev/null +++ b/tests/resources/smtp/certs/tls_privatekey.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQC3BLQXOXdKNoK4 +oReBn991ZsOiP+oOmHI0IAo169DT4PRJM2RYI7ThoVIfTNwvzOgP2CiQ2izSLs5T +3/9mFlCZ7HhXoQV1mfxsXBRY3+xxS9YcQVdWO0eV07aKxmAjEf0u7BG/soCCQhK5 +ZqGp6s5UOF7tJ5BRXQAkY0SHdxP09muBJj3fUHEHuf7mTx6hKTxnWC0HNDMvraRU +akcMp/DzJfybrmjAX14tQGs3ODGZd8LGm4fCPuGZWFLT088p439GwVLw+xg7tujS +7yMQpqW166aNFPtHJQRuKip+jpkU4aMUc8ir+4pdL3M3V+mjI2He6vmEzOdCGuqX +AEz0qUzsZ0EnSlndp3RiNPwgv8acYpHzknZZl0uppB1RmjudXrsWGp0OU0Tgx3CL +KRMyN+YtFcYBWEDMkcrJEo3CaQjZqp5Ltqp42C+4MHBjzRD1ratvmnLUDD4vzIKR +QG4duhOgOnFJVEM03wOtY2aaxefGw4bBEM4PLgYgIrLGU/5EVIzzFZJDrL2S1LY5 +8MGb4SpzD7elkqF3PbCDEuRQEnxfV6GQmsID/otO6HLqbqYyMB+hI4VTt+7zyPFp +Y+C8/sYrIf4sz+3165yhvPaSzmu1KPNXgNzkMJL3j86lVEfAd1oVI9skoE30S/yC +c2sDLH9oE8agcE8b9BSJTt1xQ0g+3QIDAQABAoICABq5oxqpF5RMtXYEgAw7rkPU +h8jPkHwlIrgd3Z/WGZ53APUXfhWo0ScJiZZsgNKyF0kJBZNxaI4gq5xv3zmnFIoF +j+Ur7EIqBERGheoceMhqjI9/syMycNeeHM/S/ALjA5ewfT8C7+UVhOpx5DWNxidi +O+phlp9q9zRZEo69grqIqVYooWxUsMyyCljTQOPDw8BLjfe5VagmsRJqmolslLDM +4UBSjZVZ18S/3Wgo2oVQia660244BHWCAkZQbbXuNI2+eUAbSoSdxw3WQcaSrywL +hzyezbqr2yPDIIVuiUgVUt0Ps0P57VCCN07jlYhvCEGnClysFzD+ATefoZ0wg7za +dQu2E+d166rAjnssyhzcHMn3pxgSdtXD+dQR/xfIGbPABucCupEFqKmhLdMm9+ud +lHay87qzMpIa8cITJwEQROfXqWAhNUU98pKCOx1SVXBqQC7QVqGQ5solDf0eMSVh +ngQ6Dz2WUI2ty75LteiFwlyTgnU9nyPN0NXsrMEET2BHWre7ufTQqiULtQ7+9BwH +AMxEKvrQHjMUjdfbXuzdyc5w5mPYJZfFVSQ1HMslx66h9yCpRIsBZvUGvoaP8Tpe +nQ66FTYRbiOkkdJ7k8DtrnhsJI1oOGjnvj/rvZ8D2pvrlJcIH2AyN3MOL8Jp5Oj1 +nCFt77TwpF92pgl0g9gBAoIBAQDcarmP54QboaIQ9S2gE/4gSVC5i44iDJuSRdI8 +K081RQcWiNzqQXTRc5nqJ7KzLyPiGlg+6rWsBKLos5l4t+MdhhH+KUvk/OtT/g8V +0NZBNXLIbSb8j8ix4v3/f2qKHN3Co6QOlxb3gFvobKDdoKqUNiSH1zTZ8/Y/BzkM +jqWKhTdaLz6eyzhKfOTA4LO8kJ3VF8HUM1N9/e8Gjorl+gZpJUXUQS0+AIi8W76C +OwDrVb3BPGVnApQJfWF78h4g20RwXrx/GYUW2vOMcLjXXDV5U7+nobPUoJnLxoZC +16o88y0Ivan8dBNXsc1epyPvvEqp6MJbAyyVuNeuRJcgYA0BAoIBAQDUkGRV7fLG +wCr5rNysUO+FKzVtTJnf9KEsqAqUmmVnG4oubxAJJtiB5n2+DT+CtO8Nrtz05BbR +uxfWm+lbEw6lVMj63bywtp0NdULg7/2t+oq2Svv16KrZIRJttXMkdEiFFmkVAEhX +l8Fyl6PJPfSMwbPdXEUPUAaNrXweVFffXczHc4W2G212ZzDB0z7QQSgEntbTDFB/ +2Cg5dvuojlM9zw0fuEyLwItZs7n16j/ONZLgBHyroMU9ZPxbnLrVyoZlqtob+RWm +Ju2fSIL9QqG6O4td1TqcUBGvFQYjGvKA+q5fsG26NBJ0Ac48cNK6PS4lMkN3Av2J +ccloYaMEHAXdAoIBAE8WMCy1Ok6byUXiYxOL+OPmyoM40q/e7DcovE2AkLQhZ3Cr +fPDEucCphPFiexkV8f8fysgQeU0WgMmUH54UBPbD81LJyISKR3nkr875Ftdg8SV/ +HL0EblN9ifuR4U1bHCrJgoUFq2T09oVH7NR44Ju7bZIcIseNZK6qzcp2qGkycXD3 +gLWDX1hCxeV6+qLPFQKvuomEPRH4+jnVDXuFIaW6jPqixDP6BxXmqU2bFDJcmnBq +VkwGvc1F4qORdUP+yOi05VeJdZqEx1x92aTUXg+BgEQKnjbNxUE7o1L6hQfHjUIU +o5iEoagWkQTEXf2YBwY+EPaNBgNWxnSuAbfJHwECggEBALOF95ezTVWauzD/U6ic ++o3n/kl/Zn4FJ5KFodn7xCSe18d7uXlhO34KYqx+l+MWWMefpbGWacdcUjfImf93 +SulLgCqP12sP7/iLzp4XUpL7hOeM0NvRU2nqSpwpoUNqik0Mrlc0U+TWoGTduVCf +aMjwV65e3VyfY8mIeclLxqM5n1fcM1OoOnzDjiRE+0n7nYa5eAnq3pn6v4449TZY +belH03e0ucFWLtrltesBmj3YdWGJqJlzQOInRhNBfXJOh8+ZynfRmP0o54udPDQV +cG3PGFd5XPTjkuvhv7sqaSGRlm/um92lWOhtFfdp+i+cuDpmByCef+7zEP19aKZx +3GkCggEAFTs7KNMfvIEaLH0yQUFeq2gLmtcMofmOmeoIECycN1rG7iJo07lJLIs0 +bVODH8Z0kX8llu3cjGMAH/6R2uugJSxkmFiZKrngTzKmxDPvTCKWR4RFwXH9j8IO +cPq7FtKN4SgrPy9ciAPdkcGmu3zz/sBKOaoPwvU2PdBRT+v/aoz+GCLXAvzFlKVe +9/7zdg87ilo8+AtV+71EJeR3kyBPKS9JrWYUKfiams12+uuH4/53rMFZfNCAaZ3Z +1sdXEO4o3Loc5TX4DbO9FVdBSBe6klEXx4T0QJboO6uBvTBnnRL2SQriJQQFwYT6 +XzVV5pwOxkIDBWDIqMUfwJDChBKfpw== +-----END PRIVATE KEY----- diff --git a/tests/resources/smtp/config/if-blocks.toml b/tests/resources/smtp/config/if-blocks.toml new file mode 100644 index 00000000..05bd7507 --- /dev/null +++ b/tests/resources/smtp/config/if-blocks.toml @@ -0,0 +1,40 @@ +durations = [ + {if = "sender", eq = "jdoe", then = "5d"}, + {any-of = [{if = "priority", eq = -1}, {if = "rcpt", starts-with = "jane"}], then = "1h"}, + {else = false} +] + +string-list = [ + {if = "sender", eq = "jdoe", then = ["From", "To", "Date"]}, + {any-of = [{if = "priority", eq = -1}, {if = "rcpt", starts-with = "jane"}], then = "Other-ID"}, + {else = []} +] + +string-list-bis = [ + {if = "sender", eq = "jdoe", then = ["From", "To", "Date"]}, + {any-of = [{if = "priority", eq = -1}, {if = "rcpt", starts-with = "jane"}], then = []}, + {else = ["ID-Bis"]} +] + +single-value = "hello world" + +bad-multi-value = [ + {if = "sender", eq = "jdoe", then = 100}, + {any-of = [{if = "priority", eq = -1}, {if = "rcpt", starts-with = "jane"}], then = [1, 2, 3]}, + {else = 2} +] + +bad-if-without-then = [ + {if = "sender", eq = "jdoe"}, + {else = 1} +] + +bad-if-without-else = [ + {if = "sender", eq = "jdoe", then = 1} +] + +bad-multiple-else = [ + {if = "sender", eq = "jdoe", then = 1}, + {else = 1}, + {else = 2} +] diff --git a/tests/resources/smtp/config/lists.toml b/tests/resources/smtp/config/lists.toml new file mode 100644 index 00000000..24b562d1 --- /dev/null +++ b/tests/resources/smtp/config/lists.toml @@ -0,0 +1,23 @@ +[list] +local-domains = ["example.org", "example.net"] +spammer-domains = "thatdomain.net" +local-users = "file://{LIST1}" +power-users = ["file://{LIST1}", "file://{LIST2}"] + +[remote."lmtp"] +address = 192.168.0.1 +port = 25 +protocol = "lmtp" +lookup = true + +[remote."lmtp".auth] +username = "hello" +secret = "world" + +[remote."lmtp".cache] +entries = 1000 +ttl = {positive = 10, negative = 5} + +[remote."lmtp".tls] +implicit = true +allow-invalid-certs = true diff --git a/tests/resources/smtp/config/rules-eval.toml b/tests/resources/smtp/config/rules-eval.toml new file mode 100644 index 00000000..147464a1 --- /dev/null +++ b/tests/resources/smtp/config/rules-eval.toml @@ -0,0 +1,168 @@ +[envelope] +rcpt-domain = "example.org" +rcpt = "user@example.org" +sender-domain = "foo.net" +sender = "bill@foo.net" +local-ip = "192.168.9.3" +remote-ip = "A:B:C::D:E" +mx = "mx.somedomain.com" +authenticated-as = "john@foobar.org" +priority = -4 +listener = 123 +helo-domain = "hi-domain.net" + +[rule] +"eq-true" = {if = "rcpt-domain", eq = "example.org"} +"eq-false" = {if = "rcpt-domain", eq = "example.com"} +"listener-eq-true" = {if = "listener", eq = "smtp"} +"listener-eq-false" = {if = "listener", eq = "smtps"} +"ip-eq-true" = {if = "local-ip", eq = "192.168.9.0/24"} +"ip-eq-false" = {if = "remote-ip", eq = "A:B:C::D:F/128"} +"ne-true" = {if = "authenticated-as", ne = ""} +"ne-false" = {if = "authenticated-as", ne = "john@foobar.org"} +"starts-with-true" = {if = "mx", starts-with = "mx.some"} +"starts-with-false" = {if = "mx", starts-with = "enchilada"} +"ends-with-true" = {if = "sender", ends-with = "@foo.net"} +"ends-with-false" = {if = "sender", ends-with = "chimichanga"} +"in-list-true" = {if = "sender-domain", in-list = "list/domains"} +"in-list-false" = {if = "rcpt-domain", in-list = "list/domains"} +"not-in-list-true" = {if = "rcpt-domain", not-in-list = "list/domains"} +"not-in-list-false" = {if = "sender-domain", not-in-list = "list/domains"} +"regex-true" = {if = "sender", matches = "^(.+)@(.+)$"} +"regex-false" = {if = "mx", matches = "/^\\S+@\\S+\\.\\S+$/"} + +"any-of-true" = { any-of = [ + {if = "authenticated-as", ne = "john@foobar.org"}, + {if = "rcpt-domain", eq = "example.org"}, + {if = "mx", starts-with = "mx.some"}, +]} +"any-of-false" = { any-of = [ + {if = "authenticated-as", eq = "something else"}, + {if = "rcpt-domain", eq = "something else"}, + {if = "mx", starts-with = "something else"}, +]} +"all-of-true" = { all-of = [ + {if = "rcpt-domain", eq = "example.org"}, + {if = "listener", eq = "smtp"}, + {if = "mx", starts-with = "mx.some"} +]} +"all-of-false" = { all-of = [ + {if = "rcpt-domain", eq = "example.org"}, + {if = "listener", eq = "smtp"}, + {if = "mx", starts-with = "something else"} +]} +"none-of-true" = { none-of = [ + {if = "authenticated-as", eq = "something else"}, + {if = "rcpt-domain", eq = "something else"}, + {if = "mx", starts-with = "something else"}, +]} +"none-of-false" = { none-of = [ + {if = "rcpt-domain", eq = "example.org"}, + {if = "listener", eq = "smtp"}, + {if = "mx", starts-with = "mx.some"} +]} +nested-any-of-true = { any-of = [ + { all-of = [ + {if = "rcpt-domain", eq = "example.org"}, + {if = "listener", eq = "smtp"}, + {if = "mx", starts-with = "something else"} + ]}, + { none-of = [ + {if = "rcpt-domain", eq = "example.org"}, + {if = "listener", eq = "smtp"}, + {if = "mx", starts-with = "mx.some"} + ]}, + { any-of = [ + {if = "authenticated-as", ne = "john@foobar.org"}, + {if = "rcpt-domain", eq = "example.org"}, + {if = "mx", starts-with = "mx.some"}, + ]} +]} +nested-any-of-false = { any-of = [ + { none-of = [ + {if = "rcpt-domain", eq = "example.org"}, + {if = "listener", eq = "smtp"}, + {if = "mx", starts-with = "mx.some"} + ]}, + { all-of = [ + {if = "rcpt-domain", eq = "example.org"}, + {if = "listener", eq = "smtp"}, + {if = "mx", starts-with = "something else"} + ]}, + { any-of = [ + {if = "authenticated-as", eq = "something else"}, + {if = "rcpt-domain", eq = "something else"}, + {if = "mx", starts-with = "something else"}, + ]} +]} +nested-all-of-true = { all-of = [ + { any-of = [ + {if = "authenticated-as", ne = "john@foobar.org"}, + {if = "rcpt-domain", eq = "example.org"}, + {if = "mx", starts-with = "mx.some"}, + ]}, + { all-of = [ + {if = "rcpt-domain", eq = "example.org"}, + {if = "listener", eq = "smtp"}, + {if = "mx", starts-with = "mx.some"} + ]}, + { none-of = [ + {if = "authenticated-as", eq = "something else"}, + {if = "rcpt-domain", eq = "something else"}, + {if = "mx", starts-with = "something else"}, + ]} +]} +nested-all-of-false = { all-of = [ + { any-of = [ + {if = "authenticated-as", ne = "john@foobar.org"}, + {if = "rcpt-domain", eq = "example.org"}, + {if = "mx", starts-with = "mx.some"}, + ]}, + { all-of = [ + {if = "rcpt-domain", eq = "example.org"}, + {if = "listener", eq = "smtp"}, + {if = "mx", starts-with = "mx.some"} + ]}, + { none-of = [ + {if = "rcpt-domain", eq = "example.org"}, + {if = "listener", eq = "smtp"}, + {if = "mx", starts-with = "mx.some"} + ]} +]} +nested-none-of-true = { none-of = [ + { none-of = [ + {if = "rcpt-domain", eq = "example.org"}, + {if = "listener", eq = "smtp"}, + {if = "mx", starts-with = "mx.some"} + ]}, + { all-of = [ + {if = "rcpt-domain", eq = "example.org"}, + {if = "listener", eq = "smtp"}, + {if = "mx", starts-with = "something else"} + ]}, + { any-of = [ + {if = "authenticated-as", eq = "something else"}, + {if = "rcpt-domain", eq = "something else"}, + {if = "mx", starts-with = "something else"}, + ]} +]} +nested-none-of-false = { none-of = [ + { any-of = [ + {if = "authenticated-as", ne = "john@foobar.org"}, + {if = "rcpt-domain", eq = "example.org"}, + {if = "mx", starts-with = "mx.some"}, + ]}, + { all-of = [ + {if = "rcpt-domain", eq = "example.org"}, + {if = "listener", eq = "smtp"}, + {if = "mx", starts-with = "mx.some"} + ]}, + { none-of = [ + {if = "authenticated-as", eq = "something else"}, + {if = "rcpt-domain", eq = "something else"}, + {if = "mx", starts-with = "something else"}, + ]} +]} + +[list] +domains = ["mydomain1.org", "foo.net", "otherdomain.net"] diff --git a/tests/resources/smtp/config/rules.toml b/tests/resources/smtp/config/rules.toml new file mode 100644 index 00000000..83f9fa63 --- /dev/null +++ b/tests/resources/smtp/config/rules.toml @@ -0,0 +1,29 @@ +[rule] +"my-nested-rule" = { any-of = [ + {if = "rcpt-domain", eq = "example.org"}, + {if = "remote-ip", eq = "192.168.0.0/24"}, + {all-of = [ + {if = "rcpt", starts-with = "no-reply@"}, + {if = "sender", ends-with = "@domain.org"}, + {none-of = [ + {if = "priority", eq = 1}, + {if = "priority", ne = -2}, + ]} + ]} +]} + +[rule."simple"] +if = "listener" +eq = "smtp" + +[rule."is-authenticated"] +if = "authenticated-as" +ne = "" + +[[rule."expanded".all-of]] +if = "sender-domain" +starts-with = "example" + +[[rule."expanded".all-of]] +if = "sender" +in-list = "test-list" diff --git a/tests/resources/smtp/config/servers.toml b/tests/resources/smtp/config/servers.toml new file mode 100644 index 00000000..c01a8e47 --- /dev/null +++ b/tests/resources/smtp/config/servers.toml @@ -0,0 +1,51 @@ +[server] +hostname = "mx.example.org" +greeting = "Stalwart SMTP - hi there!" +protocol = "smtp" + +[server.listener."smtp"] +bind = ["127.0.0.1:9925"] +tls.implicit = false + +[server.listener."smtps"] +bind = ["127.0.0.1:9465", "127.0.0.1:9466"] +max-connections = 1024 +tls.implicit = true +tls.ciphers = ["TLS13_CHACHA20_POLY1305_SHA256", "TLS13_AES_256_GCM_SHA384"] +socket.ttl = 4096 + +[server.listener."submission"] +greeting = "Stalwart SMTP submission at your service" +hostname = "submit.example.org" +bind = "127.0.0.1:9991" +#tls.sni = [{subject = "submit.example.org", certificate = "other"}, +# {subject = "submission.example.org", certificate = "other"}] +socket.backlog = 2048 + +[server.tls] +enable = true +implicit = true +timeout = 300 +certificate = "default" +#sni = [{subject = "other.domain.org", certificate = "default"}] +protocols = ["TLSv1.2", "TLSv1.3"] +ciphers = [] +ignore_client_order = true + +[server.socket] +reuse-addr = true +reuse-port = true +backlog = 1024 +ttl = 3600 +send-buffer-size = 65535 +recv-buffer-size = 65535 +linger = 1 +tos = 1 + +[certificate."default"] +cert = "file://{CERT}" +private-key = "file://{PK}" + +[certificate."other"] +cert = "file://{CERT}" +private-key = "file://{PK}" diff --git a/tests/resources/smtp/config/throttle.toml b/tests/resources/smtp/config/throttle.toml new file mode 100644 index 00000000..3e9dcbaa --- /dev/null +++ b/tests/resources/smtp/config/throttle.toml @@ -0,0 +1,10 @@ +[[throttle]] +match = {if = "remote-ip", eq = "127.0.0.1"} +key = ["remote-ip", "authenticated-as"] +concurrency = 100 +rate = "50/30s" + +[[throttle]] +key = "sender-domain" +concurrency = 10000 + diff --git a/tests/resources/smtp/config/toml-parser.toml b/tests/resources/smtp/config/toml-parser.toml new file mode 100644 index 00000000..2418ff78 --- /dev/null +++ b/tests/resources/smtp/config/toml-parser.toml @@ -0,0 +1,61 @@ +[database] +enabled = true # ignore +ports = [ 8000, 8001, 8002 ] # ignore +data = [ ["delta", "phi"], [3.14] ] +temp_targets = { cpu = 79.5, case = 72.0 } + +[servers] +"127.0.0.1" = "value" # ignore +"character encoding" = "value" + +[servers.alpha] +ip = "10.0.0.1" +role = "frontend" + +[servers.beta] +ip = "10.0.0.2" +role = "backend" + +[[products]] +name = "Hammer" +sku = 738594937 + +[[products]] # empty table within the array + +[[products]] # ignore +name = "Nail" +sku = 284758393 # ignore +color = "gray" + +[strings."my \"string\" test"] +str1 = "I'm a string." +str2 = "You can \"quote\" me." +str3 = "Name\tTabs\nNew Line." +lines = ''' +The first newline is +trimmed in raw strings. +All other whitespace +is preserved. +''' + +[arrays] +integers = [ 1, 2, 3 ] +colors = [ "red", "yellow", "green" ] +nested_arrays_of_ints = [ [ 1, 2 ], [3, 4, 5] ] +nested_mixed_array = [ [ 1, 2 ], ["a", "b", "c"] ] +string_array = [ "all", 'strings', """are the same""", '''type''' ] + +# Mixed-type arrays are allowed +numbers = [ 0.1, 0.2, 0.5, 1, 2, 5 ] +integers2 = [ + 1, 2, 3 # this is ok +] +integers3 = [ + 4, + # comment in the middle + 5, # this is ok +] +contributors = [ + "Foo Bar " , + { name = "Baz Qux", email = "bazqux@example.com", url = "https://example.com/bazqux" } +] diff --git a/tests/resources/smtp/dane/dns.txt b/tests/resources/smtp/dane/dns.txt new file mode 100644 index 00000000..4167f96f --- /dev/null +++ b/tests/resources/smtp/dane/dns.txt @@ -0,0 +1,3 @@ +_25._tcp.internet.nl 2 1 1 E1AE9C3DE848ECE1BA72E0D991AE4D0D9EC547C6BAD1DDDAB9D6BEB0A7E0E0D8 +_25._tcp.internet.nl 3 1 1 D6FEA64D4E68CAEAB7CBB2E0F905D7F3CA3308B12FD88C5B469F08AD7E05C7C7 +_25._tcp.mail.ietf.org 3 1 1 0C72AC70B745AC19998811B131D662C9AC69DBDBE7CB23E5B514B56664C5D3D6 diff --git a/tests/resources/smtp/dane/internet.nl.0.cert b/tests/resources/smtp/dane/internet.nl.0.cert new file mode 100644 index 00000000..55830fc4 Binary files /dev/null and b/tests/resources/smtp/dane/internet.nl.0.cert differ diff --git a/tests/resources/smtp/dane/internet.nl.1.cert b/tests/resources/smtp/dane/internet.nl.1.cert new file mode 100644 index 00000000..993b2eb3 Binary files /dev/null and b/tests/resources/smtp/dane/internet.nl.1.cert differ diff --git a/tests/resources/smtp/dane/mail.ietf.org.0.cert b/tests/resources/smtp/dane/mail.ietf.org.0.cert new file mode 100644 index 00000000..8928c782 Binary files /dev/null and b/tests/resources/smtp/dane/mail.ietf.org.0.cert differ diff --git a/tests/resources/smtp/dane/mail.ietf.org.1.cert b/tests/resources/smtp/dane/mail.ietf.org.1.cert new file mode 100644 index 00000000..371f36a4 Binary files /dev/null and b/tests/resources/smtp/dane/mail.ietf.org.1.cert differ diff --git a/tests/resources/smtp/dane/mail.ietf.org.2.cert b/tests/resources/smtp/dane/mail.ietf.org.2.cert new file mode 100644 index 00000000..9ceb8ee8 Binary files /dev/null and b/tests/resources/smtp/dane/mail.ietf.org.2.cert differ diff --git a/tests/resources/smtp/dane/mail.ietf.org.3.cert b/tests/resources/smtp/dane/mail.ietf.org.3.cert new file mode 100644 index 00000000..53a86ac2 Binary files /dev/null and b/tests/resources/smtp/dane/mail.ietf.org.3.cert differ diff --git a/tests/resources/smtp/dsn/delay.eml b/tests/resources/smtp/dsn/delay.eml new file mode 100644 index 00000000..f63fc195 --- /dev/null +++ b/tests/resources/smtp/dsn/delay.eml @@ -0,0 +1,47 @@ +From: "Mail Delivery Subsystem" +To: sender@foobar.org +Auto-Submitted: auto-generated +Subject: Warning: Delay in message delivery +Content-Type: multipart/report; report-type="delivery-status"; + boundary="mime_boundary" + + +--mime_boundary +Content-Type: text/plain +Content-Transfer-Encoding: 7bit + +There was a temporary problem delivering your message to the following recipients: + + (connection to 'mx.domain.org' failed: Connection timeout) + + +--mime_boundary +Content-Type: message/delivery-status +Content-Transfer-Encoding: 7bit + +Reporting-MTA: dns;mx.example.org +Arrival-Date: + +Original-Recipient: rfc822;jdoe@example.org +Final-Recipient: rfc822;john.doe@example.org +Action: delayed +Status: 4.0.0 +Remote-MTA: dns;mx.domain.org +Will-Retry-Until: + + +--mime_boundary +Content-Type: message/rfc822 +Content-Transfer-Encoding: 7bit + +Disclose-recipients: prohibited +From: Message Router Submission Agent +Subject: Status of: Re: Battery current sense +To: owner-ups-mib@CS.UTK.EDU +Message-id: <01HEGJ0WNBY28Y95LN@mr.timeplex.com> +MIME-version: 1.0 +Content-Type: text/plain + + +--mime_boundary-- + diff --git a/tests/resources/smtp/dsn/failure.eml b/tests/resources/smtp/dsn/failure.eml new file mode 100644 index 00000000..5c6b9a66 --- /dev/null +++ b/tests/resources/smtp/dsn/failure.eml @@ -0,0 +1,46 @@ +From: "Mail Delivery Subsystem" +To: sender@foobar.org +Auto-Submitted: auto-generated +Subject: Failed to deliver message +Content-Type: multipart/report; report-type="delivery-status"; + boundary="mime_boundary" + + +--mime_boundary +Content-Type: text/plain +Content-Transfer-Encoding: 7bit + +Your message could not be delivered to the following recipients: + + (host 'mx.example.org' rejected command 'RCPT TO:' with code 550 (5.1.2) 'User does not exist') + + +--mime_boundary +Content-Type: message/delivery-status +Content-Transfer-Encoding: 7bit + +Reporting-MTA: dns;mx.example.org +Arrival-Date: + +Final-Recipient: rfc822;foobar@example.org +Action: failed +Status: 5.1.2 +Diagnostic-Code: smtp;550 User does not exist +Remote-MTA: dns;mx.example.org + + +--mime_boundary +Content-Type: message/rfc822 +Content-Transfer-Encoding: 7bit + +Disclose-recipients: prohibited +From: Message Router Submission Agent +Subject: Status of: Re: Battery current sense +To: owner-ups-mib@CS.UTK.EDU +Message-id: <01HEGJ0WNBY28Y95LN@mr.timeplex.com> +MIME-version: 1.0 +Content-Type: text/plain + + +--mime_boundary-- + diff --git a/tests/resources/smtp/dsn/mixed.eml b/tests/resources/smtp/dsn/mixed.eml new file mode 100644 index 00000000..979d1ada --- /dev/null +++ b/tests/resources/smtp/dsn/mixed.eml @@ -0,0 +1,65 @@ +From: "Mail Delivery Subsystem" +To: sender@foobar.org +Auto-Submitted: auto-generated +Subject: Partially delivered message +Content-Type: multipart/report; report-type="delivery-status"; + boundary="mime_boundary" + + +--mime_boundary +Content-Type: text/plain +Content-Transfer-Encoding: 7bit + +Your message has been partially delivered: + + ----- Delivery to the following addresses was succesful ----- + (delivered to 'mx2.example.org' with code 250 (2.1.5) 'Message accepted for delivery') + + ----- There was a temporary problem delivering to these addresses ----- + (connection to 'mx.domain.org' failed: Connection timeout) + + ----- Delivery to the following addresses failed ----- + (host 'mx.example.org' rejected command 'RCPT TO:' with code 550 (5.1.2) 'User does not exist') + + +--mime_boundary +Content-Type: message/delivery-status +Content-Transfer-Encoding: 7bit + +Reporting-MTA: dns;mx.example.org +Arrival-Date: + +Final-Recipient: rfc822;foobar@example.org +Action: failed +Status: 5.1.2 +Diagnostic-Code: smtp;550 User does not exist +Remote-MTA: dns;mx.example.org + +Final-Recipient: rfc822;jane@example.org +Action: delivered +Status: 2.1.5 +Remote-MTA: dns;mx2.example.org + +Original-Recipient: rfc822;jdoe@example.org +Final-Recipient: rfc822;john.doe@example.org +Action: delayed +Status: 4.0.0 +Remote-MTA: dns;mx.domain.org +Will-Retry-Until: + + +--mime_boundary +Content-Type: message/rfc822 +Content-Transfer-Encoding: 7bit + +Disclose-recipients: prohibited +From: Message Router Submission Agent +Subject: Status of: Re: Battery current sense +To: owner-ups-mib@CS.UTK.EDU +Message-id: <01HEGJ0WNBY28Y95LN@mr.timeplex.com> +MIME-version: 1.0 +Content-Type: text/plain + + +--mime_boundary-- + diff --git a/tests/resources/smtp/dsn/original.txt b/tests/resources/smtp/dsn/original.txt new file mode 100644 index 00000000..170f3086 --- /dev/null +++ b/tests/resources/smtp/dsn/original.txt @@ -0,0 +1,10 @@ +Disclose-recipients: prohibited +Date: Fri, 08 Jul 1994 09:21:25 -0400 (EDT) +From: Message Router Submission Agent +Subject: Status of: Re: Battery current sense +To: owner-ups-mib@CS.UTK.EDU +Message-id: <01HEGJ0WNBY28Y95LN@mr.timeplex.com> +MIME-version: 1.0 +Content-Type: text/plain + + diff --git a/tests/resources/smtp/dsn/success.eml b/tests/resources/smtp/dsn/success.eml new file mode 100644 index 00000000..73ad5ae7 --- /dev/null +++ b/tests/resources/smtp/dsn/success.eml @@ -0,0 +1,45 @@ +From: "Mail Delivery Subsystem" +To: sender@foobar.org +Auto-Submitted: auto-generated +Subject: Successfully delivered message +Content-Type: multipart/report; report-type="delivery-status"; + boundary="mime_boundary" + + +--mime_boundary +Content-Type: text/plain +Content-Transfer-Encoding: 7bit + +Your message has been successfully delivered to the following recipients: + + (delivered to 'mx2.example.org' with code 250 (2.1.5) 'Message accepted for delivery') + + +--mime_boundary +Content-Type: message/delivery-status +Content-Transfer-Encoding: 7bit + +Reporting-MTA: dns;mx.example.org +Arrival-Date: + +Final-Recipient: rfc822;jane@example.org +Action: delivered +Status: 2.1.5 +Remote-MTA: dns;mx2.example.org + + +--mime_boundary +Content-Type: message/rfc822 +Content-Transfer-Encoding: 7bit + +Disclose-recipients: prohibited +From: Message Router Submission Agent +Subject: Status of: Re: Battery current sense +To: owner-ups-mib@CS.UTK.EDU +Message-id: <01HEGJ0WNBY28Y95LN@mr.timeplex.com> +MIME-version: 1.0 +Content-Type: text/plain + + +--mime_boundary-- + diff --git a/tests/resources/smtp/lists/test-list1.txt b/tests/resources/smtp/lists/test-list1.txt new file mode 100644 index 00000000..4e191fb4 --- /dev/null +++ b/tests/resources/smtp/lists/test-list1.txt @@ -0,0 +1,2 @@ +user1@domain.org +user2@domain.org diff --git a/tests/resources/smtp/lists/test-list2.txt b/tests/resources/smtp/lists/test-list2.txt new file mode 100644 index 00000000..6eeca467 --- /dev/null +++ b/tests/resources/smtp/lists/test-list2.txt @@ -0,0 +1,3 @@ +user3@example.net +user4@example.net +user5@example.net diff --git a/tests/resources/smtp/messages/arc.eml b/tests/resources/smtp/messages/arc.eml new file mode 100644 index 00000000..17488ada --- /dev/null +++ b/tests/resources/smtp/messages/arc.eml @@ -0,0 +1,34 @@ +ARC-Seal: i=2; a=rsa-sha256; s=rsa; d=manchego.org; cv=pass; + b=wpAAy6QusmF4O8SeziNaKxXL6EleeBYxQ0HrXl2cDgzHLOvYG0N1Wpz0bpVbA8VgteD2X8XCW + yrdlZ5dIPTcCvgfLGLXLRTIcYUdKyfFh5IVEciaUOUsxlSRPpekENZKzdHFkL4j1mAAvpDNJ7Ft + OFIp0ku5dACn80g7D4cSEU0=; +ARC-Message-Signature: i=2; a=rsa-sha256; s=rsa; d=manchego.org; c=relaxed/relaxed; + h=Subject:To:From:DKIM-Signature; t=1674137914; bh=4ET7siw2kYV7jcN+fzsuYng/ + sr/BmIzzEjh43dVAv40=; b=V3tMBI1RsyJJY7HUABcebHf0mDJ9odbPm++ZMY5AsCaUYNoSsAm + wCf5wYlJQ26KmsluOYXoPwML0a/xvnMXPv6Rs4Z9k4IwzpzhGLsijDXymGPsW3hgq/6ivVTPkwU + +pGSCC70rHNrAFFk5P67Ly0tbGYjJ0wZVHBzqL8IJBXK4=; +ARC-Authentication-Results: i=2; manchego.org; + dkim=pass header.d=manchego.org header.s=rsa header.b=IN4oMvqq +Authentication-Results: manchego.org; + dkim=pass header.d=manchego.org header.s=rsa header.b=IN4oMvqq +ARC-Seal: i=1; a=ed25519-sha256; s=ed; d=scamorza.org; cv=none; + b=k/MAHECtaer9v4oczoe00a6XMjrxU4QUVVPlZI8XYegbiOgDSaeR6IrwBSKVcN0ELYU+HXlNW + RuUGkRuZXQODA==; +ARC-Message-Signature: i=1; a=ed25519-sha256; s=ed; d=scamorza.org; c=relaxed/relaxed; + h=Subject:To:From:DKIM-Signature; t=1674137914; bh=4ET7siw2kYV7jcN+fzsuYng/ + sr/BmIzzEjh43dVAv40=; b=ZVPqB/5+mbOEKIgBsq+S71Sfj2JZUlGmYEA0Ygbj0S1VmTAnsVu + FQSInMY4/qcIeqU23BtzMgCFVZfAg5i3zDw==; +ARC-Authentication-Results: i=1; scamorza.org; + dkim=pass header.d=manchego.org header.s=rsa header.b=IN4oMvqq +Authentication-Results: scamorza.org; + dkim=pass header.d=manchego.org header.s=rsa header.b=IN4oMvqq +DKIM-Signature: v=1; a=rsa-sha256; s=rsa; d=manchego.org; c=relaxed/relaxed; + h=Subject:To:From; t=1674137914; bh=4ET7siw2kYV7jcN+fzsuYng/sr/BmIzzEjh43dV + Av40=; b=IN4oMvqqxWCEyC38F7fZecYJcnq+7zP3G/xjcI64M3/Dzys2lmQeLYAXipwwYvEa5a + VwCcJ7XUX0kSxtr6igC8FIJEDI6UmdvJgMEj/hnEjXR8m4GPrphigjJy7hagaQymBT9WhlzsDPI + QRlUVoW0y5v1aDp3KF9bLVCKTELJPM=; +From: queso@manchego.org +To: affumicata@scamorza.org +Subject: Say cheese + +We need to settle which one of us is tastier. diff --git a/tests/resources/smtp/messages/dkim.eml b/tests/resources/smtp/messages/dkim.eml new file mode 100644 index 00000000..49c27606 --- /dev/null +++ b/tests/resources/smtp/messages/dkim.eml @@ -0,0 +1,14 @@ +DKIM-Signature: v=1; a=rsa-sha256; s=default; d=example.com; c=relaxed/relaxed; r=y; + h=Subject:To:From; t=1674122129; bh=Xcxymouf0VhlJ7c/vHLAM3LPTUR4LKFKX7PRNni + WCEs=; b=m5lYqx81xqAIo4ZBC9FDiFIBrRnnep+taSsutc5MkbBQvf9/Lb54AXHhruEdO2EGkG + xUxL1c8QDH3eLz84fPTUgZue84tAsAa0q4gJFIYM5q2/GJvJ6cBvsXKZj82FjRTIz4wlLjzkW7p + NdR9C5CID2PO4sW+GymS45F8hwqSj4=; +DKIM-Signature: v=1; a=ed25519-sha256; s=ed; d=example.com; c=relaxed/relaxed; + h=Subject:To:From; t=1674122129; bh=Xcxymouf0VhlJ7c/vHLAM3LPTUR4LKFKX7PRNni + WCEs=; b=t8z1AsaxeWek+gMSVojbs2QJu+orzeR4CiHVquJYvXzv+Eb52Wq0fEmaOxoyY1teVL + Odp57Vq/zTLjMMZ2hbBQ==; +From: bill@example.com +To: jdoe@example.com +Subject: TPS Report + +I'm going to need those TPS reports ASAP. So, if you could do that, that'd be great. diff --git a/tests/resources/smtp/messages/invalid_arc.eml b/tests/resources/smtp/messages/invalid_arc.eml new file mode 100644 index 00000000..6e65fd78 --- /dev/null +++ b/tests/resources/smtp/messages/invalid_arc.eml @@ -0,0 +1,25 @@ +ARC-Seal: i=1; a=rsa-sha256; s=rsa; d=manchego.org; cv=fail; + b=wpAAy6QusmF4O8SeziNaKxXL6EleeBYxQ0HrXl2cDgzHLOvYG0N1Wpz0bpVbA8VgteD2X8XCW + yrdlZ5dIPTcCvgfLGLXLRTIcYUdKyfFh5IVEciaUOUsxlSRPpekENZKzdHFkL4j1mAAvpDNJ7Ft + OFIp0ku5dACn80g7D4cSEU0=; +ARC-Message-Signature: i=1; a=rsa-sha256; s=rsa; d=manchego.org; c=relaxed/relaxed; + h=Subject:To:From:DKIM-Signature; t=1674137914; bh=4ET7siw2kYV7jcN+fzsuYng/ + sr/BmIzzEjh43dVAv40=; b=V3tMBI1RsyJJY7HUABcebHf0mDJ9odbPm++ZMY5AsCaUYNoSsAm + wCf5wYlJQ26KmsluOYXoPwML0a/xvnMXPv6Rs4Z9k4IwzpzhGLsijDXymGPsW3hgq/6ivVTPkwU + +pGSCC70rHNrAFFk5P67Ly0tbGYjJ0wZVHBzqL8IJBXK4=; +ARC-Authentication-Results: i=1; manchego.org; + dkim=pass header.d=manchego.org header.s=rsa header.b=IN4oMvqq +DKIM-Signature: v=1; a=rsa-sha256; s=default; d=example.com; c=relaxed/relaxed; r=y; + h=Subject:To:From; t=1674122129; bh=Xcxymouf0VhlJ7c/vHLAM3LPTUR4LKFKX7PRNni + WCEs=; b=m5lYqx81xqAIo4ZBC9FDiFIBrRnnep+taSsutc5MkbBQvf9/Lb54AXHhruEdO2EGkG + xUxL1c8QDH3eLz84fPTUgZue84tAsAa0q4gJFIYM5q2/GJvJ6cBvsXKZj82FjRTIz4wlLjzkW7p + NdR9C5CID2PO4sW+GymS45F8hwqSj4=; +DKIM-Signature: v=1; a=ed25519-sha256; s=ed; d=example.com; c=relaxed/relaxed; + h=Subject:To:From; t=1674122129; bh=Xcxymouf0VhlJ7c/vHLAM3LPTUR4LKFKX7PRNni + WCEs=; b=t8z1AsaxeWek+gMSVojbs2QJu+orzeR4CiHVquJYvXzv+Eb52Wq0fEmaOxoyY1teVL + Odp57Vq/zTLjMMZ2hbBQ==; +From: bill@example.com +To: jdoe@example.com +Subject: TPS Report + +I'm going to need those TPS reports ASAP. So, if you could do that, that'd be great. diff --git a/tests/resources/smtp/messages/invalid_dkim.eml b/tests/resources/smtp/messages/invalid_dkim.eml new file mode 100644 index 00000000..4e6034d1 --- /dev/null +++ b/tests/resources/smtp/messages/invalid_dkim.eml @@ -0,0 +1,14 @@ +DKIM-Signature: v=1; a=rsa-sha256; s=default; d=example.com; c=relaxed/relaxed; r=y; + h=Subject:To:From; t=1674122129; bh=Xcxymouf0VhlJ7c/vHLAM3LPTUR4LKFKX7PRNni + WCEs=; b=m5lYqx81xqAIo4ZBC9FDiFIBrRnnep+taSsutc5MkbBQvf9/Lb54AXHhruEdO2EGkG + xUxL1c8QDH3eLz84fPTUgZue84tAsAa0q4gJFIYM5q2/GJvJ6cBvsXKZj82FjRTIz4wlLjzkW7p + NdR9C5CID2PO4sW+GymS45F8hwqSj4=; +DKIM-Signature: v=1; a=ed25519-sha256; s=ed; d=example.com; c=relaxed/relaxed; + h=Subject:To:From; t=1674122129; bh=Xcxymouf0VhlJ7c/vHLAM3LPTUR4LKFKX7PRNni + WCEs=; b=t8z1AsaxeWek+gMSVojbs2QJu+orzeR4CiHVquJYvXzv+Eb52Wq0fEmaOxoyY1teVL + Odp57Vq/zTLjMMZ2hbBQ==; +From: bill@example.com +To: jdoe@example.com +Subject: TPS Report + +Body hash will not match. diff --git a/tests/resources/smtp/messages/loop.eml b/tests/resources/smtp/messages/loop.eml new file mode 100644 index 00000000..fadbd66d --- /dev/null +++ b/tests/resources/smtp/messages/loop.eml @@ -0,0 +1,23 @@ +Received: from client1.football.example.com [192.0.2.1] + by submitserver.example.com with SUBMISSION; + Fri, 11 Jul 2003 21:01:54 -0700 (PDT) +Received: from client1.football.example.com [192.0.2.1] + by submitserver.example.com with SUBMISSION; + Fri, 11 Jul 2003 21:01:54 -0700 (PDT) +Received: from client1.football.example.com [192.0.2.1] + by submitserver.example.com with SUBMISSION; + Fri, 11 Jul 2003 21:01:54 -0700 (PDT) +Received: from client1.football.example.com [192.0.2.1] + by submitserver.example.com with SUBMISSION; + Fri, 11 Jul 2003 21:01:54 -0700 (PDT) +From: Joe SixPack +To: Suzie Q +Subject: Is dinner ready? +Date: Fri, 11 Jul 2003 21:00:37 -0700 (PDT) +Message-ID: <20030712040037.46341.5F8J@football.example.com> + +Hi. + +We lost the game. Are you hungry yet? + +Joe. diff --git a/tests/resources/smtp/messages/multipart.eml b/tests/resources/smtp/messages/multipart.eml new file mode 100644 index 00000000..6e735299 --- /dev/null +++ b/tests/resources/smtp/messages/multipart.eml @@ -0,0 +1,34 @@ +From: Hendrik +To: Harrie +Date: Sat, 11 Oct 2010 00:31:44 +0200 +Subject: One Two Three Four +Content-Type: multipart/mixed; boundary=AA + +This is a multi-part message in MIME format. +--AA +Content-Type: multipart/mixed; boundary=BB + +This is a multi-part message in MIME format. +--BB +Content-Type: text/plain; charset="us-ascii" + +This is the first message part containing +plain text. + +--BB +Content-Type: text/plain; charset="us-ascii" + +This is another plain text message part. + +--BB-- +This is the end of MIME multipart. + +--AA +Content-Type: text/html; charset="us-ascii" + + +This is a piece of HTML text. + + +--AA-- +This is the end of MIME multipart. diff --git a/tests/resources/smtp/messages/no_dkim.eml b/tests/resources/smtp/messages/no_dkim.eml new file mode 100644 index 00000000..af0c7e38 --- /dev/null +++ b/tests/resources/smtp/messages/no_dkim.eml @@ -0,0 +1,14 @@ +Received: from client1.football.example.com [192.0.2.1] + by submitserver.example.com with SUBMISSION; + Fri, 11 Jul 2003 21:01:54 -0700 (PDT) +From: Joe SixPack +To: Suzie Q +Subject: Is dinner ready? +Date: Fri, 11 Jul 2003 21:00:37 -0700 (PDT) +Message-ID: <20030712040037.46341.5F8J@football.example.com> + +Hi. + +We lost the game. Are you hungry yet? + +Joe. diff --git a/tests/resources/smtp/messages/no_msgid.eml b/tests/resources/smtp/messages/no_msgid.eml new file mode 100644 index 00000000..e1adc0ba --- /dev/null +++ b/tests/resources/smtp/messages/no_msgid.eml @@ -0,0 +1,9 @@ +From: Joe SixPack +To: Suzie Q +Subject: Is dinner ready? + +Hi. + +We lost the game. Are you hungry yet? + +Joe. diff --git a/tests/resources/smtp/pipe/pipe_me.sh b/tests/resources/smtp/pipe/pipe_me.sh new file mode 100644 index 00000000..d3d8c6e8 --- /dev/null +++ b/tests/resources/smtp/pipe/pipe_me.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +if [[ $1 == "hello" ]] && [[ $2 == "world" ]]; then + echo "X-My-Header: true" + while read line + do + echo "$line" + done < /dev/stdin + exit 0; +else + echo "Invalid parameters!" + exit 1; +fi + diff --git a/tests/resources/smtp/reports/arf1.eml b/tests/resources/smtp/reports/arf1.eml new file mode 100644 index 00000000..a40fda24 --- /dev/null +++ b/tests/resources/smtp/reports/arf1.eml @@ -0,0 +1,44 @@ +From: +Date: Thu, 8 Mar 2005 17:40:36 EDT +Subject: FW: Earn money +To: +MIME-Version: 1.0 +Content-Type: multipart/report; report-type=feedback-report; + boundary="part1_13d.2e68ed54_boundary" + +--part1_13d.2e68ed54_boundary +Content-Type: text/plain; charset="US-ASCII" +Content-Transfer-Encoding: 7bit + +This is an email abuse report for an email message received from IP +192.0.2.1 on Thu, 8 Mar 2005 14:00:00 EDT. For more information +about this format please see http://www.mipassoc.org/arf/. + +--part1_13d.2e68ed54_boundary +Content-Type: message/feedback-report + +Feedback-Type: abuse +User-Agent: SomeGenerator/1.0 +Version: 1 + +--part1_13d.2e68ed54_boundary +Content-Type: message/rfc822 +Content-Disposition: inline + +Received: from mailserver.example.net + (mailserver.example.net [192.0.2.1]) + by example.com with ESMTP id M63d4137594e46; + Thu, 08 Mar 2005 14:00:00 -0400 +From: +To: +Subject: Earn money +MIME-Version: 1.0 +Content-type: text/plain +Message-ID: 8787KJKJ3K4J3K4J3K4J3.mail@example.net +Date: Thu, 02 Sep 2004 12:31:03 -0500 + +Spam Spam Spam +Spam Spam Spam +Spam Spam Spam +Spam Spam Spam +--part1_13d.2e68ed54_boundary-- diff --git a/tests/resources/smtp/reports/arf2.eml b/tests/resources/smtp/reports/arf2.eml new file mode 100644 index 00000000..d95f2f2c --- /dev/null +++ b/tests/resources/smtp/reports/arf2.eml @@ -0,0 +1,55 @@ +From: +Date: Thu, 8 Mar 2005 17:40:36 EDT +Subject: FW: Earn money +To: +MIME-Version: 1.0 +Content-Type: multipart/report; report-type=feedback-report; + boundary="part1_13d.2e68ed54_boundary" + +--part1_13d.2e68ed54_boundary +Content-Type: text/plain; charset="US-ASCII" +Content-Transfer-Encoding: 7bit + +This is an email abuse report for an email message received from IP +192.0.2.1 on Thu, 8 Mar 2005 14:00:00 EDT. For more information +about this format please see http://www.mipassoc.org/arf/. + +--part1_13d.2e68ed54_boundary +Content-Type: message/feedback-report + +Feedback-Type: abuse +User-Agent: SomeGenerator/1.0 +Version: 1 +Original-Mail-From: +Original-Rcpt-To: +Arrival-Date: Thu, 8 Mar 2005 14:00:00 EDT +Reporting-MTA: dns; mail.example.com +Source-IP: 192.0.2.1 +Authentication-Results: mail.example.com; + spf=fail smtp.mail=somespammer@example.com +Reported-Domain: example.net +Reported-Uri: http://example.net/earn_money.html +Reported-Uri: mailto:user@example.com +Removal-Recipient: user@example.com + +--part1_13d.2e68ed54_boundary +Content-Type: message/rfc822 +Content-Disposition: inline + +From: +Received: from mailserver.example.net (mailserver.example.net + [192.0.2.1]) by example.com with ESMTP id M63d4137594e46; + Thu, 08 Mar 2005 14:00:00 -0400 + +To: +Subject: Earn money +MIME-Version: 1.0 +Content-type: text/plain +Message-ID: 8787KJKJ3K4J3K4J3K4J3.mail@example.net +Date: Thu, 02 Sep 2004 12:31:03 -0500 + +Spam Spam Spam +Spam Spam Spam +Spam Spam Spam +Spam Spam Spam +--part1_13d.2e68ed54_boundary-- diff --git a/tests/resources/smtp/reports/arf3.eml b/tests/resources/smtp/reports/arf3.eml new file mode 100644 index 00000000..d0f8cbf6 --- /dev/null +++ b/tests/resources/smtp/reports/arf3.eml @@ -0,0 +1,58 @@ +From: arf-daemon@example.com +To: recipient@example.net +Subject: This is a test +Date: Wed, 14 Apr 2010 12:17:45 -0700 (PDT) +MIME-Version: 1.0 +Content-Type: multipart/report; report-type=feedback-report; + boundary="part1_13d.2e68ed54_boundary" + +--part1_13d.2e68ed54_boundary +Content-Type: text/plain; charset="US-ASCII" +Content-Transfer-Encoding: 7bit + +This is an email abuse report for an email message received +from IP 192.0.2.1 on Wed, 14 Apr 2010 12:15:31 PDT. For more +information about this format please see +http://www.mipassoc.org/arf/. + +--part1_13d.2e68ed54_boundary +Content-Type: message/feedback-report + +Feedback-Type: auth-failure +User-Agent: SomeDKIMFilter/1.0 +Version: 1 +Original-Mail-From: +Original-Rcpt-To: +Received-Date: Wed, 14 Apr 2010 12:15:31 -0700 (PDT) +Source-IP: 192.0.2.1 +Authentication-Results: mail.example.com; dkim=fail + header.d=example.net +Reported-Domain: example.net +DKIM-Domain: example.net +Auth-Failure: bodyhash + +--part1_13d.2e68ed54_boundary +Content-Type: message/rfc822 + +DKIM-Signature: v=1; c=relaxed/simple; a=rsa-sha256; + s=testkey; d=example.net; h=From:To:Subject:Date; + bh=2jUSOH9NhtVGCQWNr9BrIAPreKQjO6Sn7XIkfJVOzv8=; + b=AuUoFEfDxTDkHlLXSZEpZj79LICEps6eda7W3deTVFOk4yAUoqOB + 4nujc7YopdG5dWLSdNg6xNAZpOPr+kHxt1IrE+NahM6L/LbvaHut + KVdkLLkpVaVVQPzeRDI009SO2Il5Lu7rDNH6mZckBdrIx0orEtZV + 4bmp/YzhwvcubU4= +Received: from smtp-out.example.net by mail.example.com + with SMTP id o3F52gxO029144; + Wed, 14 Apr 2010 12:15:31 -0700 (PDT) +Received: from internal-client-001.example.com + by mail.example.com + with SMTP id o3F3BwdY028431; + Wed, 14 Apr 2010 12:12:09 -0700 (PDT) +From: randomuser@example.net +To: user@example.com +Date: Wed, 14 Apr 2010 12:12:09 -0700 (PDT) +Subject: This is a test + +Hi, just making sure DKIM is working! + +--part1_13d.2e68ed54_boundary-- diff --git a/tests/resources/smtp/reports/arf4.eml b/tests/resources/smtp/reports/arf4.eml new file mode 100644 index 00000000..cb1883a7 --- /dev/null +++ b/tests/resources/smtp/reports/arf4.eml @@ -0,0 +1,73 @@ +Return-Path: +Received: by box.mydomain.name (Postfix, from userid 116) + id CF8FA658E0; Tue, 5 Oct 2021 17:37:02 +1300 (NZDT) +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=box.mydomain.name; + s=mail; t=1633408622; + bh=yDlkGfe4dFwlsFeoKaIHG6xiRgQs2/PqPnLtiPm5ewk=; + h=From:To:Date:Subject:From; + b=TwXvJJFoFwJDcb6IKMKsxp2BiRDsrjLOESyQPh/Cc4tRZltVAud/k6f0XP4l5a/T8 + kh0iDOGImc0O1WZNFt0MUcwLsfW4qbYjCBtthQDnbPApvv6MJDASwau+wipu5Nrkjc + flg+nMaD97pVgR0LevMoVIWoiy1f5PNC/z0xkY2wnvyoGn91WuDsdocOqyoPo4RmIT + A/f3M4CjOv/QmMEAWBIsa7kAZwf+rNmzDahFOtp2vFLqHt0iZi5vs40fa6O/I0snTM + fRkv2GMZAug7NMU8MN/MhuO87FV6ATZXvB0Kxvsy9z0zZYK7tM1OYHjiCYot45erG3 + dlKrYiXsfd3BQ== +From: OpenDMARC Filter +To: postmaster@vericty.interpublication.org +Date: Tue, 5 Oct 2021 17:37:02 +1300 (NZDT) +Subject: FW: Wir kaufen dein Auto! +MIME-Version: 1.0 +Content-Type: multipart/report; + report-type=feedback-report; + boundary="box.mydomain.name:8BE2660E72" +Message-Id: <20211005043702.CF8FA658E0@box.mydomain.name> + +--box.mydomain.name:8BE2660E72 +Content-Type: text/plain + +This is an authentication failure report for an email message received from IP +148.163.85.135 on Tue, 5 Oct 2021 17:37:02 +1300 (NZDT). + +--box.mydomain.name:8BE2660E72 +Content-Type: message/feedback-report + +Feedback-Type: auth-failure +Version: 1 +User-Agent: OpenDMARC-Filter/1.3.2 +Auth-Failure: dmarc +Authentication-Results: box.mydomain.name; dmarc=fail header.from=interpublication.org +Original-Envelope-Id: 8BE2660E72 +Original-Mail-From: info@interpublication.org +Source-IP: 148.163.85.135 (sainay.interpublication.org) +Reported-Domain: interpublication.org + +--box.mydomain.name:8BE2660E72 +Content-Type: text/rfc822-headers + +Authentication-Results: box.mydomain.name; + dkim=fail reason="signature verification failed" (2048-bit key; unprotected) header.d=interpublication.org header.i=@interpublication.org header.b="PrsTNnuH"; + dkim-atps=neutral +Received: from dslb-002-202-150-127.002.202.pools.vodafone-ip.de (dslb-188-099-080-029.188.099.pools.vodafone-ip.de [188.99.80.29]) + by sainay.interpublication.org (Postfix) with ESMTPA id 6BB23A2D3 + for ; Tue, 5 Oct 2021 00:36:52 -0400 (EDT) +DKIM-Filter: OpenDKIM Filter v2.11.0 sainay.interpublication.org 6BB23A2D3 +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; + d=interpublication.org; s=default; t=1633408612; + bh=q1/OPSn+VXteY2+DHXqOIgs5LsNCJisEcQIKVW9it6I=; + h=From:Subject:To:Reply-To:Date:From; + b=PrsTNnuH8D0Ch3gcWqGmXiYc2Kvu1CHGJBsqS521uYazd3G/urp7MHQvmNwK0r1gS + DR3A3KwGejI5uuqzxDCqz28Mq6AkdTkOjFyXw65MLlsKTQddWTgciVnoqJempa6yzw + PSM5550XqVFqqkNxEcYBUBYEwUdy1tY8rc4zhq8cIrsonQVxJJSbc3cdonICM1kLBV + WASv16p3376ZBcKqFLc8UQ58YQKaFm51VZGEjtabfmWbgOQ7VikFFECDG3aRt8fZa6 + D03MrzUSngwPUdcRQZuqS/sApW/a9N2YwdbR51OFzPBr4ypUEIw/qprgBG4BfQQKeS + 1PhinNvVtgQpQ== +From: "Rolf Bader" +Subject: Wir kaufen dein Auto! +To: "address" +Content-Type: multipart/alternative; boundary="TD6gM3Blv=_XBZYNFT7dCsH1DHHOKUuSyA" +MIME-Version: 1.0 +Reply-To: "Rolf Bader" +Organization: AutoTEAM24 +Date: Tue, 5 Oct 2021 06:36:51 +0200 + +--box.mydomain.name:8BE2660E72-- + diff --git a/tests/resources/smtp/reports/arf5.eml b/tests/resources/smtp/reports/arf5.eml new file mode 100644 index 00000000..50dfd02c --- /dev/null +++ b/tests/resources/smtp/reports/arf5.eml @@ -0,0 +1,87 @@ +Message-ID: <433689.81121.example@mta.mail.receiver.example> +From: "SomeISP Antispam Feedback" +To: arf-failure@sender.example +Subject: FW: You have a new bill from your bank +Date: Sat, 8 Oct 2011 15:15:59 -0500 (CDT) +MIME-Version: 1.0 +Content-Type: multipart/report; + boundary="------------Boundary-00=_3BCR4Y7kX93yP9uUPRhg"; + report-type=feedback-report +Content-Transfer-Encoding: 7bit + +--------------Boundary-00=_3BCR4Y7kX93yP9uUPRhg +Content-Type: text/plain; charset="us-ascii" +Content-Disposition: inline +Content-Transfer-Encoding: 7bit + +This is an authentication failure report for an email message +received from a.sender.example on 8 Oct 2011 20:15:58 +0000 (GMT). +For more information about this format, please see [RFC6591]. + +--------------Boundary-00=_3BCR4Y7kX93yP9uUPRhg +Content-Type: message/feedback-report +Content-Transfer-Encoding: 7bit + +Feedback-Type: auth-failure +User-Agent: Someisp!Mail-Feedback/1.0 +Version: 1 +Original-Mail-From: anexample.reply@a.sender.example +Original-Envelope-Id: o3F52gxO029144 +Authentication-Results: mta1011.mail.tp2.receiver.example; + dkim=fail (bodyhash) header.d=sender.example +Auth-Failure: bodyhash +DKIM-Canonicalized-Body: VGhpcyBpcyBhIG1lc3NhZ2UgYm9keSB0 + aGF0IGdvdCBtb2RpZmllZCBpbiB0cmFuc2l0LgoKQXQgdGhlIHNhbWU + gdGltZSB0aGF0IHRoZSBib2R5aGFzaCBmYWlscyB0byB2ZXJpZnksIH + RoZQptZXNzYWdlIGNvbnRlbnQgaXMgY2xlYXJseSBhYnVzaXZlIG9yI + HBoaXNoeSwgYXMgdGhlClN1YmplY3QgYWxyZWFkeSBoaW50cy4gIElu + ZGVlZCwgdGhpcyBib2R5IGFsc28gY29udGFpbnMKdGhlIGZvbGxvd2l + uZyB0ZXh0OgoKICAgUGxlYXNlIGVudGVyIHlvdXIgZnVsbCBiYW5rIG + NyZWRlbnRpYWxzIGF0CiAgIGh0dHA6Ly93d3cuc2VuZGVyLmV4YW1wb + GUvCgpXZSBhcmUgaW1wbHlpbmcgdGhhdCwgYWx0aG91Z2ggbXVsdGlw + bGUgZmFpbHVyZXMKcmVxdWlyZSBtdWx0aXBsZSByZXBvcnRzLCBhIHN + pbmdsZSBmYWlsdXJlIGNhbiBiZQpyZXBvcnRlZCBhbG9uZyB3aXRoIH + BoaXNoaW5nIGluIGEgc2luZ2xlIHJlcG9ydC4K +DKIM-Domain: sender.example +DKIM-Identity: @sender.example +DKIM-Selector: testkey +Arrival-Date: 8 Oct 2011 20:15:58 +0000 (GMT) +Source-IP: 192.0.2.1 +Reported-Domain: a.sender.example +Reported-URI: http://www.sender.example/ + +--------------Boundary-00=_3BCR4Y7kX93yP9uUPRhg +Content-Type: text/rfc822-headers +Content-Transfer-Encoding: 7bit + +Authentication-Results: mta1011.mail.tp2.receiver.example; + dkim=fail (bodyhash) header.d=sender.example; + spf=pass smtp.mailfrom=anexample.reply@a.sender.example +Received: from smtp-out.sender.example + by mta1011.mail.tp2.receiver.example + with SMTP id oB85W8xV000169; + Sat, 08 Oct 2011 13:15:58 -0700 (PDT) +DKIM-Signature: v=1; c=relaxed/simple; a=rsa-sha256; + s=testkey; d=sender.example; h=From:To:Subject:Date; + bh=2jUSOH9NhtVGCQWNr9BrIAPreKQjO6Sn7XIkfJVOzv8=; + b=AuUoFEfDxTDkHlLXSZEpZj79LICEps6eda7W3deTVFOk4yAUoqOB + 4nujc7YopdG5dWLSdNg6xNAZpOPr+kHxt1IrE+NahM6L/LbvaHut + KVdkLLkpVaVVQPzeRDI009SO2Il5Lu7rDNH6mZckBdrIx0orEtZV + 4bmp/YzhwvcubU4= +Received: from mail.sender.example + by smtp-out.sender.example + with SMTP id o3F52gxO029144; + Sat, 08 Oct 2011 13:15:31 -0700 (PDT) + Received: from internal-client-001.sender.example + by mail.sender.example + with SMTP id o3F3BwdY028431; + Sat, 08 Oct 2011 13:15:24 -0700 (PDT) +Date: Sat, 8 Oct 2011 16:15:24 -0400 (EDT) +Reply-To: anexample.reply@a.sender.example +From: anexample@a.sender.example +To: someuser@receiver.example +Subject: You have a new bill from your bank +Message-ID: <87913910.1318094604546@out.sender.example> + +--------------Boundary-00=_3BCR4Y7kX93yP9uUPRhg-- + diff --git a/tests/resources/smtp/reports/dmarc1.eml b/tests/resources/smtp/reports/dmarc1.eml new file mode 100644 index 00000000..9a2ed1d7 --- /dev/null +++ b/tests/resources/smtp/reports/dmarc1.eml @@ -0,0 +1,66 @@ +Received: from mail.stalw.art ([mail.stalw.art]) + by 127.0.0.1 (Stalwart JMAP) with LMTP; + Mon, 28 Nov 2022 10:51:56 +0000 +Received: from mail-qv1-xf4a.google.com (mail-qv1-xf4a.google.com [IPv6:2607:f8b0:4864:20::f4a]) + (using TLSv1.3 with cipher TLS_AES_128_GCM_SHA256 (128/128 bits) + key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) + (No client certificate requested) + by mail.stalw.art (Postfix) with ESMTPS id 1145E7CC0B + for ; Mon, 28 Nov 2022 10:51:53 +0000 (UTC) +Received: by mail-qv1-xf4a.google.com with SMTP id 71-20020a0c804d000000b004b2fb260447so12985969qva.10 + for ; Mon, 28 Nov 2022 02:51:52 -0800 (PST) +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; + d=google.com; s=20210112; + h=content-transfer-encoding:content-disposition:to:from:subject + :message-id:date:mime-version:from:to:cc:subject:date:message-id + :reply-to; + bh=sMF/38UFRhmUYFRJST4vLBu/U1BXgsdCUE02HF8nXx8=; + b=I7WONP7tMsULp4eKjJeeKtM+nDYqMSIYMxqNHqCP1bTsnUiW2xM278I2+F8EjtFNYf + XOgusNn8kqbSnA4w1+q4G87zTF4K3tGnxNpuUMQ7GzcofBKtr7VPv9XFqvTPJ+N8YSwe + 926ec7xi71BpSHAgqp5Wqocj8ruIVjcCZ37hYrG0C4s+FVBtbaU3EeyPpkESaaY2vE5y + Qa2KsrMsyJXlbyW/sFJ7AGDDuXwyGkTa+btP/xIiQM2HlBKy7vNOFZKkxInOuQsXJgZy + 3H7ivlpD3hMrszwU77o5jBArVwN0RIkUSosAPQf+pzgvRlkseRlDrmzKQutvYWIaTP3/ + FHPA== +X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; + d=1e100.net; s=20210112; + h=content-transfer-encoding:content-disposition:to:from:subject + :message-id:date:mime-version:x-gm-message-state:from:to:cc:subject + :date:message-id:reply-to; + bh=sMF/38UFRhmUYFRJST4vLBu/U1BXgsdCUE02HF8nXx8=; + b=D3oClvT5AKcTpEjjffHQqPPQ9j5mmtExiviSq7iBYkoq+322LtR2hGqGxtvlAwRDsQ + VIfuKVExygw3c9bckjzKtJYX128HGK35gHnmsrzqvCC93JlRaC/55kcM9Bhks0xJnl7i + yNFHPZ0DY/jasdUdQ1QqnI+8qiPy+/12JvD+/TGlaDuS+RWYFU4/ky46S3vMXwXmRt6D + IGggXoW7snSaM4s88DzMUl0U7DH823UPQrUxnA5Oxscwn9M1ENJUWD/3EJo5ZEUMw0ll + Y8AlyhjWFgVqs1Y4V/LVWeXdF10fpm78+jm8QyIZYZJjh4I33AekdsWVM71ZNNYGL0+8 + +GDg== +X-Gm-Message-State: ANoB5pn0zRZSWXdFXd9G0tawbSeUYuhxToVkIYoLf8OJzoBLcIc0wKcB + AfU9Coz5vAuiM1mASWJhbg== +X-Google-Smtp-Source: AA0mqf6WWsnqMD4cHE40jB89/zblmT7yNKeHKlsvvCmlYANKmpKLTQTaCm5qCA0mVmxR/PTQogsNndWH/qe0ug== +MIME-Version: 1.0 +X-Received: by 2002:ac8:5182:0:b0:39c:cb6a:300b with SMTP id + c2-20020ac85182000000b0039ccb6a300bmr48409299qtn.181.1669632711968; Mon, 28 + Nov 2022 02:51:51 -0800 (PST) +Date: Sun, 27 Nov 2022 15:59:59 -0800 +Message-ID: <5264580628977113351@google.com> +Subject: Report domain: stalw.art Submitter: google.com Report-ID: 5264580628977113351 +From: noreply-dmarc-support@google.com +To: domains@stalw.art +Content-Type: application/zip; + name="google.com!stalw.art!1669507200!1669593599.zip" +Content-Disposition: attachment; + filename="google.com!stalw.art!1669507200!1669593599.zip" +Content-Transfer-Encoding: base64 + +UEsDBAoAAAAIAHFUfFWAeOSU8QEAAKkEAAAuAAAAZ29vZ2xlLmNvbSFzdGFsdy5hcnQhMTY2OTUw +NzIwMCExNjY5NTkzNTk5LnhtbKVUwZKjIBC9z1ekck9Qk5hoMcye9gt2zxbB1lBBoACTmb9fHNCw +ma257El83f2632sUv70PYnUDY7mSr+t8m61XIJlquexf179//dyc1qs38oI7gPZM2ZW8rFbYgFbG +NQM42lJHJ8yjyvSNpAOQXqlewJapAaMFDDkwUC6IVJ5BfGzagRq2saOe6H6kZSEv1rw7QxumpKPM +NVx2ilyc07ZGKJZuH6WIIirtHQwq9mV5OGWe62t9II4yeEsORbn3uWVxqo7HPN/tDjlGj3BI91Kh +MVT2UYyHztBzSfKyrA7Zsch8s4DMcZBtiFa7Q1X5UeRMhv5mW7qlnmKtBGcfjR7PgtsLLIMo744k +1lFx31LjPFlAQpi2Vz4Qg1E4RNDq7hObngHSfg8SMNLx3c6AnRHNHMknVdPhc8p/TeR9ZMrMwxl1 +X+RbNRoGDdekoFle77uqZlme1+f9jtW1t/iRMJcwNUrfFKNwmOHYF25UjN64dg5MbnCrleXOX+A4 +f4okeZMZmlrrExZfovAuBhZzEq1PPf2mZoWYtyAd77j/fJayC9AWTNMZNaQbSuHI86Ua09FdGgN2 +FO5B+DTs98uP93piiJLiS6IWBDCnDLmB4FdujaayKLz2GV8MSDvjxJr/niIx2t/IJ9FTcrhPGD3+ +On8AUEsBAgoACgAAAAgAcVR8VYB45JTxAQAAqQQAAC4AAAAAAAAAAAAAAAAAAAAAAGdvb2dsZS5j +b20hc3RhbHcuYXJ0ITE2Njk1MDcyMDAhMTY2OTU5MzU5OS54bWxQSwUGAAAAAAEAAQBcAAAAPQIA +AAAA \ No newline at end of file diff --git a/tests/resources/smtp/reports/dmarc2.eml b/tests/resources/smtp/reports/dmarc2.eml new file mode 100644 index 00000000..3a661741 --- /dev/null +++ b/tests/resources/smtp/reports/dmarc2.eml @@ -0,0 +1,68 @@ +Received: from mail.stalw.art ([mail.stalw.art]) + by 127.0.0.1 (Stalwart JMAP) with LMTP; + Thu, 10 Nov 2022 03:27:19 +0000 +Received: from mx0.backschues.net (lnxs001.backschues.net [85.183.142.13]) + (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) + key-exchange X25519 server-signature RSA-PSS (2048 bits) server-digest SHA256) + (No client certificate requested) + by mail.stalw.art (Postfix) with ESMTPS id 6DD117CC0B + for ; Thu, 10 Nov 2022 03:27:16 +0000 (UTC) +Received: from mx0.backschues.net (localhost [127.0.0.1]) + by mx0.backschues.net with SMTP id 4N76hg4lNgz9ryP + for ; Thu, 10 Nov 2022 04:27:15 +0100 (CET) +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=backschues.net; + s=mail-2014-01; t=1668050835; + h=from:from:reply-to:subject:subject:date:date:message-id:message-id: + to:to:cc:mime-version:mime-version:content-type:content-type; + bh=LTj1tdFz9JQFL/mVJASN0b9hGcolcCtY5v0bhnChJYY=; + b=AtRegYc51PTYqDOy/6fB4xETTWAbVc2ivf8AfF4ygu3+6+oqBPyloTuOnEt7xYmjLFnll/ + SMZFFpRETsMlkiVg/1O0VpPRpIpiTbh4dwtUrRyo1Uw/cDJv5auz4rBMxcRNnDKypHwUKs + BUahHWsVKH/TL5SzV79kqyjlYAs1HdJvS+wRINYBaptkeT6UeHGZakL21NnQUdOGt0fj4y + eJvWVtCYHZ5DUJ8K8h2W1NlTAWP8nTBoQVVDQrI5Zi1AEvnUWw+H7E8d/q2cF756/IBYso + rT56D3PYo2iuSt3aIBth1wL7/GJwc6N4JHcNpJ9XPV6xQbt+lm2b3+W59osL0Q== +DKIM-Signature: v=1; a=ed25519-sha256; c=relaxed/relaxed; d=backschues.net; + s=ed25519-mail-2018-10; t=1668050835; + h=from:from:reply-to:subject:subject:date:date:message-id:message-id: + to:to:cc:mime-version:mime-version:content-type:content-type; + bh=LTj1tdFz9JQFL/mVJASN0b9hGcolcCtY5v0bhnChJYY=; + b=y7d79OWWCrDX40k91FoBdGnUcrjN7xvWYyqskPfQmMoaSqFNSlTHH8gMXC/vXwiYIP3Oxp + d/hVvEuIIQBlwMDQ== +From: "DMARC Aggregate Report" +To: domains@stalw.art +Subject: Report Domain: stalw.art + Submitter: backschues.net + Report-ID: stalw.art.1667948400.1668034800 +Date: Thu, 10 Nov 2022 03:27:02 GMT +MIME-Version: 1.0 +Message-ID: +Content-Type: multipart/mixed; + boundary="----=_NextPart_84e1fdd0-b285-4922-9fc7-88b070204303" + +This is a multipart message in MIME format. + +------=_NextPart_84e1fdd0-b285-4922-9fc7-88b070204303 +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +This is an aggregate report from backschues.net. + +Report domain: stalw.art +Submitter: backschues.net +Report ID: stalw.art.1667948400.1668034800 + +------=_NextPart_84e1fdd0-b285-4922-9fc7-88b070204303 +Content-Type: application/gzip +Content-Transfer-Encoding: base64 +Content-Disposition: attachment; + filename="backschues.net!stalw.art!1667948400!1668034800.xml.gz" + +H4sIAAAAAAAAA5VUsXLbMAzd/RU6D9ksSo6b2heG6dKOndJZR5OQzYtEsiSVNH9fUqQoqXWHT +gIfgAfgASf8/KvvijcwVij5tK3LaluAZIoLeXna/nj5tjtui2eywS0AP1P2SjZFgQ1oZVzTg6 +OcOhowjypzaSTtgdz9HJR7DNGWXQew5fevLxhld4yGnoqOSOW5uo8d76lhOzvoQPxlkSrBYRR +jY16qLTixjnbvJTWurB8ePp8Ox0NVBfNY3R+OVYXRHBpTfa/QGCovqQcPneEiJJnzMYrI5AfJ +yZIyvCMZWrPlaktRsFadYB+NHs6dsFfIjSg/kJwH8GQRiW7KX0VPDEbRSKDV7YiFb4S0l08CR +jq97QTYCdHMkTr0HYyxy7878ooyZXhcrHqfuNRgGDRCk09Vud/fl/X+VNangyfPnhjJ1CB9FY +yikQrHMvBGu8HrxLOgXFitrHD+3FKzSyRHhblbv3TvzhKME7YJnlVAt2r5dcRRsOAgnWiFP/G +UcAXKwTStUf1yBUt4ZPgjE9PBXRsDdujcRLVqLu1QgGtLf+zrpY6XG1KJptaGaxkf0y0Fnpts +/7iRmS7KcYNukxX7zwbjWtaMicaf30qEEBaPB6P8h/gNNHLX4VQEAAA= +------=_NextPart_84e1fdd0-b285-4922-9fc7-88b070204303-- diff --git a/tests/resources/smtp/reports/dmarc3.eml b/tests/resources/smtp/reports/dmarc3.eml new file mode 100644 index 00000000..7acd93ed --- /dev/null +++ b/tests/resources/smtp/reports/dmarc3.eml @@ -0,0 +1,52 @@ +Received: from mail.stalw.art ([mail.stalw.art]) + by 127.0.0.1 (Stalwart JMAP) with LMTP; + Tue, 08 Nov 2022 23:26:41 +0000 +Received: from relay7.m.smailru.net (relay7.m.smailru.net [94.100.178.51]) + (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) + (No client certificate requested) + by mail.stalw.art (Postfix) with ESMTPS id DD4337CC09 + for ; Tue, 8 Nov 2022 23:26:38 +0000 (UTC) +DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/relaxed; d=corp.mail.ru; s=mail4; + h=Date:Message-ID:To:From:Subject:MIME-Version:Content-Type:From:Subject:Content-Type:Content-Transfer-Encoding:To:Cc; bh=fooa0+RBCZvyV2mP8Nx/UsLQ5RhazFg+SPGNtxZrCX0=; + t=1667950001;x=1668040001; + b=J9aMEkY9eVdOxjkNxaPFJ2Yk+/NCux9uOZl3iJXI0hEFaeYj9g7l+WtmXczk+YvgH3yhVhtvONUEYFsValRWWCAfmePm429N3mSuclVktk7t6RPJ4O5EcMjwrD9882vmX1xpI7ecPOzd5AD67HPt5SIA1RIa5injaOI5CWUXBBa5c0zDfmciyANAiDw0gm1axEMK4AUc61txPsX7H1qRq/FxGNITnnpYdqkkT2lR8sTl5HPwTjEsw4sYGKr5SiMpROhhbLZTM8RpojkP73bmw3UBZ9FI8iKApJUFB8i9tu0hjzHkev4uoDXgOXFYs/RAI1JkCWEp2Rjb3LpTSHT6cA==; +Received: from [10.161.4.115] (port=60844 helo=60) + by relay7.m.smailru.net with esmtp (envelope-from ) + id 1osXzK-0007VC-BD + for domains@stalw.art; Wed, 09 Nov 2022 02:26:38 +0300 +Content-Type: multipart/mixed; boundary="===============5640625649776607409==" +MIME-Version: 1.0 +Subject: Report Domain: stalw.art; Submitter: Mail.Ru; + Report-ID: 28551467700969547611667865600 +From: dmarc_support@corp.mail.ru +To: domains@stalw.art +Message-ID: +Date: Wed, 09 Nov 2022 02:26:38 +0300 +Auto-Submitted: auto-generated +Authentication-Results: relay7.m.smailru.net; auth=pass smtp.auth=dmarc_support@corp.mail.ru smtp.mailfrom=dmarc_support@corp.mail.ru; iprev=pass policy.iprev=10.161.4.115 + +--===============5640625649776607409== +MIME-Version: 1.0 +Content-Type: text/plain; charset="utf-8" +Content-Transfer-Encoding: base64 + +VGhpcyBpcyBhbiBhZ2dyZWdhdGUgcmVwb3J0IGZyb20gTWFpbC5SdS4= + +--===============5640625649776607409== +Content-Type: application/gzip +MIME-Version: 1.0 +Content-Transfer-Encoding: base64 +Content-Disposition: attachment; + filename="mail.ru!stalw.art!1667865600!1667952000.xml.gz" + +H4sICK7lamMC/21haWwucnUhc3RhbHcuYXJ0ITE2Njc4NjU2MDAhMTY2Nzk1MjAwMC54bWwAdVNB +cqMwELzvK3LzKQhYg01qouwHctkPULIYjMogqSThJL/fEQSC18kFzbRmWt0jAS/vQ/9wReeV0c+7 +LEl3D6ilaZQ+P+/G0D4edy/8F7SIzUnICweH1rhQDxhEI4LgYNy51mJA/ipUn/wdga0I4EAYbwbh +ZO1HGzv/SONsEvHEUe1cAfgenKil0UHIUCvdGt6FYJ8Y67Bfy1lcHyNCjfcdizbV8PxYFNm+PBzS +tCqrYn8os6wsD8eyKNMU2FchkAmsndBnknvCs9J8WzgjgLqZ4KrI0wjHHNi2ld3NxZpeyY/ajqde ++Q7jUYb0a+6D6N8S4QIxzAiI5qIG7oDNAQhv2ymNK1iujUZgloNfYgrAysCzKCcG9L070CENO67m +jVrN6CTWyvIiTfL8d5LlVZJVe+Jad0CaURMpsDlYTOBV9CO5jSaUt8arQO/lU8oWgUl/S9dE+GQl +OpjzyQu7Z2STPNWgDqpV9BQ5dCgadHXrzLAd1xYGdtMhxtDVDv3YB/+pYpm3wtAm9Ca/xu2xRxmM +m7bI7JrDzMCt8D5e6ZQsTm5Iv7nEleWKvboo76zQef4N+zyO/9in6fysWBqLfIjOiXBKftA6T/l2 +HGx5CGz9j/8BQWPZIPkDAAA= +--===============5640625649776607409==-- \ No newline at end of file diff --git a/tests/resources/smtp/reports/dmarc4.eml b/tests/resources/smtp/reports/dmarc4.eml new file mode 100644 index 00000000..2fe512f0 --- /dev/null +++ b/tests/resources/smtp/reports/dmarc4.eml @@ -0,0 +1,126 @@ +Received: from mail.stalw.art ([mail.stalw.art]) by 127.0.0.1 (Stalwart JMAP) with LMTP; Tue, 25 Oct 2022 04:08:22 +0000 +Received: from NAM12-MW2-obe.outbound.protection.outlook.com (mail-mw2nam12on2073.outbound.protection.outlook.com [40.107.244.73]) + (using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits)) + (No client certificate requested) + by mail.stalw.art (Postfix) with ESMTPS id A24107CC0A + for ; Tue, 25 Oct 2022 04:08:22 +0000 (UTC) +ARC-Seal: i=1; a=rsa-sha256; s=arcselector9901; d=microsoft.com; cv=none; + b=SvolQ1oIEgdfCI6dbwmJ1jS0ovWmprW6kT3q9NgrbX+CMhIsdrqyS3Q1sO16KT2wCQAyNofiEZ5tKY0e1PzzMqeR29jUWvEye9T43fCfUeLFx9b45YrfkGYwqLeDIq0Ywl+ggVmsm7X83XqI6+9EC6qMukCb0cbLazu3rW/Rbyc6d5+fq6QTFZovATGRvHz71H9t7e//hYI23XjU5Q3Enw0Qq3xPSyusWDi3t7CfGXn9i2120XlNLnPxef5PCmwy4E+OTJ5qC5WtMthOskKKuFvx8onOYmc/JjJ3VrtZwALx9C+ulzix5US6H7pFvZ2jtDbMnW4U7ir/hp5xn5adFw== +ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=microsoft.com; + s=arcselector9901; + h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-AntiSpam-MessageData-ChunkCount:X-MS-Exchange-AntiSpam-MessageData-0:X-MS-Exchange-AntiSpam-MessageData-1; + bh=65M39Uvc5w8zbmK6TxEdoblSMXlIyqTXJHNalJ80wk4=; + b=PN7QPeXJLr6tmH2CxydbDQjHqBtFKNN9HjGimeUHaIeSr82WHf4R295QbVX7gxw6sFE7Z9lZMTrMSqbRVI7rhbx+SEkxCfAothf9207FDX6t37Zt0wd/5EwR6dzfbcNJBL+U0/iG4J03L5b1geWY+e68mHKYH4/ybGcr+SBKuv/LgfZNtOfbQ3ioiKvFcpSDqd/qGUs4U9l2tVlXgbcKkct04sCuPciqgLEuIGirPLLbDUaBRJc51ZZB6CeporySRdHp6uFXyy3VBvvLVuwDNnnPrW4BUL05AuutzK7rc8ZQEpWf7r0gUEg2ArSrvs6Znnfe97oRa01L2SeFwuZsMA== +ARC-Authentication-Results: i=1; mx.microsoft.com 1; spf=none; dmarc=none + action=none header.from=microsoft.com; dkim=none (message not signed); + arc=none +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; + d=notification.microsoft.com; s=selector1; + h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-SenderADCheck; + bh=65M39Uvc5w8zbmK6TxEdoblSMXlIyqTXJHNalJ80wk4=; + b=NjqsA7D6sEq1WgCZ/E1f5/B+XUXe5F4uv6CF2KQYVuyRnxItdox09LCWqZQ+fNQ6BbJ4Ne05Cb1BPbPP9yvb8Y6B1s2QvuxkUb69UFbAhoFgsRT6A4K76ykKQQyiPoYpxlO6FEyy+gel4y7c9XRLiWW6OxMIBcjBGB5ziP7mGFaJx4qXJ2mROfO7uZfrCu5pzOimkjPw6extWv4i0Kl3XKvBtXZnsr9eoC10mJvEAp7E2cpnaZnP46RQc9cmXzlmvhKPvCQCUWipJN9f1BTTvFjJ9ff6ehmN9RSzCckj3SZGw9XAnd0WYqh4evt6Y1RxQ4iQDSaZHNRpyMOtmkWc/w== +Authentication-Results: dkim=none (message not signed) + header.d=none;dmarc=none action=none header.from=microsoft.com; +Received: from BN9PR03CA0046.namprd03.prod.outlook.com (2603:10b6:408:fb::21) + by SJ0PR18MB3916.namprd18.prod.outlook.com (2603:10b6:a03:2c9::21) with + Microsoft SMTP Server (version=TLS1_2, + cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.5746.21; Tue, 25 Oct + 2022 04:08:19 +0000 +Received: from BN7NAM10FT048.eop-nam10.prod.protection.outlook.com + (2603:10b6:408:fb:cafe::d7) by BN9PR03CA0046.outlook.office365.com + (2603:10b6:408:fb::21) with Microsoft SMTP Server (version=TLS1_2, + cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.5746.27 via Frontend + Transport; Tue, 25 Oct 2022 04:08:19 +0000 +Received: from nam10.map.protection.outlook.com (2a01:111:f400:fe53::30) by + BN7NAM10FT048.mail.protection.outlook.com (2a01:111:e400:7e8f::199) with + Microsoft SMTP Server (version=TLS1_2, + cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.5746.16 via Frontend + Transport; Tue, 25 Oct 2022 04:08:19 +0000 +Message-ID: <725cbfbe133940149987cfc528387235@microsoft.com> +X-Sender: XATTRDIRECT=Originating XATTRORGID=xorgid:96f9e21d-a1c4-44a3-99e4-37191ac61848 +MIME-Version: 1.0 +From: "DMARC Aggregate Report" +To: +Subject: =?utf-8?B?UmVwb3J0IERvbWFpbjogc3RhbHcuYXJ0IFN1Ym1pdHRlcjogcHJvdGVjdGlvbi5vdXRsb29rLmNvbSBSZXBvcnQtSUQ6IDcyNWNiZmJlMTMzOTQwMTQ5OTg3Y2ZjNTI4Mzg3MjM1?= +Content-Type: multipart/mixed; + boundary="_mpm_a4bcd9a515b44b9d8eceb05d7333675fpiotk5m200exchangecorpm_" +Date: Tue, 25 Oct 2022 04:08:19 +0000 +X-EOPAttributedMessage: 0 +X-MS-PublicTrafficType: Email +X-MS-TrafficTypeDiagnostic: BN7NAM10FT048:EE_|SJ0PR18MB3916:EE_ +X-MS-Office365-Filtering-Correlation-Id: 7f843e40-ccc6-4c17-1ce7-08dab63e8cd1 +X-MS-Exchange-SenderADCheck: 2 +X-MS-Exchange-AntiSpam-Relay: 0 +X-Microsoft-Antispam: BCL:0; +X-Microsoft-Antispam-Message-Info: + PSiI3L4DKj/cyRBl8/bmbyQMrr1DvsYEB1+aTn/3Y39oHnyJ5HXcxu6jNUl32WcPW6Gfqmhc6P1RFE5L/9ev0cWnqh4GgIs2qmHicLexPmMjP8viPdjb1N7TSSOv1hhXMT+gVLx889X5sltd4qpfIAWhoxNonjQpVIgt4VOVnbCWTu1hyOjOVplq0rKqIF04BQGHZnBRfkcD1No+mZrvx8RLWIwInU3fpPeGz77Wn3TIvHtzypR/d22WpZ8eHk3aIxxdjwp5WLg4unpiJaieyQN7BRhD/v6b3pLFVJP8Ii2+FGjTsKASczEL4dHnIoIrHYE0wwaFFPcSNzovLhzYguDV42EGS8Fm7soiew4ch+hICM0LPNTGTZIDe7wm2eSwhN2tkJK4QCfh1DON39jXninVr88ZlzMcDXnXpgvWHHiur8az7Gvs9zHH/1tFMsPVSh7BS+8fHEcBYpdtihrP22GcjbOd98IiTAs/dVzSy0TUg6WEgJO6oUklGjqVbi99CrNZI1BtLP4vH4aSlz9JYg4et6SxiJlKyoSzqUr2NN9/pyFdQ//5d/EEjKJz8CAcQmCjjPEObGFttT3maY2+zsa2THodZgpfMyDbA3WUKxE= +X-Forefront-Antispam-Report: + CIP:255.255.255.255;CTRY:;LANG:en;SCL:1;SRV:;IPV:NLI;SFV:NSPM;H:nam10.map.protection.outlook.com;PTR:;CAT:NONE;SFS:(13230022)(396003)(39860400002)(346002)(34036004)(366004)(376002)(136003)(47540400005)(451199015)(2616005)(52230400001)(121820200001)(83380400001)(166002)(86362001)(41300700001)(2906002)(4001150100001)(8936002)(316002)(5660300002)(235185007)(41320700001)(508600001)(6486002)(6512007)(6506007)(24736004)(108616005)(68406010)(85236043)(8676002)(10290500003)(6916009)(36736006)(36756003)(66899015);DIR:OUT;SFP:1101; +X-OriginatorOrg: dmarcrep.onmicrosoft.com +X-MS-Exchange-CrossTenant-OriginalArrivalTime: 25 Oct 2022 04:08:19.1682 + (UTC) +X-MS-Exchange-CrossTenant-Network-Message-Id: 7f843e40-ccc6-4c17-1ce7-08dab63e8cd1 +X-MS-Exchange-CrossTenant-AuthSource: BN7NAM10FT048.eop-nam10.prod.protection.outlook.com +X-MS-Exchange-CrossTenant-AuthAs: Internal +X-MS-Exchange-CrossTenant-Id: 96f9e21d-a1c4-44a3-99e4-37191ac61848 +X-MS-Exchange-CrossTenant-FromEntityHeader: Internet +X-MS-Exchange-Transport-CrossTenantHeadersStamped: SJ0PR18MB3916 + +This is a multi-part message in MIME format. + +--_mpm_a4bcd9a515b44b9d8eceb05d7333675fpiotk5m200exchangecorpm_ +Content-Type: multipart/related; + boundary="_rv_a4bcd9a515b44b9d8eceb05d7333675fpiotk5m200exchangecorpm_" + +--_rv_a4bcd9a515b44b9d8eceb05d7333675fpiotk5m200exchangecorpm_ +Content-Type: multipart/alternative; + boundary="_av_a4bcd9a515b44b9d8eceb05d7333675fpiotk5m200exchangecorpm_" + +--_av_a4bcd9a515b44b9d8eceb05d7333675fpiotk5m200exchangecorpm_ + + +--_av_a4bcd9a515b44b9d8eceb05d7333675fpiotk5m200exchangecorpm_ +Content-Type: text/html; charset=us-ascii +Content-Transfer-Encoding: base64 + +PGRpdiBzdHlsZSA9ImZvbnQtZmFtaWx5OlNlZ29lIFVJOyBmb250LXNpemU6MTRweDsiPlRoaXMgaX +MgYSBETUFSQyBhZ2dyZWdhdGUgcmVwb3J0IGZyb20gTWljcm9zb2Z0IENvcnBvcmF0aW9uLiBGb3Ig +RW1haWxzIHJlY2VpdmVkIGJldHdlZW4gMjAyMi0xMC0yMyAwMDowMDowMCBVVEMgdG8gMjAyMi0xMC +0yNCAwMDowMDowMCBVVEMuPC8gZGl2PjxiciAvPjxiciAvPllvdSdyZSByZWNlaXZpbmcgdGhpcyBl +bWFpbCBiZWNhdXNlIHlvdSBoYXZlIGluY2x1ZGVkIHlvdXIgZW1haWwgYWRkcmVzcyBpbiB0aGUgJ3 +J1YScgdGFnIG9mIHlvdXIgRE1BUkMgcmVjb3JkIGluIEROUyBmb3Igc3RhbHcuYXJ0LiBQbGVhc2Ug +cmVtb3ZlIHlvdXIgZW1haWwgYWRkcmVzcyBmcm9tIHRoZSAncnVhJyB0YWcgaWYgeW91IGRvbid0IH +dhbnQgdG8gcmVjZWl2ZSB0aGlzIGVtYWlsLjxiciAvPjxiciAvPjxkaXYgc3R5bGUgPSJmb250LWZh +bWlseTpTZWdvZSBVSTsgZm9udC1zaXplOjEycHg7IGNvbG9yOiM2NjY2NjY7Ij5QbGVhc2UgZG8gbm +90IHJlc3BvbmQgdG8gdGhpcyBlLW1haWwuIFRoaXMgbWFpbGJveCBpcyBub3QgbW9uaXRvcmVkIGFu +ZCB5b3Ugd2lsbCBub3QgcmVjZWl2ZSBhIHJlc3BvbnNlLiBGb3IgYW55IGZlZWRiYWNrL3N1Z2dlc3 +Rpb25zLCBraW5kbHkgbWFpbCB0byBkbWFyY3JlcG9ydGZlZWRiYWNrQG1pY3Jvc29mdC5jb20uPGJy +IC8+PGJyIC8+TWljcm9zb2Z0IHJlc3BlY3RzIHlvdXIgcHJpdmFjeS4gUmV2aWV3IG91ciBPbmxpbm +UgU2VydmljZXMgPGEgaHJlZiA9Imh0dHBzOi8vcHJpdmFjeS5taWNyb3NvZnQuY29tL2VuLXVzL3By +aXZhY3lzdGF0ZW1lbnQiPlByaXZhY3kgU3RhdGVtZW50PC9hPi48YnIgLz5PbmUgTWljcm9zb2Z0IF +dheSwgUmVkbW9uZCwgV0EsIFVTQSA5ODA1Mi48LyBkaXYgPg== + +--_av_a4bcd9a515b44b9d8eceb05d7333675fpiotk5m200exchangecorpm_-- + +--_rv_a4bcd9a515b44b9d8eceb05d7333675fpiotk5m200exchangecorpm_-- + +--_mpm_a4bcd9a515b44b9d8eceb05d7333675fpiotk5m200exchangecorpm_ +Content-Type: application/gzip +Content-Transfer-Encoding: base64 +Content-ID: <3ff45643-7977-4f3c-a97d-14b9e7faa5e7> +Content-Description: protection.outlook.com!stalw.art!1666483200!1666569600.xml.gz +Content-Disposition: attachment; filename="protection.outlook.com!stalw.art!1666483200!1666569600.xml.gz"; + +H4sIAAAAAAAEAM1VzY7bIBi8V+o7RLnXxHZ+VyzbB2jVQy+9WQTjBMUGBDjZvn0/G4JJsu3usZcE5h +vzDcNg45fXrp2dubFCyed5ni3mL+TzJ9xwXu8pO82gLO3Tq62f50fn9BNCl8slu5SZMgdULBY5+vX9 +20925B2dR7J4n/xFSOuoZHwO7WYzHCQQUIDRdTJWDNfKuKrjjtbU0REEGJasJO04+dG7VqlTxlSHUU +QDCzqJltQdNcyv87UTzCirGucf8ITADq1ETTbFiu2bPc/Lcrdc5MvdbrthDVsV23K7KcoVRhM3PAzi +eGWoPFybA7bnBwF7Wq/Xy20JBmDkkUjgsh7Lq/VuPZSHeVgP3S0YW944gbVqBftd6X7fCnvkkxwFO5 +METG4vGTUO1vNIqNP6JDpiMPKDK2p1M4LDf8A0kUpyjPQVsFfERkgzR/JhA8MgYI0iAMCvV/+mULCc +KRNFG3WZvLGqN4xXQpPVIiuKMsuLXZbvltA3ViKZqV6CBIz8IOKhKz/Ttgc/61gZLBJWKyvcEDW/oR +RJiYNDDQQFGJNZwYsmVCbHkt3e94VDjFvEoubSiUZA2tNEnHmrNK+cIiqNdlp4ZDdGdURw1wx3LSGP +eKQfOa258WCSjBS+6nwUh2nvjpXhtm9dIvjekRCzSctN7rxpvOXMKTOS4MziPOH4PkRTa4fkj5PJ3p +um/7GEv12/Ww1wVuIkrNFUFsU/tfiovaMlTeIH3WCQFdINAYD24+TDPiRvCvSQkIEfLji8CsJHhfwB +wJC79XYGAAA= + +--_mpm_a4bcd9a515b44b9d8eceb05d7333675fpiotk5m200exchangecorpm_-- diff --git a/tests/resources/smtp/reports/dmarc5.eml b/tests/resources/smtp/reports/dmarc5.eml new file mode 100644 index 00000000..3463d8bf --- /dev/null +++ b/tests/resources/smtp/reports/dmarc5.eml @@ -0,0 +1,54 @@ +Received: from mail.stalw.art ([mail.stalw.art]) by 127.0.0.1 (Stalwart JMAP) with LMTP; Tue, 20 Sep 2022 10:28:19 +0000 +Received: from a14-92.smtp-out.amazonses.com (a14-92.smtp-out.amazonses.com [54.240.14.92]) + (using TLSv1.2 with cipher ECDHE-RSA-AES128-SHA256 (128/128 bits)) + (No client certificate requested) + by mail.stalw.art (Postfix) with ESMTPS id 1337D7E19D + for ; Tue, 20 Sep 2022 10:28:18 +0000 (UTC) +DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/simple; + s=a66wkfbz3zwxdt2n5p6d7lj2ja7sdwuc; d=amazonses.com; t=1663669697; + h=From:To:Message-ID:Subject:MIME-Version:Content-Type:Date; + bh=h9v7dueDYfUxVokuKSLTqLuOwisdgdDRQ6TLwJOzXes=; + b=dHR5EJhoY9s8g2/Y4K4rHdz44k67r7fyC4wr2AWZmemrVBoxYHJPwa295S2VJQtY + kxTxppN2GEcNxhUMw8TXBrRwNKdoOLU38ZtrAN1a4hWVxmlwky1dtjXETQ/qJ257Nzg + bsXkAo4S1RABFmkQQJ0zSPZGkMW+lpZTBCDzlOHU= +DKIM-Signature: v=1; a=rsa-sha256; q=dns/txt; c=relaxed/simple; + s=6gbrjpgwjskckoa6a5zn6fwqkn67xbtw; d=amazonses.com; t=1663669697; + h=From:To:Message-ID:Subject:MIME-Version:Content-Type:Date:Feedback-ID; + bh=h9v7dueDYfUxVokuKSLTqLuOwisdgdDRQ6TLwJOzXes=; + b=UDIvc6rvbihyGbzGRsmSSSzVNFgpfb3V3j0UivcNjlX2y63vjLinol463Z/+3Xh3 + BmxAOiLHF/DbVnqqNg5ygdxsa7MBHXEJ5we3W8vQr37xNk5DqhV7HPBSFttWP5sy0dg + rdjyfMIjqJ1J/2+aM4opFA/6EWif7TGmjo7N1KKM= +From: postmaster@amazonses.com +To: domains@stalw.art +Message-ID: <010001835a70fc8d-a3d7eff5-7adb-41cc-87bd-a646d9776a69-000000@email.amazonses.com> +Subject: Dmarc Aggregate Report Domain: {stalw.art} Submitter: {Amazon SES} + Date: {2022-09-19} Report-ID: {6b06c366-0631-4ca0-8337-f5aecf137918} +MIME-Version: 1.0 +Content-Type: multipart/mixed; + boundary="----=_Part_42492_694130218.1663669697673" +Date: Tue, 20 Sep 2022 10:28:17 +0000 +Feedback-ID: 1.us-east-1.CTa/CO4t1eWkL0VlHBu5/eINCZhxZraAIsQC/FZHIgk=:AmazonSES +X-SES-Outgoing: 2022.09.20-54.240.14.92 + +------=_Part_42492_694130218.1663669697673 +Content-Type: text/plain; charset=us-ascii +Content-Transfer-Encoding: 7bit + +This MIME email was sent through Amazon SES. +------=_Part_42492_694130218.1663669697673 +Content-Type: application/octet-stream; + name=amazonses.com!stalw.art!1663545600!1663632000.xml.gz +Content-Transfer-Encoding: base64 +Content-Disposition: attachment; + filename=amazonses.com!stalw.art!1663545600!1663632000.xml.gz + +H4sIAAAAAAAAAG1TwXLbIBA9O1/RyV1CWLHszlDSHHJMe8itFw1GK5uJBAwgp+3XlwXJVjK9SOzb +1b59vBV7/D0OXy7gvDL62z0tq/tHfsd6gO4o5Bu/27A5yauSMrIEEXdgjQvtCEF0IogIbZhxp1aL +EfjTy9Ovnz+K1+dXRq4gVsAo1MCt8WEUPoD7Lkbx12gPvpRmZCTnsXLurzreHKtG1k1TVE1Niwcp +quJQ1/ui3wmQPa33X+mBkVs9fh1HgtYJfUq0G3aEk9KcNk29e9g1VcVIRlISdJdSTb2tMIUxNiEf +ulwpVpKZNYOSf1o7HQflzzCTm6hCcx/E8F4KF2KjjGBSdG9q5I6RfEiQt31C8I2A5dpoYMSmyC+h +z7GVgVOcEw8I9IbHKD5xyP9MFO9SGpdnc+Y9i/ZmchJaZfm22pd0T0t6OJRb7HtLpUppJh0ZGcmH +hM0scBHDFC8p9UblykdvVcAdyTOvkbkGZffR5picbyCJ7GdwvoSblA8k0YWsgKkOdFC9iiu52HiB +wVhoe2dGnher1AP6uU6k2jOIDlwGVj6t4UT2iYSJKZxbB34awsy6jHu1fUV8sz0tNH7FrfAeVykF +WediO/nUHcuycdHd5Zf8BxMenbqzAwAA +------=_Part_42492_694130218.1663669697673-- diff --git a/tests/resources/smtp/reports/tls1.eml b/tests/resources/smtp/reports/tls1.eml new file mode 100644 index 00000000..e54c9e8e --- /dev/null +++ b/tests/resources/smtp/reports/tls1.eml @@ -0,0 +1,41 @@ +From: tlsrpt@mail.sender.example.com +Date: Fri, May 09 2017 16:54:30 -0800 +To: mts-sts-tlsrpt@example.net +Subject: Report Domain: example.net + Submitter: mail.sender.example.com + Report-ID: <735ff.e317+bf22029@example.net> +TLS-Report-Domain: example.net +TLS-Report-Submitter: mail.sender.example.com +MIME-Version: 1.0 +Content-Type: multipart/report; report-type="tlsrpt"; + boundary="----=_NextPart_000_024E_01CC9B0A.AFE54C00" +Content-Language: en-us + +This is a multipart message in MIME format. + +------=_NextPart_000_024E_01CC9B0A.AFE54C00 +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +This is an aggregate TLS report from mail.sender.example.com + +------=_NextPart_000_024E_01CC9B0A.AFE54C00 +Content-Type: application/tlsrpt+gzip +Content-Transfer-Encoding: base64 +Content-Disposition: attachment; + filename="mail.sender.example!example.com!1013662812!1013749130.json.gz" + +H4sICCpFtWMAA3JwdDAxLmpzb24uMQCtVVtr2zAYfe+vEN7bmFzZjt3EMLbRhu1hdCUJI+soRpGU +VMy2jCSHZCX/fZIvzVLHpc0WDDHSOfqOznfxwxkwP0fIFc75b6y5yGGOM+bEwLkUWYHzLZw772oU +xZpBifOV3X6o1qp1pbHU0O5qXlN95EUQDSDyZgjF1XPbnFIxWE778H4QhyPz3DoVfNfEJiLXmGjI +86WwDKUVlKwQUvN89ZE0Ujcu2+CsSFkruYZATi0nRFE48C8I9AMawMEFwXARMQRHg4hhxIZksHgk +FiLlhDNleD8fde/vvMdsD7x4sgf1tmCN3L/u/xSltDS3OAh1AFszqUxmYjCdTdfekYMqVCYoi4Fm +ylrSC9rE4K2bYZ66rWnbJ6Z1OXiT4JU5exgNEHI6oLv+m1FhQuXWgZdEM+rgvVC634le6V1RByu7 +w2COKrMMy57caaFxClVJCFNqWZpX8287g4gyt+LCwI+OqK95SyOwlKxDClDwrKSWR5k2b+qoB12x +FVUyVab6sdgIM12x5MS2K9sUXDLal1plOtFUC8w0hryoWxF5MV0MY7wg1PSt58dxb8lJRhhfVwfU +mWtnR7bxXllk9vqMdlzzEOrgd90jXmZMNah0qmAutMlvYWfDP3kTnOaN/0pv9ke1OgIXuZ4XuGH0 +Sj/NFXoImFJu578pYTtkZVZ9DWy4e60LFZ+f18NUuZ1p2+wklgc+AE7Be9AMW0AABH4AaPAGTBv7 +r4WetuaDbueenN41Tjmtv2FNM708td5o6Iaea8rNjfwT8jA8oQzgApNfZfF/GiV4Bm7HimRYVXBa +RZ+HaJR8T8aTSXIz+Tb/kdx8mn1Jvo6vP5u/8fxyPL4aXx3JzcHKfsbW63dnuz+8byfQUQgAAA== + +------=_NextPart_000_024E_01CC9B0A.AFE54C00-- diff --git a/tests/resources/smtp/reports/tls2.eml b/tests/resources/smtp/reports/tls2.eml new file mode 100644 index 00000000..afb86871 --- /dev/null +++ b/tests/resources/smtp/reports/tls2.eml @@ -0,0 +1,64 @@ +From: tlsrpt@mail.sender.example.com +Date: Fri, May 09 2017 16:54:30 -0800 +To: mts-sts-tlsrpt@example.net +Subject: Report Domain: example.net + Submitter: mail.sender.example.com + Report-ID: <735ff.e317+bf22029@example.net> +TLS-Report-Domain: example.net +TLS-Report-Submitter: mail.sender.example.com +MIME-Version: 1.0 +Content-Type: multipart/report; report-type="tlsrpt"; + boundary="----=_NextPart_000_024E_01CC9B0A.AFE54C00" +Content-Language: en-us + +This is a multipart message in MIME format. + +------=_NextPart_000_024E_01CC9B0A.AFE54C00 +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +This is an aggregate TLS report from mail.sender.example.com + +------=_NextPart_000_024E_01CC9B0A.AFE54C00 +Content-Type: application/tlsrpt +Content-Disposition: attachment; + filename="mail.sender.example!example.com!1013662812!1013749130.json" + +{ + "report-id": "2020-01-01T00:00:00Z_example.com", + "date-range": { + "start-datetime": "2020-01-01T00:00:00Z", + "end-datetime": "2020-01-07T23:59:59Z" + }, + "organization-name": "Google Inc.", + "contact-info": "smtp-tls-reporting@google.com", + "policies": [ + { + "policy": { + "policy-type": "sts", + "policy-string": [ + "version: STSv1", + "mode: enforce", + "mx: demo.example.com", + "max_age: 604800" + ], + "policy-domain": "example.com" + }, + "summary": { + "total-successful-session-count": 23, + "total-failure-session-count": 1 + }, + "failure-details": [ + { + "result-type": "certificate-host-mismatch", + "sending-mta-ip": "123.123.123.123", + "receiving-ip": "234.234.234.234", + "receiving-mx-hostname": "demo.example.com", + "failed-session-count": 1 + } + ] + } + ] +} + +------=_NextPart_000_024E_01CC9B0A.AFE54C00-- diff --git a/tests/src/jmap/mod.rs b/tests/src/jmap/mod.rs index 1a807664..8fa1f083 100644 --- a/tests/src/jmap/mod.rs +++ b/tests/src/jmap/mod.rs @@ -32,6 +32,7 @@ hostname = 'jmap.example.org' bind = ['127.0.0.1:8899'] url = 'https://127.0.0.1:8899' protocol = 'jmap' +max-connections = 512 [server.socket] reuse-addr = true @@ -54,7 +55,6 @@ set.max-objects = 100000 [jmap.protocol.request] max-concurrent = 8 -max-concurrent-total = 512 [jmap.protocol.upload] max-size = 5000000 diff --git a/tests/src/lib.rs b/tests/src/lib.rs index 3f52fa41..b301b616 100644 --- a/tests/src/lib.rs +++ b/tests/src/lib.rs @@ -1,10 +1,11 @@ use std::path::PathBuf; #[cfg(test)] -pub mod jmap; - +//pub mod jmap; #[cfg(test)] -pub mod store; +//pub mod store; +#[cfg(test)] +pub mod smtp; pub fn add_test_certs(config: &str) -> String { let mut cert_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); diff --git a/tests/src/smtp/config.rs b/tests/src/smtp/config.rs new file mode 100644 index 00000000..90ab779b --- /dev/null +++ b/tests/src/smtp/config.rs @@ -0,0 +1,132 @@ +use std::{fs, path::PathBuf}; + +use tokio::net::TcpSocket; + +use utils::config::{Config, Listener, Server, ServerProtocol}; + +use super::add_test_certs; + +#[test] +fn parse_servers() { + let mut file = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + file.push("resources"); + file.push("smtp"); + file.push("config"); + file.push("servers.toml"); + + let toml = add_test_certs(&fs::read_to_string(file).unwrap()); + + // Parse servers + let config = Config::parse(&toml).unwrap(); + let servers = config.parse_servers().unwrap().inner; + let expected_servers = vec![ + Server { + id: "smtp".to_string(), + internal_id: 0, + hostname: "mx.example.org".to_string(), + data: "Stalwart SMTP - hi there!".to_string(), + protocol: ServerProtocol::Smtp, + listeners: vec![Listener { + socket: TcpSocket::new_v4().unwrap(), + addr: "127.0.0.1:9925".parse().unwrap(), + ttl: 3600.into(), + backlog: 1024.into(), + }], + tls: None, + tls_implicit: false, + max_connections: 8192, + }, + Server { + id: "smtps".to_string(), + internal_id: 1, + hostname: "mx.example.org".to_string(), + data: "Stalwart SMTP - hi there!".to_string(), + protocol: ServerProtocol::Smtp, + listeners: vec![ + Listener { + socket: TcpSocket::new_v4().unwrap(), + addr: "127.0.0.1:9465".parse().unwrap(), + ttl: 4096.into(), + backlog: 1024.into(), + }, + Listener { + socket: TcpSocket::new_v4().unwrap(), + addr: "127.0.0.1:9466".parse().unwrap(), + ttl: 4096.into(), + backlog: 1024.into(), + }, + ], + tls: None, + tls_implicit: true, + max_connections: 1024, + }, + Server { + id: "submission".to_string(), + internal_id: 2, + hostname: "submit.example.org".to_string(), + data: "Stalwart SMTP submission at your service".to_string(), + protocol: ServerProtocol::Smtp, + listeners: vec![Listener { + socket: TcpSocket::new_v4().unwrap(), + addr: "127.0.0.1:9991".parse().unwrap(), + ttl: 3600.into(), + backlog: 2048.into(), + }], + tls: None, + tls_implicit: true, + max_connections: 8192, + }, + ]; + + for (server, expected_server) in servers.into_iter().zip(expected_servers) { + assert_eq!( + server.id, expected_server.id, + "failed for {}", + expected_server.id + ); + assert_eq!( + server.internal_id, expected_server.internal_id, + "failed for {}", + expected_server.id + ); + assert_eq!( + server.hostname, expected_server.hostname, + "failed for {}", + expected_server.id + ); + assert_eq!( + server.data, expected_server.data, + "failed for {}", + expected_server.id + ); + assert_eq!( + server.protocol, expected_server.protocol, + "failed for {}", + expected_server.id + ); + assert_eq!( + server.tls_implicit, expected_server.tls_implicit, + "failed for {}", + expected_server.id + ); + for (listener, expected_listener) in + server.listeners.into_iter().zip(expected_server.listeners) + { + assert_eq!( + listener.addr, expected_listener.addr, + "failed for {}", + expected_server.id + ); + assert_eq!( + listener.ttl, expected_listener.ttl, + "failed for {}", + expected_server.id + ); + assert_eq!( + listener.backlog, expected_listener.backlog, + "failed for {}", + expected_server.id + ); + } + } +} diff --git a/tests/src/smtp/inbound/auth.rs b/tests/src/smtp/inbound/auth.rs new file mode 100644 index 00000000..f61aef60 --- /dev/null +++ b/tests/src/smtp/inbound/auth.rs @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::sync::Arc; + +use ahash::AHashSet; +use smtp_proto::{AUTH_LOGIN, AUTH_PLAIN}; + +use crate::smtp::{ + session::{TestSession, VerifyResponse}, + ParseTestConfig, TestConfig, +}; +use smtp::{ + config::ConfigContext, + core::{Core, Session, State}, + lookup::Lookup, +}; + +#[tokio::test] +async fn auth() { + let mut core = Core::test(); + let mut ctx = ConfigContext::default(); + ctx.lookup.insert( + "plain".to_string(), + Arc::new(Lookup::Local(AHashSet::from_iter([ + "john:secret".to_string(), + "jane:p4ssw0rd".to_string(), + ]))), + ); + + let mut config = &mut core.session.config.auth; + + config.require = r"[{if = 'remote-ip', eq = '10.0.0.1', then = true}, + {else = false}]" + .parse_if(&ctx); + config.lookup = r"[{if = 'remote-ip', eq = '10.0.0.1', then = 'plain'}, + {else = false}]" + .parse_if::>(&ctx) + .map_if_block(&ctx.lookup, "", "") + .unwrap(); + config.errors_max = r"[{if = 'remote-ip', eq = '10.0.0.1', then = 2}, + {else = 3}]" + .parse_if(&ctx); + config.errors_wait = "'100ms'".parse_if(&ctx); + config.mechanisms = format!( + "[{{if = 'remote-ip', eq = '10.0.0.1', then = {}}}, + {{else = 0}}]", + AUTH_PLAIN | AUTH_LOGIN + ) + .as_str() + .parse_if(&ctx); + core.session.config.extensions.future_release = + r"[{if = 'authenticated-as', ne = '', then = '1d'}, + {else = false}]" + .parse_if(&ConfigContext::default()); + + // EHLO should not avertise plain text auth without TLS + let mut session = Session::test(core); + session.data.remote_ip = "10.0.0.1".parse().unwrap(); + session.eval_session_params().await; + session.stream.tls = false; + session + .ehlo("mx.foobar.org") + .await + .assert_not_contains(" PLAIN") + .assert_not_contains(" LOGIN"); + + // EHLO should advertise AUTH for 10.0.0.1 + session.stream.tls = true; + session + .ehlo("mx.foobar.org") + .await + .assert_contains("AUTH ") + .assert_contains(" PLAIN") + .assert_contains(" LOGIN") + .assert_not_contains("FUTURERELEASE"); + + // Invalid password should be rejected + session + .cmd("AUTH PLAIN AGpvaG4AY2hpbWljaGFuZ2Fz", "535 5.7.8") + .await; + + // Session should be disconnected after second invalid auth attempt + session + .ingest(b"AUTH PLAIN AGpvaG4AY2hpbWljaGFuZ2Fz\r\n") + .await + .unwrap_err(); + session.response().assert_code("421 4.3.0"); + + // Should not be able to send without authenticating + session.state = State::default(); + session.mail_from("bill@foobar.org", "503 5.5.1").await; + + // Successful PLAIN authentication + session.data.auth_errors = 0; + session + .cmd("AUTH PLAIN AGpvaG4Ac2VjcmV0", "235 2.7.0") + .await; + session.mail_from("bill@foobar.org", "250").await; + session.data.mail_from.take(); + + // Should not be able to authenticate twice + session + .cmd("AUTH PLAIN AGpvaG4Ac2VjcmV0", "503 5.5.1") + .await; + + // FUTURERELEASE extension should be available after authenticating + session + .ehlo("mx.foobar.org") + .await + .assert_not_contains("AUTH ") + .assert_not_contains(" PLAIN") + .assert_not_contains(" LOGIN") + .assert_contains("FUTURERELEASE 86400"); + + // Successful LOGIN authentication + session.data.authenticated_as.clear(); + session.cmd("AUTH LOGIN", "334").await; + session.cmd("amFuZQ==", "334").await; + session.cmd("cDRzc3cwcmQ=", "235 2.7.0").await; + + // Login should not be advertised to 10.0.0.2 + session.data.remote_ip = "10.0.0.2".parse().unwrap(); + session.eval_session_params().await; + session.stream.tls = true; + session + .ehlo("mx.foobar.org") + .await + .assert_not_contains("AUTH ") + .assert_not_contains(" PLAIN") + .assert_not_contains(" LOGIN"); + session + .cmd("AUTH PLAIN AGpvaG4Ac2VjcmV0", "503 5.5.1") + .await; +} diff --git a/tests/src/smtp/inbound/basic.rs b/tests/src/smtp/inbound/basic.rs new file mode 100644 index 00000000..50d57542 --- /dev/null +++ b/tests/src/smtp/inbound/basic.rs @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use crate::smtp::{ + session::{TestSession, VerifyResponse}, + TestConfig, +}; +use smtp::core::{Core, Session}; + +#[tokio::test] +async fn basic_commands() { + let mut session = Session::test(Core::test()); + + // STARTTLS should be available on clear text connections + session.stream.tls = false; + session + .ehlo("mx.foobar.org") + .await + .assert_contains("STARTTLS"); + assert!(!session.ingest(b"STARTTLS\r\n").await.unwrap()); + session.response().assert_contains("220 2.0.0"); + + // STARTTLS should not be offered on TLS connections + session.stream.tls = true; + session + .ehlo("mx.foobar.org") + .await + .assert_not_contains("STARTTLS"); + session.cmd("STARTTLS", "504 5.7.4").await; + + // Test NOOP + session.cmd("NOOP", "250").await; + + // Test RSET + session.cmd("RSET", "250").await; + + // Test HELP + session.cmd("HELP QUIT", "250").await; + + // Test LHLO on SMTP channel + session.cmd("LHLO domain.org", "502").await; + + // Test QUIT + session.ingest(b"QUIT\r\n").await.unwrap_err(); + session.response().assert_code("221"); +} diff --git a/tests/src/smtp/inbound/data.rs b/tests/src/smtp/inbound/data.rs new file mode 100644 index 00000000..24730963 --- /dev/null +++ b/tests/src/smtp/inbound/data.rs @@ -0,0 +1,214 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::sync::Arc; + +use ahash::AHashSet; + +use crate::smtp::{ + inbound::{TestMessage, TestQueueEvent}, + session::{load_test_message, TestSession, VerifyResponse}, + ParseTestConfig, TestConfig, TestCore, +}; +use smtp::{ + config::{ConfigContext, IfBlock}, + core::{Core, Session}, + lookup::Lookup, +}; + +#[tokio::test] +async fn data() { + let mut core = Core::test(); + + // Create temp dir for queue + let mut qr = core.init_test_queue("smtp_data_test"); + + let mut config = &mut core.session.config.rcpt; + config.lookup_domains = IfBlock::new(Some(Arc::new(Lookup::Local(AHashSet::from_iter([ + "foobar.org".to_string(), + "domain.net".to_string(), + "test.com".to_string(), + ]))))); + config.lookup_addresses = IfBlock::new(Some(Arc::new(Lookup::Local(AHashSet::from_iter([ + "bill@foobar.org".to_string(), + "john@foobar.org".to_string(), + "jane@domain.net".to_string(), + "mike@test.com".to_string(), + ]))))); + + let mut config = &mut core.session.config; + config.data.add_auth_results = "[{if = 'remote-ip', eq = '10.0.0.3', then = true}, + {else = false}]" + .parse_if(&ConfigContext::default()); + config.data.add_date = config.data.add_auth_results.clone(); + config.data.add_message_id = config.data.add_auth_results.clone(); + config.data.add_received = config.data.add_auth_results.clone(); + config.data.add_return_path = config.data.add_auth_results.clone(); + config.data.add_received_spf = config.data.add_auth_results.clone(); + config.data.max_received_headers = IfBlock::new(3); + config.data.max_messages = r"[{if = 'remote-ip', eq = '10.0.0.1', then = 1}, + {else = 100}]" + .parse_if(&ConfigContext::default()); + + core.queue.config.quota = r"[[queue.quota]] + match = {if = 'sender', eq = 'john@doe.org'} + key = ['sender'] + messages = 1 + + [[queue.quota]] + match = {if = 'rcpt-domain', eq = 'foobar.org'} + key = ['rcpt-domain'] + size = 450 + + [[queue.quota]] + match = {if = 'rcpt', eq = 'jane@domain.net'} + key = ['rcpt'] + size = 450 + " + .parse_quota(&ConfigContext::default()); + + // Test queue message builder + let mut session = Session::test(core); + session.data.remote_ip = "10.0.0.1".parse().unwrap(); + session.eval_session_params().await; + session.test_builder().await; + + // Send DATA without RCPT + session.ehlo("mx.doe.org").await; + session.ingest(b"DATA\r\n").await.unwrap(); + session.response().assert_code("503 5.5.1"); + + // Send broken message + session + .send_message( + "john@doe.org", + &["bill@foobar.org"], + "From: john", + "550 5.7.7", + ) + .await; + + // Naive Loop detection + session + .send_message( + "john@doe.org", + &["bill@foobar.org"], + "test:loop", + "450 4.4.6", + ) + .await; + + // No headers should be added to messages from 10.0.0.1 + session + .send_message("john@doe.org", &["bill@foobar.org"], "test:no_msgid", "250") + .await; + assert_eq!( + qr.read_event().await.unwrap_message().read_message(), + load_test_message("no_msgid", "messages") + ); + + // Maximum one message per session is allowed for 10.0.0.1 + session.mail_from("john@doe.org", "250").await; + session.rcpt_to("bill@foobar.org", "250").await; + session.ingest(b"DATA\r\n").await.unwrap(); + session.response().assert_code("451 4.4.5"); + session.rset().await; + + // Headers should be added to messages from 10.0.0.3 + session.data.remote_ip = "10.0.0.3".parse().unwrap(); + session.eval_session_params().await; + session + .send_message("john@doe.org", &["mike@test.com"], "test:no_msgid", "250") + .await; + qr.read_event() + .await + .unwrap_message() + .read_lines() + .assert_contains("From: ") + .assert_contains("To: ") + .assert_contains("Subject: ") + .assert_contains("Date: ") + .assert_contains("Message-ID: ") + .assert_contains("Return-Path: ") + .assert_contains("Received: ") + .assert_contains("Authentication-Results: ") + .assert_contains("Received-SPF: "); + + // Only one message is allowed in the queue from john@doe.org + let mut queued_messages = vec![]; + session.data.remote_ip = "10.0.0.2".parse().unwrap(); + session.eval_session_params().await; + session + .send_message("john@doe.org", &["bill@foobar.org"], "test:no_dkim", "250") + .await; + queued_messages.push(qr.read_event().await); + session + .send_message( + "john@doe.org", + &["bill@foobar.org"], + "test:no_dkim", + "452 4.3.1", + ) + .await; + + // Release quota + queued_messages.clear(); + + // Only 1500 bytes are allowed in the queue to domain foobar.org + session + .send_message( + "jane@foobar.org", + &["bill@foobar.org"], + "test:no_dkim", + "250", + ) + .await; + queued_messages.push(qr.read_event().await); + session + .send_message( + "jane@foobar.org", + &["bill@foobar.org"], + "test:no_dkim", + "452 4.3.1", + ) + .await; + + // Only 1500 bytes are allowed in the queue to recipient jane@domain.net + session + .send_message( + "jane@foobar.org", + &["jane@domain.net"], + "test:no_dkim", + "250", + ) + .await; + queued_messages.push(qr.read_event().await); + session + .send_message( + "jane@foobar.org", + &["jane@domain.net"], + "test:no_dkim", + "452 4.3.1", + ) + .await; +} diff --git a/tests/src/smtp/inbound/dmarc.rs b/tests/src/smtp/inbound/dmarc.rs new file mode 100644 index 00000000..1f14228a --- /dev/null +++ b/tests/src/smtp/inbound/dmarc.rs @@ -0,0 +1,302 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; + +use ahash::AHashSet; +use mail_auth::{ + common::{parse::TxtRecordParser, verify::DomainKey}, + dkim::DomainKeyReport, + dmarc::Dmarc, + report::DmarcResult, + spf::Spf, +}; + +use crate::smtp::{ + inbound::{sign::TextConfigContext, TestMessage, TestQueueEvent, TestReportingEvent}, + session::{TestSession, VerifyResponse}, + ParseTestConfig, TestConfig, TestCore, +}; +use smtp::{ + config::{AggregateFrequency, ConfigContext, IfBlock, Rate, VerifyStrategy}, + core::{Core, Session}, + lookup::Lookup, +}; + +#[tokio::test] +async fn dmarc() { + let mut core = Core::test(); + let ctx = ConfigContext::default().parse_signatures(); + + // Create temp dir for queue + let mut qr = core.init_test_queue("smtp_dmarc_test"); + + // Add SPF, DKIM and DMARC records + core.resolvers.dns.txt_add( + "mx.example.com", + Spf::parse(b"v=spf1 ip4:10.0.0.1 ip4:10.0.0.2 -all").unwrap(), + Instant::now() + Duration::from_secs(5), + ); + core.resolvers.dns.txt_add( + "example.com", + Spf::parse(b"v=spf1 ip4:10.0.0.1 -all ra=spf-failures rr=e:f:s:n").unwrap(), + Instant::now() + Duration::from_secs(5), + ); + core.resolvers.dns.txt_add( + "foobar.com", + Spf::parse(b"v=spf1 ip4:10.0.0.1 -all").unwrap(), + Instant::now() + Duration::from_secs(5), + ); + core.resolvers.dns.txt_add( + "ed._domainkey.example.com", + DomainKey::parse( + concat!( + "v=DKIM1; k=ed25519; ", + "p=11qYAYKxCrfVS/7TyWQHOg7hcvPapiMlrwIaaPcHURo=" + ) + .as_bytes(), + ) + .unwrap(), + Instant::now() + Duration::from_secs(5), + ); + core.resolvers.dns.txt_add( + "default._domainkey.example.com", + DomainKey::parse( + concat!( + "v=DKIM1; t=s; p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQ", + "KBgQDwIRP/UC3SBsEmGqZ9ZJW3/DkMoGeLnQg1fWn7/zYt", + "IxN2SnFCjxOCKG9v3b4jYfcTNh5ijSsq631uBItLa7od+v", + "/RtdC2UzJ1lWT947qR+Rcac2gbto/NMqJ0fzfVjH4OuKhi", + "tdY9tf6mcwGjaNBcWToIMmPSPDdQPNUYckcQ2QIDAQAB", + ) + .as_bytes(), + ) + .unwrap(), + Instant::now() + Duration::from_secs(5), + ); + core.resolvers.dns.txt_add( + "_report._domainkey.example.com", + DomainKeyReport::parse(b"ra=dkim-failures; rp=100; rr=d:o:p:s:u:v:x;").unwrap(), + Instant::now() + Duration::from_secs(5), + ); + core.resolvers.dns.txt_add( + "_dmarc.example.com", + Dmarc::parse( + concat!( + "v=DMARC1; p=reject; sp=quarantine; np=None; aspf=s; adkim=s; fo=1;", + "rua=mailto:dmarc-feedback@example.com;", + "ruf=mailto:dmarc-failures@example.com" + ) + .as_bytes(), + ) + .unwrap(), + Instant::now() + Duration::from_secs(5), + ); + + // Create report channels + let mut rr = core.init_test_report(); + + let mut config = &mut core.session.config.rcpt; + config.lookup_domains = IfBlock::new(Some(Arc::new(Lookup::Local(AHashSet::from_iter([ + "example.com".to_string(), + ]))))); + config.lookup_addresses = IfBlock::new(Some(Arc::new(Lookup::Local(AHashSet::from_iter([ + "jdoe@example.com".to_string(), + ]))))); + + let mut config = &mut core.session.config; + config.data.add_auth_results = IfBlock::new(true); + config.data.add_date = IfBlock::new(true); + config.data.add_message_id = IfBlock::new(true); + config.data.add_received = IfBlock::new(true); + config.data.add_return_path = IfBlock::new(true); + config.data.add_received_spf = IfBlock::new(true); + + let mut config = &mut core.report.config; + config.dkim.send = IfBlock::new(Some(Rate { + requests: 1, + period: Duration::from_secs(1), + })); + config.dmarc.send = config.dkim.send.clone(); + config.spf.send = config.dkim.send.clone(); + config.dmarc_aggregate.send = IfBlock::new(AggregateFrequency::Daily); + + let mut config = &mut core.mail_auth; + config.spf.verify_ehlo = "[{if = 'remote-ip', eq = '10.0.0.2', then = 'strict'}, + { else = 'relaxed' }]" + .parse_if(&ConfigContext::default()); + config.spf.verify_mail_from = config.spf.verify_ehlo.clone(); + config.dmarc.verify = IfBlock::new(VerifyStrategy::Strict); + config.arc.verify = config.dmarc.verify.clone(); + config.dkim.verify = "[{if = 'sender-domain', eq = 'test.net', then = 'relaxed'}, + { else = 'strict' }]" + .parse_if(&ConfigContext::default()); + + let mut config = &mut core.report.config; + config.spf.sign = "['rsa']" + .parse_if::>(&ctx) + .map_if_block(&ctx.signers, "", "") + .unwrap(); + config.dmarc.sign = config.spf.sign.clone(); + config.dkim.sign = config.spf.sign.clone(); + + // SPF must pass + let core = Arc::new(core); + let mut session = Session::test(core.clone()); + session.data.remote_ip = "10.0.0.2".parse().unwrap(); + session.eval_session_params().await; + session.ehlo("mx.example.com").await; + session.mail_from("bill@example.com", "550 5.7.23").await; + + // Expect SPF auth failure report + let message = qr.read_event().await.unwrap_message(); + assert_eq!( + message.recipients.last().unwrap().address, + "spf-failures@example.com" + ); + message + .read_lines() + .assert_contains("DKIM-Signature: v=1; a=rsa-sha256; s=rsa; d=example.com;") + .assert_contains("To: spf-failures@example.com") + .assert_contains("Feedback-Type: auth-failure") + .assert_contains("Auth-Failure: spf"); + + // Second DKIM failure report should be rate limited + session.mail_from("bill@example.com", "550 5.7.23").await; + qr.assert_empty_queue(); + + // Invalid DKIM signatures should be rejected + session.data.remote_ip = "10.0.0.1".parse().unwrap(); + session.eval_session_params().await; + session + .send_message( + "bill@example.com", + &["jdoe@example.com"], + "test:invalid_dkim", + "550 5.7.20", + ) + .await; + + // Expect DKIM auth failure report + let message = qr.read_event().await.unwrap_message(); + assert_eq!( + message.recipients.last().unwrap().address, + "dkim-failures@example.com" + ); + message + .read_lines() + .assert_contains("DKIM-Signature: v=1; a=rsa-sha256; s=rsa; d=example.com;") + .assert_contains("To: dkim-failures@example.com") + .assert_contains("Feedback-Type: auth-failure") + .assert_contains("Auth-Failure: bodyhash"); + + // Second DKIM failure report should be rate limited + session + .send_message( + "bill@example.com", + &["jdoe@example.com"], + "test:invalid_dkim", + "550 5.7.20", + ) + .await; + qr.assert_empty_queue(); + + // Invalid ARC should be rejected + session + .send_message( + "bill@example.com", + &["jdoe@example.com"], + "test:invalid_arc", + "550 5.7.29", + ) + .await; + qr.assert_empty_queue(); + + // Unaligned DMARC should be rejected + core.resolvers.dns.txt_add( + "test.net", + Spf::parse(b"v=spf1 -all").unwrap(), + Instant::now() + Duration::from_secs(5), + ); + session + .send_message( + "joe@test.net", + &["jdoe@example.com"], + "test:invalid_dkim", + "550 5.7.1", + ) + .await; + + // Expect DMARC auth failure report + let message = qr.read_event().await.unwrap_message(); + assert_eq!( + message.recipients.last().unwrap().address, + "dmarc-failures@example.com" + ); + message + .read_lines() + .assert_contains("DKIM-Signature: v=1; a=rsa-sha256; s=rsa; d=example.com;") + .assert_contains("To: dmarc-failures@example.com") + .assert_contains("Feedback-Type: auth-failure") + .assert_contains("Auth-Failure: dmarc") + .assert_contains("dmarc=3Dnone"); + + // Expect DMARC aggregate report + let report = rr.read_report().await.unwrap_dmarc(); + assert_eq!(report.domain, "example.com"); + assert_eq!(report.interval, AggregateFrequency::Daily); + assert_eq!(report.dmarc_record.rua().len(), 1); + assert_eq!(report.report_record.dmarc_spf_result(), DmarcResult::Fail); + + // Second DMARC failure report should be rate limited + session + .send_message( + "joe@test.net", + &["jdoe@example.com"], + "test:invalid_dkim", + "550 5.7.1", + ) + .await; + qr.assert_empty_queue(); + + // Messagess passing DMARC should be accepted + session + .send_message( + "bill@example.com", + &["jdoe@example.com"], + "test:dkim", + "250", + ) + .await; + qr.read_event() + .await + .unwrap_message() + .read_lines() + .assert_contains("dkim=pass") + .assert_contains("spf=pass") + .assert_contains("dmarc=pass") + .assert_contains("Received-SPF: pass"); +} diff --git a/tests/src/smtp/inbound/dnsrbl.rs b/tests/src/smtp/inbound/dnsrbl.rs new file mode 100644 index 00000000..e66bf7ae --- /dev/null +++ b/tests/src/smtp/inbound/dnsrbl.rs @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::time::{Duration, Instant}; + +use smtp::{ + config::IfBlock, + core::{Core, Session}, +}; + +use crate::smtp::{inbound::TestQueueEvent, session::TestSession, TestConfig, TestCore}; + +#[tokio::test] +async fn dnsrbl() { + /*tracing::subscriber::set_global_default( + tracing_subscriber::FmtSubscriber::builder() + .with_max_level(tracing::Level::TRACE) + .finish(), + ) + .unwrap();*/ + + let mut core = Core::test(); + for entry in [ + "1.0.0.10.zen.spamhaus.org", + "2.0.0.10.b.barracudacentral.org", + "spammer.com.dbl.spamhaus.org", + "spammer.net.dbl.spamhaus.org", + "spammer.org.dbl.spamhaus.org", + ] { + core.resolvers.dns.ipv4_add( + entry, + vec!["127.0.0.2".parse().unwrap()], + Instant::now() + Duration::from_secs(10), + ); + } + core.resolvers.dns.ipv4_add( + "shouldwork.org.dbl.spamhaus.org", + vec!["127.255.255.254".parse().unwrap()], + Instant::now() + Duration::from_secs(10), + ); + core.resolvers.dns.ptr_add( + "10.0.0.3".parse().unwrap(), + vec!["spammer.org.".to_string()], + Instant::now() + Duration::from_secs(10), + ); + + let mut qr = core.init_test_queue("smtp_dnsrbl_test"); + let mut config = &mut core.mail_auth.dnsbl; + config.ip_lookup = vec![ + "zen.spamhaus.org".to_string(), + "bl.spamcop.net".to_string(), + "b.barracudacentral.org".to_string(), + ]; + config.domain_lookup = vec!["dbl.spamhaus.org".to_string()]; + config.verify = IfBlock::new(u32::MAX); + core.session.config.rcpt.relay = IfBlock::new(true); + + // DNSRBL codes other than 127.0.0.0/8 should not be interpreted as block + let mut session = Session::test(core); + session.data.remote_ip = "10.0.0.4".parse().unwrap(); + session.eval_session_params().await; + session.cmd("EHLO shouldwork.org", "250").await; + + // Reject blocked EHLO domains + session + .cmd( + "EHLO spammer.com", + "554 5.7.1 Service unavailable; Domain 'spammer.com' blocked", + ) + .await; + + // Reject blocked return paths + session.ehlo("foobar.org").await; + session + .mail_from( + "list@spammer.com", + "554 5.7.1 Service unavailable; Domain 'spammer.com' blocked", + ) + .await; + + // Reject blocked From addresses + session + .send_message( + "bill@foobar.org", + &["jane@example.org"], + concat!( + "From: Mr. Spammer \r\n", + "To: jane@example.org\r\n", + "Subject: Adwords is expensive, please let me spam you.\r\n\r\n", + "Buy my spammer product\r\n" + ), + "554 5.7.1 Service unavailable; Domain 'spammer.net' blocked", + ) + .await; + + // Reject blocked IPs + session.data.remote_ip = "10.0.0.1".parse().unwrap(); + session.data.iprev.take(); + session.reset_dnsbl_error(); + session.verify_ip_dnsbl().await; + session.ehlo("foobar.org").await; + session + .mail_from( + "bill@foobar.org", + "554 5.7.1 Service unavailable; IP address 10.0.0.1 blocked", + ) + .await; + + session.data.remote_ip = "10.0.0.2".parse().unwrap(); + session.data.iprev.take(); + session.reset_dnsbl_error(); + session.verify_ip_dnsbl().await; + session.ehlo("foobar.org").await; + session + .mail_from( + "bill@foobar.org", + "554 5.7.1 Service unavailable; IP address 10.0.0.2 blocked", + ) + .await; + + // Reject blocked PTR domains + session.data.remote_ip = "10.0.0.3".parse().unwrap(); + session.data.iprev.take(); + session.reset_dnsbl_error(); + session.verify_ip_dnsbl().await; + session.ehlo("foobar.org").await; + session + .mail_from( + "bill@foobar.org", + "554 5.7.1 Service unavailable; Domain 'spammer.org.' blocked", + ) + .await; + + // Non-blocked IPs should work + session.data.remote_ip = "10.0.0.4".parse().unwrap(); + session.data.iprev.take(); + session.reset_dnsbl_error(); + session.verify_ip_dnsbl().await; + session.ehlo("foobar.org").await; + session + .send_message("bill@foobar.org", &["jane@example.org"], "test:dkim", "250") + .await; + qr.read_event().await.unwrap_message(); +} diff --git a/tests/src/smtp/inbound/ehlo.rs b/tests/src/smtp/inbound/ehlo.rs new file mode 100644 index 00000000..c15c0465 --- /dev/null +++ b/tests/src/smtp/inbound/ehlo.rs @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::time::{Duration, Instant}; + +use mail_auth::{common::parse::TxtRecordParser, spf::Spf, SpfResult}; + +use crate::smtp::{ + session::{TestSession, VerifyResponse}, + ParseTestConfig, TestConfig, +}; +use smtp::{ + config::{ConfigContext, IfBlock}, + core::{Core, Session}, +}; + +#[tokio::test] +async fn ehlo() { + let mut core = Core::test(); + core.resolvers.dns.txt_add( + "mx1.foobar.org", + Spf::parse(b"v=spf1 ip4:10.0.0.1 -all").unwrap(), + Instant::now() + Duration::from_secs(5), + ); + core.resolvers.dns.txt_add( + "mx2.foobar.org", + Spf::parse(b"v=spf1 ip4:10.0.0.2 -all").unwrap(), + Instant::now() + Duration::from_secs(5), + ); + + let mut config = &mut core.session.config; + config.data.max_message_size = r"[{if = 'remote-ip', eq = '10.0.0.1', then = 1024}, + {else = 2048}]" + .parse_if(&ConfigContext::default()); + config.extensions.future_release = r"[{if = 'remote-ip', eq = '10.0.0.1', then = '1h'}, + {else = false}]" + .parse_if(&ConfigContext::default()); + config.extensions.mt_priority = r"[{if = 'remote-ip', eq = '10.0.0.1', then = 'nsep'}, + {else = false}]" + .parse_if(&ConfigContext::default()); + core.mail_auth.spf.verify_ehlo = r"[{if = 'remote-ip', eq = '10.0.0.2', then = 'strict'}, + {else = 'relaxed'}]" + .parse_if(&ConfigContext::default()); + config.ehlo.reject_non_fqdn = IfBlock::new(true); + + // Reject non-FQDN domains + let mut session = Session::test(core); + session.data.remote_ip = "10.0.0.1".parse().unwrap(); + session.stream.tls = false; + session.eval_session_params().await; + session.cmd("EHLO domain", "550 5.5.0").await; + + // EHLO capabilities evaluation + session + .cmd("EHLO mx1.foobar.org", "250") + .await + .assert_contains("SIZE 1024") + .assert_contains("MT-PRIORITY NSEP") + .assert_contains("FUTURERELEASE 3600") + .assert_contains("STARTTLS"); + + // SPF should be a Pass for 10.0.0.1 + assert_eq!( + session.data.spf_ehlo.as_ref().unwrap().result(), + SpfResult::Pass + ); + + // Test SPF strict mode + session.data.helo_domain = String::new(); + session.data.remote_ip = "10.0.0.2".parse().unwrap(); + session.stream.tls = true; + session.eval_session_params().await; + session.ingest(b"EHLO mx1.foobar.org\r\n").await.unwrap(); + session.response().assert_code("550 5.7.23"); + + // EHLO capabilities evaluation + session.ingest(b"EHLO mx2.foobar.org\r\n").await.unwrap(); + assert_eq!( + session.data.spf_ehlo.as_ref().unwrap().result(), + SpfResult::Pass + ); + session + .response() + .assert_code("250") + .assert_contains("SIZE 2048") + .assert_not_contains("MT-PRIORITY") + .assert_not_contains("FUTURERELEASE") + .assert_not_contains("STARTTLS"); +} diff --git a/tests/src/smtp/inbound/limits.rs b/tests/src/smtp/inbound/limits.rs new file mode 100644 index 00000000..b94985d4 --- /dev/null +++ b/tests/src/smtp/inbound/limits.rs @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::time::{Duration, Instant}; + +use tokio::sync::watch; + +use crate::smtp::{ + session::{TestSession, VerifyResponse}, + ParseTestConfig, TestConfig, +}; +use smtp::{ + config::ConfigContext, + core::{Core, Session}, +}; + +#[tokio::test] +async fn limits() { + let mut core = Core::test(); + let mut config = &mut core.session.config; + config.transfer_limit = r"[{if = 'remote-ip', eq = '10.0.0.1', then = 10}, + {else = 1024}]" + .parse_if(&ConfigContext::default()); + config.timeout = r"[{if = 'remote-ip', eq = '10.0.0.2', then = '500ms'}, + {else = '30m'}]" + .parse_if(&ConfigContext::default()); + config.duration = r"[{if = 'remote-ip', eq = '10.0.0.3', then = '500ms'}, + {else = '60m'}]" + .parse_if(&ConfigContext::default()); + let (_tx, rx) = watch::channel(true); + + // Exceed max line length + let mut session = Session::test(core); + session.data.remote_ip = "10.0.0.1".parse().unwrap(); + let mut buf = vec![b'A'; 2049]; + session.ingest(&buf).await.unwrap(); + session.ingest(b"\r\n").await.unwrap(); + session.response().assert_code("554 5.3.4"); + + // Invalid command + buf.extend_from_slice(b"\r\n"); + session.ingest(&buf).await.unwrap(); + session.response().assert_code("500 5.5.1"); + + // Exceed transfer quota + session.eval_session_params().await; + session.write_rx("MAIL FROM:\r\n"); + session.handle_conn_().await; + session.response().assert_code("451 4.7.28"); + + // Loitering + session.data.remote_ip = "10.0.0.3".parse().unwrap(); + session.data.valid_until = Instant::now(); + session.eval_session_params().await; + tokio::time::sleep(Duration::from_millis(600)).await; + session.write_rx("MAIL FROM:\r\n"); + session.handle_conn_().await; + session.response().assert_code("453 4.3.2"); + + // Timeout + session.data.remote_ip = "10.0.0.2".parse().unwrap(); + session.data.valid_until = Instant::now(); + session.eval_session_params().await; + session.write_rx("MAIL FROM:\r\n"); + session.handle_conn_().await; + session.response().assert_code("221 2.0.0"); +} diff --git a/tests/src/smtp/inbound/mail.rs b/tests/src/smtp/inbound/mail.rs new file mode 100644 index 00000000..e4ac5c70 --- /dev/null +++ b/tests/src/smtp/inbound/mail.rs @@ -0,0 +1,310 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{ + sync::Arc, + time::{Duration, Instant, SystemTime}, +}; + +use mail_auth::{common::parse::TxtRecordParser, spf::Spf, IprevResult, SpfResult}; +use smtp_proto::{MAIL_BY_NOTIFY, MAIL_BY_RETURN, MAIL_REQUIRETLS}; + +use crate::smtp::{ + session::{TestSession, VerifyResponse}, + ParseTestConfig, TestConfig, +}; +use smtp::{ + config::{ConfigContext, IfBlock, VerifyStrategy}, + core::{Core, Session}, +}; + +#[tokio::test] +async fn mail() { + let mut core = Core::test(); + core.resolvers.dns.txt_add( + "foobar.org", + Spf::parse(b"v=spf1 ip4:10.0.0.1 -all").unwrap(), + Instant::now() + Duration::from_secs(5), + ); + core.resolvers.dns.txt_add( + "mx1.foobar.org", + Spf::parse(b"v=spf1 ip4:10.0.0.1 -all").unwrap(), + Instant::now() + Duration::from_secs(5), + ); + core.resolvers.dns.ptr_add( + "10.0.0.1".parse().unwrap(), + vec!["mx1.foobar.org.".to_string()], + Instant::now() + Duration::from_secs(5), + ); + core.resolvers.dns.ipv4_add( + "mx1.foobar.org.", + vec!["10.0.0.1".parse().unwrap()], + Instant::now() + Duration::from_secs(5), + ); + core.resolvers.dns.ptr_add( + "10.0.0.2".parse().unwrap(), + vec!["mx2.foobar.org.".to_string()], + Instant::now() + Duration::from_secs(5), + ); + + let mut config = &mut core.session.config; + config.ehlo.require = IfBlock::new(true); + core.mail_auth.spf.verify_ehlo = IfBlock::new(VerifyStrategy::Relaxed); + core.mail_auth.spf.verify_mail_from = r"[{if = 'remote-ip', eq = '10.0.0.2', then = 'strict'}, + {else = 'relaxed'}]" + .parse_if(&ConfigContext::default()); + core.mail_auth.iprev.verify = r"[{if = 'remote-ip', eq = '10.0.0.2', then = 'strict'}, + {else = 'relaxed'}]" + .parse_if(&ConfigContext::default()); + config.extensions.future_release = r"[{if = 'remote-ip', eq = '10.0.0.2', then = '1d'}, + {else = false}]" + .parse_if(&ConfigContext::default()); + config.extensions.deliver_by = r"[{if = 'remote-ip', eq = '10.0.0.2', then = '1d'}, + {else = false}]" + .parse_if(&ConfigContext::default()); + config.extensions.requiretls = r"[{if = 'remote-ip', eq = '10.0.0.2', then = true}, + {else = false}]" + .parse_if(&ConfigContext::default()); + config.extensions.mt_priority = r"[{if = 'remote-ip', eq = '10.0.0.2', then = 'nsep'}, + {else = false}]" + .parse_if(&ConfigContext::default()); + config.data.max_message_size = r"[{if = 'remote-ip', eq = '10.0.0.2', then = 2048}, + {else = 1024}]" + .parse_if(&ConfigContext::default()); + + config.throttle.mail_from = r"[[throttle]] + match = {if = 'remote-ip', eq = '10.0.0.1'} + key = 'sender' + rate = '2/1s' + " + .parse_throttle(&ConfigContext::default()); + + // Be rude and do not say EHLO + let core = Arc::new(core); + let mut session = Session::test(core.clone()); + session.data.remote_ip = "10.0.0.1".parse().unwrap(); + session.eval_session_params().await; + session + .ingest(b"MAIL FROM:\r\n") + .await + .unwrap(); + session.response().assert_code("503 5.5.1"); + + // Both IPREV and SPF should pass + session.ingest(b"EHLO mx1.foobar.org\r\n").await.unwrap(); + session.response().assert_code("250"); + session + .ingest(b"MAIL FROM:\r\n") + .await + .unwrap(); + session.response().assert_code("250"); + assert_eq!( + session.data.spf_ehlo.as_ref().unwrap().result(), + SpfResult::Pass + ); + assert_eq!( + session.data.spf_mail_from.as_ref().unwrap().result(), + SpfResult::Pass + ); + assert_eq!( + session.data.iprev.as_ref().unwrap().result(), + &IprevResult::Pass + ); + + // Multiple MAIL FROMs should not be allowed + session + .ingest(b"MAIL FROM:\r\n") + .await + .unwrap(); + session.response().assert_code("503 5.5.1"); + + // Test rate limit + for n in 0..2 { + session.rset().await; + session + .ingest(b"MAIL FROM:\r\n") + .await + .unwrap(); + session + .response() + .assert_code(if n == 0 { "250" } else { "451 4.4.5" }); + } + + // Test disabled extensions + for param in [ + "HOLDFOR=123", + "HOLDUNTIL=49374347", + "MT-PRIORITY=3", + "BY=120;R", + "REQUIRETLS", + ] { + session + .ingest(format!("MAIL FROM: {param}\r\n").as_bytes()) + .await + .unwrap(); + session.response().assert_code("501 5.5.4"); + } + + // Test size with a large value + session + .ingest(b"MAIL FROM: SIZE=1512\r\n") + .await + .unwrap(); + session.response().assert_code("552 5.3.4"); + + // Test strict IPREV + session.data.remote_ip = "10.0.0.2".parse().unwrap(); + session.data.iprev = None; + session.eval_session_params().await; + session + .ingest(b"MAIL FROM:\r\n") + .await + .unwrap(); + session.response().assert_code("550 5.7.25"); + session.data.iprev = None; + core.resolvers.dns.ipv4_add( + "mx2.foobar.org.", + vec!["10.0.0.2".parse().unwrap()], + Instant::now() + Duration::from_secs(5), + ); + + // Test strict SPF + session + .ingest(b"MAIL FROM:\r\n") + .await + .unwrap(); + session.response().assert_code("550 5.7.23"); + core.resolvers.dns.txt_add( + "foobar.org", + Spf::parse(b"v=spf1 ip4:10.0.0.1 ip4:10.0.0.2 -all").unwrap(), + Instant::now() + Duration::from_secs(5), + ); + session + .ingest(b"MAIL FROM:\r\n") + .await + .unwrap(); + session.response().assert_code("250"); + let mail_from = session.data.mail_from.as_ref().unwrap(); + assert_eq!(mail_from.domain, "foobar.org"); + assert_eq!(mail_from.address, "Jane@FooBar.org"); + assert_eq!(mail_from.address_lcase, "jane@foobar.org"); + session.rset().await; + + // Test SIZE extension + session + .ingest(b"MAIL FROM: SIZE=1023\r\n") + .await + .unwrap(); + session.response().assert_code("250"); + session.rset().await; + + // Test MT-PRIORITY extension + session + .ingest(b"MAIL FROM: MT-PRIORITY=-3\r\n") + .await + .unwrap(); + session.response().assert_code("250"); + assert_eq!(session.data.priority, -3); + session.rset().await; + + // Test REQUIRETLS extension + session + .ingest(b"MAIL FROM: REQUIRETLS\r\n") + .await + .unwrap(); + session.response().assert_code("250"); + assert!((session.data.mail_from.as_ref().unwrap().flags & MAIL_REQUIRETLS) != 0); + session.rset().await; + + // Test DELIVERBY extension with by-mode=R + session + .ingest(b"MAIL FROM: BY=120;R\r\n") + .await + .unwrap(); + session.response().assert_code("250"); + assert!((session.data.mail_from.as_ref().unwrap().flags & MAIL_BY_RETURN) != 0); + assert_eq!(session.data.delivery_by, 120); + session.rset().await; + + // Test DELIVERBY extension with by-mode=N + session + .ingest(b"MAIL FROM: BY=-456;N\r\n") + .await + .unwrap(); + session.response().assert_code("250"); + assert!((session.data.mail_from.as_ref().unwrap().flags & MAIL_BY_NOTIFY) != 0); + assert_eq!(session.data.delivery_by, -456); + session.rset().await; + + // Test DELIVERBY extension with invalid by-mode=R + session + .ingest(b"MAIL FROM: BY=-1;R\r\n") + .await + .unwrap(); + session.response().assert_code("501 5.5.4"); + session.rset().await; + + session + .ingest(b"MAIL FROM: BY=99999;R\r\n") + .await + .unwrap(); + session.response().assert_code("501 5.5.4"); + session.rset().await; + + // Test FUTURERELEASE extension with HOLDFOR + session + .ingest(b"MAIL FROM: HOLDFOR=1234\r\n") + .await + .unwrap(); + session.response().assert_code("250"); + assert_eq!(session.data.future_release, 1234); + session.rset().await; + + // Test FUTURERELEASE extension with invalid HOLDFOR falue + session + .ingest(b"MAIL FROM: HOLDFOR=99999\r\n") + .await + .unwrap(); + session.response().assert_code("501 5.5.4"); + session.rset().await; + + // Test FUTURERELEASE extension with HOLDUNTIL + let now = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .map_or(0, |d| d.as_secs()); + session + .ingest(format!("MAIL FROM: HOLDUNTIL={}\r\n", now + 10).as_bytes()) + .await + .unwrap(); + session.response().assert_code("250"); + assert_eq!(session.data.future_release, 10); + session.rset().await; + + // Test FUTURERELEASE extension with invalud HOLDUNTIL value + session + .ingest(format!("MAIL FROM: HOLDUNTIL={}\r\n", now + 99999).as_bytes()) + .await + .unwrap(); + session.response().assert_code("501 5.5.4"); + session.rset().await; +} diff --git a/tests/src/smtp/inbound/mod.rs b/tests/src/smtp/inbound/mod.rs new file mode 100644 index 00000000..1c30f880 --- /dev/null +++ b/tests/src/smtp/inbound/mod.rs @@ -0,0 +1,204 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::time::Duration; + +use tokio::sync::mpsc::error::TryRecvError; + +use smtp::{ + queue::{self, Message, OnHold, Schedule, WorkerResult}, + reporting::{self, DmarcEvent, TlsEvent}, +}; + +use super::{QueueReceiver, ReportReceiver}; + +pub mod auth; +pub mod basic; +pub mod data; +pub mod dmarc; +pub mod dnsrbl; +pub mod ehlo; +pub mod limits; +pub mod mail; +pub mod rcpt; +pub mod scripts; +pub mod sign; +pub mod throttle; +pub mod vrfy; + +impl QueueReceiver { + pub async fn read_event(&mut self) -> queue::Event { + match tokio::time::timeout(Duration::from_millis(100), self.queue_rx.recv()).await { + Ok(Some(event)) => event, + Ok(None) => panic!("Channel closed."), + Err(_) => panic!("No queue event received."), + } + } + + pub async fn try_read_event(&mut self) -> Option { + match tokio::time::timeout(Duration::from_millis(100), self.queue_rx.recv()).await { + Ok(Some(event)) => Some(event), + Ok(None) => panic!("Channel closed."), + Err(_) => None, + } + } + pub fn assert_empty_queue(&mut self) { + match self.queue_rx.try_recv() { + Err(TryRecvError::Empty) => (), + Ok(event) => panic!("Expected empty queue but got {event:?}"), + Err(err) => panic!("Queue error: {err:?}"), + } + } +} + +impl ReportReceiver { + pub async fn read_report(&mut self) -> reporting::Event { + match tokio::time::timeout(Duration::from_millis(100), self.report_rx.recv()).await { + Ok(Some(event)) => event, + Ok(None) => panic!("Channel closed."), + Err(_) => panic!("No report event received."), + } + } + + pub async fn try_read_report(&mut self) -> Option { + match tokio::time::timeout(Duration::from_millis(100), self.report_rx.recv()).await { + Ok(Some(event)) => Some(event), + Ok(None) => panic!("Channel closed."), + Err(_) => None, + } + } + pub fn assert_no_reports(&mut self) { + match self.report_rx.try_recv() { + Err(TryRecvError::Empty) => (), + Ok(event) => panic!("Expected no reports but got {event:?}"), + Err(err) => panic!("Report error: {err:?}"), + } + } +} + +pub trait TestQueueEvent { + fn unwrap_message(self) -> Box; + fn unwrap_schedule(self) -> Schedule>; + fn unwrap_result(self) -> WorkerResult; + fn unwrap_done(self); + fn unwrap_on_hold(self) -> OnHold>; + fn unwrap_retry(self) -> Schedule>; +} + +impl TestQueueEvent for queue::Event { + fn unwrap_message(self) -> Box { + match self { + queue::Event::Queue(message) => message.inner, + e => panic!("Unexpected event: {e:?}"), + } + } + + fn unwrap_schedule(self) -> Schedule> { + match self { + queue::Event::Queue(message) => message, + e => panic!("Unexpected event: {e:?}"), + } + } + + fn unwrap_result(self) -> WorkerResult { + match self { + queue::Event::Done(result) => result, + queue::Event::Queue(message) => { + panic!("Unexpected message: {}", message.inner.read_message()); + } + e => panic!("Unexpected event: {e:?}"), + } + } + + fn unwrap_done(self) { + match self { + queue::Event::Done(WorkerResult::Done) => (), + queue::Event::Queue(message) => { + panic!("Unexpected message: {}", message.inner.read_message()); + } + e => panic!("Unexpected event: {e:?}"), + } + } + + fn unwrap_on_hold(self) -> OnHold> { + match self { + queue::Event::Done(WorkerResult::OnHold(value)) => value, + queue::Event::Queue(message) => { + panic!("Unexpected message: {}", message.inner.read_message()); + } + e => panic!("Unexpected event: {e:?}"), + } + } + + fn unwrap_retry(self) -> Schedule> { + match self { + queue::Event::Done(WorkerResult::Retry(value)) => value, + queue::Event::Queue(message) => { + panic!("Unexpected message: {}", message.inner.read_message()); + } + e => panic!("Unexpected event: {e:?}"), + } + } +} + +pub trait TestReportingEvent { + fn unwrap_dmarc(self) -> Box; + fn unwrap_tls(self) -> Box; +} + +impl TestReportingEvent for reporting::Event { + fn unwrap_dmarc(self) -> Box { + match self { + reporting::Event::Dmarc(event) => event, + e => panic!("Unexpected event: {e:?}"), + } + } + + fn unwrap_tls(self) -> Box { + match self { + reporting::Event::Tls(event) => event, + e => panic!("Unexpected event: {e:?}"), + } + } +} + +pub trait TestMessage { + fn read_message(&self) -> String; + fn read_lines(&self) -> Vec; +} + +impl TestMessage for Message { + fn read_message(&self) -> String { + let mut buf = vec![0u8; self.size]; + let mut file = std::fs::File::open(&self.path).unwrap(); + std::io::Read::read_exact(&mut file, &mut buf).unwrap(); + String::from_utf8(buf).unwrap() + } + + fn read_lines(&self) -> Vec { + self.read_message() + .split('\n') + .map(|l| l.to_string()) + .collect() + } +} diff --git a/tests/src/smtp/inbound/rcpt.rs b/tests/src/smtp/inbound/rcpt.rs new file mode 100644 index 00000000..a02256de --- /dev/null +++ b/tests/src/smtp/inbound/rcpt.rs @@ -0,0 +1,147 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{sync::Arc, time::Duration}; + +use ahash::AHashSet; +use smtp_proto::{RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_SUCCESS}; + +use crate::smtp::{ + session::{TestSession, VerifyResponse}, + ParseTestConfig, TestConfig, +}; +use smtp::{ + config::{ConfigContext, IfBlock}, + core::{Core, Session, State}, + lookup::Lookup, +}; + +#[tokio::test] +async fn rcpt() { + let mut core = Core::test(); + + let list_addresses = Lookup::Local(AHashSet::from_iter([ + "jane@foobar.org".to_string(), + "bill@foobar.org".to_string(), + "mike@foobar.org".to_string(), + "john@foobar.org".to_string(), + ])); + let list_domains = Lookup::Local(AHashSet::from_iter(["foobar.org".to_string()])); + + let mut config = &mut core.session.config.rcpt; + let mut config_ext = &mut core.session.config.extensions; + config.lookup_domains = IfBlock::new(Some(Arc::new(list_domains))); + config.lookup_addresses = IfBlock::new(Some(Arc::new(list_addresses))); + config.max_recipients = r"[{if = 'remote-ip', eq = '10.0.0.1', then = 3}, + {else = 5}]" + .parse_if(&ConfigContext::default()); + config.relay = r"[{if = 'remote-ip', eq = '10.0.0.1', then = false}, + {else = true}]" + .parse_if(&ConfigContext::default()); + config_ext.dsn = r"[{if = 'remote-ip', eq = '10.0.0.1', then = false}, + {else = true}]" + .parse_if(&ConfigContext::default()); + config.errors_max = r"[{if = 'remote-ip', eq = '10.0.0.1', then = 3}, + {else = 100}]" + .parse_if(&ConfigContext::default()); + config.errors_wait = r"[{if = 'remote-ip', eq = '10.0.0.1', then = '5ms'}, + {else = '1s'}]" + .parse_if(&ConfigContext::default()); + core.session.config.throttle.rcpt_to = r"[[throttle]] + match = {if = 'remote-ip', eq = '10.0.0.1'} + key = 'sender' + rate = '2/1s' + " + .parse_throttle(&ConfigContext::default()); + + // RCPT without MAIL FROM + let mut session = Session::test(core); + session.data.remote_ip = "10.0.0.1".parse().unwrap(); + session.eval_session_params().await; + session.ehlo("mx1.foobar.org").await; + session.rcpt_to("jane@foobar.org", "503 5.5.1").await; + + // Relaying is disabled for 10.0.0.1 + session.mail_from("john@example.net", "250").await; + session.rcpt_to("external@domain.com", "550 5.1.2").await; + + // DSN is disabled for 10.0.0.1 + session + .ingest(b"RCPT TO: NOTIFY=SUCCESS,FAILURE,DELAY\r\n") + .await + .unwrap(); + session.response().assert_code("501 5.5.4"); + + // Send to non-existing user + session.rcpt_to("tom@foobar.org", "550 5.1.2").await; + + // Exceeding max number of errors + session + .ingest(b"RCPT TO:\r\n") + .await + .unwrap_err(); + session.response().assert_code("421 4.3.0"); + + // Rate limit + session.data.rcpt_errors = 0; + session.state = State::default(); + session.rcpt_to("Jane@FooBar.org", "250").await; + session.rcpt_to("Bill@FooBar.org", "250").await; + session.rcpt_to("Mike@FooBar.org", "451 4.4.5").await; + + // Restore rate limit + tokio::time::sleep(Duration::from_millis(1100)).await; + session.rcpt_to("Mike@FooBar.org", "250").await; + session.rcpt_to("john@foobar.org", "451 4.5.3").await; + + // Check recipients + assert_eq!(session.data.rcpt_to.len(), 3); + for (rcpt, expected) in + session + .data + .rcpt_to + .iter() + .zip(["Jane@FooBar.org", "Bill@FooBar.org", "Mike@FooBar.org"]) + { + assert_eq!(rcpt.address, expected); + assert_eq!(rcpt.domain, "foobar.org"); + assert_eq!(rcpt.address_lcase, expected.to_lowercase()); + } + + // Relaying should be allowed for 10.0.0.2 + session.data.remote_ip = "10.0.0.2".parse().unwrap(); + session.eval_session_params().await; + session.rset().await; + session.mail_from("john@example.net", "250").await; + session.rcpt_to("external@domain.com", "250").await; + + // DSN is enabled for 10.0.0.2 + session + .ingest(b"RCPT TO: NOTIFY=SUCCESS,FAILURE,DELAY ORCPT=rfc822;Jane.Doe@Foobar.org\r\n") + .await + .unwrap(); + session.response().assert_code("250"); + let rcpt = session.data.rcpt_to.last().unwrap(); + assert!((rcpt.flags & (RCPT_NOTIFY_DELAY | RCPT_NOTIFY_SUCCESS | RCPT_NOTIFY_FAILURE)) != 0); + assert_eq!(rcpt.dsn_info.as_ref().unwrap(), "Jane.Doe@Foobar.org"); +} diff --git a/tests/src/smtp/inbound/scripts.rs b/tests/src/smtp/inbound/scripts.rs new file mode 100644 index 00000000..f67c66b6 --- /dev/null +++ b/tests/src/smtp/inbound/scripts.rs @@ -0,0 +1,333 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::path::PathBuf; + +use crate::smtp::{ + inbound::{sign::TextConfigContext, TestMessage, TestQueueEvent}, + session::{TestSession, VerifyResponse}, + TestConfig, TestCore, +}; +use smtp::{ + config::{ + database::ConfigDatabase, list::ConfigList, scripts::ConfigSieve, session::ConfigSession, + ConfigContext, EnvelopeKey, IfBlock, + }, + core::{Core, Session}, +}; +use utils::config::Config; + +const CONFIG: &str = r#" +[database."sql"] +address = "sqlite://%PATH%/test.db?mode=rwc" +max-connections = 10 +min-connections = 0 +idle-timeout = "5m" + +[list] +invalid-ehlos = ["spammer.org", "spammer.net"] + +[session.data.pipe."test"] +command = [ { if = "remote-ip", eq = "10.0.0.123", then = "/bin/bash" }, + { else = false } ] +arguments = ["%CFG_PATH%/pipe_me.sh", "hello", "world"] +timeout = "10s" + +[sieve] +from-name = "Sieve Daemon" +from-addr = "sieve@foobar.org" +return-path = "" +hostname = "mx.foobar.org" +sign = ["rsa"] +use-database = "sql" + +[sieve.limits] +redirects = 3 +out-messages = 5 +received-headers = 50 +cpu = 10000 +nested-includes = 5 +duplicate-expiry = "7d" + +[sieve.scripts] +connect = ''' +require ["variables", "reject"]; + +if string "${env.remote_ip}" "10.0.0.88" { + reject "Your IP '${env.remote_ip}' is not welcomed here."; +} +''' + +ehlo = ''' +require ["variables", "extlists", "reject"]; + +if string :list "${env.helo_domain}" "list/invalid-ehlos" { + reject "551 5.1.1 Your domain '${env.helo_domain}' has been blacklisted."; +} +''' + +mail = ''' +require ["variables", "vnd.stalwart.execute", "envelope", "reject"]; + +if envelope :localpart :is "from" "spammer" { + reject "450 4.1.1 Invalid address"; +} + +execute :query "CREATE TABLE IF NOT EXISTS blocked_senders (addr TEXT PRIMARY KEY)"; +execute :query "INSERT OR IGNORE INTO blocked_senders (addr) VALUES (?)" "marketing@spam-domain.com"; + +if execute :query "SELECT EXISTS(SELECT 1 FROM blocked_senders WHERE addr=? LIMIT 1)" ["${envelope.from}"] { + reject "Your address has been blocked."; +} +''' + +rcpt = ''' +require ["variables", "vnd.stalwart.execute", "envelope", "reject"]; + +if envelope :domain :is "to" "foobar.org" { + execute :query "CREATE TABLE IF NOT EXISTS greylist (addr TEXT PRIMARY KEY)"; + + set "triplet" "${env.remote_ip}.${envelope.from}.${envelope.to}"; + + if not execute :query "SELECT EXISTS(SELECT 1 FROM greylist WHERE addr=? LIMIT 1)" ["${triplet}"] { + execute :query "INSERT INTO greylist (addr) VALUES (?)" ["${triplet}"]; + reject "422 4.2.2 You have been greylisted '${triplet}'."; + } +} + +''' + +data = ''' +require ["envelope", "reject", "variables", "replace", "mime", "foreverypart", "editheader", "extracttext", "enotify"]; + +if envelope :localpart :is "to" "bill" { + reject "Bill cannot receive messages."; + stop; +} + +if envelope :localpart :is "to" "jane" { + set "counter" "a"; + foreverypart { + if header :mime :contenttype "content-type" "text/html" { + extracttext :upper "text_content"; + replace "${text_content}"; + } + set :length "part_num" "${counter}"; + addheader :last "X-Part-Number" "${part_num}"; + set "counter" "${counter}a"; + } +} + +if envelope :domain :is "to" "foobar.net" { + notify "mailto:john@example.net?cc=jane@example.org&subject=You%20have%20got%20mail"; +} +''' + +"#; + +#[tokio::test] +async fn sieve_scripts() { + /*tracing::subscriber::set_global_default( + tracing_subscriber::FmtSubscriber::builder() + .with_max_level(tracing::Level::TRACE) + .finish(), + ) + .unwrap();*/ + + let mut pipe_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + pipe_path.push("resources"); + pipe_path.push("smtp"); + pipe_path.push("pipe"); + + // Prepare config + let mut core = Core::test(); + let mut qr = core.init_test_queue("smtp_sieve_test"); + let mut ctx = ConfigContext::default().parse_signatures(); + let config = Config::parse( + &CONFIG + .replace("%PATH%", qr._temp_dir.temp_dir.as_path().to_str().unwrap()) + .replace("%CFG_PATH%", pipe_path.as_path().to_str().unwrap()), + ) + .unwrap(); + config.parse_lists(&mut ctx).unwrap(); + config.parse_databases(&mut ctx).unwrap(); + let pipes = config.parse_pipes(&ctx, &[EnvelopeKey::RemoteIp]).unwrap(); + core.sieve = config.parse_sieve(&mut ctx).unwrap(); + let config = &mut core.session.config; + config.connect.script = IfBlock::new(ctx.scripts.get("connect").cloned()); + config.ehlo.script = IfBlock::new(ctx.scripts.get("ehlo").cloned()); + config.mail.script = IfBlock::new(ctx.scripts.get("mail").cloned()); + config.rcpt.script = IfBlock::new(ctx.scripts.get("rcpt").cloned()); + config.data.script = IfBlock::new(ctx.scripts.get("data").cloned()); + config.rcpt.relay = IfBlock::new(true); + config.data.pipe_commands = pipes; + + // Test connect script + let mut session = Session::test(core); + session.data.remote_ip = "10.0.0.88".parse().unwrap(); + assert!(!session.init_conn().await); + session + .response() + .assert_contains("503 5.5.3 Your IP '10.0.0.88' is not welcomed here"); + session.data.remote_ip = "10.0.0.5".parse().unwrap(); + assert!(session.init_conn().await); + session + .response() + .assert_contains("220 mx.example.org at your service"); + + // Test EHLO script + session + .cmd( + "EHLO spammer.org", + "551 5.1.1 Your domain 'spammer.org' has been blacklisted", + ) + .await; + session.cmd("EHLO foobar.net", "250").await; + + // Test MAIL-FROM script + session + .mail_from("spammer@domain.com", "450 4.1.1 Invalid address") + .await; + session + .mail_from( + "marketing@spam-domain.com", + "503 5.5.3 Your address has been blocked", + ) + .await; + session.mail_from("bill@foobar.org", "250").await; + + // Test RCPT-TO script + session + .rcpt_to( + "jane@foobar.org", + "422 4.2.2 You have been greylisted '10.0.0.5.bill@foobar.org.jane@foobar.org'.", + ) + .await; + session.rcpt_to("jane@foobar.org", "250").await; + + // Expect a modified message + session.data("test:multipart", "250").await; + qr.read_event() + .await + .unwrap_message() + .read_lines() + .assert_contains("X-Part-Number: 5") + .assert_contains("THIS IS A PIECE OF HTML TEXT"); + qr.assert_empty_queue(); + + // Expect rejection for bill@foobar.net + session + .send_message( + "test@example.net", + &["bill@foobar.net"], + "test:multipart", + "503 5.5.3 Bill cannot receive messages", + ) + .await; + qr.assert_empty_queue(); + + // Expect message delivery plus a notification + session + .send_message( + "test@example.net", + &["john@foobar.net"], + "test:multipart", + "250", + ) + .await; + let notification = qr.read_event().await.unwrap_message(); + assert_eq!(notification.return_path, ""); + assert_eq!(notification.recipients.len(), 2); + assert_eq!( + notification.recipients.first().unwrap().address, + "john@example.net" + ); + assert_eq!( + notification.recipients.last().unwrap().address, + "jane@example.org" + ); + notification + .read_lines() + .assert_contains("DKIM-Signature: v=1; a=rsa-sha256; s=rsa; d=example.com;") + .assert_contains("From: \"Sieve Daemon\" ") + .assert_contains("To: ") + .assert_contains("Cc: ") + .assert_contains("Subject: You have got mail") + .assert_contains("One Two Three Four"); + qr.read_event() + .await + .unwrap_message() + .read_lines() + .assert_contains("One Two Three Four") + .assert_contains("multi-part message in MIME format") + .assert_not_contains("X-Part-Number: 5") + .assert_not_contains("THIS IS A PIECE OF HTML TEXT"); + qr.assert_empty_queue(); + + // Expect a modified message delivery plus a notification + session + .send_message( + "test@example.net", + &["jane@foobar.net"], + "test:multipart", + "250", + ) + .await; + + qr.read_event() + .await + .unwrap_message() + .read_lines() + .assert_contains("DKIM-Signature: v=1; a=rsa-sha256; s=rsa; d=example.com;") + .assert_contains("From: \"Sieve Daemon\" ") + .assert_contains("To: ") + .assert_contains("Cc: ") + .assert_contains("Subject: You have got mail") + .assert_contains("One Two Three Four"); + + qr.read_event() + .await + .unwrap_message() + .read_lines() + .assert_contains("X-Part-Number: 5") + .assert_contains("THIS IS A PIECE OF HTML TEXT") + .assert_not_contains("X-My-Header: true"); + + // Test pipes + session.data.remote_ip = "10.0.0.123".parse().unwrap(); + session + .send_message( + "test@example.net", + &["pipe@foobar.com"], + "test:no_dkim", + "250", + ) + .await; + qr.read_event() + .await + .unwrap_message() + .read_lines() + .assert_contains("X-My-Header: true") + .assert_contains("Authentication-Results"); + qr.assert_empty_queue(); +} diff --git a/tests/src/smtp/inbound/sign.rs b/tests/src/smtp/inbound/sign.rs new file mode 100644 index 00000000..43e24e84 --- /dev/null +++ b/tests/src/smtp/inbound/sign.rs @@ -0,0 +1,222 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; + +use ahash::AHashSet; +use mail_auth::{ + common::{parse::TxtRecordParser, verify::DomainKey}, + spf::Spf, +}; +use utils::config::Config; + +use crate::smtp::{ + inbound::{TestMessage, TestQueueEvent}, + session::{TestSession, VerifyResponse}, + ParseTestConfig, TestConfig, TestCore, +}; +use smtp::{ + config::{auth::ConfigAuth, ConfigContext, IfBlock, VerifyStrategy}, + core::{Core, Session}, + lookup::Lookup, +}; + +const SIGNATURES: &str = " +[signature.rsa] +private-key = ''' +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAv9XYXG3uK95115mB4nJ37nGeNe2CrARm1agrbcnSk5oIaEfM +ZLUR/X8gPzoiNHZcfMZEVR6bAytxUhc5EvZIZrjSuEEeny+fFd/cTvcm3cOUUbIa +UmSACj0dL2/KwW0LyUaza9z9zor7I5XdIl1M53qVd5GI62XBB76FH+Q0bWPZNkT4 +NclzTLspD/MTpNCCPhySM4Kdg5CuDczTH4aNzyS0TqgXdtw6A4Sdsp97VXT9fkPW +9rso3lrkpsl/9EQ1mR/DWK6PBmRfIuSFuqnLKY6v/z2hXHxF7IoojfZLa2kZr9Ae +d4l9WheQOTA19k5r2BmlRw/W9CrgCBo0Sdj+KQIDAQABAoIBAFPChEi/OvnulReB +ECQWhOUYuNKlFKQU++2YEvZJ4+bMn5UgnE7wfJ1pj2Pr9xlfALz+OMHNrjMxGbaV +KzdrT2uCkYcf78XjnhuH9gKIiXDUv4L4N+P3u6w8yOx4bFgOS9IjS53yDOPM7SC5 +g6dIg5aigHaHlffqIuFFv4yQMI/+Ai+zBKxS7wRhxK/7nnAuo28fe5MEdp57ho9/ +AGlDNsdg9zCgjwhokwFE3+AaD+bkUFm4gQ1XjkUFrlmnQn8vDQ0i9toEWhCj+UPY +iOKL63MJnr90MXTXWLHoFj99wBp//mYygbF9Lj8fa28/oa8LWp3Jhb7QeMgH46iv +3aLHbTECgYEA5M2dAw+nyMw9vYlkMejhwObKYP8Mr/6zcGMLCalYvRJM5iUAM0JI +H6sM6pV9/nv167cbKocj3xYPdtE7FPOn4132MLM8Ne1f8nPE64Qrcbj5WBXvLnU8 +hpWbwe2Z8h7UUMKx6q4F1/TXYkc3ScxYwfjM4mP/pLsAOgVzRSEEgrUCgYEA1qNQ +xaQHNWZ1O8WuTnqWd5JSsic6iURAmUcLeFDZY2PWhVoaQ8L/xMQhDYs1FIbLWArW +4Qq3Ibu8AbSejAKuaJz7Uf26PX+PYVUwAOO0qamCJ8d/qd6So7qWMDyAY2yXI39Y +1nMqRjr7bkEsggAZao7BKqA7ZtmogjOusBT38iUCgYEA06agJ8TDoKvOMRZ26PRU +YO0dKLzGL8eclcoI29cbj0rud7aiiMg3j5PbTuUat95TjsjDCIQaWrM9etvxm2AJ +Xfn9Uu96MyhyKQWOk46f4YMKpMElkARDCPw8KRhx39dE77AqhLyWCz8iPndCXbH6 +KPTOEl4OjYOuof2Is9nnIkECgYBh948RdsnXhNlzm8nwhiGRmBbou+EK8D0v+O5y +Tyy6IcKzgSnFzgZh8EdJ4EUtBk1f9SqY8wQdgIvSl3daXorusuA/TzkngsaV3YUY +ktZOLlF7CKLrjOyPkMWmZKcROmpNyH1q/IvKHHfQnizLdXIkYd4nL5WNX0F7lE1i +j1+QhQKBgB2lviBK7rJFwlFYdQUP1NAN2dKxMZk8uJS8JglHrM0+8nRI83HbTdEQ +vB0ManEKBkbS4T5n+gRtdEqKSDmWDTXDlrBfcdCHNQLwYtBpOotCqQn/AmfjcPBl +byAbwh4+HiZ5JISoRZpiZqy67aJNVoXmdtb/E9mi7ozzytpxMNql +-----END RSA PRIVATE KEY-----''' +domain = 'example.com' +selector = 'rsa' +headers = ['From', 'To', 'Date', 'Subject', 'Message-ID'] +algorithm = 'rsa-sha256' +canonicalization = 'simple/relaxed' +expire = '10d' +set-body-length = true +report = true + +[signature.ed] +public-key = '11qYAYKxCrfVS/7TyWQHOg7hcvPapiMlrwIaaPcHURo=' +private-key = 'nWGxne/9WmC6hEr0kuwsxERJxWl7MmkZcDusAxyuf2A=' +domain = 'example.com' +selector = 'ed' +headers = ['From', 'To', 'Date', 'Subject', 'Message-ID'] +algorithm = 'ed25519-sha256' +canonicalization = 'relaxed/simple' +set-body-length = false +"; + +#[tokio::test] +async fn sign_and_seal() { + let mut core = Core::test(); + + // Create temp dir for queue + let mut qr = core.init_test_queue("smtp_sign_test"); + + // Add SPF, DKIM and DMARC records + core.resolvers.dns.txt_add( + "mx.example.com", + Spf::parse(b"v=spf1 ip4:10.0.0.1 ip4:10.0.0.2 -all").unwrap(), + Instant::now() + Duration::from_secs(5), + ); + core.resolvers.dns.txt_add( + "example.com", + Spf::parse(b"v=spf1 ip4:10.0.0.1 -all").unwrap(), + Instant::now() + Duration::from_secs(5), + ); + core.resolvers.dns.txt_add( + "ed._domainkey.scamorza.org", + DomainKey::parse( + concat!( + "v=DKIM1; k=ed25519; ", + "p=11qYAYKxCrfVS/7TyWQHOg7hcvPapiMlrwIaaPcHURo=" + ) + .as_bytes(), + ) + .unwrap(), + Instant::now() + Duration::from_secs(5), + ); + core.resolvers.dns.txt_add( + "rsa._domainkey.manchego.org", + DomainKey::parse( + concat!( + "v=DKIM1; t=s; p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQ", + "KBgQDwIRP/UC3SBsEmGqZ9ZJW3/DkMoGeLnQg1fWn7/zYt", + "IxN2SnFCjxOCKG9v3b4jYfcTNh5ijSsq631uBItLa7od+v", + "/RtdC2UzJ1lWT947qR+Rcac2gbto/NMqJ0fzfVjH4OuKhi", + "tdY9tf6mcwGjaNBcWToIMmPSPDdQPNUYckcQ2QIDAQAB", + ) + .as_bytes(), + ) + .unwrap(), + Instant::now() + Duration::from_secs(5), + ); + + let mut config = &mut core.session.config.rcpt; + config.lookup_domains = IfBlock::new(Some(Arc::new(Lookup::Local(AHashSet::from_iter([ + "example.com".to_string(), + ]))))); + config.lookup_addresses = IfBlock::new(Some(Arc::new(Lookup::Local(AHashSet::from_iter([ + "jdoe@example.com".to_string(), + ]))))); + + let mut config = &mut core.session.config; + config.data.add_auth_results = IfBlock::new(true); + config.data.add_date = IfBlock::new(true); + config.data.add_message_id = IfBlock::new(true); + config.data.add_received = IfBlock::new(true); + config.data.add_return_path = IfBlock::new(true); + config.data.add_received_spf = IfBlock::new(true); + + let mut config = &mut core.mail_auth; + let ctx = ConfigContext::default().parse_signatures(); + config.spf.verify_ehlo = IfBlock::new(VerifyStrategy::Relaxed); + config.spf.verify_mail_from = config.spf.verify_ehlo.clone(); + config.dkim.verify = config.spf.verify_ehlo.clone(); + config.arc.verify = config.spf.verify_ehlo.clone(); + config.dmarc.verify = config.spf.verify_ehlo.clone(); + config.dkim.sign = "['rsa']" + .parse_if::>(&ctx) + .map_if_block(&ctx.signers, "", "") + .unwrap(); + config.arc.seal = "'ed'" + .parse_if::>(&ctx) + .map_if_block(&ctx.sealers, "", "") + .unwrap(); + + // Test DKIM signing + let mut session = Session::test(core); + session.data.remote_ip = "10.0.0.2".parse().unwrap(); + session.eval_session_params().await; + session.ehlo("mx.example.com").await; + session + .send_message( + "bill@foobar.org", + &["jdoe@example.com"], + "test:no_dkim", + "250", + ) + .await; + qr.read_event() + .await + .unwrap_message() + .read_lines() + .assert_contains( + "DKIM-Signature: v=1; a=rsa-sha256; s=rsa; d=example.com; c=simple/relaxed;", + ); + + // Test ARC verify and seal + session + .send_message("bill@foobar.org", &["jdoe@example.com"], "test:arc", "250") + .await; + qr.read_event() + .await + .unwrap_message() + .read_lines() + .assert_contains("ARC-Seal: i=3; a=ed25519-sha256; s=ed; d=example.com; cv=pass;") + .assert_contains( + "ARC-Message-Signature: i=3; a=ed25519-sha256; s=ed; d=example.com; c=relaxed/simple;", + ); +} + +pub trait TextConfigContext { + fn parse_signatures(self) -> ConfigContext; +} + +impl TextConfigContext for ConfigContext { + fn parse_signatures(mut self) -> Self { + Config::parse(SIGNATURES) + .unwrap() + .parse_signatures(&mut self) + .unwrap(); + self + } +} diff --git a/tests/src/smtp/inbound/throttle.rs b/tests/src/smtp/inbound/throttle.rs new file mode 100644 index 00000000..f3935c69 --- /dev/null +++ b/tests/src/smtp/inbound/throttle.rs @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::time::Duration; + +use crate::smtp::{session::TestSession, ParseTestConfig, TestConfig}; +use smtp::{ + config::ConfigContext, + core::{Core, Session, SessionAddress}, +}; + +#[tokio::test] +async fn throttle_inbound() { + let mut core = Core::test(); + let mut config = &mut core.session.config; + config.throttle.connect = r"[[throttle]] + match = {if = 'remote-ip', eq = '10.0.0.1'} + key = 'remote-ip' + concurrency = 2 + rate = '3/1s' + " + .parse_throttle(&ConfigContext::default()); + config.throttle.mail_from = r"[[throttle]] + key = 'sender' + rate = '2/1s' + " + .parse_throttle(&ConfigContext::default()); + config.throttle.rcpt_to = r"[[throttle]] + key = ['remote-ip', 'rcpt'] + rate = '2/1s' + " + .parse_throttle(&ConfigContext::default()); + + // Test connection concurrency limit + let mut session = Session::test(core); + session.data.remote_ip = "10.0.0.1".parse().unwrap(); + assert!( + session.is_allowed().await, + "Concurrency limiter too strict." + ); + assert!( + session.is_allowed().await, + "Concurrency limiter too strict." + ); + assert!(!session.is_allowed().await, "Concurrency limiter failed."); + + // Test connection rate limit + session.in_flight.clear(); // Manually reset concurrency limiter + assert!(session.is_allowed().await, "Rate limiter too strict."); + assert!(!session.is_allowed().await, "Rate limiter failed."); + session.in_flight.clear(); + tokio::time::sleep(Duration::from_millis(1100)).await; + assert!( + session.is_allowed().await, + "Rate limiter did not restore quota." + ); + + // Test mail from rate limit + session.data.mail_from = SessionAddress { + address: "sender@test.org".to_string(), + address_lcase: "sender@test.org".to_string(), + domain: "test.org".to_string(), + flags: 0, + dsn_info: None, + } + .into(); + assert!(session.is_allowed().await, "Rate limiter too strict."); + assert!(session.is_allowed().await, "Rate limiter too strict."); + assert!(!session.is_allowed().await, "Rate limiter failed."); + session.data.mail_from = SessionAddress { + address: "other-sender@test.org".to_string(), + address_lcase: "other-sender@test.org".to_string(), + domain: "test.org".to_string(), + flags: 0, + dsn_info: None, + } + .into(); + assert!(session.is_allowed().await, "Rate limiter failed."); + + // Test recipient rate limit + session.data.rcpt_to.push(SessionAddress { + address: "recipient@example.org".to_string(), + address_lcase: "recipient@example.org".to_string(), + domain: "example.org".to_string(), + flags: 0, + dsn_info: None, + }); + assert!(session.is_allowed().await, "Rate limiter too strict."); + assert!(session.is_allowed().await, "Rate limiter too strict."); + assert!(!session.is_allowed().await, "Rate limiter failed."); + session.data.remote_ip = "10.0.0.2".parse().unwrap(); + assert!(session.is_allowed().await, "Rate limiter too strict."); +} diff --git a/tests/src/smtp/inbound/vrfy.rs b/tests/src/smtp/inbound/vrfy.rs new file mode 100644 index 00000000..5df6b7e6 --- /dev/null +++ b/tests/src/smtp/inbound/vrfy.rs @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::sync::Arc; + +use ahash::AHashSet; + +use crate::smtp::{ + session::{TestSession, VerifyResponse}, + ParseTestConfig, TestConfig, +}; +use smtp::{ + config::ConfigContext, + core::{Core, Session}, + lookup::Lookup, +}; + +#[tokio::test] +async fn vrfy_expn() { + let mut core = Core::test(); + let mut ctx = ConfigContext::default(); + ctx.lookup.insert( + "vrfy".to_string(), + Arc::new(Lookup::Local(AHashSet::from_iter([ + "john@foobar.org:john@foobar.org".to_string(), + "john:john@foobar.org".to_string(), + ]))), + ); + ctx.lookup.insert( + "expn".to_string(), + Arc::new(Lookup::Local(AHashSet::from_iter([ + "sales:john@foobar.org,bill@foobar.org,jane@foobar.org".to_string(), + "support:mike@foobar.org".to_string(), + ]))), + ); + + let mut config = &mut core.session.config.rcpt; + + config.lookup_vrfy = r"[{if = 'remote-ip', eq = '10.0.0.1', then = 'vrfy'}, + {else = false}]" + .parse_if::>(&ctx) + .map_if_block(&ctx.lookup, "", "") + .unwrap(); + config.lookup_expn = r"[{if = 'remote-ip', eq = '10.0.0.1', then = 'expn'}, + {else = false}]" + .parse_if::>(&ctx) + .map_if_block(&ctx.lookup, "", "") + .unwrap(); + + // EHLO should not avertise VRFY/EXPN to 10.0.0.2 + let mut session = Session::test(core); + session.data.remote_ip = "10.0.0.2".parse().unwrap(); + session.eval_session_params().await; + session + .ehlo("mx.foobar.org") + .await + .assert_not_contains("EXPN") + .assert_not_contains("VRFY"); + session.cmd("VRFY john", "252 2.5.1").await; + session.cmd("EXPN sales", "252 2.5.1").await; + + // EHLO should advertise VRFY/EXPN for 10.0.0.1 + session.data.remote_ip = "10.0.0.1".parse().unwrap(); + session.eval_session_params().await; + session + .ehlo("mx.foobar.org") + .await + .assert_contains("EXPN") + .assert_contains("VRFY"); + + // Successful VRFY + session.cmd("VRFY john", "250 john@foobar.org").await; + + // Successful EXPN + session + .cmd("EXPN sales", "250") + .await + .assert_contains("250-john@foobar.org") + .assert_contains("250-bill@foobar.org") + .assert_contains("250 jane@foobar.org"); + + // Non-existent VRFY + session.cmd("VRFY bill", "550 5.1.2").await; + + // Non-existent EXPN + session.cmd("EXPN procurement", "550 5.1.2").await; +} diff --git a/tests/src/smtp/lookup/imap.rs b/tests/src/smtp/lookup/imap.rs new file mode 100644 index 00000000..d9f330a7 --- /dev/null +++ b/tests/src/smtp/lookup/imap.rs @@ -0,0 +1,234 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::sync::Arc; + +use mail_parser::decoders::base64::base64_decode; +use mail_send::Credentials; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::{TcpListener, TcpStream}, + sync::watch, +}; +use tokio_rustls::TlsAcceptor; + +use smtp::{ + config::{remote::ConfigHost, ConfigContext}, + lookup::{Item, LookupResult}, +}; +use utils::{ + config::Config, + listener::limiter::{ConcurrencyLimiter, InFlight}, +}; + +use crate::smtp::lookup::{TestItem, TestLookupResult}; + +use super::dummy_tls_acceptor; + +const REMOTE: &str = " +[remote.imap] +address = 127.0.0.1 +port = 9998 +concurrency = 5 +protocol = 'imap' + +[remote.imap.limits] +errors = 3 +requests = 5 + +[remote.imap.cache] +entries = 500 +ttl = {positive = '10s', negative = '5s'} + +[remote.imap.tls] +implicit = true +allow-invalid-certs = true +"; + +#[tokio::test] +async fn lookup_imap() { + // Enable logging + /*tracing::subscriber::set_global_default( + tracing_subscriber::FmtSubscriber::builder() + .with_max_level(tracing::Level::DEBUG) + .finish(), + ) + .unwrap();*/ + + // Spawn mock LMTP server + let shutdown = spawn_mock_imap_server(5); + + // Spawn lookup client + let mut ctx = ConfigContext::default(); + let config = Config::parse(REMOTE).unwrap(); + config.parse_remote_hosts(&mut ctx).unwrap(); + let lookup = ctx.hosts.remove("imap").unwrap().spawn(&config); + + // Basic lookup + let tests = vec![ + ( + Item::Authenticate(Credentials::Plain { + username: "john".to_string(), + secret: "ok".to_string(), + }), + LookupResult::True, + ), + ( + Item::Authenticate(Credentials::Plain { + username: "john".to_string(), + secret: "bad".to_string(), + }), + LookupResult::False, + ), + ]; + + for (item, expected) in &tests { + assert_eq!(&lookup.lookup(item.clone()).await.unwrap(), expected); + } + + // Concurrent requests + let mut requests = Vec::new(); + for n in 0..100 { + let (item, expected) = &tests[n % tests.len()]; + let item = item.append(n); + let item_clone = item.clone(); + let lookup = lookup.clone(); + requests.push(( + tokio::spawn(async move { lookup.lookup(item).await }), + item_clone, + expected.append(n), + )); + } + for (result, item, expected_result) in requests { + let result = result.await.unwrap(); + assert_eq!(result, Some(expected_result), "Failed for {item:?}"); + } + + // Shutdown + shutdown.send(false).ok(); + + // Verify that caching works + TcpStream::connect("127.0.0.1:9998").await.unwrap_err(); + + let mut requests = Vec::new(); + for n in 0..100 { + let (item, expected) = &tests[n % tests.len()]; + let item = item.append(n); + let item_clone = item.clone(); + let lookup = lookup.clone(); + requests.push(( + tokio::spawn(async move { lookup.lookup(item).await }), + item_clone, + expected.append(n), + )); + } + for (result, item, expected_result) in requests { + let result = result.await.unwrap(); + assert_eq!(result, Some(expected_result), "Failed for {item:?}"); + } +} + +pub fn spawn_mock_imap_server(max_concurrency: u64) -> watch::Sender { + let (tx, mut rx) = watch::channel(true); + + tokio::spawn(async move { + let listener = TcpListener::bind("127.0.0.1:9998") + .await + .unwrap_or_else(|e| { + panic!("Failed to bind mock SMTP server to 127.0.0.1:9998: {e}"); + }); + let acceptor = dummy_tls_acceptor(); + let limited = ConcurrencyLimiter::new(max_concurrency); + loop { + tokio::select! { + stream = listener.accept() => { + match stream { + Ok((stream, _)) => { + let acceptor = acceptor.clone(); + let in_flight = limited.is_allowed(); + tokio::spawn(accept_smtp(stream, acceptor, in_flight)); + } + Err(err) => { + panic!("Something went wrong: {err}" ); + } + } + }, + _ = rx.changed() => { + break; + } + }; + } + }); + + tx +} + +async fn accept_smtp(stream: TcpStream, acceptor: Arc, in_flight: Option) { + let mut stream = acceptor.accept(stream).await.unwrap(); + stream + .write_all(b"* OK Clueless host service ready\r\n") + .await + .unwrap(); + + if in_flight.is_none() { + eprintln!("WARNING: Concurrency exceeded!"); + } + + let mut buf_u8 = vec![0u8; 1024]; + + loop { + let br = if let Ok(br) = stream.read(&mut buf_u8).await { + br + } else { + break; + }; + let buf = std::str::from_utf8(&buf_u8[0..br]).unwrap(); + let (op, buf) = buf.split_once(' ').unwrap(); + + //print!("-> {}", buf); + let response = if buf.starts_with("CAPABILITY") { + format!( + "* CAPABILITY IMAP4rev2 IMAP4rev1 AUTH=PLAIN\r\n{op} OK CAPABILITY completed\r\n", + ) + } else if buf.starts_with("NOOP") { + format!("{op} OK NOOP completed\r\n") + } else if buf.starts_with("AUTHENTICATE PLAIN") { + let buf = base64_decode(buf.rsplit_once(' ').unwrap().1.as_bytes()).unwrap(); + if String::from_utf8_lossy(&buf).contains("ok") { + format!("{op} OK Great success!\r\n") + } else { + format!("{op} BAD No soup for you!\r\n") + } + } else if buf.starts_with("LOGOUT") { + format!("* BYE\r\n{op} OK LOGOUT completed\r\n") + } else { + panic!("Unknown command: {}", buf.trim()); + }; + //print!("<- {}", response); + stream.write_all(response.as_bytes()).await.unwrap(); + + if buf.contains("bye") || buf.starts_with("LOGOUT") { + return; + } + } +} diff --git a/tests/src/smtp/lookup/mod.rs b/tests/src/smtp/lookup/mod.rs new file mode 100644 index 00000000..b5488c45 --- /dev/null +++ b/tests/src/smtp/lookup/mod.rs @@ -0,0 +1,202 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{io::BufReader, sync::Arc}; + +use mail_send::Credentials; +use rustls::{Certificate, PrivateKey, ServerConfig}; +use rustls_pemfile::{certs, pkcs8_private_keys}; +use tokio_rustls::TlsAcceptor; + +use ::smtp::lookup::{Item, LookupResult}; + +pub mod imap; +pub mod smtp; +pub mod sql; + +const CERT: &str = "-----BEGIN CERTIFICATE----- +MIIFCTCCAvGgAwIBAgIUCgHGQYUqtelbHGVSzCVwBL3fyEUwDQYJKoZIhvcNAQEL +BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTIyMDUxNjExNDAzNFoXDTIzMDUx +NjExNDAzNFowFDESMBAGA1UEAwwJbG9jYWxob3N0MIICIjANBgkqhkiG9w0BAQEF +AAOCAg8AMIICCgKCAgEAtwS0Fzl3SjaCuKEXgZ/fdWbDoj/qDphyNCAKNevQ0+D0 +STNkWCO04aFSH0zcL8zoD9gokNos0i7OU9//ZhZQmex4V6EFdZn8bFwUWN/scUvW +HEFXVjtHldO2isZgIxH9LuwRv7KAgkISuWahqerOVDhe7SeQUV0AJGNEh3cT9PZr +gSY931BxB7n+5k8eoSk8Z1gtBzQzL62kVGpHDKfw8yX8m65owF9eLUBrNzgxmXfC +xpuHwj7hmVhS09PPKeN/RsFS8PsYO7bo0u8jEKalteumjRT7RyUEbioqfo6ZFOGj +FHPIq/uKXS9zN1fpoyNh3ur5hMznQhrqlwBM9KlM7GdBJ0pZ3ad0YjT8IL/GnGKR +85J2WZdLqaQdUZo7nV67FhqdDlNE4MdwiykTMjfmLRXGAVhAzJHKyRKNwmkI2aqe +S7aqeNgvuDBwY80Q9a2rb5py1Aw+L8yCkUBuHboToDpxSVRDNN8DrWNmmsXnxsOG +wRDODy4GICKyxlP+RFSM8xWSQ6y9ktS2OfDBm+Eqcw+3pZKhdz2wgxLkUBJ8X1eh +kJrCA/6LTuhy6m6mMjAfoSOFU7fu88jxaWPgvP7GKyH+LM/t9eucobz2ks5rtSjz +V4Dc5DCS94/OpVRHwHdaFSPbJKBN9Ev8gnNrAyx/aBPGoHBPG/QUiU7dcUNIPt0C +AwEAAaNTMFEwHQYDVR0OBBYEFI167IxBmErB11EqiPPqFLa31ZaMMB8GA1UdIwQY +MBaAFI167IxBmErB11EqiPPqFLa31ZaMMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI +hvcNAQELBQADggIBALU00IOiH5ubEauVCmakms5ermNTZfculnhnDfWTLMeh2+a7 +G4cqADErfMhm/mmLbrw33t9s6tCAhQltvewKR40ST9uMPSyiQbYaCXd5DXnuI6Ox +JtNW+UOWIaMf8abnkdLvREOvb8dVQS1i3xq14tAjY5XgpGwCPP8m54b7N3Q7soLn +e5PDhPNTnhRIn2RLuYoZmQmMA5fcqEUDYff4epUww7PhrM1QckZligI3566NlGOf +j1G9JrivBtY0eaJtamIFnGMBT0ThDudxVja2Nv0C2Elry0p4T/o4nc4M67BJ/y1R +vjNLAgFhbxssemU3lZqSd+pykpJBwDBjFSPrZZmQcbk7H6Uz8V1xr/xuzfw6fA13 +NWZ5vLgP/DQ13sM+XFlxThKfbPMPVe/UCTvfGtNW+3XyBgPntEkR+fNEawQmzbYl +R+X1ymT9MZnEZqRMf7/UD/SYek1aUJefoew3upjMgxYVvh4F8dqJ+39F+xoFzIA2 +1dDAEMzXtjA3zKhZ2cycZbEzpJvYA3eGLuR16Suqfi4kPvfwK0mOhCxQmpayt7/X +vuEzW6dPCH8Hgbb0WvsSppGOvhdbDaZFNfFc5eNSxhyKzu3H3ACNImZRtZE+yixx +0fR8+xz9kDLf8xupV+X9heyFGHSyYU2Lveaevtr2Ij3weLRgJ6LbNALoeKXk +-----END CERTIFICATE----- +"; +const PK: &str = "-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQC3BLQXOXdKNoK4 +oReBn991ZsOiP+oOmHI0IAo169DT4PRJM2RYI7ThoVIfTNwvzOgP2CiQ2izSLs5T +3/9mFlCZ7HhXoQV1mfxsXBRY3+xxS9YcQVdWO0eV07aKxmAjEf0u7BG/soCCQhK5 +ZqGp6s5UOF7tJ5BRXQAkY0SHdxP09muBJj3fUHEHuf7mTx6hKTxnWC0HNDMvraRU +akcMp/DzJfybrmjAX14tQGs3ODGZd8LGm4fCPuGZWFLT088p439GwVLw+xg7tujS +7yMQpqW166aNFPtHJQRuKip+jpkU4aMUc8ir+4pdL3M3V+mjI2He6vmEzOdCGuqX +AEz0qUzsZ0EnSlndp3RiNPwgv8acYpHzknZZl0uppB1RmjudXrsWGp0OU0Tgx3CL +KRMyN+YtFcYBWEDMkcrJEo3CaQjZqp5Ltqp42C+4MHBjzRD1ratvmnLUDD4vzIKR +QG4duhOgOnFJVEM03wOtY2aaxefGw4bBEM4PLgYgIrLGU/5EVIzzFZJDrL2S1LY5 +8MGb4SpzD7elkqF3PbCDEuRQEnxfV6GQmsID/otO6HLqbqYyMB+hI4VTt+7zyPFp +Y+C8/sYrIf4sz+3165yhvPaSzmu1KPNXgNzkMJL3j86lVEfAd1oVI9skoE30S/yC +c2sDLH9oE8agcE8b9BSJTt1xQ0g+3QIDAQABAoICABq5oxqpF5RMtXYEgAw7rkPU +h8jPkHwlIrgd3Z/WGZ53APUXfhWo0ScJiZZsgNKyF0kJBZNxaI4gq5xv3zmnFIoF +j+Ur7EIqBERGheoceMhqjI9/syMycNeeHM/S/ALjA5ewfT8C7+UVhOpx5DWNxidi +O+phlp9q9zRZEo69grqIqVYooWxUsMyyCljTQOPDw8BLjfe5VagmsRJqmolslLDM +4UBSjZVZ18S/3Wgo2oVQia660244BHWCAkZQbbXuNI2+eUAbSoSdxw3WQcaSrywL +hzyezbqr2yPDIIVuiUgVUt0Ps0P57VCCN07jlYhvCEGnClysFzD+ATefoZ0wg7za +dQu2E+d166rAjnssyhzcHMn3pxgSdtXD+dQR/xfIGbPABucCupEFqKmhLdMm9+ud +lHay87qzMpIa8cITJwEQROfXqWAhNUU98pKCOx1SVXBqQC7QVqGQ5solDf0eMSVh +ngQ6Dz2WUI2ty75LteiFwlyTgnU9nyPN0NXsrMEET2BHWre7ufTQqiULtQ7+9BwH +AMxEKvrQHjMUjdfbXuzdyc5w5mPYJZfFVSQ1HMslx66h9yCpRIsBZvUGvoaP8Tpe +nQ66FTYRbiOkkdJ7k8DtrnhsJI1oOGjnvj/rvZ8D2pvrlJcIH2AyN3MOL8Jp5Oj1 +nCFt77TwpF92pgl0g9gBAoIBAQDcarmP54QboaIQ9S2gE/4gSVC5i44iDJuSRdI8 +K081RQcWiNzqQXTRc5nqJ7KzLyPiGlg+6rWsBKLos5l4t+MdhhH+KUvk/OtT/g8V +0NZBNXLIbSb8j8ix4v3/f2qKHN3Co6QOlxb3gFvobKDdoKqUNiSH1zTZ8/Y/BzkM +jqWKhTdaLz6eyzhKfOTA4LO8kJ3VF8HUM1N9/e8Gjorl+gZpJUXUQS0+AIi8W76C +OwDrVb3BPGVnApQJfWF78h4g20RwXrx/GYUW2vOMcLjXXDV5U7+nobPUoJnLxoZC +16o88y0Ivan8dBNXsc1epyPvvEqp6MJbAyyVuNeuRJcgYA0BAoIBAQDUkGRV7fLG +wCr5rNysUO+FKzVtTJnf9KEsqAqUmmVnG4oubxAJJtiB5n2+DT+CtO8Nrtz05BbR +uxfWm+lbEw6lVMj63bywtp0NdULg7/2t+oq2Svv16KrZIRJttXMkdEiFFmkVAEhX +l8Fyl6PJPfSMwbPdXEUPUAaNrXweVFffXczHc4W2G212ZzDB0z7QQSgEntbTDFB/ +2Cg5dvuojlM9zw0fuEyLwItZs7n16j/ONZLgBHyroMU9ZPxbnLrVyoZlqtob+RWm +Ju2fSIL9QqG6O4td1TqcUBGvFQYjGvKA+q5fsG26NBJ0Ac48cNK6PS4lMkN3Av2J +ccloYaMEHAXdAoIBAE8WMCy1Ok6byUXiYxOL+OPmyoM40q/e7DcovE2AkLQhZ3Cr +fPDEucCphPFiexkV8f8fysgQeU0WgMmUH54UBPbD81LJyISKR3nkr875Ftdg8SV/ +HL0EblN9ifuR4U1bHCrJgoUFq2T09oVH7NR44Ju7bZIcIseNZK6qzcp2qGkycXD3 +gLWDX1hCxeV6+qLPFQKvuomEPRH4+jnVDXuFIaW6jPqixDP6BxXmqU2bFDJcmnBq +VkwGvc1F4qORdUP+yOi05VeJdZqEx1x92aTUXg+BgEQKnjbNxUE7o1L6hQfHjUIU +o5iEoagWkQTEXf2YBwY+EPaNBgNWxnSuAbfJHwECggEBALOF95ezTVWauzD/U6ic ++o3n/kl/Zn4FJ5KFodn7xCSe18d7uXlhO34KYqx+l+MWWMefpbGWacdcUjfImf93 +SulLgCqP12sP7/iLzp4XUpL7hOeM0NvRU2nqSpwpoUNqik0Mrlc0U+TWoGTduVCf +aMjwV65e3VyfY8mIeclLxqM5n1fcM1OoOnzDjiRE+0n7nYa5eAnq3pn6v4449TZY +belH03e0ucFWLtrltesBmj3YdWGJqJlzQOInRhNBfXJOh8+ZynfRmP0o54udPDQV +cG3PGFd5XPTjkuvhv7sqaSGRlm/um92lWOhtFfdp+i+cuDpmByCef+7zEP19aKZx +3GkCggEAFTs7KNMfvIEaLH0yQUFeq2gLmtcMofmOmeoIECycN1rG7iJo07lJLIs0 +bVODH8Z0kX8llu3cjGMAH/6R2uugJSxkmFiZKrngTzKmxDPvTCKWR4RFwXH9j8IO +cPq7FtKN4SgrPy9ciAPdkcGmu3zz/sBKOaoPwvU2PdBRT+v/aoz+GCLXAvzFlKVe +9/7zdg87ilo8+AtV+71EJeR3kyBPKS9JrWYUKfiams12+uuH4/53rMFZfNCAaZ3Z +1sdXEO4o3Loc5TX4DbO9FVdBSBe6klEXx4T0QJboO6uBvTBnnRL2SQriJQQFwYT6 +XzVV5pwOxkIDBWDIqMUfwJDChBKfpw== +-----END PRIVATE KEY----- +"; + +pub fn dummy_tls_acceptor() -> Arc { + // Init server config builder with safe defaults + let config = ServerConfig::builder() + .with_safe_defaults() + .with_no_client_auth(); + + // load TLS key/cert files + let cert_file = &mut BufReader::new(CERT.as_bytes()); + let key_file = &mut BufReader::new(PK.as_bytes()); + + // convert files to key/cert objects + let cert_chain = certs(cert_file) + .unwrap() + .into_iter() + .map(Certificate) + .collect(); + let mut keys: Vec = pkcs8_private_keys(key_file) + .unwrap() + .into_iter() + .map(PrivateKey) + .collect(); + + // exit if no keys could be parsed + if keys.is_empty() { + panic!("Could not locate PKCS 8 private keys."); + } + + Arc::new(TlsAcceptor::from(Arc::new( + config.with_single_cert(cert_chain, keys.remove(0)).unwrap(), + ))) +} + +pub trait TestItem { + fn append(&self, append: usize) -> Self; +} + +impl TestItem for Item { + fn append(&self, append: usize) -> Self { + match self { + Item::IsAccount(str) => Item::IsAccount(format!("{append}{str}")), + Item::Authenticate(str) => Item::Authenticate(match str { + Credentials::Plain { username, secret } => Credentials::Plain { + username: username.to_string(), + secret: format!("{append}{secret}"), + }, + Credentials::OAuthBearer { token } => Credentials::OAuthBearer { + token: format!("{append}{token}"), + }, + Credentials::XOauth2 { username, secret } => Credentials::XOauth2 { + username: username.to_string(), + secret: format!("{append}{secret}"), + }, + }), + Item::Verify(str) => Item::Verify(format!("{append}{str}")), + Item::Expand(str) => Item::Expand(format!("{append}{str}")), + } + } +} + +pub trait TestLookupResult { + fn append(&self, append: usize) -> Self; +} + +impl TestLookupResult for LookupResult { + fn append(&self, append: usize) -> Self { + match self { + LookupResult::True => LookupResult::True, + LookupResult::False => LookupResult::False, + LookupResult::Values(v) => { + let mut r = Vec::with_capacity(v.len()); + for (pos, val) in v.iter().enumerate() { + r.push(if pos == 0 { + format!("{append}{val}") + } else { + val.to_string() + }); + } + LookupResult::Values(r) + } + } + } +} diff --git a/tests/src/smtp/lookup/smtp.rs b/tests/src/smtp/lookup/smtp.rs new file mode 100644 index 00000000..9c7f5434 --- /dev/null +++ b/tests/src/smtp/lookup/smtp.rs @@ -0,0 +1,295 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::sync::Arc; + +use mail_parser::decoders::base64::base64_decode; +use mail_send::Credentials; +use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::{TcpListener, TcpStream}, + sync::watch, +}; +use tokio_rustls::TlsAcceptor; + +use smtp::{ + config::{remote::ConfigHost, ConfigContext}, + lookup::{Item, LookupResult}, +}; +use utils::{ + config::Config, + listener::limiter::{ConcurrencyLimiter, InFlight}, +}; + +use crate::smtp::lookup::{TestItem, TestLookupResult}; + +use super::dummy_tls_acceptor; + +const REMOTE: &str = " +[remote.lmtp] +address = 127.0.0.1 +port = 9999 +protocol = 'lmtp' +concurrency = 5 + +[remote.lmtp.limits] +errors = 3 +requests = 5 + +[remote.lmtp.cache] +entries = 500 +ttl = {positive = '10s', negative = '5s'} + +[remote.lmtp.tls] +implicit = true +allow-invalid-certs = true +"; + +#[tokio::test] +async fn lookup_smtp() { + // Spawn mock LMTP server + let shutdown = spawn_mock_lmtp_server(5); + + // Spawn lookup client + let mut ctx = ConfigContext::default(); + let config = Config::parse(REMOTE).unwrap(); + config.parse_remote_hosts(&mut ctx).unwrap(); + let lookup = ctx.hosts.remove("lmtp").unwrap().spawn(&config); + + // Basic lookup + let tests = vec![ + ( + Item::IsAccount("john-ok@domain".to_string()), + LookupResult::True, + ), + ( + Item::IsAccount("john-bad@domain".to_string()), + LookupResult::False, + ), + ( + Item::Verify("john-ok@domain".to_string()), + LookupResult::Values(vec!["john-ok@domain".to_string()]), + ), + ( + Item::Verify("doesnot@exist.org".to_string()), + LookupResult::False, + ), + ( + Item::Expand("sales-ok,item1,item2,item3".to_string()), + LookupResult::Values(vec![ + "sales-ok".to_string(), + "item1".to_string(), + "item2".to_string(), + "item3".to_string(), + ]), + ), + (Item::Expand("other".to_string()), LookupResult::False), + ( + Item::Authenticate(Credentials::Plain { + username: "john".to_string(), + secret: "ok".to_string(), + }), + LookupResult::True, + ), + ( + Item::Authenticate(Credentials::Plain { + username: "john".to_string(), + secret: "bad".to_string(), + }), + LookupResult::False, + ), + ]; + + for (item, expected) in &tests { + assert_eq!(&lookup.lookup(item.clone()).await.unwrap(), expected); + } + + // Concurrent requests + let mut requests = Vec::new(); + for n in 0..100 { + let (item, expected) = &tests[n % tests.len()]; + let item = item.append(n); + let item_clone = item.clone(); + let lookup = lookup.clone(); + requests.push(( + tokio::spawn(async move { lookup.lookup(item).await }), + item_clone, + expected.append(n), + )); + } + for (result, item, expected_result) in requests { + let result = result.await.unwrap(); + assert_eq!(result, Some(expected_result), "Failed for {item:?}"); + } + + // Shutdown + shutdown.send(false).ok(); + + // Verify that caching works + TcpStream::connect("127.0.0.1:9999").await.unwrap_err(); + + let mut requests = Vec::new(); + for n in 0..100 { + let (item, expected) = &tests[n % tests.len()]; + if !matches!(item, Item::Verify(_) | Item::Expand(_)) { + let item = item.append(n); + let item_clone = item.clone(); + let lookup = lookup.clone(); + requests.push(( + tokio::spawn(async move { lookup.lookup(item).await }), + item_clone, + expected.append(n), + )); + } + } + for (result, item, expected_result) in requests { + let result = result.await.unwrap(); + assert_eq!(result, Some(expected_result), "Failed for {item:?}"); + } +} + +pub fn spawn_mock_lmtp_server(max_concurrency: u64) -> watch::Sender { + let (tx, mut rx) = watch::channel(true); + + tokio::spawn(async move { + let listener = TcpListener::bind("127.0.0.1:9999") + .await + .unwrap_or_else(|e| { + panic!("Failed to bind mock SMTP server to 127.0.0.1:9999: {e}"); + }); + let acceptor = dummy_tls_acceptor(); + let limited = ConcurrencyLimiter::new(max_concurrency); + loop { + tokio::select! { + stream = listener.accept() => { + match stream { + Ok((stream, _)) => { + let acceptor = acceptor.clone(); + let in_flight = limited.is_allowed(); + tokio::spawn(accept_smtp(stream, acceptor, in_flight)); + } + Err(err) => { + panic!("Something went wrong: {err}" ); + } + } + }, + _ = rx.changed() => { + break; + } + }; + } + }); + + tx +} + +async fn accept_smtp(stream: TcpStream, acceptor: Arc, in_flight: Option) { + let mut stream = acceptor.accept(stream).await.unwrap(); + stream + .write_all(b"220 [127.0.0.1] Clueless host service ready\r\n") + .await + .unwrap(); + + if in_flight.is_none() { + eprintln!("WARNING: Concurrency exceeded!"); + } + + let mut buf_u8 = vec![0u8; 1024]; + + loop { + let br = if let Ok(br) = stream.read(&mut buf_u8).await { + br + } else { + break; + }; + let buf = std::str::from_utf8(&buf_u8[0..br]).unwrap(); + //print!("-> {}", buf); + let response = if buf.starts_with("LHLO") { + "250-mx.foobar.org\r\n250 AUTH PLAIN\r\n".to_string() + } else if buf.starts_with("MAIL FROM") { + if buf.contains("<>") || buf.contains("ok@") { + "250 OK\r\n".to_string() + } else { + "552-I do not\r\n552 like that MAIL FROM.\r\n".to_string() + } + } else if buf.starts_with("RCPT TO") { + if buf.contains("ok") { + "250 OK\r\n".to_string() + } else { + "550-I refuse to\r\n550 accept that recipient.\r\n".to_string() + } + } else if buf.starts_with("VRFY") { + if buf.contains("ok") { + format!("250 {}\r\n", buf.split_once(' ').unwrap().1) + } else { + "550-I refuse to\r\n550 verify that recipient.\r\n".to_string() + } + } else if buf.starts_with("EXPN") { + if buf.contains("ok") { + let parts = buf + .split_once(' ') + .unwrap() + .1 + .split(',') + .filter_map(|s| { + if !s.is_empty() { + s.to_string().into() + } else { + None + } + }) + .collect::>(); + let mut buf = String::with_capacity(16); + for (pos, part) in parts.iter().enumerate() { + buf.push_str("250"); + buf.push(if pos == parts.len() - 1 { ' ' } else { '-' }); + buf.push_str(part); + buf.push_str("\r\n"); + } + + buf + } else { + "550-I refuse to\r\n550 accept that recipient.\r\n".to_string() + } + } else if buf.starts_with("AUTH PLAIN") { + let buf = base64_decode(buf.rsplit_once(' ').unwrap().1.as_bytes()).unwrap(); + if String::from_utf8_lossy(&buf).contains("ok") { + "235 Great success!\r\n".to_string() + } else { + "535 No soup for you\r\n".to_string() + } + } else if buf.starts_with("QUIT") { + "250 Arrivederci!\r\n".to_string() + } else if buf.starts_with("RSET") { + "250 Your wish is my command.\r\n".to_string() + } else { + panic!("Unknown command: {}", buf.trim()); + }; + //print!("<- {}", response); + stream.write_all(response.as_bytes()).await.unwrap(); + + if buf.contains("bye") || buf.starts_with("QUIT") { + return; + } + } +} diff --git a/tests/src/smtp/lookup/sql.rs b/tests/src/smtp/lookup/sql.rs new file mode 100644 index 00000000..d024c19e --- /dev/null +++ b/tests/src/smtp/lookup/sql.rs @@ -0,0 +1,205 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::time::Duration; + +use smtp_proto::{AUTH_LOGIN, AUTH_PLAIN}; +use utils::config::Config; + +use crate::smtp::{ + make_temp_dir, + session::{TestSession, VerifyResponse}, + ParseTestConfig, TestConfig, +}; +use smtp::{ + config::{database::ConfigDatabase, ConfigContext, IfBlock}, + core::{Core, Session}, + lookup::SqlDatabase, +}; + +const CONFIG: &str = r#" +[database."sql"] +address = "sqlite://%PATH%/test.db?mode=rwc" +max-connections = 10 +min-connections = 0 +idle-timeout = "5m" + +[database."sql".lookup] +auth = "SELECT secret FROM users WHERE email=?" +rcpt = "SELECT EXISTS(SELECT 1 FROM users WHERE email=? LIMIT 1)" +vrfy = "SELECT email FROM users WHERE email LIKE '%' || ? || '%' LIMIT 5" +expn = "SELECT member FROM mailing_lists WHERE id = ?" +domains = "SELECT EXISTS(SELECT 1 FROM domains WHERE name=? LIMIT 1)" +is_ip_allowed = "SELECT EXISTS(SELECT 1 FROM allowed_ips WHERE addr=? LIMIT 1)" + +[database."sql".cache] +enable = ["rcpt", "domains"] +entries = 1000 +ttl = {positive = "1d", negative = "1h"} +"#; + +#[tokio::test] +async fn lookup_sql() { + // Enable logging + /*tracing::subscriber::set_global_default( + tracing_subscriber::FmtSubscriber::builder() + .with_max_level(tracing::Level::DEBUG) + .finish(), + ) + .unwrap();*/ + + // Parse settings + let mut core = Core::test(); + let _temp_dir = make_temp_dir("sql_lookup_test", true); + let mut ctx = ConfigContext::default(); + let config = + Config::parse(&CONFIG.replace("%PATH%", _temp_dir.temp_dir.as_path().to_str().unwrap())) + .unwrap(); + config.parse_databases(&mut ctx).unwrap(); + + // Create test records + if let SqlDatabase::SqlLite(db) = ctx.databases.get("sql").unwrap() { + for query in [ + "CREATE TABLE users (email TEXT PRIMARY KEY, secret TEXT NOT NULL);", + "CREATE TABLE mailing_lists (id TEXT NOT NULL, member TEXT NOT NULL, PRIMARY KEY (id, member));", + "CREATE TABLE domains (name TEXT PRIMARY KEY, description TEXT);", + "CREATE TABLE allowed_ips (addr TEXT PRIMARY KEY);", + "INSERT INTO allowed_ips (addr) VALUES ('10.0.0.50');", + "INSERT INTO domains (name, description) VALUES ('foobar.org', 'Main domain');", + "INSERT INTO domains (name, description) VALUES ('foobar.net', 'Secondary domain');", + "INSERT INTO users (email, secret) VALUES ('jane@foobar.org', 's3cr3tp4ss');", + "INSERT INTO users (email, secret) VALUES ('john@foobar.org', 'mypassword');", + "INSERT INTO users (email, secret) VALUES ('bill@foobar.org', '123456');", + "INSERT INTO mailing_lists (id, member) VALUES ('sales@foobar.org', 'jane@foobar.org');", + "INSERT INTO mailing_lists (id, member) VALUES ('sales@foobar.org', 'john@foobar.org');", + "INSERT INTO mailing_lists (id, member) VALUES ('sales@foobar.org', 'bill@foobar.org');", + "INSERT INTO mailing_lists (id, member) VALUES ('support@foobar.org', 'mike@foobar.net');", + ] { + sqlx::query(query).execute(db).await.unwrap(); + } + } else { + panic!("Unexpected database type"); + } + + // Enable AUTH + let mut config = &mut core.session.config.auth; + config.lookup = r"'db/sql/auth'" + .parse_if::>(&ctx) + .map_if_block(&ctx.lookup, "", "") + .unwrap(); + config.mechanisms = IfBlock::new(AUTH_PLAIN | AUTH_LOGIN); + config.errors_wait = IfBlock::new(Duration::from_millis(5)); + + // Enable VRFY/EXPN/RCPT + let mut config = &mut core.session.config.rcpt; + config.lookup_addresses = r"'db/sql/rcpt'" + .parse_if::>(&ctx) + .map_if_block(&ctx.lookup, "", "") + .unwrap(); + config.lookup_domains = r"'db/sql/domains'" + .parse_if::>(&ctx) + .map_if_block(&ctx.lookup, "", "") + .unwrap(); + config.lookup_expn = r"'db/sql/expn'" + .parse_if::>(&ctx) + .map_if_block(&ctx.lookup, "", "") + .unwrap(); + config.lookup_vrfy = r"'db/sql/vrfy'" + .parse_if::>(&ctx) + .map_if_block(&ctx.lookup, "", "") + .unwrap(); + config.relay = IfBlock::new(false); + config.errors_wait = IfBlock::new(Duration::from_millis(5)); + + // Enable REQUIRETLS based on SQL lookup + core.session.config.extensions.requiretls = + r"[{if = 'remote-ip', in-list = 'db/sql/is_ip_allowed', then = true}, + {else = false}]" + .parse_if(&ctx); + let mut session = Session::test(core); + session.data.remote_ip = "10.0.0.50".parse().unwrap(); + session.eval_session_params().await; + session.stream.tls = true; + session + .ehlo("mx.foobar.org") + .await + .assert_contains("REQUIRETLS"); + session.data.remote_ip = "10.0.0.1".parse().unwrap(); + session.eval_session_params().await; + session + .ehlo("mx1.foobar.org") + .await + .assert_not_contains("REQUIRETLS"); + + // Test RCPT + session.mail_from("john@example.net", "250").await; + + // External domain + session.rcpt_to("user@otherdomain.org", "550 5.1.2").await; + + // Non-existant user + session.rcpt_to("jack@foobar.org", "550 5.1.2").await; + + // Valid users + session.rcpt_to("jane@foobar.org", "250").await; + session.rcpt_to("john@foobar.org", "250").await; + session.rcpt_to("bill@foobar.org", "250").await; + + // Test EXPN + session + .cmd("EXPN sales@foobar.org", "250") + .await + .assert_contains("jane@foobar.org") + .assert_contains("john@foobar.org") + .assert_contains("bill@foobar.org"); + session + .cmd("EXPN support@foobar.org", "250") + .await + .assert_contains("mike@foobar.net"); + session.cmd("EXPN marketing@foobar.org", "550 5.1.2").await; + + // Test VRFY + session + .cmd("VRFY john", "250") + .await + .assert_contains("john@foobar.org"); + session + .cmd("VRFY jane", "250") + .await + .assert_contains("jane@foobar.org"); + session.cmd("VRFY tim", "550 5.1.2").await; + + // Test AUTH + session + .cmd( + "AUTH PLAIN AGphbmVAZm9vYmFyLm9yZwB3cm9uZ3Bhc3M=", + "535 5.7.8", + ) + .await; + session + .cmd( + "AUTH PLAIN AGphbmVAZm9vYmFyLm9yZwBzM2NyM3RwNHNz", + "235 2.7.0", + ) + .await; +} diff --git a/tests/src/smtp/lookup/utils.rs b/tests/src/smtp/lookup/utils.rs new file mode 100644 index 00000000..3920917d --- /dev/null +++ b/tests/src/smtp/lookup/utils.rs @@ -0,0 +1,102 @@ +use std::time::{Duration, Instant}; + +use mail_auth::{IpLookupStrategy, MX}; + +use smtp::{config::IfBlock, core::Core, outbound::RemoteHost}; + +use super::ToRemoteHost; + +#[tokio::test] +async fn lookup_ip() { + let ipv6 = vec![ + "a:b::1".parse().unwrap(), + "a:b::2".parse().unwrap(), + "a:b::3".parse().unwrap(), + "a:b::4".parse().unwrap(), + ]; + let ipv4 = vec![ + "10.0.0.1".parse().unwrap(), + "10.0.0.2".parse().unwrap(), + "10.0.0.3".parse().unwrap(), + "10.0.0.4".parse().unwrap(), + ]; + let mut core = Core::test(); + core.queue.config.source_ip.ipv4 = IfBlock::new(ipv4.clone()); + core.queue.config.source_ip.ipv6 = IfBlock::new(ipv6.clone()); + core.resolvers.dns.ipv4_add( + "mx.foobar.org", + vec![ + "172.168.0.100".parse().unwrap(), + "172.168.0.101".parse().unwrap(), + ], + Instant::now() + Duration::from_secs(10), + ); + core.resolvers.dns.ipv6_add( + "mx.foobar.org", + vec!["e:f::a".parse().unwrap(), "e:f::b".parse().unwrap()], + Instant::now() + Duration::from_secs(10), + ); + + // Ipv4 strategy + core.queue.config.ip_strategy = IfBlock::new(IpLookupStrategy::Ipv4thenIpv6); + let (source_ips, remote_ips) = core + .resolve_host(&RemoteHost::MX("mx.foobar.org"), &"envelope", 2) + .await + .unwrap(); + assert!(ipv4.contains(&match source_ips.unwrap() { + std::net::IpAddr::V4(v4) => v4, + _ => unreachable!(), + })); + assert!(remote_ips.contains(&"172.168.0.100".parse().unwrap())); + + // Ipv6 strategy + core.queue.config.ip_strategy = IfBlock::new(IpLookupStrategy::Ipv6thenIpv4); + let (source_ips, remote_ips) = core + .resolve_host(&RemoteHost::MX("mx.foobar.org"), &"envelope", 2) + .await + .unwrap(); + assert!(ipv6.contains(&match source_ips.unwrap() { + std::net::IpAddr::V6(v6) => v6, + _ => unreachable!(), + })); + assert!(remote_ips.contains(&"e:f::a".parse().unwrap())); +} + +#[test] +fn to_remote_hosts() { + let mx = vec![ + MX { + exchanges: vec!["mx1".to_string(), "mx2".to_string()], + preference: 10, + }, + MX { + exchanges: vec![ + "mx3".to_string(), + "mx4".to_string(), + "mx5".to_string(), + "mx6".to_string(), + ], + preference: 20, + }, + MX { + exchanges: vec!["mx7".to_string(), "mx8".to_string()], + preference: 10, + }, + MX { + exchanges: vec!["mx9".to_string(), "mxA".to_string()], + preference: 10, + }, + ]; + let hosts = mx.to_remote_hosts("domain", 7).unwrap(); + assert_eq!(hosts.len(), 7); + for host in hosts { + if let RemoteHost::MX(host) = host { + assert!((*host.as_bytes().last().unwrap() - b'0') <= 8); + } + } + let mx = vec![MX { + exchanges: vec![".".to_string()], + preference: 0, + }]; + assert!(mx.to_remote_hosts("domain", 10).is_none()); +} diff --git a/tests/src/smtp/management/mod.rs b/tests/src/smtp/management/mod.rs new file mode 100644 index 00000000..57076c72 --- /dev/null +++ b/tests/src/smtp/management/mod.rs @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::time::Duration; + +use hyper::header::AUTHORIZATION; +use serde::{de::DeserializeOwned, Deserialize}; + +pub mod queue; +pub mod report; + +#[derive(Deserialize)] +#[serde(untagged)] +pub enum Response { + Data { data: T }, + Error { error: String, details: String }, +} + +pub async fn send_manage_request(query: &str) -> Result, String> { + send_manage_request_raw(query).await.map(|result| { + serde_json::from_str::>(&result).unwrap_or_else(|err| panic!("{err}: {result}")) + }) +} + +pub async fn send_manage_request_raw(query: &str) -> Result { + reqwest::Client::builder() + .timeout(Duration::from_millis(500)) + .danger_accept_invalid_certs(true) + .build() + .unwrap() + .get(format!("https://127.0.0.1:9980{query}")) + .header(AUTHORIZATION, "Basic YWRtaW46c2VjcmV0") + .send() + .await + .map_err(|err| err.to_string())? + .bytes() + .await + .map(|bytes| String::from_utf8(bytes.to_vec()).unwrap()) + .map_err(|err| err.to_string()) +} + +impl Response { + pub fn unwrap_data(self) -> T { + match self { + Response::Data { data } => data, + Response::Error { error, details } => { + panic!("Expected data, found error {error:?}: {details:?}") + } + } + } + + pub fn unwrap_error(self) -> (String, String) { + match self { + Response::Error { error, details } => (error, details), + Response::Data { .. } => panic!("Expected error, found data."), + } + } +} diff --git a/tests/src/smtp/management/queue.rs b/tests/src/smtp/management/queue.rs new file mode 100644 index 00000000..67952e23 --- /dev/null +++ b/tests/src/smtp/management/queue.rs @@ -0,0 +1,462 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; + +use ahash::{AHashMap, AHashSet, HashMap, HashSet}; +use hyper::{header::AUTHORIZATION, StatusCode}; +use mail_auth::MX; +use mail_parser::DateTime; +use utils::config::ServerProtocol; + +use crate::smtp::{ + inbound::TestQueueEvent, management::send_manage_request, outbound::start_test_server, + session::TestSession, TestConfig, TestCore, +}; +use smtp::{ + config::IfBlock, + core::{management::Message, Core, Session}, + lookup::Lookup, + queue::{ + manager::{Queue, SpawnQueue}, + QueueId, Status, + }, +}; + +#[tokio::test] +#[serial_test::serial] +async fn manage_queue() { + /*tracing::subscriber::set_global_default( + tracing_subscriber::FmtSubscriber::builder() + .with_max_level(tracing::Level::DEBUG) + .finish(), + ) + .unwrap();*/ + + // Start remote test server + let mut core = Core::test(); + core.session.config.rcpt.relay = IfBlock::new(true); + let mut remote_qr = core.init_test_queue("smtp_manage_queue_remote"); + let _rx_remote = start_test_server(core.into(), &[ServerProtocol::Smtp]); + + // Add mock DNS entries + let mut core = Core::test(); + core.resolvers.dns.mx_add( + "foobar.org", + vec![MX { + exchanges: vec!["mx1.foobar.org".to_string()], + preference: 10, + }], + Instant::now() + Duration::from_secs(10), + ); + + core.resolvers.dns.ipv4_add( + "mx1.foobar.org", + vec!["127.0.0.1".parse().unwrap()], + Instant::now() + Duration::from_secs(10), + ); + + // Start local management interface + core.session.config.rcpt.relay = IfBlock::new(true); + core.session.config.rcpt.max_recipients = IfBlock::new(100); + core.session.config.extensions.future_release = IfBlock::new(Some(Duration::from_secs(86400))); + core.session.config.extensions.dsn = IfBlock::new(true); + core.queue.config.retry = IfBlock::new(vec![Duration::from_secs(1000)]); + core.queue.config.notify = IfBlock::new(vec![Duration::from_secs(2000)]); + core.queue.config.expire = IfBlock::new(Duration::from_secs(3000)); + core.queue.config.management_lookup = Arc::new(Lookup::Local(AHashSet::from_iter([ + "admin:secret".to_string(), + ]))); + let local_qr = core.init_test_queue("smtp_manage_queue_local"); + let core = Arc::new(core); + local_qr.queue_rx.spawn(core.clone(), Queue::default()); + let _rx_manage = start_test_server(core.clone(), &[ServerProtocol::Http]); + + // Send test messages + let envelopes = HashMap::from_iter([ + ( + "a", + ( + "bill1@foobar.net", + vec![ + "rcpt1@example1.org", + "rcpt1@example2.org", + "rcpt1@example2.org", + ], + ), + ), + ( + "b", + ( + "bill2@foobar.net", + vec!["rcpt3@example1.net", "rcpt4@example1.net"], + ), + ), + ( + "c", + ( + "bill3@foobar.net", + vec![ + "rcpt5@example1.com", + "rcpt6@example2.com", + "rcpt7@example2.com", + "rcpt8@example3.com", + "rcpt9@example4.com", + ], + ), + ), + ("d", ("bill4@foobar.net", vec!["delay@foobar.org"])), + ("e", ("bill5@foobar.net", vec!["john@foobar.org"])), + ("f", ("", vec!["success@foobar.org", "delay@foobar.org"])), + ]); + let mut session = Session::test(core.clone()); + session.data.remote_ip = "10.0.0.1".parse().unwrap(); + session.eval_session_params().await; + session.ehlo("foobar.net").await; + for test_num in 0..6 { + let env_id = char::from(b'a' + test_num).to_string(); + let hold_for = ((test_num + 1) as u32) * 100; + let (sender, recipients) = envelopes.get(env_id.as_str()).unwrap(); + session + .send_message( + &if env_id != "f" { + format!("<{sender}> ENVID={env_id} HOLDFOR={hold_for}") + } else { + format!("<{sender}> ENVID={env_id}") + }, + recipients, + "test:no_dkim", + "250", + ) + .await; + } + + // Expect delivery to success@foobar.org + tokio::time::sleep(Duration::from_millis(100)).await; + assert_eq!( + remote_qr + .read_event() + .await + .unwrap_message() + .recipients + .into_iter() + .map(|r| r.address) + .collect::>(), + vec!["success@foobar.org".to_string()] + ); + + // Fetch and validate messages + let ids = send_manage_request::>("/queue/list") + .await + .unwrap() + .unwrap_data(); + assert_eq!(ids.len(), 6); + let mut id_map = AHashMap::new(); + let mut id_map_rev = AHashMap::new(); + let mut test_search = String::new(); + for (message, id) in get_messages(&ids).await.into_iter().zip(ids) { + let message = message.unwrap(); + let env_id = message.env_id.as_ref().unwrap().clone(); + + // Validate return path and recipients + let (sender, recipients) = envelopes.get(env_id.as_str()).unwrap(); + assert_eq!(&message.return_path, sender); + 'outer: for recipient in recipients { + for domain in &message.domains { + for rcpt in &domain.recipients { + if &rcpt.address == recipient { + continue 'outer; + } + } + } + panic!("Recipient {recipient} not found in message."); + } + + // Validate status and datetimes + let created = message.created.to_timestamp(); + let hold_for = (env_id.as_bytes().first().unwrap() - b'a' + 1) as i64 * 100; + let next_retry = created + hold_for; + let next_notify = created + 2000 + hold_for; + let expires = created + 3000 + hold_for; + for domain in &message.domains { + if env_id == "c" { + test_search = domain.next_retry.as_ref().unwrap().to_rfc3339(); + } + if env_id != "f" { + assert_eq!(domain.retry_num, 0); + assert_timestamp( + domain.next_retry.as_ref().unwrap(), + next_retry, + "retry", + &message, + ); + assert_timestamp( + domain.next_notify.as_ref().unwrap(), + next_notify, + "notify", + &message, + ); + assert_timestamp(&domain.expires, expires, "expires", &message); + for rcpt in &domain.recipients { + assert_eq!(&rcpt.status, &Status::Scheduled, "{message:#?}"); + } + } else { + assert_eq!(domain.retry_num, 1); + for rcpt in &domain.recipients { + if rcpt.address == "success@foobar.org" { + assert!( + matches!(&rcpt.status, Status::Completed(_)), + "{:?}", + rcpt.status + ); + } else { + assert!( + matches!(&rcpt.status, Status::TemporaryFailure(_)), + "{:?}", + rcpt.status + ); + } + } + } + } + + id_map.insert(env_id.clone(), id); + id_map_rev.insert(id, env_id); + } + assert_eq!(id_map.len(), 6); + + // Test list search + for (query, expected_ids) in [ + ("/queue/list?from=bill1@foobar.net".to_string(), vec!["a"]), + ("/queue/list?to=foobar.org".to_string(), vec!["d", "e", "f"]), + ( + "/queue/list?from=bill3@foobar.net&to=rcpt5@example1.com".to_string(), + vec!["c"], + ), + (format!("/queue/list?before={test_search}"), vec!["a", "b"]), + ( + format!("/queue/list?after={test_search}"), + vec!["d", "e", "f", "c"], + ), + ] { + let expected_ids = HashSet::from_iter(expected_ids.into_iter().map(|s| s.to_string())); + let ids = send_manage_request::>(&query) + .await + .unwrap() + .unwrap_data() + .into_iter() + .map(|id| id_map_rev.get(&id).unwrap().clone()) + .collect::>(); + assert_eq!(ids, expected_ids, "failed for {query}"); + } + + // Retry delivery + assert_eq!( + send_manage_request::>(&format!( + "/queue/retry?id={},{}", + id_map.get("e").unwrap(), + id_map.get("f").unwrap() + )) + .await + .unwrap() + .unwrap_data(), + vec![true, true] + ); + assert_eq!( + send_manage_request::>(&format!( + "/queue/retry?id={}&filter=example1.org&at=2200-01-01T00:00:00Z", + id_map.get("a").unwrap(), + )) + .await + .unwrap() + .unwrap_data(), + vec![true] + ); + + // Expect delivery to john@foobar.org + tokio::time::sleep(Duration::from_millis(100)).await; + assert_eq!( + remote_qr + .read_event() + .await + .unwrap_message() + .recipients + .into_iter() + .map(|r| r.address) + .collect::>(), + vec!["john@foobar.org".to_string()] + ); + + // Message 'e' should be gone, 'f' should have retry_num == 2 + // while 'a' should have a retry time of 2200-01-01T00:00:00Z for example1.org + let mut messages = get_messages(&[ + *id_map.get("e").unwrap(), + *id_map.get("f").unwrap(), + *id_map.get("a").unwrap(), + ]) + .await + .into_iter(); + assert_eq!(messages.next().unwrap(), None); + assert_eq!( + messages + .next() + .unwrap() + .unwrap() + .domains + .first() + .unwrap() + .retry_num, + 2 + ); + for domain in messages.next().unwrap().unwrap().domains { + let next_retry = domain.next_retry.as_ref().unwrap().to_rfc3339(); + let matched = + ["2200-01-01T00:00:00Z", "2199-12-31T23:59:59Z"].contains(&next_retry.as_str()); + if domain.name == "example1.org" { + assert!(matched, "{next_retry}"); + } else { + assert!(!matched, "{next_retry}"); + } + } + + // Cancel deliveries + for (id, filter) in [ + ("a", "example2.org"), + ("b", "example1.net"), + ("c", "rcpt6@example2.com"), + ("d", ""), + ] { + assert_eq!( + send_manage_request::>(&format!( + "/queue/cancel?id={}{}{}", + id_map.get(id).unwrap(), + if !filter.is_empty() { "&filter=" } else { "" }, + filter + )) + .await + .unwrap() + .unwrap_data(), + vec![true], + "failed for {id}: {filter}" + ); + } + assert_eq!( + send_manage_request::>("/queue/list") + .await + .unwrap() + .unwrap_data() + .len(), + 3 + ); + for (message, id) in get_messages(&[ + *id_map.get("a").unwrap(), + *id_map.get("b").unwrap(), + *id_map.get("c").unwrap(), + *id_map.get("d").unwrap(), + ]) + .await + .into_iter() + .zip(["a", "b", "c", "d"]) + { + if ["b", "d"].contains(&id) { + assert_eq!(message, None); + } else { + let message = message.unwrap(); + assert!(!message.domains.is_empty()); + for domain in message.domains { + match id { + "a" => { + if domain.name == "example2.org" { + assert_eq!(&domain.status, &Status::Completed("".to_string())); + for rcpt in &domain.recipients { + assert!(matches!(&rcpt.status, Status::Completed(_))); + } + } else { + assert_eq!(&domain.status, &Status::Scheduled); + for rcpt in &domain.recipients { + assert!(matches!(&rcpt.status, Status::Scheduled)); + } + } + } + "c" => { + assert_eq!(&domain.status, &Status::Scheduled); + if domain.name == "example2.com" { + for rcpt in &domain.recipients { + if rcpt.address == "rcpt6@example2.com" { + assert!(matches!(&rcpt.status, Status::Completed(_))); + } else { + assert!(matches!(&rcpt.status, Status::Scheduled)); + } + } + } else { + for rcpt in &domain.recipients { + assert!(matches!(&rcpt.status, Status::Scheduled)); + } + } + } + _ => unreachable!(), + } + } + } + } + + // Test authentication error + assert_eq!( + reqwest::Client::builder() + .timeout(Duration::from_millis(500)) + .danger_accept_invalid_certs(true) + .build() + .unwrap() + .get("https://127.0.0.1:9980/list") + .header(AUTHORIZATION, "Basic YWRtaW46aGVsbG93b3JsZA==") + .send() + .await + .unwrap() + .status(), + StatusCode::UNAUTHORIZED + ); +} + +fn assert_timestamp(timestamp: &DateTime, expected: i64, ctx: &str, message: &Message) { + let timestamp = timestamp.to_timestamp(); + let diff = timestamp - expected; + if ![-2, -1, 0, 1, 2].contains(&diff) { + panic!("Got timestamp {timestamp}, expected {expected} (diff {diff} for {ctx}) for {message:?}"); + } +} + +async fn get_messages(ids: &[QueueId]) -> Vec> { + send_manage_request(&format!( + "/queue/status?id={}", + ids.iter() + .map(|id| id.to_string()) + .collect::>() + .join(",") + )) + .await + .unwrap() + .unwrap_data() +} diff --git a/tests/src/smtp/management/report.rs b/tests/src/smtp/management/report.rs new file mode 100644 index 00000000..18daf5d3 --- /dev/null +++ b/tests/src/smtp/management/report.rs @@ -0,0 +1,220 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::sync::Arc; + +use ahash::{AHashMap, AHashSet, HashSet}; +use mail_auth::{ + common::parse::TxtRecordParser, + dmarc::Dmarc, + mta_sts::TlsRpt, + report::{ + tlsrpt::{FailureDetails, ResultType}, + ActionDisposition, DmarcResult, Record, + }, +}; +use tokio::sync::mpsc; +use utils::config::ServerProtocol; + +use crate::smtp::{ + make_temp_dir, management::send_manage_request, outbound::start_test_server, TestConfig, +}; +use smtp::{ + config::{AggregateFrequency, IfBlock}, + core::{management::Report, Core}, + lookup::Lookup, + reporting::{ + scheduler::{Scheduler, SpawnReport}, + DmarcEvent, TlsEvent, + }, +}; + +#[tokio::test] +#[serial_test::serial] +async fn manage_reports() { + /*tracing::subscriber::set_global_default( + tracing_subscriber::FmtSubscriber::builder() + .with_max_level(tracing::Level::DEBUG) + .finish(), + ) + .unwrap();*/ + + // Start reporting service + let mut core = Core::test(); + let temp_dir = make_temp_dir("smtp_report_management_test", true); + let config = &mut core.report.config; + config.path = IfBlock::new(temp_dir.temp_dir.clone()); + config.hash = IfBlock::new(16); + config.dmarc_aggregate.max_size = IfBlock::new(1024); + config.tls.max_size = IfBlock::new(1024); + core.queue.config.management_lookup = Arc::new(Lookup::Local(AHashSet::from_iter([ + "admin:secret".to_string(), + ]))); + let (report_tx, report_rx) = mpsc::channel(1024); + core.report.tx = report_tx; + let core = Arc::new(core); + report_rx.spawn(core.clone(), Scheduler::default()); + let _rx_manage = start_test_server(core.clone(), &[ServerProtocol::Http]); + + // Send test reporting events + core.schedule_report(DmarcEvent { + domain: "foobar.org".to_string(), + report_record: Record::new() + .with_source_ip("192.168.1.2".parse().unwrap()) + .with_action_disposition(ActionDisposition::Pass) + .with_dmarc_dkim_result(DmarcResult::Pass) + .with_dmarc_spf_result(DmarcResult::Fail) + .with_envelope_from("hello@example.org") + .with_envelope_to("other@example.org") + .with_header_from("bye@example.org"), + dmarc_record: Arc::new( + Dmarc::parse(b"v=DMARC1; p=reject; rua=mailto:reports@foobar.org").unwrap(), + ), + interval: AggregateFrequency::Daily, + }) + .await; + core.schedule_report(DmarcEvent { + domain: "foobar.net".to_string(), + report_record: Record::new() + .with_source_ip("a:b:c::e:f".parse().unwrap()) + .with_action_disposition(ActionDisposition::Reject) + .with_dmarc_dkim_result(DmarcResult::Fail) + .with_dmarc_spf_result(DmarcResult::Pass), + dmarc_record: Arc::new( + Dmarc::parse( + b"v=DMARC1; p=quarantine; rua=mailto:reports@foobar.net,mailto:reports@example.net", + ) + .unwrap(), + ), + interval: AggregateFrequency::Weekly, + }) + .await; + core.schedule_report(TlsEvent { + domain: "foobar.org".to_string(), + policy: smtp::reporting::PolicyType::None, + failure: None, + tls_record: Arc::new(TlsRpt::parse(b"v=TLSRPTv1;rua=mailto:reports@foobar.org").unwrap()), + interval: AggregateFrequency::Daily, + }) + .await; + core.schedule_report(TlsEvent { + domain: "foobar.net".to_string(), + policy: smtp::reporting::PolicyType::Sts(None), + failure: FailureDetails::new(ResultType::StsPolicyInvalid).into(), + tls_record: Arc::new(TlsRpt::parse(b"v=TLSRPTv1;rua=mailto:reports@foobar.net").unwrap()), + interval: AggregateFrequency::Weekly, + }) + .await; + + // List reports + let ids = send_manage_request::>("/report/list") + .await + .unwrap() + .unwrap_data(); + assert_eq!(ids.len(), 4); + let mut id_map = AHashMap::new(); + let mut id_map_rev = AHashMap::new(); + for (report, id) in get_reports(&ids).await.into_iter().zip(ids) { + let mut parts = id.split('!'); + let report = report.unwrap(); + let mut id_num = if parts.next().unwrap() == "t" { + assert_eq!(report.type_, "tls"); + 2 + } else { + assert_eq!(report.type_, "dmarc"); + 0 + }; + assert_eq!(parts.next().unwrap(), report.domain); + let diff = report.range_to.to_timestamp() - report.range_from.to_timestamp(); + if report.domain == "foobar.org" { + assert_eq!(diff, 86400); + } else { + assert_eq!(diff, 7 * 86400); + id_num += 1; + } + id_map.insert(char::from(b'a' + id_num).to_string(), id.clone()); + id_map_rev.insert(id, char::from(b'a' + id_num).to_string()); + } + + // Test list search + for (query, expected_ids) in [ + ("/report/list?type=dmarc", vec!["a", "b"]), + ("/report/list?type=tls", vec!["c", "d"]), + ("/report/list?domain=foobar.org", vec!["a", "c"]), + ("/report/list?domain=foobar.net", vec!["b", "d"]), + ("/report/list?domain=foobar.org&type=dmarc", vec!["a"]), + ("/report/list?domain=foobar.net&type=tls", vec!["d"]), + ] { + let expected_ids = HashSet::from_iter(expected_ids.into_iter().map(|s| s.to_string())); + let ids = send_manage_request::>(query) + .await + .unwrap() + .unwrap_data() + .into_iter() + .map(|id| id_map_rev.get(&id).unwrap().clone()) + .collect::>(); + assert_eq!(ids, expected_ids, "failed for {query}"); + } + + // Cancel reports + for id in ["a", "b"] { + assert_eq!( + send_manage_request::>(&format!( + "/report/cancel?id={}", + id_map.get(id).unwrap(), + )) + .await + .unwrap() + .unwrap_data(), + vec![true], + "failed for {id}" + ); + } + assert_eq!( + send_manage_request::>("/report/list") + .await + .unwrap() + .unwrap_data() + .len(), + 2 + ); + let mut ids = get_reports(&[ + id_map.get("a").unwrap().clone(), + id_map.get("b").unwrap().clone(), + id_map.get("c").unwrap().clone(), + id_map.get("d").unwrap().clone(), + ]) + .await + .into_iter(); + assert!(ids.next().unwrap().is_none()); + assert!(ids.next().unwrap().is_none()); + assert!(ids.next().unwrap().is_some()); + assert!(ids.next().unwrap().is_some()); +} + +async fn get_reports(ids: &[String]) -> Vec> { + send_manage_request(&format!("/report/status?id={}", ids.join(","))) + .await + .unwrap() + .unwrap_data() +} diff --git a/tests/src/smtp/mod.rs b/tests/src/smtp/mod.rs new file mode 100644 index 00000000..71808b86 --- /dev/null +++ b/tests/src/smtp/mod.rs @@ -0,0 +1,497 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{path::PathBuf, sync::Arc, time::Duration}; + +use ahash::{AHashMap, AHashSet}; +use dashmap::DashMap; +use mail_auth::{ + common::lru::{DnsCache, LruCache}, + trust_dns_resolver::config::{ResolverConfig, ResolverOpts}, + IpLookupStrategy, Resolver, +}; +use mail_send::smtp::tls::build_tls_connector; +use sieve::Runtime; +use smtp_proto::{AUTH_LOGIN, AUTH_PLAIN}; +use tokio::sync::mpsc; + +use smtp::{ + config::{ + if_block::ConfigIf, queue::ConfigQueue, throttle::ConfigThrottle, AggregateReport, + ArcAuthConfig, Auth, ConfigContext, Connect, Data, DkimAuthConfig, DmarcAuthConfig, + DnsBlConfig, Dsn, Ehlo, EnvelopeKey, Extensions, IfBlock, IpRevAuthConfig, Mail, + MailAuthConfig, QueueConfig, QueueOutboundSourceIp, QueueOutboundTimeout, QueueOutboundTls, + QueueQuotas, QueueThrottle, Rcpt, Report, ReportAnalysis, ReportConfig, SessionConfig, + SessionThrottle, SpfAuthConfig, Throttle, VerifyStrategy, + }, + core::{ + throttle::ThrottleKeyHasherBuilder, Core, QueueCore, ReportCore, Resolvers, SessionCore, + SieveConfig, SieveCore, TlsConnectors, + }, + lookup::Lookup, + outbound::dane::DnssecResolver, +}; +use utils::config::{utils::ParseValues, Config}; + +pub mod config; +pub mod inbound; +pub mod lookup; +pub mod management; +pub mod outbound; +pub mod queue; +pub mod reporting; +pub mod session; + +pub trait ParseTestConfig { + fn parse_if(&self, ctx: &ConfigContext) -> IfBlock; + fn parse_throttle(&self, ctx: &ConfigContext) -> Vec; + fn parse_quota(&self, ctx: &ConfigContext) -> QueueQuotas; + fn parse_queue_throttle(&self, ctx: &ConfigContext) -> QueueThrottle; +} + +impl ParseTestConfig for &str { + fn parse_if(&self, ctx: &ConfigContext) -> IfBlock { + Config::parse(&format!("test = {self}\n")) + .unwrap() + .parse_if_block( + "test", + ctx, + &[ + EnvelopeKey::Recipient, + EnvelopeKey::RecipientDomain, + EnvelopeKey::Sender, + EnvelopeKey::SenderDomain, + EnvelopeKey::Mx, + EnvelopeKey::HeloDomain, + EnvelopeKey::AuthenticatedAs, + EnvelopeKey::Listener, + EnvelopeKey::RemoteIp, + EnvelopeKey::LocalIp, + EnvelopeKey::Priority, + ], + ) + .unwrap() + .unwrap() + } + + fn parse_throttle(&self, ctx: &ConfigContext) -> Vec { + Config::parse(self) + .unwrap() + .parse_throttle( + "throttle", + ctx, + &[ + EnvelopeKey::Recipient, + EnvelopeKey::RecipientDomain, + EnvelopeKey::Sender, + EnvelopeKey::SenderDomain, + EnvelopeKey::Mx, + EnvelopeKey::HeloDomain, + EnvelopeKey::AuthenticatedAs, + EnvelopeKey::Listener, + EnvelopeKey::RemoteIp, + EnvelopeKey::LocalIp, + EnvelopeKey::Priority, + ], + u16::MAX, + ) + .unwrap() + } + + fn parse_quota(&self, ctx: &ConfigContext) -> QueueQuotas { + Config::parse(self).unwrap().parse_queue_quota(ctx).unwrap() + } + + fn parse_queue_throttle(&self, ctx: &ConfigContext) -> QueueThrottle { + Config::parse(self) + .unwrap() + .parse_queue_throttle(ctx) + .unwrap() + } +} + +pub trait TestConfig { + fn test() -> Self; +} + +impl TestConfig for Core { + fn test() -> Self { + Core { + worker_pool: rayon::ThreadPoolBuilder::new() + .num_threads(num_cpus::get()) + .build() + .unwrap(), + session: SessionCore::test(), + queue: QueueCore::test(), + resolvers: Resolvers { + dns: Resolver::new_system_conf().unwrap(), + dnssec: DnssecResolver::with_capacity( + ResolverConfig::cloudflare(), + ResolverOpts::default(), + ) + .unwrap(), + cache: smtp::core::DnsCache { + tlsa: LruCache::with_capacity(100), + mta_sts: LruCache::with_capacity(100), + }, + }, + mail_auth: MailAuthConfig::test(), + report: ReportCore::test(), + sieve: SieveCore::test(), + } + } +} + +impl TestConfig for SessionCore { + fn test() -> Self { + SessionCore { + config: SessionConfig::test(), + throttle: DashMap::with_capacity_and_hasher_and_shard_amount( + 10, + ThrottleKeyHasherBuilder::default(), + 16, + ), + } + } +} + +impl TestConfig for SessionConfig { + fn test() -> Self { + Self { + timeout: IfBlock::new(Duration::from_secs(10)), + duration: IfBlock::new(Duration::from_secs(10)), + transfer_limit: IfBlock::new(1024 * 1024), + throttle: SessionThrottle { + connect: vec![], + mail_from: vec![], + rcpt_to: vec![], + }, + connect: Connect { + script: IfBlock::new(None), + }, + ehlo: Ehlo { + script: IfBlock::new(None), + require: IfBlock::new(true), + reject_non_fqdn: IfBlock::new(false), + }, + extensions: Extensions { + pipelining: IfBlock::new(true), + chunking: IfBlock::new(true), + requiretls: IfBlock::new(true), + no_soliciting: IfBlock::new("domain.org".to_string().into()), + future_release: IfBlock::new(None), + deliver_by: IfBlock::new(None), + mt_priority: IfBlock::new(None), + dsn: IfBlock::new(true), + }, + auth: Auth { + lookup: IfBlock::new(None), + mechanisms: IfBlock::new(AUTH_PLAIN | AUTH_LOGIN), + require: IfBlock::new(false), + errors_max: IfBlock::new(10), + errors_wait: IfBlock::new(Duration::from_secs(1)), + }, + mail: Mail { + script: IfBlock::new(None), + }, + rcpt: Rcpt { + script: IfBlock::new(None), + relay: IfBlock::new(false), + lookup_domains: IfBlock::new(None), + lookup_addresses: IfBlock::new(None), + lookup_expn: IfBlock::new(None), + lookup_vrfy: IfBlock::new(None), + errors_max: IfBlock::new(3), + errors_wait: IfBlock::new(Duration::from_secs(1)), + max_recipients: IfBlock::new(3), + }, + data: Data { + script: IfBlock::new(None), + max_messages: IfBlock::new(10), + max_message_size: IfBlock::new(1024 * 1024), + max_received_headers: IfBlock::new(10), + add_received: IfBlock::new(true), + add_received_spf: IfBlock::new(true), + add_return_path: IfBlock::new(true), + add_auth_results: IfBlock::new(true), + add_message_id: IfBlock::new(true), + add_date: IfBlock::new(true), + pipe_commands: vec![], + }, + } + } +} + +impl TestConfig for QueueCore { + fn test() -> Self { + Self { + config: QueueConfig::test(), + throttle: DashMap::with_capacity_and_hasher_and_shard_amount( + 10, + ThrottleKeyHasherBuilder::default(), + 16, + ), + quota: DashMap::with_capacity_and_hasher_and_shard_amount( + 10, + ThrottleKeyHasherBuilder::default(), + 16, + ), + tx: mpsc::channel(1024).0, + id_seq: 0.into(), + connectors: TlsConnectors { + pki_verify: build_tls_connector(false), + dummy_verify: build_tls_connector(true), + }, + } + } +} + +impl TestConfig for QueueConfig { + fn test() -> Self { + Self { + path: Default::default(), + hash: IfBlock::new(10), + retry: IfBlock::new(vec![Duration::from_secs(10)]), + notify: IfBlock::new(vec![Duration::from_secs(20)]), + expire: IfBlock::new(Duration::from_secs(10)), + hostname: IfBlock::new("mx.example.org".to_string()), + next_hop: Default::default(), + max_mx: IfBlock::new(5), + max_multihomed: IfBlock::new(5), + source_ip: QueueOutboundSourceIp { + ipv4: IfBlock::new(vec![]), + ipv6: IfBlock::new(vec![]), + }, + ip_strategy: IfBlock::new(IpLookupStrategy::Ipv4thenIpv6), + tls: QueueOutboundTls { + dane: IfBlock::new(smtp::config::RequireOptional::Optional), + mta_sts: IfBlock::new(smtp::config::RequireOptional::Optional), + start: IfBlock::new(smtp::config::RequireOptional::Optional), + }, + dsn: Dsn { + name: IfBlock::new("Mail Delivery Subsystem".to_string()), + address: IfBlock::new("MAILER-DAEMON@example.org".to_string()), + sign: IfBlock::default(), + }, + timeout: QueueOutboundTimeout { + connect: IfBlock::new(Duration::from_secs(1)), + greeting: IfBlock::new(Duration::from_secs(1)), + tls: IfBlock::new(Duration::from_secs(1)), + ehlo: IfBlock::new(Duration::from_secs(1)), + mail: IfBlock::new(Duration::from_secs(1)), + rcpt: IfBlock::new(Duration::from_secs(1)), + data: IfBlock::new(Duration::from_secs(1)), + mta_sts: IfBlock::new(Duration::from_secs(1)), + }, + throttle: QueueThrottle { + sender: vec![], + rcpt: vec![], + host: vec![], + }, + quota: QueueQuotas { + sender: vec![], + rcpt: vec![], + rcpt_domain: vec![], + }, + management_lookup: Arc::new(Lookup::Local(AHashSet::default())), + } + } +} + +impl TestConfig for MailAuthConfig { + fn test() -> Self { + Self { + dkim: DkimAuthConfig { + verify: IfBlock::new(VerifyStrategy::Relaxed), + sign: IfBlock::default(), + }, + arc: ArcAuthConfig { + verify: IfBlock::new(VerifyStrategy::Relaxed), + seal: IfBlock::default(), + }, + spf: SpfAuthConfig { + verify_ehlo: IfBlock::new(VerifyStrategy::Relaxed), + verify_mail_from: IfBlock::new(VerifyStrategy::Relaxed), + }, + dmarc: DmarcAuthConfig { + verify: IfBlock::new(VerifyStrategy::Relaxed), + }, + iprev: IpRevAuthConfig { + verify: IfBlock::new(VerifyStrategy::Relaxed), + }, + dnsbl: DnsBlConfig { + verify: IfBlock::new(0), + ip_lookup: vec![], + domain_lookup: vec![], + }, + } + } +} + +impl TestConfig for ReportCore { + fn test() -> Self { + Self { + config: ReportConfig::test(), + tx: mpsc::channel(1024).0, + } + } +} + +impl TestConfig for ReportConfig { + fn test() -> Self { + Self { + path: Default::default(), + hash: IfBlock::new(10), + submitter: IfBlock::new("example.org".to_string()), + analysis: ReportAnalysis { + addresses: vec![], + forward: true, + store: None, + report_id: 0.into(), + }, + dkim: Report::test(), + spf: Report::test(), + dmarc: Report::test(), + dmarc_aggregate: AggregateReport::test(), + tls: AggregateReport::test(), + } + } +} + +impl TestConfig for Report { + fn test() -> Self { + Self { + name: IfBlock::default(), + address: IfBlock::default(), + subject: IfBlock::default(), + sign: IfBlock::default(), + send: IfBlock::default(), + } + } +} + +impl TestConfig for AggregateReport { + fn test() -> Self { + Self { + name: IfBlock::default(), + address: IfBlock::default(), + org_name: IfBlock::default(), + contact_info: IfBlock::default(), + send: IfBlock::default(), + sign: IfBlock::default(), + max_size: IfBlock::default(), + } + } +} + +impl TestConfig for SieveCore { + fn test() -> Self { + SieveCore { + runtime: Runtime::new(), + scripts: AHashMap::new(), + lookup: AHashMap::new(), + config: SieveConfig { + from_addr: "MAILER-DAEMON@example.org".to_string(), + from_name: "Mailer Daemon".to_string(), + return_path: "".to_string(), + sign: vec![], + db: None, + }, + } + } +} + +pub struct TempDir { + pub temp_dir: PathBuf, + pub delete: bool, +} + +pub fn make_temp_dir(name: &str, delete: bool) -> TempDir { + let mut temp_dir = std::env::temp_dir(); + temp_dir.push(name); + if !temp_dir.exists() { + let _ = std::fs::create_dir(&temp_dir); + } else if delete { + let _ = std::fs::remove_dir_all(&temp_dir); + let _ = std::fs::create_dir(&temp_dir); + } + TempDir { temp_dir, delete } +} + +impl Drop for TempDir { + fn drop(&mut self) { + if self.delete { + let _ = std::fs::remove_dir_all(&self.temp_dir); + } + } +} + +pub fn add_test_certs(config: &str) -> String { + let mut cert_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + cert_path.push("resources"); + cert_path.push("smtp"); + cert_path.push("certs"); + let mut cert = cert_path.clone(); + cert.push("tls_cert.pem"); + let mut pk = cert_path.clone(); + pk.push("tls_privatekey.pem"); + + config + .replace("{CERT}", cert.as_path().to_str().unwrap()) + .replace("{PK}", pk.as_path().to_str().unwrap()) +} + +pub struct QueueReceiver { + _temp_dir: TempDir, + pub queue_rx: mpsc::Receiver, +} + +pub struct ReportReceiver { + pub report_rx: mpsc::Receiver, +} + +pub trait TestCore { + fn init_test_queue(&mut self, test_name: &str) -> QueueReceiver; + fn init_test_report(&mut self) -> ReportReceiver; +} + +impl TestCore for Core { + fn init_test_queue(&mut self, test_name: &str) -> QueueReceiver { + let _temp_dir = make_temp_dir(test_name, true); + self.queue.config.path = IfBlock::new(_temp_dir.temp_dir.clone()); + + let (queue_tx, queue_rx) = mpsc::channel(128); + self.queue.tx = queue_tx; + + QueueReceiver { + _temp_dir, + queue_rx, + } + } + + fn init_test_report(&mut self) -> ReportReceiver { + let (report_tx, report_rx) = mpsc::channel(128); + self.report.tx = report_tx; + ReportReceiver { report_rx } + } +} diff --git a/tests/src/smtp/outbound/dane.rs b/tests/src/smtp/outbound/dane.rs new file mode 100644 index 00000000..5f9f1a43 --- /dev/null +++ b/tests/src/smtp/outbound/dane.rs @@ -0,0 +1,210 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; + +use mail_auth::{ + common::parse::TxtRecordParser, + mta_sts::{ReportUri, TlsRpt}, + report::tlsrpt::ResultType, + MX, +}; +use utils::config::ServerProtocol; + +use crate::smtp::{ + inbound::{TestMessage, TestQueueEvent, TestReportingEvent}, + outbound::start_test_server, + session::{TestSession, VerifyResponse}, + TestConfig, TestCore, +}; +use smtp::{ + config::{AggregateFrequency, IfBlock, RequireOptional}, + core::{Core, Session}, + outbound::dane::{Tlsa, TlsaEntry}, + queue::{manager::Queue, DeliveryAttempt}, + reporting::PolicyType, +}; + +#[tokio::test] +#[serial_test::serial] +async fn dane_verify() { + /*tracing::subscriber::set_global_default( + tracing_subscriber::FmtSubscriber::builder() + .with_max_level(tracing::Level::TRACE) + .finish(), + ) + .unwrap();*/ + + // Start test server + let mut core = Core::test(); + core.session.config.rcpt.relay = IfBlock::new(true); + let mut remote_qr = core.init_test_queue("smtp_dane_remote"); + let _rx = start_test_server(core.into(), &[ServerProtocol::Smtp]); + + // Add mock DNS entries + let mut core = Core::test(); + core.resolvers.dns.mx_add( + "foobar.org", + vec![MX { + exchanges: vec!["mx.foobar.org".to_string()], + preference: 10, + }], + Instant::now() + Duration::from_secs(10), + ); + core.resolvers.dns.ipv4_add( + "mx.foobar.org", + vec!["127.0.0.1".parse().unwrap()], + Instant::now() + Duration::from_secs(10), + ); + core.resolvers.dns.txt_add( + "_smtp._tls.foobar.org", + TlsRpt::parse(b"v=TLSRPTv1; rua=mailto:reports@foobar.org").unwrap(), + Instant::now() + Duration::from_secs(10), + ); + + // Fail on missing TLSA record + let mut local_qr = core.init_test_queue("smtp_dane_local"); + let mut rr = core.init_test_report(); + core.session.config.rcpt.relay = IfBlock::new(true); + core.queue.config.tls.dane = IfBlock::new(RequireOptional::Require); + core.report.config.tls.send = IfBlock::new(AggregateFrequency::Weekly); + + let core = Arc::new(core); + let mut queue = Queue::default(); + let mut session = Session::test(core.clone()); + session.data.remote_ip = "10.0.0.1".parse().unwrap(); + session.eval_session_params().await; + session.ehlo("mx.test.org").await; + session + .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") + .await; + DeliveryAttempt::from(local_qr.read_event().await.unwrap_message()) + .try_deliver(core.clone(), &mut queue) + .await; + local_qr + .read_event() + .await + .unwrap_message() + .read_lines() + .assert_contains(" (DANE failed to authenticate") + .assert_contains("No TLSA records found"); + local_qr.read_event().await.unwrap_done(); + + // Expect TLS failure report + let report = rr.read_report().await.unwrap_tls(); + assert_eq!(report.domain, "foobar.org"); + assert_eq!(report.policy, PolicyType::Tlsa(None)); + assert_eq!( + report.failure.as_ref().unwrap().result_type, + ResultType::DaneRequired + ); + assert_eq!( + report.failure.as_ref().unwrap().receiving_mx_hostname, + Some("mx.foobar.org".to_string()) + ); + assert_eq!( + report.tls_record.rua, + vec![ReportUri::Mail("reports@foobar.org".to_string())] + ); + + // DANE failure with no matching certificates + let tlsa = Arc::new(Tlsa { + entries: vec![TlsaEntry { + is_end_entity: true, + is_sha256: true, + is_spki: true, + data: vec![1, 2, 3], + }], + has_end_entities: true, + has_intermediates: false, + }); + core.resolvers.tlsa_add( + "_25._tcp.mx.foobar.org", + tlsa.clone(), + Instant::now() + Duration::from_secs(10), + ); + session + .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") + .await; + DeliveryAttempt::from(local_qr.read_event().await.unwrap_message()) + .try_deliver(core.clone(), &mut queue) + .await; + local_qr + .read_event() + .await + .unwrap_message() + .read_lines() + .assert_contains(" (DANE failed to authenticate") + .assert_contains("No matching certificates found"); + local_qr.read_event().await.unwrap_done(); + + // Expect TLS failure report + let report = rr.read_report().await.unwrap_tls(); + assert_eq!(report.policy, PolicyType::Tlsa(tlsa.into())); + assert_eq!( + report.failure.as_ref().unwrap().result_type, + ResultType::ValidationFailure + ); + remote_qr.assert_empty_queue(); + + // DANE successful delivery + let tlsa = Arc::new(Tlsa { + entries: vec![TlsaEntry { + is_end_entity: true, + is_sha256: true, + is_spki: true, + data: vec![ + 73, 186, 44, 106, 13, 198, 100, 180, 0, 44, 158, 188, 15, 195, 39, 198, 61, 254, + 215, 237, 100, 26, 15, 155, 219, 235, 120, 64, 128, 172, 17, 0, + ], + }], + has_end_entities: true, + has_intermediates: false, + }); + core.resolvers.tlsa_add( + "_25._tcp.mx.foobar.org", + tlsa.clone(), + Instant::now() + Duration::from_secs(10), + ); + session + .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") + .await; + DeliveryAttempt::from(local_qr.read_event().await.unwrap_message()) + .try_deliver(core.clone(), &mut queue) + .await; + local_qr.read_event().await.unwrap_done(); + remote_qr + .read_event() + .await + .unwrap_message() + .read_lines() + .assert_contains("using TLSv1.3 with cipher"); + + // Expect TLS success report + let report = rr.read_report().await.unwrap_tls(); + assert_eq!(report.policy, PolicyType::Tlsa(tlsa.into())); + assert!(report.failure.is_none()); +} diff --git a/tests/src/smtp/outbound/extensions.rs b/tests/src/smtp/outbound/extensions.rs new file mode 100644 index 00000000..b0995a36 --- /dev/null +++ b/tests/src/smtp/outbound/extensions.rs @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; + +use mail_auth::MX; +use smtp_proto::{MAIL_REQUIRETLS, MAIL_RET_HDRS, MAIL_SMTPUTF8, RCPT_NOTIFY_NEVER}; +use utils::config::ServerProtocol; + +use crate::smtp::{ + inbound::{TestMessage, TestQueueEvent}, + outbound::start_test_server, + session::{TestSession, VerifyResponse}, + TestConfig, TestCore, +}; +use smtp::{ + config::IfBlock, + core::{Core, Session}, + queue::{manager::Queue, DeliveryAttempt}, +}; + +#[tokio::test] +#[serial_test::serial] +async fn extensions() { + /*tracing::subscriber::set_global_default( + tracing_subscriber::FmtSubscriber::builder() + .with_max_level(tracing::Level::TRACE) + .finish(), + ) + .unwrap();*/ + + // Start test server + let mut core = Core::test(); + core.session.config.rcpt.relay = IfBlock::new(true); + core.session.config.data.max_message_size = IfBlock::new(1500); + core.session.config.extensions.dsn = IfBlock::new(true); + core.session.config.extensions.requiretls = IfBlock::new(true); + let mut remote_qr = core.init_test_queue("smtp_ext_remote"); + let _rx = start_test_server(core.into(), &[ServerProtocol::Smtp]); + + // Add mock DNS entries + let mut core = Core::test(); + core.resolvers.dns.mx_add( + "foobar.org", + vec![MX { + exchanges: vec!["mx.foobar.org".to_string()], + preference: 10, + }], + Instant::now() + Duration::from_secs(10), + ); + core.resolvers.dns.ipv4_add( + "mx.foobar.org", + vec!["127.0.0.1".parse().unwrap()], + Instant::now() + Duration::from_secs(10), + ); + + // Successful delivery with DSN + let mut local_qr = core.init_test_queue("smtp_ext_local"); + core.session.config.rcpt.relay = IfBlock::new(true); + core.session.config.extensions.dsn = IfBlock::new(true); + let core = Arc::new(core); + let mut queue = Queue::default(); + let mut session = Session::test(core.clone()); + session.data.remote_ip = "10.0.0.1".parse().unwrap(); + session.eval_session_params().await; + session.ehlo("mx.test.org").await; + session + .send_message( + "john@test.org", + &[" NOTIFY=SUCCESS,FAILURE"], + "test:no_dkim", + "250", + ) + .await; + DeliveryAttempt::from(local_qr.read_event().await.unwrap_message()) + .try_deliver(core.clone(), &mut queue) + .await; + + local_qr + .read_event() + .await + .unwrap_message() + .read_lines() + .assert_contains(" (delivered to") + .assert_contains("Final-Recipient: rfc822;bill@foobar.org") + .assert_contains("Action: delivered"); + local_qr.read_event().await.unwrap_done(); + remote_qr + .read_event() + .await + .unwrap_message() + .read_lines() + .assert_contains("using TLSv1.3 with cipher"); + + // Test SIZE extension + session + .send_message("john@test.org", &["bill@foobar.org"], "test:arc", "250") + .await; + DeliveryAttempt::from(local_qr.read_event().await.unwrap_message()) + .try_deliver(core.clone(), &mut queue) + .await; + local_qr + .read_event() + .await + .unwrap_message() + .read_lines() + .assert_contains(" (host 'mx.foobar.org' rejected command 'MAIL FROM:") + .assert_contains("Action: failed") + .assert_contains("Diagnostic-Code: smtp;552") + .assert_contains("Status: 5.3.4"); + local_qr.read_event().await.unwrap_done(); + remote_qr.assert_empty_queue(); + + // Test DSN, SMTPUTF8 and REQUIRETLS extensions + session + .send_message( + " ENVID=abc123 RET=HDRS REQUIRETLS SMTPUTF8", + &[" NOTIFY=NEVER"], + "test:no_dkim", + "250", + ) + .await; + DeliveryAttempt::from(local_qr.read_event().await.unwrap_message()) + .try_deliver(core.clone(), &mut queue) + .await; + local_qr.read_event().await.unwrap_done(); + let message = remote_qr.read_event().await.unwrap_message(); + assert_eq!(message.env_id, Some("abc123".to_string())); + assert!((message.flags & MAIL_RET_HDRS) != 0); + assert!((message.flags & MAIL_REQUIRETLS) != 0); + assert!((message.flags & MAIL_SMTPUTF8) != 0); + assert!((message.recipients.last().unwrap().flags & RCPT_NOTIFY_NEVER) != 0); +} diff --git a/tests/src/smtp/outbound/lmtp.rs b/tests/src/smtp/outbound/lmtp.rs new file mode 100644 index 00000000..865c7d87 --- /dev/null +++ b/tests/src/smtp/outbound/lmtp.rs @@ -0,0 +1,201 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; + +use crate::smtp::{ + inbound::{TestMessage, TestQueueEvent}, + outbound::start_test_server, + session::{TestSession, VerifyResponse}, + ParseTestConfig, TestConfig, TestCore, +}; +use smtp::{ + config::{remote::ConfigHost, ConfigContext, IfBlock}, + core::{Core, Session}, + queue::{manager::Queue, DeliveryAttempt, Event, WorkerResult}, +}; +use utils::config::{Config, ServerProtocol}; + +const REMOTE: &str = " +[remote.lmtp] +address = lmtp.foobar.org +port = 9924 +protocol = 'lmtp' +concurrency = 5 + +[remote.lmtp.tls] +implicit = true +allow-invalid-certs = true +"; + +#[tokio::test] +#[serial_test::serial] +async fn lmtp_delivery() { + /*tracing::subscriber::set_global_default( + tracing_subscriber::FmtSubscriber::builder() + .with_max_level(tracing::Level::TRACE) + .finish(), + ) + .unwrap();*/ + + // Start test server + let mut core = Core::test(); + core.session.config.rcpt.relay = IfBlock::new(true); + core.session.config.extensions.dsn = IfBlock::new(true); + let mut remote_qr = core.init_test_queue("lmtp_delivery_remote"); + let _rx = start_test_server(core.into(), &[ServerProtocol::Lmtp]); + + // Add mock DNS entries + let mut core = Core::test(); + core.resolvers.dns.ipv4_add( + "lmtp.foobar.org", + vec!["127.0.0.1".parse().unwrap()], + Instant::now() + Duration::from_secs(10), + ); + + // Multiple delivery attempts + let mut local_qr = core.init_test_queue("lmtp_delivery_local"); + + let mut ctx = ConfigContext::default(); + let config = Config::parse(REMOTE).unwrap(); + config.parse_remote_hosts(&mut ctx).unwrap(); + core.queue.config.next_hop = "[{if = 'rcpt-domain', eq = 'foobar.org', then = 'lmtp'}, + {else = false}]" + .parse_if::>(&ctx) + .into_relay_host(&ctx) + .unwrap(); + core.session.config.rcpt.relay = IfBlock::new(true); + core.session.config.rcpt.max_recipients = IfBlock::new(100); + core.session.config.extensions.dsn = IfBlock::new(true); + let mut config = &mut core.queue.config; + config.retry = IfBlock::new(vec![Duration::from_millis(100)]); + config.notify = "[{if = 'rcpt-domain', eq = 'foobar.org', then = ['100ms', '200ms']}, + {else = ['100ms']}]" + .parse_if(&ctx); + config.expire = "[{if = 'rcpt-domain', eq = 'foobar.org', then = '400ms'}, + {else = '500ms'}]" + .parse_if(&ctx); + config.timeout.data = IfBlock::new(Duration::from_millis(50)); + + let core = Arc::new(core); + let mut queue = Queue::default(); + let mut session = Session::test(core.clone()); + session.data.remote_ip = "10.0.0.1".parse().unwrap(); + session.eval_session_params().await; + session.ehlo("mx.test.org").await; + session + .send_message( + "john@test.org", + &[ + " NOTIFY=SUCCESS,DELAY,FAILURE", + " NOTIFY=SUCCESS,DELAY,FAILURE", + " NOTIFY=SUCCESS,DELAY,FAILURE", + " NOTIFY=SUCCESS,DELAY,FAILURE", + " NOTIFY=SUCCESS,DELAY,FAILURE", + " NOTIFY=SUCCESS,DELAY,FAILURE", + ], + "test:no_dkim", + "250", + ) + .await; + DeliveryAttempt::from(local_qr.read_event().await.unwrap_message()) + .try_deliver(core.clone(), &mut queue) + .await; + let mut dsn = Vec::new(); + loop { + match local_qr.try_read_event().await { + Some(Event::Queue(message)) => { + dsn.push(message.inner); + } + Some(Event::Done(wr)) => match wr { + WorkerResult::Done => { + break; + } + WorkerResult::Retry(retry) => { + queue.schedule(retry); + } + WorkerResult::OnHold(_) => unreachable!(), + }, + None | Some(Event::Stop) => break, + Some(Event::Manage(_)) => unreachable!(), + } + + if !queue.scheduled.is_empty() { + tokio::time::sleep(queue.wake_up_time()).await; + DeliveryAttempt::from(queue.next_due().unwrap()) + .try_deliver(core.clone(), &mut queue) + .await; + } + } + assert!(queue.scheduled.is_empty()); + assert_eq!(dsn.len(), 4); + + let mut dsn = dsn.into_iter(); + + dsn.next() + .unwrap() + .read_lines() + .assert_contains(" (delivered to") + .assert_contains(" (delivered to") + .assert_contains(" (delivered to") + .assert_contains(" (failed to lookup") + .assert_contains(" (host 'lmtp.foobar.org' rejected command"); + + dsn.next() + .unwrap() + .read_lines() + .assert_contains(" (host 'lmtp.foobar.org' rejected") + .assert_contains("Action: delayed"); + + dsn.next() + .unwrap() + .read_lines() + .assert_contains(" (host 'lmtp.foobar.org' rejected") + .assert_contains("Action: delayed"); + + dsn.next() + .unwrap() + .read_lines() + .assert_contains(" (host 'lmtp.foobar.org' rejected") + .assert_contains("Action: failed"); + + assert_eq!( + remote_qr + .read_event() + .await + .unwrap_message() + .recipients + .into_iter() + .map(|r| r.address) + .collect::>(), + vec![ + "bill@foobar.org".to_string(), + "jane@foobar.org".to_string(), + "john@foobar.org".to_string() + ] + ); + remote_qr.assert_empty_queue(); +} diff --git a/tests/src/smtp/outbound/mod.rs b/tests/src/smtp/outbound/mod.rs new file mode 100644 index 00000000..83bace1e --- /dev/null +++ b/tests/src/smtp/outbound/mod.rs @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::sync::Arc; + +use tokio::sync::watch; + +use ::smtp::core::{Core, HttpAdminSessionManager, SmtpSessionManager}; +use utils::config::{Config, ServerProtocol}; + +use super::add_test_certs; + +pub mod dane; +pub mod extensions; +pub mod lmtp; +pub mod mta_sts; +pub mod smtp; +pub mod throttle; + +const SERVER: &str = " +[server] +hostname = 'mx.example.org' +greeting = 'Test SMTP instance' +protocol = 'smtp' + +[server.listener.smtp-debug] +bind = ['127.0.0.1:9925'] + +[server.listener.lmtp-debug] +bind = ['127.0.0.1:9924'] +protocol = 'lmtp' +tls.implicit = true + +[server.listener.management-debug] +bind = ['127.0.0.1:9980'] +protocol = 'http' + +[server.socket] +reuse-addr = true + +[server.tls] +enable = true +implicit = false +certificate = 'default' + +[certificate.default] +cert = 'file://{CERT}' +private-key = 'file://{PK}' +"; + +pub fn start_test_server(core: Arc, protocols: &[ServerProtocol]) -> watch::Sender { + // Spawn listeners + let config = Config::parse(&add_test_certs(SERVER)).unwrap(); + let servers = config.parse_servers().unwrap(); + + // Start servers + let smtp_manager = SmtpSessionManager::new(core.clone()); + let smtp_admin_manager = HttpAdminSessionManager::new(core); + servers.spawn(&config, |server, shutdown_rx| { + if protocols.contains(&server.protocol) { + match &server.protocol { + ServerProtocol::Smtp | ServerProtocol::Lmtp => { + server.spawn(smtp_manager.clone(), shutdown_rx) + } + ServerProtocol::Http => server.spawn(smtp_admin_manager.clone(), shutdown_rx), + ServerProtocol::Imap | ServerProtocol::Jmap => unreachable!(), + }; + } + }) +} diff --git a/tests/src/smtp/outbound/mta_sts.rs b/tests/src/smtp/outbound/mta_sts.rs new file mode 100644 index 00000000..030ec3ae --- /dev/null +++ b/tests/src/smtp/outbound/mta_sts.rs @@ -0,0 +1,236 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; + +use mail_auth::{ + common::parse::TxtRecordParser, + mta_sts::{MtaSts, ReportUri, TlsRpt}, + report::tlsrpt::ResultType, + MX, +}; +use utils::config::ServerProtocol; + +use crate::smtp::{ + inbound::{TestMessage, TestQueueEvent, TestReportingEvent}, + outbound::start_test_server, + session::{TestSession, VerifyResponse}, + TestConfig, TestCore, +}; +use smtp::{ + config::{AggregateFrequency, IfBlock, RequireOptional}, + core::{Core, Session}, + outbound::mta_sts::{lookup::STS_TEST_POLICY, Policy}, + queue::{manager::Queue, DeliveryAttempt}, + reporting::PolicyType, +}; + +#[tokio::test] +#[serial_test::serial] +async fn mta_sts_verify() { + /*tracing::subscriber::set_global_default( + tracing_subscriber::FmtSubscriber::builder() + .with_max_level(tracing::Level::TRACE) + .finish(), + ) + .unwrap();*/ + + // Start test server + let mut core = Core::test(); + core.session.config.rcpt.relay = IfBlock::new(true); + let mut remote_qr = core.init_test_queue("smtp_mta_sts_remote"); + let _rx = start_test_server(core.into(), &[ServerProtocol::Smtp]); + + // Add mock DNS entries + let mut core = Core::test(); + core.resolvers.dns.mx_add( + "foobar.org", + vec![MX { + exchanges: vec!["mx.foobar.org".to_string()], + preference: 10, + }], + Instant::now() + Duration::from_secs(10), + ); + core.resolvers.dns.ipv4_add( + "mx.foobar.org", + vec!["127.0.0.1".parse().unwrap()], + Instant::now() + Duration::from_secs(10), + ); + core.resolvers.dns.txt_add( + "_smtp._tls.foobar.org", + TlsRpt::parse(b"v=TLSRPTv1; rua=mailto:reports@foobar.org").unwrap(), + Instant::now() + Duration::from_secs(10), + ); + + // Fail on missing MTA-STS record + let mut local_qr = core.init_test_queue("smtp_mta_sts_local"); + let mut rr = core.init_test_report(); + core.session.config.rcpt.relay = IfBlock::new(true); + core.queue.config.tls.mta_sts = IfBlock::new(RequireOptional::Require); + core.report.config.tls.send = IfBlock::new(AggregateFrequency::Weekly); + + let core = Arc::new(core); + let mut queue = Queue::default(); + let mut session = Session::test(core.clone()); + session.data.remote_ip = "10.0.0.1".parse().unwrap(); + session.eval_session_params().await; + session.ehlo("mx.test.org").await; + session + .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") + .await; + DeliveryAttempt::from(local_qr.read_event().await.unwrap_message()) + .try_deliver(core.clone(), &mut queue) + .await; + local_qr + .read_event() + .await + .unwrap_message() + .read_lines() + .assert_contains(" (MTA-STS failed to authenticate") + .assert_contains("Record not found"); + local_qr.read_event().await.unwrap_done(); + + // Expect TLS failure report + let report = rr.read_report().await.unwrap_tls(); + assert_eq!(report.domain, "foobar.org"); + assert_eq!(report.policy, PolicyType::Sts(None)); + assert_eq!( + report.failure.as_ref().unwrap().result_type, + ResultType::Other + ); + assert_eq!( + report.tls_record.rua, + vec![ReportUri::Mail("reports@foobar.org".to_string())] + ); + + // MTA-STS policy fetch failure + core.resolvers.dns.txt_add( + "_mta-sts.foobar.org", + MtaSts::parse(b"v=STSv1; id=policy_will_fail;").unwrap(), + Instant::now() + Duration::from_secs(10), + ); + session + .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") + .await; + DeliveryAttempt::from(local_qr.read_event().await.unwrap_message()) + .try_deliver(core.clone(), &mut queue) + .await; + local_qr + .read_event() + .await + .unwrap_message() + .read_lines() + .assert_contains(" (MTA-STS failed to authenticate") + .assert_contains("No 'mx' entries found"); + local_qr.read_event().await.unwrap_done(); + + // Expect TLS failure report + let report = rr.read_report().await.unwrap_tls(); + assert_eq!(report.policy, PolicyType::Sts(None)); + assert_eq!( + report.failure.as_ref().unwrap().result_type, + ResultType::StsPolicyInvalid + ); + + // MTA-STS policy does not authorize mx.foobar.org + let policy = concat!( + "version: STSv1\n", + "mode: enforce\n", + "mx: mail.foobar.net\n", + "max_age: 604800\n" + ); + STS_TEST_POLICY.lock().extend_from_slice(policy.as_bytes()); + session + .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") + .await; + DeliveryAttempt::from(local_qr.read_event().await.unwrap_message()) + .try_deliver(core.clone(), &mut queue) + .await; + local_qr + .read_event() + .await + .unwrap_message() + .read_lines() + .assert_contains(" (MTA-STS failed to authenticate") + .assert_contains("not authorized by policy"); + local_qr.read_event().await.unwrap_done(); + + // Expect TLS failure report + let report = rr.read_report().await.unwrap_tls(); + assert_eq!( + report.policy, + PolicyType::Sts( + Arc::new(Policy::parse(policy, "policy_will_fail".to_string()).unwrap()).into() + ) + ); + assert_eq!( + report.failure.as_ref().unwrap().receiving_mx_hostname, + Some("mx.foobar.org".to_string()) + ); + assert_eq!( + report.failure.as_ref().unwrap().result_type, + ResultType::ValidationFailure + ); + remote_qr.assert_empty_queue(); + + // MTA-STS successful validation + core.resolvers.dns.txt_add( + "_mta-sts.foobar.org", + MtaSts::parse(b"v=STSv1; id=policy_will_work;").unwrap(), + Instant::now() + Duration::from_secs(10), + ); + let policy = concat!( + "version: STSv1\n", + "mode: enforce\n", + "mx: *.foobar.org\n", + "max_age: 604800\n" + ); + STS_TEST_POLICY.lock().clear(); + STS_TEST_POLICY.lock().extend_from_slice(policy.as_bytes()); + session + .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") + .await; + DeliveryAttempt::from(local_qr.read_event().await.unwrap_message()) + .try_deliver(core.clone(), &mut queue) + .await; + local_qr.read_event().await.unwrap_done(); + remote_qr + .read_event() + .await + .unwrap_message() + .read_lines() + .assert_contains("using TLSv1.3 with cipher"); + + // Expect TLS success report + let report = rr.read_report().await.unwrap_tls(); + assert_eq!( + report.policy, + PolicyType::Sts( + Arc::new(Policy::parse(policy, "policy_will_work".to_string()).unwrap()).into() + ) + ); + assert!(report.failure.is_none()); +} diff --git a/tests/src/smtp/outbound/smtp.rs b/tests/src/smtp/outbound/smtp.rs new file mode 100644 index 00000000..7824d3fc --- /dev/null +++ b/tests/src/smtp/outbound/smtp.rs @@ -0,0 +1,248 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; + +use mail_auth::MX; +use utils::config::ServerProtocol; + +use crate::smtp::{ + inbound::{TestMessage, TestQueueEvent}, + outbound::start_test_server, + session::{TestSession, VerifyResponse}, + ParseTestConfig, TestConfig, TestCore, +}; +use smtp::{ + config::{ConfigContext, IfBlock}, + core::{Core, Session}, + queue::{manager::Queue, DeliveryAttempt, Event, WorkerResult}, +}; + +#[tokio::test] +#[serial_test::serial] +async fn smtp_delivery() { + /*tracing::subscriber::set_global_default( + tracing_subscriber::FmtSubscriber::builder() + .with_max_level(tracing::Level::DEBUG) + .finish(), + ) + .unwrap();*/ + + // Start test server + let mut core = Core::test(); + core.session.config.rcpt.relay = IfBlock::new(true); + core.session.config.extensions.dsn = IfBlock::new(true); + let mut remote_qr = core.init_test_queue("smtp_delivery_remote"); + let _rx = start_test_server(core.into(), &[ServerProtocol::Smtp]); + + // Add mock DNS entries + let mut core = Core::test(); + core.resolvers.dns.mx_add( + "foobar.org", + vec![ + MX { + exchanges: vec!["mx1.foobar.org".to_string()], + preference: 10, + }, + MX { + exchanges: vec!["mx2.foobar.org".to_string()], + preference: 20, + }, + ], + Instant::now() + Duration::from_secs(10), + ); + core.resolvers.dns.mx_add( + "foobar.net", + vec![MX { + exchanges: vec!["mx1.foobar.net".to_string(), "mx2.foobar.net".to_string()], + preference: 10, + }], + Instant::now() + Duration::from_secs(10), + ); + core.resolvers.dns.ipv4_add( + "mx1.foobar.org", + vec!["127.0.0.1".parse().unwrap()], + Instant::now() + Duration::from_secs(10), + ); + core.resolvers.dns.ipv4_add( + "mx2.foobar.org", + vec!["127.0.0.1".parse().unwrap()], + Instant::now() + Duration::from_secs(10), + ); + core.resolvers.dns.ipv4_add( + "mx1.foobar.net", + vec!["127.0.0.1".parse().unwrap()], + Instant::now() + Duration::from_secs(10), + ); + core.resolvers.dns.ipv4_add( + "mx2.foobar.net", + vec!["127.0.0.1".parse().unwrap()], + Instant::now() + Duration::from_secs(10), + ); + + // Multiple delivery attempts + let mut local_qr = core.init_test_queue("smtp_delivery_local"); + core.session.config.rcpt.relay = IfBlock::new(true); + core.session.config.rcpt.max_recipients = IfBlock::new(100); + core.session.config.extensions.dsn = IfBlock::new(true); + let mut config = &mut core.queue.config; + config.retry = IfBlock::new(vec![Duration::from_millis(100)]); + config.notify = "[{if = 'rcpt-domain', eq = 'foobar.org', then = ['100ms', '200ms']}, + {else = ['100ms']}]" + .parse_if(&ConfigContext::default()); + config.expire = "[{if = 'rcpt-domain', eq = 'foobar.org', then = '650ms'}, + {else = '750ms'}]" + .parse_if(&ConfigContext::default()); + + let core = Arc::new(core); + let mut queue = Queue::default(); + let mut session = Session::test(core.clone()); + session.data.remote_ip = "10.0.0.1".parse().unwrap(); + session.eval_session_params().await; + session.ehlo("mx.test.org").await; + session + .send_message( + "john@test.org", + &[ + " NOTIFY=SUCCESS,DELAY,FAILURE", + " NOTIFY=SUCCESS,DELAY,FAILURE", + " NOTIFY=SUCCESS,DELAY,FAILURE", + " NOTIFY=SUCCESS,DELAY,FAILURE", + " NOTIFY=SUCCESS,DELAY,FAILURE", + " NOTIFY=SUCCESS,DELAY,FAILURE", + " NOTIFY=SUCCESS,DELAY,FAILURE", + ], + "test:no_dkim", + "250", + ) + .await; + let message = local_qr.read_event().await.unwrap_message(); + let num_domains = message.domains.len(); + assert_eq!(num_domains, 3); + DeliveryAttempt::from(message) + .try_deliver(core.clone(), &mut queue) + .await; + let mut dsn = Vec::new(); + let mut domain_retries = vec![0; num_domains]; + loop { + match local_qr.try_read_event().await { + Some(Event::Queue(message)) => { + dsn.push(message.inner); + } + Some(Event::Done(wr)) => match wr { + WorkerResult::Done => { + break; + } + WorkerResult::Retry(retry) => { + for (idx, domain) in retry.inner.domains.iter().enumerate() { + domain_retries[idx] = domain.retry.inner; + } + queue.schedule(retry); + } + WorkerResult::OnHold(_) => unreachable!(), + }, + None | Some(Event::Stop) => break, + Some(Event::Manage(_)) => unreachable!(), + } + + if !queue.scheduled.is_empty() { + tokio::time::sleep(queue.wake_up_time()).await; + DeliveryAttempt::from(queue.next_due().unwrap()) + .try_deliver(core.clone(), &mut queue) + .await; + } + } + assert_eq!(domain_retries[0], 0, "retries {domain_retries:?}"); + assert!(domain_retries[1] >= 5, "retries {domain_retries:?}"); + assert!(domain_retries[2] >= 5, "retries {domain_retries:?}"); + assert!( + domain_retries[1] >= domain_retries[2], + "retries {domain_retries:?}" + ); + + assert!(queue.scheduled.is_empty()); + assert_eq!(dsn.len(), 5); + + let mut dsn = dsn.into_iter(); + + dsn.next() + .unwrap() + .read_lines() + .assert_contains(" (delivered to") + .assert_contains(" (delivered to") + .assert_contains(" (failed to lookup") + .assert_contains(" (host ") + .assert_contains(" (host "); + + dsn.next() + .unwrap() + .read_lines() + .assert_contains(" (host ") + .assert_contains(" (host ") + .assert_contains("Action: delayed"); + + dsn.next() + .unwrap() + .read_lines() + .assert_contains(" (host ") + .assert_contains("Action: delayed"); + + dsn.next() + .unwrap() + .read_lines() + .assert_contains(" (host "); + + dsn.next() + .unwrap() + .read_lines() + .assert_contains(" (host ") + .assert_contains("Action: failed"); + + assert_eq!( + remote_qr + .read_event() + .await + .unwrap_message() + .recipients + .into_iter() + .map(|r| r.address) + .collect::>(), + vec!["ok@foobar.net".to_string()] + ); + assert_eq!( + remote_qr + .read_event() + .await + .unwrap_message() + .recipients + .into_iter() + .map(|r| r.address) + .collect::>(), + vec!["ok@foobar.org".to_string()] + ); + + remote_qr.assert_empty_queue(); +} diff --git a/tests/src/smtp/outbound/throttle.rs b/tests/src/smtp/outbound/throttle.rs new file mode 100644 index 00000000..d5aaef64 --- /dev/null +++ b/tests/src/smtp/outbound/throttle.rs @@ -0,0 +1,315 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{ + net::{IpAddr, Ipv4Addr}, + sync::Arc, + time::{Duration, Instant}, +}; + +use mail_auth::MX; + +use crate::smtp::{ + inbound::TestQueueEvent, queue::manager::new_message, session::TestSession, ParseTestConfig, + TestConfig, TestCore, +}; +use smtp::{ + config::{ConfigContext, IfBlock}, + core::{Core, Session}, + queue::{manager::Queue, DeliveryAttempt, Message, QueueEnvelope}, +}; + +const THROTTLE: &str = " +[[queue.throttle]] +match = {if = 'sender-domain', eq = 'foobar.org'} +key = 'sender-domain' +concurrency = 1 + +[[queue.throttle]] +match = {if = 'sender-domain', eq = 'foobar.net'} +key = 'sender-domain' +rate = '1/30m' + +[[queue.throttle]] +match = {if = 'rcpt-domain', eq = 'example.org'} +key = 'rcpt-domain' +concurrency = 1 + +[[queue.throttle]] +match = {if = 'rcpt-domain', eq = 'example.net'} +key = 'rcpt-domain' +rate = '1/40m' + +[[queue.throttle]] +match = {if = 'mx', eq = 'mx.test.org'} +key = 'mx' +concurrency = 1 + +[[queue.throttle]] +match = {if = 'mx', eq = 'mx.test.net'} +key = 'mx' +rate = '1/50m' +"; + +#[tokio::test] +async fn throttle_outbound() { + /*tracing::subscriber::set_global_default( + tracing_subscriber::FmtSubscriber::builder() + .with_max_level(tracing::Level::TRACE) + .finish(), + ) + .unwrap();*/ + + // Build test message + let mut test_message = new_message(0); + test_message.return_path_domain = "foobar.org".to_string(); + let mut core = Core::test(); + let mut local_qr = core.init_test_queue("smtp_throttle_outbound"); + core.session.config.rcpt.relay = IfBlock::new(true); + core.queue.config.throttle = THROTTLE.parse_queue_throttle(&ConfigContext::default()); + core.queue.config.retry = IfBlock::new(vec![Duration::from_secs(86400)]); + core.queue.config.notify = IfBlock::new(vec![Duration::from_secs(86400)]); + core.queue.config.expire = IfBlock::new(Duration::from_secs(86400)); + + let core = Arc::new(core); + let mut queue = Queue::default(); + let mut session = Session::test(core.clone()); + session.data.remote_ip = "10.0.0.1".parse().unwrap(); + session.eval_session_params().await; + session.ehlo("mx.test.org").await; + session + .send_message("john@foobar.org", &["bill@test.org"], "test:no_dkim", "250") + .await; + + // Throttle sender + let span = tracing::info_span!("test"); + let mut in_flight = vec![]; + let throttle = &core.queue.config.throttle; + for t in &throttle.sender { + core.queue + .is_allowed( + t, + &QueueEnvelope::test(&test_message, "", ""), + &mut in_flight, + &span, + ) + .await + .unwrap(); + } + assert!(!in_flight.is_empty()); + + // Expect concurrency throttle for sender domain 'foobar.org' + DeliveryAttempt::from(local_qr.read_event().await.unwrap_message()) + .try_deliver(core.clone(), &mut queue) + .await; + local_qr.assert_empty_queue(); + in_flight.clear(); + assert!(!queue.on_hold.is_empty()); + queue.next_on_hold().unwrap(); + + // Expect rate limit throttle for sender domain 'foobar.net' + test_message.return_path_domain = "foobar.net".to_string(); + for t in &throttle.sender { + core.queue + .is_allowed( + t, + &QueueEnvelope::test(&test_message, "", ""), + &mut in_flight, + &span, + ) + .await + .unwrap(); + } + assert!(in_flight.is_empty()); + session + .send_message("john@foobar.net", &["bill@test.org"], "test:no_dkim", "250") + .await; + DeliveryAttempt::from(local_qr.read_event().await.unwrap_message()) + .try_deliver(core.clone(), &mut queue) + .await; + local_qr.assert_empty_queue(); + assert!([1799, 1800].contains( + &queue + .scheduled + .pop() + .unwrap() + .due + .duration_since(Instant::now()) + .as_secs() + )); + + // Expect concurrency throttle for recipient domain 'example.org' + test_message.return_path_domain = "test.net".to_string(); + for t in &throttle.rcpt { + core.queue + .is_allowed( + t, + &QueueEnvelope::test(&test_message, "example.org", ""), + &mut in_flight, + &span, + ) + .await + .unwrap(); + } + assert!(!in_flight.is_empty()); + session + .send_message( + "john@test.net", + &["jane@example.org"], + "test:no_dkim", + "250", + ) + .await; + DeliveryAttempt::from(local_qr.read_event().await.unwrap_message()) + .try_deliver(core.clone(), &mut queue) + .await; + local_qr.read_event().await.unwrap_on_hold(); + in_flight.clear(); + + // Expect rate limit throttle for recipient domain 'example.org' + for t in &throttle.rcpt { + core.queue + .is_allowed( + t, + &QueueEnvelope::test(&test_message, "example.net", ""), + &mut in_flight, + &span, + ) + .await + .unwrap(); + } + assert!(in_flight.is_empty()); + session + .send_message( + "john@test.net", + &["jane@example.net"], + "test:no_dkim", + "250", + ) + .await; + DeliveryAttempt::from(local_qr.read_event().await.unwrap_message()) + .try_deliver(core.clone(), &mut queue) + .await; + assert!([2399, 2400].contains( + &local_qr + .read_event() + .await + .unwrap_retry() + .due + .duration_since(Instant::now()) + .as_secs() + )); + + // Expect concurrency throttle for mx 'mx.test.org' + core.resolvers.dns.mx_add( + "test.org", + vec![MX { + exchanges: vec!["mx.test.org".to_string()], + preference: 10, + }], + Instant::now() + Duration::from_secs(10), + ); + core.resolvers.dns.ipv4_add( + "mx.test.org", + vec!["127.0.0.1".parse().unwrap()], + Instant::now() + Duration::from_secs(10), + ); + for t in &throttle.host { + core.queue + .is_allowed( + t, + &QueueEnvelope::test(&test_message, "test.org", "mx.test.org"), + &mut in_flight, + &span, + ) + .await + .unwrap(); + } + assert!(!in_flight.is_empty()); + session + .send_message("john@test.net", &["jane@test.org"], "test:no_dkim", "250") + .await; + DeliveryAttempt::from(local_qr.read_event().await.unwrap_message()) + .try_deliver(core.clone(), &mut queue) + .await; + local_qr.read_event().await.unwrap_on_hold(); + in_flight.clear(); + + // Expect rate limit throttle for mx 'mx.test.net' + core.resolvers.dns.mx_add( + "test.net", + vec![MX { + exchanges: vec!["mx.test.net".to_string()], + preference: 10, + }], + Instant::now() + Duration::from_secs(10), + ); + core.resolvers.dns.ipv4_add( + "mx.test.net", + vec!["127.0.0.1".parse().unwrap()], + Instant::now() + Duration::from_secs(10), + ); + for t in &throttle.host { + core.queue + .is_allowed( + t, + &QueueEnvelope::test(&test_message, "example.net", "mx.test.net"), + &mut in_flight, + &span, + ) + .await + .unwrap(); + } + assert!(in_flight.is_empty()); + session + .send_message("john@test.net", &["jane@test.net"], "test:no_dkim", "250") + .await; + DeliveryAttempt::from(local_qr.read_event().await.unwrap_message()) + .try_deliver(core.clone(), &mut queue) + .await; + assert!([2999, 3000].contains( + &local_qr + .read_event() + .await + .unwrap_retry() + .due + .duration_since(Instant::now()) + .as_secs() + )); +} + +pub trait TestQueueEnvelope<'x> { + fn test(message: &'x Message, domain: &'x str, mx: &'x str) -> Self; +} + +impl<'x> TestQueueEnvelope<'x> for QueueEnvelope<'x> { + fn test(message: &'x Message, domain: &'x str, mx: &'x str) -> Self { + QueueEnvelope { + message, + domain, + mx, + remote_ip: IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), + local_ip: IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), + } + } +} diff --git a/tests/src/smtp/queue/dsn.rs b/tests/src/smtp/queue/dsn.rs new file mode 100644 index 00000000..32296274 --- /dev/null +++ b/tests/src/smtp/queue/dsn.rs @@ -0,0 +1,235 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{ + fs, + path::PathBuf, + time::{Duration, Instant, SystemTime}, +}; + +use smtp_proto::{Response, RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_SUCCESS}; +use tokio::{fs::File, io::AsyncReadExt}; + +use crate::smtp::{ + inbound::{sign::TextConfigContext, TestQueueEvent}, + ParseTestConfig, TestConfig, TestCore, +}; +use smtp::{ + config::ConfigContext, + core::Core, + queue::{ + DeliveryAttempt, Domain, Error, ErrorDetails, HostResponse, Message, Recipient, Schedule, + Status, + }, +}; + +#[tokio::test] +async fn generate_dsn() { + let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + path.push("resources"); + path.push("smtp"); + path.push("dsn"); + path.push("original.txt"); + let size = fs::metadata(&path).unwrap().len() as usize; + + let flags = RCPT_NOTIFY_FAILURE | RCPT_NOTIFY_DELAY | RCPT_NOTIFY_SUCCESS; + let message = Box::new(Message { + size, + id: 0, + path, + created: SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .map_or(0, |d| d.as_secs()), + return_path: "sender@foobar.org".to_string(), + return_path_lcase: "".to_string(), + return_path_domain: "foobar.org".to_string(), + recipients: vec![Recipient { + domain_idx: 0, + address: "foobar@example.org".to_string(), + address_lcase: "foobar@example.org".to_string(), + status: Status::PermanentFailure(HostResponse { + hostname: ErrorDetails { + entity: "mx.example.org".to_string(), + details: "RCPT TO:".to_string(), + }, + response: Response { + code: 550, + esc: [5, 1, 2], + message: "User does not exist".to_string(), + }, + }), + flags: 0, + orcpt: None, + }], + domains: vec![Domain { + domain: "example.org".to_string(), + retry: Schedule::now(), + notify: Schedule::now(), + expires: Instant::now() + Duration::from_secs(10), + status: Status::TemporaryFailure(Error::ConnectionError(ErrorDetails { + entity: "mx.domain.org".to_string(), + details: "Connection timeout".to_string(), + })), + changed: false, + }], + flags: 0, + env_id: None, + priority: 0, + + queue_refs: vec![], + }); + let mut attempt = DeliveryAttempt { + span: tracing::span!(tracing::Level::INFO, "hi"), + message, + in_flight: vec![], + }; + + // Load config + let mut core = Core::test(); + let ctx = ConfigContext::default().parse_signatures(); + let mut config = &mut core.queue.config.dsn; + config.sign = "['rsa']" + .parse_if::>(&ctx) + .map_if_block(&ctx.signers, "", "") + .unwrap(); + + // Create temp dir for queue + let mut qr = core.init_test_queue("smtp_dsn_test"); + + // Disabled DSN + core.queue.send_dsn(&mut attempt).await; + qr.assert_empty_queue(); + + // Failure DSN + attempt.message.recipients[0].flags = flags; + core.queue.send_dsn(&mut attempt).await; + compare_dsn(qr.read_event().await.unwrap_message(), "failure.eml").await; + + // Success DSN + attempt.message.recipients.push(Recipient { + domain_idx: 0, + address: "jane@example.org".to_string(), + address_lcase: "jane@example.org".to_string(), + status: Status::Completed(HostResponse { + hostname: "mx2.example.org".to_string(), + response: Response { + code: 250, + esc: [2, 1, 5], + message: "Message accepted for delivery".to_string(), + }, + }), + flags, + orcpt: None, + }); + core.queue.send_dsn(&mut attempt).await; + compare_dsn(qr.read_event().await.unwrap_message(), "success.eml").await; + + // Delay DSN + attempt.message.recipients.push(Recipient { + domain_idx: 0, + address: "john.doe@example.org".to_string(), + address_lcase: "john.doe@example.org".to_string(), + status: Status::Scheduled, + flags, + orcpt: "jdoe@example.org".to_string().into(), + }); + core.queue.send_dsn(&mut attempt).await; + compare_dsn(qr.read_event().await.unwrap_message(), "delay.eml").await; + + // Mixed DSN + for rcpt in &mut attempt.message.recipients { + rcpt.flags = flags; + } + attempt.message.domains[0].notify.due = Instant::now(); + core.queue.send_dsn(&mut attempt).await; + compare_dsn(qr.read_event().await.unwrap_message(), "mixed.eml").await; + + // Load queue + let queue = core.queue.read_queue().await; + assert_eq!(queue.scheduled.len(), 4); +} + +async fn compare_dsn(message: Box, test: &str) { + let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + path.push("resources"); + path.push("smtp"); + path.push("dsn"); + path.push(test); + + let mut bytes = vec![0u8; message.size]; + File::open(&message.path) + .await + .unwrap() + .read_exact(&mut bytes) + .await + .unwrap(); + + let dsn = remove_ids(bytes); + let dsn_expected = fs::read_to_string(&path).unwrap(); + + //fs::write(&path, dsn.as_bytes()).unwrap(); + assert_eq!(dsn, dsn_expected, "Failed for {}", path.display()); +} + +fn remove_ids(message: Vec) -> String { + let old_message = String::from_utf8(message).unwrap(); + let mut message = String::with_capacity(old_message.len()); + let mut found_dkim = false; + let mut skip = false; + + let mut boundary = ""; + for line in old_message.split("\r\n") { + if skip { + if line.chars().next().unwrap().is_ascii_whitespace() { + continue; + } else { + skip = false; + } + } + if line.starts_with("Date:") || line.starts_with("Message-ID:") { + continue; + } else if !found_dkim && line.starts_with("DKIM-Signature:") { + found_dkim = true; + skip = true; + continue; + } else if line.starts_with("--") { + message.push_str(&line.replace(boundary, "mime_boundary")); + } else if let Some((_, boundary_)) = line.split_once("boundary=\"") { + boundary = boundary_.split_once('"').unwrap().0; + message.push_str(&line.replace(boundary, "mime_boundary")); + } else if line.starts_with("Arrival-Date:") { + message.push_str("Arrival-Date: "); + } else if line.starts_with("Will-Retry-Until:") { + message.push_str("Will-Retry-Until: "); + } else { + message.push_str(line); + } + message.push_str("\r\n"); + } + + if !found_dkim { + panic!("No DKIM signature found in: {old_message}"); + } + + message +} diff --git a/tests/src/smtp/queue/manager.rs b/tests/src/smtp/queue/manager.rs new file mode 100644 index 00000000..028411e26 --- /dev/null +++ b/tests/src/smtp/queue/manager.rs @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::time::{Duration, Instant}; + +use mail_auth::trust_dns_resolver::proto::op::ResponseCode; + +use smtp::queue::{manager::Queue, Domain, Message, Schedule, Status}; + +#[test] +fn queue_due() { + let mut queue = Queue::default(); + + let mut message = new_message(0); + message.domains.push(domain("c", 3, 8, 9)); + queue.schedule(Schedule { + due: message.next_delivery_event(), + inner: message, + }); + + let mut message = new_message(1); + message.domains.push(domain("b", 2, 6, 7)); + queue.schedule(Schedule { + due: message.next_delivery_event(), + inner: message, + }); + + let mut message = new_message(2); + message.domains.push(domain("a", 1, 4, 5)); + queue.schedule(Schedule { + due: message.next_delivery_event(), + inner: message, + }); + + for domain in vec!["a", "b", "c"].into_iter() { + let wake_up = queue.wake_up_time(); + assert!( + (900..=1000).contains(&wake_up.as_millis()), + "{}", + wake_up.as_millis() + ); + std::thread::sleep(wake_up); + queue.next_due().unwrap().domain(domain); + } + + assert!(queue.next_due().is_none()); +} + +#[test] +fn delivery_events() { + let mut message = new_message(0); + + message.domains.push(domain("a", 1, 2, 3)); + message.domains.push(domain("b", 4, 5, 6)); + message.domains.push(domain("c", 7, 8, 9)); + + for t in 0..2 { + assert_eq!(message.next_event().unwrap(), message.domain("a").retry.due); + assert_eq!(message.next_delivery_event(), message.domain("a").retry.due); + assert_eq!( + message + .next_event_after(message.domain("a").expires) + .unwrap(), + message.domain("b").retry.due + ); + assert_eq!( + message + .next_event_after(message.domain("b").expires) + .unwrap(), + message.domain("c").retry.due + ); + assert_eq!( + message + .next_event_after(message.domain("c").notify.due) + .unwrap(), + message.domain("c").expires + ); + assert!(message + .next_event_after(message.domain("c").expires) + .is_none()); + + if t == 0 { + message.domains.reverse(); + } else { + message.domains.swap(0, 1); + } + } + + message.domain_mut("a").set_status( + mail_auth::Error::DnsRecordNotFound(ResponseCode::BADCOOKIE), + &[], + ); + assert_eq!(message.next_event().unwrap(), message.domain("b").retry.due); + assert_eq!(message.next_delivery_event(), message.domain("b").retry.due); + + message.domain_mut("b").set_status( + mail_auth::Error::DnsRecordNotFound(ResponseCode::BADCOOKIE), + &[], + ); + assert_eq!(message.next_event().unwrap(), message.domain("c").retry.due); + assert_eq!(message.next_delivery_event(), message.domain("c").retry.due); + + message.domain_mut("c").set_status( + mail_auth::Error::DnsRecordNotFound(ResponseCode::BADCOOKIE), + &[], + ); + assert!(message.next_event().is_none()); +} + +pub fn new_message(id: u64) -> Box { + Box::new(Message { + size: 0, + id, + path: Default::default(), + created: 0, + return_path: "sender@foobar.org".to_string(), + return_path_lcase: "".to_string(), + return_path_domain: "foobar.org".to_string(), + recipients: vec![], + domains: vec![], + flags: 0, + env_id: None, + priority: 0, + queue_refs: vec![], + }) +} + +fn domain(domain: &str, retry: u64, notify: u64, expires: u64) -> Domain { + Domain { + domain: domain.to_string(), + retry: Schedule::later(Duration::from_secs(retry)), + notify: Schedule::later(Duration::from_secs(notify)), + expires: Instant::now() + Duration::from_secs(expires), + status: Status::Scheduled, + changed: false, + } +} + +pub trait TestMessage { + fn domain(&self, name: &str) -> &Domain; + fn domain_mut(&mut self, name: &str) -> &mut Domain; +} + +impl TestMessage for Message { + fn domain(&self, name: &str) -> &Domain { + self.domains.iter().find(|d| d.domain == name).unwrap() + } + + fn domain_mut(&mut self, name: &str) -> &mut Domain { + self.domains.iter_mut().find(|d| d.domain == name).unwrap() + } +} diff --git a/tests/src/smtp/queue/mod.rs b/tests/src/smtp/queue/mod.rs new file mode 100644 index 00000000..fdef57ee --- /dev/null +++ b/tests/src/smtp/queue/mod.rs @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +pub mod dsn; +pub mod manager; +pub mod retry; +pub mod serialize; diff --git a/tests/src/smtp/queue/retry.rs b/tests/src/smtp/queue/retry.rs new file mode 100644 index 00000000..2a33d705 --- /dev/null +++ b/tests/src/smtp/queue/retry.rs @@ -0,0 +1,237 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; + +use crate::smtp::{ + inbound::{TestMessage, TestQueueEvent}, + session::{TestSession, VerifyResponse}, + ParseTestConfig, TestConfig, TestCore, +}; +use smtp::{ + config::{ConfigContext, IfBlock}, + core::{Core, Session}, + queue::{manager::Queue, DeliveryAttempt, Event, WorkerResult}, +}; + +#[tokio::test] +async fn queue_retry() { + /*tracing::subscriber::set_global_default( + tracing_subscriber::FmtSubscriber::builder() + .with_max_level(tracing::Level::DEBUG) + .finish(), + ) + .unwrap();*/ + + let mut core = Core::test(); + + // Create temp dir for queue + let mut qr = core.init_test_queue("smtp_queue_retry_test"); + + let mut config = &mut core.session.config.rcpt; + config.relay = IfBlock::new(true); + let mut config = &mut core.session.config.extensions; + config.deliver_by = IfBlock::new(Some(Duration::from_secs(86400))); + config.future_release = IfBlock::new(Some(Duration::from_secs(86400))); + let mut config = &mut core.queue.config; + config.retry = IfBlock::new(vec![ + Duration::from_millis(100), + Duration::from_millis(200), + Duration::from_millis(300), + ]); + config.notify = "[{if = 'sender-domain', eq = 'test.org', then = ['150ms', '200ms']}, + {else = ['15h', '22h']}]" + .parse_if(&ConfigContext::default()); + config.expire = "[{if = 'sender-domain', eq = 'test.org', then = '600ms'}, + {else = '1d'}]" + .parse_if(&ConfigContext::default()); + + // Create test message + let core = Arc::new(core); + let mut queue = Queue::default(); + let mut session = Session::test(core.clone()); + session.data.remote_ip = "10.0.0.1".parse().unwrap(); + session.eval_session_params().await; + session.ehlo("mx.test.org").await; + session + .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") + .await; + let attempt = DeliveryAttempt::from(qr.read_event().await.unwrap_message()); + + // Expect a failed DSN + let path = attempt.message.path.clone(); + attempt.try_deliver(core.clone(), &mut queue).await; + let message = qr.read_event().await.unwrap_message(); + assert_eq!(message.return_path, ""); + assert_eq!(message.domains.first().unwrap().domain, "test.org"); + assert_eq!(message.recipients.first().unwrap().address, "john@test.org"); + message + .read_lines() + .assert_contains("Content-Type: multipart/report") + .assert_contains("Final-Recipient: rfc822;bill@foobar.org") + .assert_contains("Action: failed"); + qr.read_event().await.unwrap_done(); + assert!(!path.exists()); + + // Expect a failed DSN for foobar.org, followed by two delayed DSN and + // a final failed DSN for _dns_error.org. + session + .send_message( + "john@test.org", + &["bill@foobar.org", "jane@_dns_error.org"], + "test:no_dkim", + "250", + ) + .await; + let attempt = DeliveryAttempt::from(qr.read_event().await.unwrap_message()); + let path = attempt.message.path.clone(); + let mut dsn = Vec::new(); + let mut num_retries = 0; + attempt.try_deliver(core.clone(), &mut queue).await; + loop { + match qr.try_read_event().await { + Some(Event::Queue(message)) => { + dsn.push(message.inner); + } + Some(Event::Done(wr)) => match wr { + WorkerResult::Done => break, + WorkerResult::Retry(retry) => { + queue.schedule(retry); + num_retries += 1; + } + WorkerResult::OnHold(_) => unreachable!(), + }, + None | Some(Event::Stop) => break, + Some(Event::Manage(_)) => unreachable!(), + } + + if !queue.scheduled.is_empty() { + tokio::time::sleep(queue.wake_up_time()).await; + DeliveryAttempt::from(queue.next_due().unwrap()) + .try_deliver(core.clone(), &mut queue) + .await; + } + } + assert!(queue.scheduled.is_empty()); + assert_eq!(num_retries, 3); + assert_eq!(dsn.len(), 4); + assert!(!path.exists()); + let mut dsn = dsn.into_iter(); + + dsn.next() + .unwrap() + .read_lines() + .assert_contains(" (failed to lookup 'foobar.org'") + .assert_contains("Final-Recipient: rfc822;bill@foobar.org") + .assert_contains("Action: failed"); + + dsn.next() + .unwrap() + .read_lines() + .assert_contains(" (failed to lookup '_dns_error.org'") + .assert_contains("Final-Recipient: rfc822;jane@_dns_error.org") + .assert_contains("Action: delayed"); + + dsn.next() + .unwrap() + .read_lines() + .assert_contains(" (failed to lookup '_dns_error.org'") + .assert_contains("Final-Recipient: rfc822;jane@_dns_error.org") + .assert_contains("Action: delayed"); + + dsn.next() + .unwrap() + .read_lines() + .assert_contains(" (failed to lookup '_dns_error.org'") + .assert_contains("Final-Recipient: rfc822;jane@_dns_error.org") + .assert_contains("Action: failed"); + + // Test FUTURERELEASE + DELIVERBY (RETURN) + session.data.remote_ip = "10.0.0.2".parse().unwrap(); + session.eval_session_params().await; + session + .send_message( + " HOLDFOR=60 BY=3600;R", + &["john@test.net"], + "test:no_dkim", + "250", + ) + .await; + let now = Instant::now(); + let schedule = qr.read_event().await.unwrap_schedule(); + assert!([59, 60].contains(&schedule.due.duration_since(now).as_secs())); + assert!([59, 60].contains( + &schedule + .inner + .next_delivery_event() + .duration_since(now) + .as_secs() + )); + assert!([3599, 3600].contains( + &schedule + .inner + .domains + .first() + .unwrap() + .expires + .duration_since(now) + .as_secs() + )); + assert!([54059, 54060].contains( + &schedule + .inner + .domains + .first() + .unwrap() + .notify + .due + .duration_since(now) + .as_secs() + )); + + // Test DELIVERBY (NOTIFY) + session + .send_message( + " BY=3600;N", + &["john@test.net"], + "test:no_dkim", + "250", + ) + .await; + let now = Instant::now(); + let schedule = qr.read_event().await.unwrap_schedule(); + assert!([3599, 3600].contains( + &schedule + .inner + .domains + .first() + .unwrap() + .notify + .due + .duration_since(now) + .as_secs() + )); +} diff --git a/tests/src/smtp/queue/serialize.rs b/tests/src/smtp/queue/serialize.rs new file mode 100644 index 00000000..5f2ba15b --- /dev/null +++ b/tests/src/smtp/queue/serialize.rs @@ -0,0 +1,211 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{ + path::PathBuf, + time::{Duration, Instant}, +}; + +use smtp_proto::{Response, MAIL_REQUIRETLS, MAIL_SMTPUTF8, RCPT_CONNEG, RCPT_NOTIFY_FAILURE}; + +use smtp::{ + core::Core, + queue::{ + Domain, Error, ErrorDetails, HostResponse, Message, Recipient, Schedule, Status, + RCPT_STATUS_CHANGED, + }, +}; + +use crate::smtp::{inbound::TestQueueEvent, TestConfig, TestCore}; + +#[tokio::test] +async fn queue_serialize() { + let mut core = Core::test(); + + // Create temp dir for queue + let mut qr = core.init_test_queue("smtp_queue_serialize_test"); + + // Create test message + let message = Message { + size: 0, + id: 0, + path: PathBuf::new(), + created: 123456, + return_path: "sender@FooBar.org".to_string(), + return_path_lcase: "sender@foobar.org".to_string(), + return_path_domain: "foobar.org".to_string(), + recipients: vec![ + Recipient { + domain_idx: 0, + address: "FOOBAR@example.org".to_string(), + address_lcase: "foobar@example.org".to_string(), + status: Status::Scheduled, + flags: RCPT_CONNEG, + orcpt: None, + }, + Recipient { + domain_idx: 1, + address: "FOOBAR@example.org".to_string(), + address_lcase: "foobar@example.org".to_string(), + status: Status::Scheduled, + flags: RCPT_NOTIFY_FAILURE, + orcpt: None, + }, + ], + domains: vec![ + Domain { + domain: "example.org".to_string(), + retry: Schedule::now(), + notify: Schedule::now(), + expires: Instant::now() + Duration::from_secs(10), + status: Status::Scheduled, + changed: false, + }, + Domain { + domain: "example.com".to_string(), + retry: Schedule::now(), + notify: Schedule::now(), + expires: Instant::now() + Duration::from_secs(10), + status: Status::Scheduled, + changed: false, + }, + ], + flags: MAIL_REQUIRETLS | MAIL_SMTPUTF8, + env_id: "hello".to_string().into(), + priority: -1, + + queue_refs: vec![], + }; + + // Queue message + assert!( + core.queue + .queue_message( + Box::new(message), + (&b"From: test@foobar.org\r\n"[..]).into(), + b"Subject: test\r\n\n\ntest", + &tracing::info_span!("hi") + ) + .await + ); + let mut message = qr.read_event().await.unwrap_message(); + + // Deserialize + assert_msg_eq( + &message, + &Message::from_path(message.path.clone()).await.unwrap(), + ); + + // Write update + message.recipients[0].status = Status::PermanentFailure(HostResponse { + hostname: ErrorDetails { + entity: "mx.example.org".to_string(), + details: "RCPT TO:".to_string(), + }, + response: Response { + code: 550, + esc: [5, 1, 2], + message: "User does not exist\nplease contact support for details\n".to_string(), + }, + }); + message.recipients[0].flags |= RCPT_STATUS_CHANGED; + + message.recipients[1].status = Status::Completed(HostResponse { + hostname: "smtp.foo.bar".to_string(), + response: Response { + code: 250, + esc: [2, 1, 5], + message: "Great success!".to_string(), + }, + }); + message.recipients[1].flags |= RCPT_STATUS_CHANGED; + + message.domains[0].status = Status::TemporaryFailure(Error::UnexpectedResponse(HostResponse { + hostname: ErrorDetails { + entity: "mx2.example.org".to_string(), + details: "DATA".to_string(), + }, + response: Response { + code: 450, + esc: [4, 3, 1], + message: "Can't accept mail at this moment".to_string(), + }, + })); + message.domains[0].changed = true; + + message.domains[1].status = Status::TemporaryFailure(Error::ConnectionError(ErrorDetails { + entity: "mx.domain.org".to_string(), + details: "Connection timeout".to_string(), + })); + message.domains[1].changed = true; + message.domains[1].notify = Schedule::later(Duration::from_secs(30)); + message.domains[1].notify.inner = 321; + message.domains[1].retry = Schedule::later(Duration::from_secs(62)); + message.domains[1].retry.inner = 678; + + // Save changes + message.save_changes().await; + assert!(message.serialize_changes().is_empty()); + assert_msg_eq( + &message, + &Message::from_path(message.path.clone()).await.unwrap(), + ); + + // Remove + message.remove().await; + assert!(!message.path.exists()); +} + +fn assert_msg_eq(msg: &Message, other: &Message) { + assert_eq!(msg.id, other.id); + assert_eq!(msg.created, other.created); + assert_eq!(msg.path, other.path); + assert_eq!(msg.return_path, other.return_path); + assert_eq!(msg.return_path_lcase, other.return_path_lcase); + assert_eq!(msg.return_path_domain, other.return_path_domain); + assert_eq!(msg.recipients, other.recipients); + assert_eq!(msg.domains.len(), other.domains.len()); + for (domain, other) in msg.domains.iter().zip(other.domains.iter()) { + assert_eq!(domain.domain, other.domain); + assert_eq!(domain.retry.inner, other.retry.inner); + assert_eq!(domain.notify.inner, other.notify.inner); + assert_eq!(domain.status, other.status); + assert_instant_eq(domain.expires, other.expires); + assert_instant_eq(domain.retry.due, other.retry.due); + assert_instant_eq(domain.notify.due, other.notify.due); + } + assert_eq!(msg.flags, other.flags); + assert_eq!(msg.env_id, other.env_id); + assert_eq!(msg.priority, other.priority); + assert_eq!(msg.size, other.size); +} + +fn assert_instant_eq(instant: Instant, other: Instant) { + let dur = if instant > other { + instant - other + } else { + other - instant + } + .as_secs(); + assert!(dur <= 1, "dur {dur}"); +} diff --git a/tests/src/smtp/reporting/analyze.rs b/tests/src/smtp/reporting/analyze.rs new file mode 100644 index 00000000..b1614901 --- /dev/null +++ b/tests/src/smtp/reporting/analyze.rs @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{fs, sync::Arc, time::Duration}; + +use crate::smtp::{ + inbound::TestQueueEvent, make_temp_dir, session::TestSession, TestConfig, TestCore, +}; +use smtp::{ + config::{AddressMatch, IfBlock}, + core::{Core, Session}, +}; + +#[tokio::test] +async fn report_analyze() { + let mut core = Core::test(); + + // Create temp dir for queue + let mut qr = core.init_test_queue("smtp_analyze_report_test"); + let report_dir = make_temp_dir("smtp_report_incoming", true); + + let mut config = &mut core.session.config.rcpt; + config.relay = IfBlock::new(true); + let mut config = &mut core.session.config.data; + config.max_messages = IfBlock::new(1024); + let mut config = &mut core.report.config.analysis; + config.addresses = vec![ + AddressMatch::StartsWith("reports@".to_string()), + AddressMatch::EndsWith("@dmarc.foobar.org".to_string()), + AddressMatch::Equals("feedback@foobar.org".to_string()), + ]; + config.forward = false; + config.store = report_dir.temp_dir.clone().into(); + + // Create test message + let core = Arc::new(core); + let mut session = Session::test(core.clone()); + session.data.remote_ip = "10.0.0.1".parse().unwrap(); + session.eval_session_params().await; + session.ehlo("mx.test.org").await; + + let addresses = [ + "reports@foobar.org", + "rep@dmarc.foobar.org", + "feedback@foobar.org", + ]; + let mut ac = 0; + let mut total_reports_received = 0; + for (test, num_tests) in [("arf", 5), ("dmarc", 5), ("tls", 2)] { + for num_test in 1..=num_tests { + total_reports_received += 1; + session + .send_message( + "john@test.org", + &[addresses[ac % addresses.len()]], + &format!("report:{test}{num_test}"), + "250", + ) + .await; + qr.assert_empty_queue(); + ac += 1; + } + } + tokio::time::sleep(Duration::from_millis(200)).await; + + let mut total_reports = 0; + for entry in fs::read_dir(&report_dir.temp_dir).unwrap() { + let path = entry.unwrap().path(); + assert_ne!(fs::metadata(&path).unwrap().len(), 0); + total_reports += 1; + } + assert_eq!(total_reports, total_reports_received); + + // Test delivery to non-report addresses + session + .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") + .await; + qr.read_event().await.unwrap_message(); +} diff --git a/tests/src/smtp/reporting/dmarc.rs b/tests/src/smtp/reporting/dmarc.rs new file mode 100644 index 00000000..69d2d3f1 --- /dev/null +++ b/tests/src/smtp/reporting/dmarc.rs @@ -0,0 +1,189 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{ + net::IpAddr, + sync::Arc, + time::{Duration, Instant}, +}; + +use mail_auth::{ + common::parse::TxtRecordParser, + dmarc::Dmarc, + report::{ActionDisposition, Disposition, DmarcResult, Record, Report}, +}; + +use crate::smtp::{ + inbound::{sign::TextConfigContext, TestMessage, TestQueueEvent}, + make_temp_dir, + session::VerifyResponse, + ParseTestConfig, TestConfig, TestCore, +}; +use smtp::{ + config::{AggregateFrequency, ConfigContext, IfBlock}, + core::Core, + reporting::{ + dmarc::GenerateDmarcReport, + scheduler::{ReportType, Scheduler}, + DmarcEvent, + }, +}; + +#[tokio::test] +async fn report_dmarc() { + /*tracing::subscriber::set_global_default( + tracing_subscriber::FmtSubscriber::builder() + .with_max_level(tracing::Level::DEBUG) + .finish(), + ) + .unwrap();*/ + + // Create scheduler + let mut core = Core::test(); + let ctx = ConfigContext::default().parse_signatures(); + let temp_dir = make_temp_dir("smtp_report_dmarc_test", true); + let config = &mut core.report.config; + config.path = IfBlock::new(temp_dir.temp_dir.clone()); + config.hash = IfBlock::new(16); + config.dmarc_aggregate.sign = "['rsa']" + .parse_if::>(&ctx) + .map_if_block(&ctx.signers, "", "") + .unwrap(); + config.dmarc_aggregate.max_size = IfBlock::new(4096); + config.submitter = IfBlock::new("mx.example.org".to_string()); + config.dmarc_aggregate.address = IfBlock::new("reports@example.org".to_string()); + config.dmarc_aggregate.org_name = IfBlock::new("Foobar, Inc.".to_string().into()); + config.dmarc_aggregate.contact_info = + IfBlock::new("https://foobar.org/contact".to_string().into()); + let mut scheduler = Scheduler::default(); + + // Authorize external report for foobar.org + core.resolvers.dns.txt_add( + "foobar.org._report._dmarc.foobar.net", + Dmarc::parse(b"v=DMARC1;").unwrap(), + Instant::now() + Duration::from_secs(10), + ); + + // Create temp dir for queue + let mut qr = core.init_test_queue("smtp_report_dmarc_test"); + let core = Arc::new(core); + + // Schedule two events with a same policy and another one with a different policy + let dmarc_record = Arc::new( + Dmarc::parse( + b"v=DMARC1; p=quarantine; rua=mailto:reports@foobar.net,mailto:reports@example.net", + ) + .unwrap(), + ); + assert_eq!(dmarc_record.rua().len(), 2); + for _ in 0..2 { + scheduler + .schedule_dmarc( + Box::new(DmarcEvent { + domain: "foobar.org".to_string(), + report_record: Record::new() + .with_source_ip("192.168.1.2".parse().unwrap()) + .with_action_disposition(ActionDisposition::Pass) + .with_dmarc_dkim_result(DmarcResult::Pass) + .with_dmarc_spf_result(DmarcResult::Fail) + .with_envelope_from("hello@example.org") + .with_envelope_to("other@example.org") + .with_header_from("bye@example.org"), + dmarc_record: dmarc_record.clone(), + interval: AggregateFrequency::Weekly, + }), + &core, + ) + .await; + } + scheduler + .schedule_dmarc( + Box::new(DmarcEvent { + domain: "foobar.org".to_string(), + report_record: Record::new() + .with_source_ip("a:b:c::e:f".parse().unwrap()) + .with_action_disposition(ActionDisposition::Reject) + .with_dmarc_dkim_result(DmarcResult::Fail) + .with_dmarc_spf_result(DmarcResult::Pass), + dmarc_record: dmarc_record.clone(), + interval: AggregateFrequency::Weekly, + }), + &core, + ) + .await; + assert_eq!(scheduler.reports.len(), 1); + tokio::time::sleep(Duration::from_millis(200)).await; + let report_path; + match scheduler.reports.into_iter().next().unwrap() { + (ReportType::Dmarc(domain), ReportType::Dmarc(path)) => { + report_path = path.path.clone(); + core.generate_dmarc_report(domain, path); + } + _ => unreachable!(), + } + + // Expect report + let message = qr.read_event().await.unwrap_message(); + qr.assert_empty_queue(); + assert_eq!(message.recipients.len(), 1); + assert_eq!( + message.recipients.last().unwrap().address, + "reports@foobar.net" + ); + assert_eq!(message.return_path, "reports@example.org"); + message + .read_lines() + .assert_contains("DKIM-Signature: v=1; a=rsa-sha256; s=rsa; d=example.com;") + .assert_contains("To: ") + .assert_contains("Report Domain: foobar.org") + .assert_contains("Submitter: mx.example.org"); + + // Verify generated report + let report = Report::parse_rfc5322(message.read_message().as_bytes()).unwrap(); + assert_eq!(report.domain(), "foobar.org"); + assert_eq!(report.email(), "reports@example.org"); + assert_eq!(report.org_name(), "Foobar, Inc."); + assert_eq!( + report.extra_contact_info().unwrap(), + "https://foobar.org/contact" + ); + assert_eq!(report.p(), Disposition::Quarantine); + assert_eq!(report.records().len(), 2); + for record in report.records() { + let source_ip = record.source_ip().unwrap(); + if source_ip == "192.168.1.2".parse::().unwrap() { + assert_eq!(record.count(), 2); + assert_eq!(record.action_disposition(), ActionDisposition::Pass); + assert_eq!(record.envelope_from(), "hello@example.org"); + assert_eq!(record.header_from(), "bye@example.org"); + assert_eq!(record.envelope_to().unwrap(), "other@example.org"); + } else if source_ip == "a:b:c::e:f".parse::().unwrap() { + assert_eq!(record.count(), 1); + assert_eq!(record.action_disposition(), ActionDisposition::Reject); + } else { + panic!("unexpected ip {source_ip}"); + } + } + + assert!(!report_path.exists()); +} diff --git a/tests/src/smtp/reporting/mod.rs b/tests/src/smtp/reporting/mod.rs new file mode 100644 index 00000000..5cbbeabc --- /dev/null +++ b/tests/src/smtp/reporting/mod.rs @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +pub mod analyze; +pub mod dmarc; +pub mod scheduler; +pub mod tls; diff --git a/tests/src/smtp/reporting/scheduler.rs b/tests/src/smtp/reporting/scheduler.rs new file mode 100644 index 00000000..207c901b --- /dev/null +++ b/tests/src/smtp/reporting/scheduler.rs @@ -0,0 +1,274 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::sync::Arc; + +use mail_auth::{ + common::parse::TxtRecordParser, + dmarc::{Dmarc, URI}, + mta_sts::TlsRpt, + report::{ActionDisposition, Alignment, Disposition, DmarcResult, PolicyPublished, Record}, +}; +use tokio::fs; + +use crate::smtp::{make_temp_dir, TestConfig}; +use smtp::{ + config::{AggregateFrequency, IfBlock}, + core::Core, + reporting::{ + dmarc::DmarcFormat, + scheduler::{ReportType, Scheduler}, + DmarcEvent, PolicyType, TlsEvent, + }, +}; + +#[tokio::test] +async fn report_scheduler() { + /*tracing::subscriber::set_global_default( + tracing_subscriber::FmtSubscriber::builder() + .with_max_level(tracing::Level::DEBUG) + .finish(), + ) + .unwrap();*/ + + // Create scheduler + let mut core = Core::test(); + let temp_dir = make_temp_dir("smtp_report_scheduler_test", true); + let config = &mut core.report.config; + config.path = IfBlock::new(temp_dir.temp_dir.clone()); + config.hash = IfBlock::new(16); + config.dmarc_aggregate.max_size = IfBlock::new(500); + config.tls.max_size = IfBlock::new(550); + let mut scheduler = Scheduler::default(); + + // Schedule two events with a same policy and another one with a different policy + let dmarc_record = + Arc::new(Dmarc::parse(b"v=DMARC1; p=quarantine; rua=mailto:dmarc@foobar.org").unwrap()); + scheduler + .schedule_dmarc( + Box::new(DmarcEvent { + domain: "foobar.org".to_string(), + report_record: Record::new() + .with_source_ip("192.168.1.2".parse().unwrap()) + .with_action_disposition(ActionDisposition::Pass) + .with_dmarc_dkim_result(DmarcResult::Pass) + .with_dmarc_spf_result(DmarcResult::Fail) + .with_envelope_from("hello@example.org") + .with_envelope_to("other@example.org") + .with_header_from("bye@example.org"), + dmarc_record: dmarc_record.clone(), + interval: AggregateFrequency::Weekly, + }), + &core, + ) + .await; + + // No records should be added once the 550 bytes max size is reached + for _ in 0..10 { + scheduler + .schedule_dmarc( + Box::new(DmarcEvent { + domain: "foobar.org".to_string(), + report_record: Record::new() + .with_source_ip("192.168.1.2".parse().unwrap()) + .with_action_disposition(ActionDisposition::Pass) + .with_dmarc_dkim_result(DmarcResult::Pass) + .with_dmarc_spf_result(DmarcResult::Fail) + .with_envelope_from("hello@example.org") + .with_envelope_to("other@example.org") + .with_header_from("bye@example.org"), + dmarc_record: dmarc_record.clone(), + interval: AggregateFrequency::Weekly, + }), + &core, + ) + .await; + } + let dmarc_record = + Arc::new(Dmarc::parse(b"v=DMARC1; p=reject; rua=mailto:dmarc@foobar.org").unwrap()); + scheduler + .schedule_dmarc( + Box::new(DmarcEvent { + domain: "foobar.org".to_string(), + report_record: Record::new() + .with_source_ip("a:b:c::e:f".parse().unwrap()) + .with_action_disposition(ActionDisposition::Reject) + .with_dmarc_dkim_result(DmarcResult::Fail) + .with_dmarc_spf_result(DmarcResult::Pass), + dmarc_record: dmarc_record.clone(), + interval: AggregateFrequency::Weekly, + }), + &core, + ) + .await; + + // Schedule TLS event + let tls_record = Arc::new(TlsRpt::parse(b"v=TLSRPTv1;rua=mailto:reports@foobar.org").unwrap()); + scheduler + .schedule_tls( + Box::new(TlsEvent { + domain: "foobar.org".to_string(), + policy: PolicyType::Tlsa(None), + failure: None, + tls_record: tls_record.clone(), + interval: AggregateFrequency::Daily, + }), + &core, + ) + .await; + scheduler + .schedule_tls( + Box::new(TlsEvent { + domain: "foobar.org".to_string(), + policy: PolicyType::Tlsa(None), + failure: None, + tls_record: tls_record.clone(), + interval: AggregateFrequency::Daily, + }), + &core, + ) + .await; + scheduler + .schedule_tls( + Box::new(TlsEvent { + domain: "foobar.org".to_string(), + policy: PolicyType::Sts(None), + failure: None, + tls_record: tls_record.clone(), + interval: AggregateFrequency::Daily, + }), + &core, + ) + .await; + scheduler + .schedule_tls( + Box::new(TlsEvent { + domain: "foobar.org".to_string(), + policy: PolicyType::None, + failure: None, + tls_record: tls_record.clone(), + interval: AggregateFrequency::Daily, + }), + &core, + ) + .await; + + // Verify sizes and counts + let mut total_tls = 0; + let mut total_tls_policies = 0; + let mut total_dmarc_policies = 0; + for report in scheduler.reports.values() { + match report { + ReportType::Dmarc(r) => { + assert!(r.size <= 550, "{}", r.size); + assert_eq!(fs::metadata(&r.path).await.unwrap().len() as usize, r.size); + assert_eq!(r.deliver_at, AggregateFrequency::Weekly); + total_dmarc_policies += 1; + } + ReportType::Tls(r) => { + total_tls += 1; + total_tls_policies += r.path.len(); + assert!(r.size <= 550); + assert_eq!(r.deliver_at, AggregateFrequency::Daily); + let mut sizes = 0; + for p in &r.path { + sizes += fs::metadata(&p.inner).await.unwrap().len() as usize; + } + assert_eq!(r.size, sizes); + } + } + } + assert_eq!(total_tls, 1); + assert_eq!(total_tls_policies, 3); + assert_eq!(total_dmarc_policies, 2); + + // Verify deserialized report queue + let mut scheduler_deser = core.report.read_reports().await; + for (key, value) in scheduler.reports { + let a = Some(value); + let b = scheduler_deser.reports.remove(&key); + match (&a, &b) { + (Some(ReportType::Tls(a)), Some(ReportType::Tls(b))) => { + assert_eq!(a.created, b.created); + assert_eq!(a.size, b.size); + assert_eq!(a.deliver_at, b.deliver_at); + assert_eq!(a.path.len(), b.path.len()); + for p in &a.path { + assert!(b.path.contains(p)); + } + for p in &b.path { + assert!(a.path.contains(p)); + } + } + _ => { + assert_eq!(a, b, "failed for {key:?}"); + } + } + } + assert_eq!(scheduler.main.len(), scheduler_deser.main.len()); +} + +#[test] +fn report_strip_json() { + let mut d = DmarcFormat { + rua: vec![ + URI { + uri: "hello".to_string(), + max_size: 0, + }, + URI { + uri: "world".to_string(), + max_size: 0, + }, + ], + policy: PolicyPublished { + domain: "example.org".to_string(), + version_published: None, + adkim: Alignment::Relaxed, + aspf: Alignment::Strict, + p: Disposition::Quarantine, + sp: Disposition::Reject, + testing: false, + fo: None, + }, + records: vec![Record::default() + .with_count(1) + .with_envelope_from("domain.net") + .with_envelope_to("other.org")], + }; + let mut s = serde_json::to_string(&d).unwrap(); + s.truncate(s.len() - 2); + + let r = Record::default() + .with_count(2) + .with_envelope_from("otherdomain.net") + .with_envelope_to("otherother.org"); + let rs = serde_json::to_string(&r).unwrap(); + + d.records.push(r); + + assert_eq!( + serde_json::from_str::(&format!("{s},{rs}]}}")).unwrap(), + d + ); +} diff --git a/tests/src/smtp/reporting/tls.rs b/tests/src/smtp/reporting/tls.rs new file mode 100644 index 00000000..55e33c55 --- /dev/null +++ b/tests/src/smtp/reporting/tls.rs @@ -0,0 +1,262 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{io::Read, sync::Arc, time::Duration}; + +use mail_auth::{ + common::parse::TxtRecordParser, + flate2::read::GzDecoder, + mta_sts::TlsRpt, + report::tlsrpt::{FailureDetails, PolicyType, ResultType, TlsReport}, +}; + +use crate::smtp::{ + inbound::{sign::TextConfigContext, TestMessage, TestQueueEvent}, + make_temp_dir, + session::VerifyResponse, + ParseTestConfig, TestConfig, TestCore, +}; +use smtp::{ + config::{AggregateFrequency, ConfigContext, IfBlock}, + core::Core, + reporting::{ + scheduler::{ReportType, Scheduler}, + tls::{GenerateTlsReport, TLS_HTTP_REPORT}, + TlsEvent, + }, +}; + +#[tokio::test] +async fn report_tls() { + /*tracing::subscriber::set_global_default( + tracing_subscriber::FmtSubscriber::builder() + .with_max_level(tracing::Level::DEBUG) + .finish(), + ) + .unwrap();*/ + + // Create scheduler + let mut core = Core::test(); + let ctx = ConfigContext::default().parse_signatures(); + let temp_dir = make_temp_dir("smtp_report_tls_test", true); + let config = &mut core.report.config; + config.path = IfBlock::new(temp_dir.temp_dir.clone()); + config.hash = IfBlock::new(16); + config.tls.sign = "['rsa']" + .parse_if::>(&ctx) + .map_if_block(&ctx.signers, "", "") + .unwrap(); + config.tls.max_size = IfBlock::new(4096); + config.submitter = IfBlock::new("mx.example.org".to_string()); + config.tls.address = IfBlock::new("reports@example.org".to_string()); + config.tls.org_name = IfBlock::new("Foobar, Inc.".to_string().into()); + config.tls.contact_info = IfBlock::new("https://foobar.org/contact".to_string().into()); + let mut scheduler = Scheduler::default(); + + // Create temp dir for queue + let mut qr = core.init_test_queue("smtp_report_tls_test"); + let core = Arc::new(core); + + // Schedule TLS reports to be delivered via email + let tls_record = Arc::new(TlsRpt::parse(b"v=TLSRPTv1;rua=mailto:reports@foobar.org").unwrap()); + + for _ in 0..2 { + // Add two successful records + scheduler + .schedule_tls( + Box::new(TlsEvent { + domain: "foobar.org".to_string(), + policy: smtp::reporting::PolicyType::None, + failure: None, + tls_record: tls_record.clone(), + interval: AggregateFrequency::Daily, + }), + &core, + ) + .await; + } + + for (policy, rt) in [ + ( + smtp::reporting::PolicyType::None, + ResultType::CertificateExpired, + ), + ( + smtp::reporting::PolicyType::Tlsa(None), + ResultType::TlsaInvalid, + ), + ( + smtp::reporting::PolicyType::Sts(None), + ResultType::StsPolicyFetchError, + ), + ( + smtp::reporting::PolicyType::Sts(None), + ResultType::StsPolicyInvalid, + ), + ] { + scheduler + .schedule_tls( + Box::new(TlsEvent { + domain: "foobar.org".to_string(), + policy, + failure: FailureDetails::new(rt).into(), + tls_record: tls_record.clone(), + interval: AggregateFrequency::Daily, + }), + &core, + ) + .await; + } + + // Wait for flush + tokio::time::sleep(Duration::from_millis(200)).await; + + assert_eq!(scheduler.reports.len(), 1); + let mut report_path = Vec::new(); + match scheduler.reports.into_iter().next().unwrap() { + (ReportType::Tls(domain), ReportType::Tls(path)) => { + for p in &path.path { + report_path.push(p.inner.clone()); + } + core.generate_tls_report(domain, path); + } + _ => unreachable!(), + } + + // Expect report + let message = qr.read_event().await.unwrap_message(); + assert_eq!( + message.recipients.last().unwrap().address, + "reports@foobar.org" + ); + assert_eq!(message.return_path, "reports@example.org"); + message + .read_lines() + .assert_contains("DKIM-Signature: v=1; a=rsa-sha256; s=rsa; d=example.com;") + .assert_contains("To: ") + .assert_contains("Report Domain: foobar.org") + .assert_contains("Submitter: mx.example.org"); + + // Verify generated report + let report = TlsReport::parse_rfc5322(message.read_message().as_bytes()).unwrap(); + assert_eq!(report.organization_name.unwrap(), "Foobar, Inc."); + assert_eq!(report.contact_info.unwrap(), "https://foobar.org/contact"); + assert_eq!(report.policies.len(), 3); + let mut seen = [false; 3]; + for policy in report.policies { + match policy.policy.policy_type { + PolicyType::Tlsa => { + seen[0] = true; + assert_eq!(policy.summary.total_failure, 1); + assert_eq!(policy.summary.total_success, 0); + assert_eq!(policy.policy.policy_domain, "foobar.org"); + assert_eq!(policy.failure_details.len(), 1); + assert_eq!( + policy.failure_details.first().unwrap().result_type, + ResultType::TlsaInvalid + ); + } + PolicyType::Sts => { + seen[1] = true; + assert_eq!(policy.summary.total_failure, 2); + assert_eq!(policy.summary.total_success, 0); + assert_eq!(policy.policy.policy_domain, "foobar.org"); + assert_eq!(policy.failure_details.len(), 2); + assert!(policy + .failure_details + .iter() + .any(|d| d.result_type == ResultType::StsPolicyFetchError)); + assert!(policy + .failure_details + .iter() + .any(|d| d.result_type == ResultType::StsPolicyInvalid)); + } + PolicyType::NoPolicyFound => { + seen[2] = true; + assert_eq!(policy.summary.total_failure, 1); + assert_eq!(policy.summary.total_success, 2); + assert_eq!(policy.policy.policy_domain, "foobar.org"); + assert_eq!(policy.failure_details.len(), 1); + assert_eq!( + policy.failure_details.first().unwrap().result_type, + ResultType::CertificateExpired + ); + } + PolicyType::Other => unreachable!(), + } + } + + assert!(seen[0]); + assert!(seen[1]); + assert!(seen[2]); + + for path in report_path { + assert!(!path.exists()); + } + + // Schedule TLS reports to be delivered via https + let mut scheduler = Scheduler::default(); + let tls_record = Arc::new(TlsRpt::parse(b"v=TLSRPTv1;rua=https://127.0.0.1/tls").unwrap()); + + for _ in 0..2 { + // Add two successful records + scheduler + .schedule_tls( + Box::new(TlsEvent { + domain: "foobar.org".to_string(), + policy: smtp::reporting::PolicyType::None, + failure: None, + tls_record: tls_record.clone(), + interval: AggregateFrequency::Daily, + }), + &core, + ) + .await; + } + + let mut report_path = Vec::new(); + match scheduler.reports.into_iter().next().unwrap() { + (ReportType::Tls(domain), ReportType::Tls(path)) => { + for p in &path.path { + report_path.push(p.inner.clone()); + } + core.generate_tls_report(domain, path); + } + _ => unreachable!(), + } + tokio::time::sleep(Duration::from_millis(200)).await; + + // Uncompress report + let gz_report = TLS_HTTP_REPORT.lock(); + let mut file = GzDecoder::new(&gz_report[..]); + let mut buf = Vec::new(); + file.read_to_end(&mut buf).unwrap(); + let report = TlsReport::parse_json(&buf).unwrap(); + assert_eq!(report.organization_name.unwrap(), "Foobar, Inc."); + assert_eq!(report.contact_info.unwrap(), "https://foobar.org/contact"); + assert_eq!(report.policies.len(), 1); + + for path in report_path { + assert!(!path.exists()); + } +} diff --git a/tests/src/smtp/session.rs b/tests/src/smtp/session.rs new file mode 100644 index 00000000..1055c5ee --- /dev/null +++ b/tests/src/smtp/session.rs @@ -0,0 +1,347 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{path::PathBuf, sync::Arc}; + +use tokio::{ + io::{AsyncRead, AsyncWrite}, + sync::watch, +}; + +use smtp::{ + core::{Core, Session, SessionAddress, SessionData, SessionParameters, State}, + inbound::IsTls, +}; +use utils::{ + config::ServerProtocol, + listener::{limiter::ConcurrencyLimiter, ServerInstance}, +}; + +use super::TestConfig; + +pub struct DummyIo { + pub tx_buf: Vec, + pub rx_buf: Vec, + pub tls: bool, +} + +impl AsyncRead for DummyIo { + fn poll_read( + mut self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + buf: &mut tokio::io::ReadBuf<'_>, + ) -> std::task::Poll> { + if !self.rx_buf.is_empty() { + buf.put_slice(&self.rx_buf); + self.rx_buf.clear(); + std::task::Poll::Ready(Ok(())) + } else { + std::task::Poll::Pending + } + } +} + +impl AsyncWrite for DummyIo { + fn poll_write( + mut self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + buf: &[u8], + ) -> std::task::Poll> { + self.tx_buf.extend_from_slice(buf); + std::task::Poll::Ready(Ok(buf.len())) + } + + fn poll_flush( + self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::task::Poll::Ready(Ok(())) + } + + fn poll_shutdown( + self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + ) -> std::task::Poll> { + std::task::Poll::Ready(Ok(())) + } +} + +impl IsTls for DummyIo { + fn is_tls(&self) -> bool { + self.tls + } + + fn write_tls_header(&self, _headers: &mut Vec) {} +} + +impl Unpin for DummyIo {} + +#[async_trait::async_trait] +pub trait TestSession { + fn test(core: impl Into>) -> Self; + fn response(&mut self) -> Vec; + fn write_rx(&mut self, data: &str); + async fn rset(&mut self); + async fn cmd(&mut self, cmd: &str, expected_code: &str) -> Vec; + async fn ehlo(&mut self, host: &str) -> Vec; + async fn mail_from(&mut self, from: &str, expected_code: &str); + async fn rcpt_to(&mut self, to: &str, expected_code: &str); + async fn data(&mut self, data: &str, expected_code: &str); + async fn send_message(&mut self, from: &str, to: &[&str], data: &str, expected_code: &str); + async fn test_builder(&self); +} + +#[async_trait::async_trait] +impl TestSession for Session { + fn test(core: impl Into>) -> Self { + Self { + state: State::default(), + instance: Arc::new(ServerInstance::test()), + core: core.into(), + span: tracing::info_span!("test"), + stream: DummyIo { + rx_buf: vec![], + tx_buf: vec![], + tls: false, + }, + data: SessionData::new("127.0.0.1".parse().unwrap(), "127.0.0.1".parse().unwrap()), + params: SessionParameters::default(), + in_flight: vec![], + } + } + + fn response(&mut self) -> Vec { + if !self.stream.tx_buf.is_empty() { + let response = std::str::from_utf8(&self.stream.tx_buf) + .unwrap() + .split("\r\n") + .filter_map(|r| { + if !r.is_empty() { + r.to_string().into() + } else { + None + } + }) + .collect::>(); + self.stream.tx_buf.clear(); + response + } else { + panic!("There was no response."); + } + } + + fn write_rx(&mut self, data: &str) { + self.stream.rx_buf.extend_from_slice(data.as_bytes()); + } + + async fn rset(&mut self) { + self.ingest(b"RSET\r\n").await.unwrap(); + self.response().assert_code("250"); + } + + async fn cmd(&mut self, cmd: &str, expected_code: &str) -> Vec { + self.ingest(format!("{cmd}\r\n").as_bytes()).await.unwrap(); + self.response().assert_code(expected_code) + } + + async fn ehlo(&mut self, host: &str) -> Vec { + self.ingest(format!("EHLO {host}\r\n").as_bytes()) + .await + .unwrap(); + self.response().assert_code("250") + } + + async fn mail_from(&mut self, from: &str, expected_code: &str) { + self.ingest( + if !from.starts_with('<') { + format!("MAIL FROM:<{from}>\r\n") + } else { + format!("MAIL FROM:{from}\r\n") + } + .as_bytes(), + ) + .await + .unwrap(); + self.response().assert_code(expected_code); + } + + async fn rcpt_to(&mut self, to: &str, expected_code: &str) { + self.ingest( + if !to.starts_with('<') { + format!("RCPT TO:<{to}>\r\n") + } else { + format!("RCPT TO:{to}\r\n") + } + .as_bytes(), + ) + .await + .unwrap(); + self.response().assert_code(expected_code); + } + + async fn data(&mut self, data: &str, expected_code: &str) { + self.ingest(b"DATA\r\n").await.unwrap(); + self.response().assert_code("354"); + if let Some(file) = data.strip_prefix("test:") { + self.ingest(load_test_message(file, "messages").as_bytes()) + .await + .unwrap(); + } else if let Some(file) = data.strip_prefix("report:") { + self.ingest(load_test_message(file, "reports").as_bytes()) + .await + .unwrap(); + } else { + self.ingest(data.as_bytes()).await.unwrap(); + } + self.ingest(b"\r\n.\r\n").await.unwrap(); + self.response().assert_code(expected_code); + } + + async fn send_message(&mut self, from: &str, to: &[&str], data: &str, expected_code: &str) { + self.mail_from(from, "250").await; + for to in to { + self.rcpt_to(to, "250").await; + } + self.data(data, expected_code).await; + } + + async fn test_builder(&self) { + let message = self + .build_message( + SessionAddress { + address: "bill@foobar.org".to_string(), + address_lcase: "bill@foobar.org".to_string(), + domain: "foobar.org".to_string(), + flags: 123, + dsn_info: "envelope1".to_string().into(), + }, + vec![ + SessionAddress { + address: "a@foobar.org".to_string(), + address_lcase: "a@foobar.org".to_string(), + domain: "foobar.org".to_string(), + flags: 1, + dsn_info: None, + }, + SessionAddress { + address: "b@test.net".to_string(), + address_lcase: "b@test.net".to_string(), + domain: "test.net".to_string(), + flags: 2, + dsn_info: None, + }, + SessionAddress { + address: "c@foobar.org".to_string(), + address_lcase: "c@foobar.org".to_string(), + domain: "foobar.org".to_string(), + flags: 3, + dsn_info: None, + }, + SessionAddress { + address: "d@test.net".to_string(), + address_lcase: "d@test.net".to_string(), + domain: "test.net".to_string(), + flags: 4, + dsn_info: None, + }, + ], + ) + .await; + assert_eq!( + message + .domains + .iter() + .map(|d| d.domain.clone()) + .collect::>(), + vec!["foobar.org".to_string(), "test.net".to_string()] + ); + let rcpts = ["a@foobar.org", "b@test.net", "c@foobar.org", "d@test.net"]; + let domain_idx = [0, 1, 0, 1]; + for rcpt in &message.recipients { + let idx = (rcpt.flags - 1) as usize; + assert_eq!(rcpts[idx], rcpt.address); + assert_eq!(domain_idx[idx], rcpt.domain_idx); + } + } +} + +pub fn load_test_message(file: &str, test: &str) -> String { + let mut test_file = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + test_file.push("resources"); + test_file.push("smtp"); + test_file.push(test); + test_file.push(format!("{file}.eml")); + std::fs::read_to_string(test_file).unwrap() +} + +pub trait VerifyResponse { + fn assert_code(self, expected_code: &str) -> Self; + fn assert_contains(self, expected_text: &str) -> Self; + fn assert_not_contains(self, expected_text: &str) -> Self; +} + +impl VerifyResponse for Vec { + fn assert_code(self, expected_code: &str) -> Self { + if self.last().expect("response").starts_with(expected_code) { + self + } else { + panic!("Expected {:?} but got {}.", expected_code, self.join("\n")); + } + } + + fn assert_contains(self, expected_text: &str) -> Self { + if self.iter().any(|line| line.contains(expected_text)) { + self + } else { + panic!("Expected {:?} but got {}.", expected_text, self.join("\n")); + } + } + + fn assert_not_contains(self, expected_text: &str) -> Self { + if !self.iter().any(|line| line.contains(expected_text)) { + self + } else { + panic!( + "Not expecting {:?} but got it {}.", + expected_text, + self.join("\n") + ); + } + } +} + +impl TestConfig for ServerInstance { + fn test() -> Self { + Self { + id: "smtp".to_string(), + listener_id: 1, + hostname: "mx.example.org".to_string(), + protocol: ServerProtocol::Smtp, + data: "220 mx.example.org at your service.\r\n".to_string(), + tls_acceptor: None, + is_tls_implicit: false, + limiter: ConcurrencyLimiter::new(100), + shutdown_rx: watch::channel(false).1, + } + } +}