diff --git a/CHANGELOG.md b/CHANGELOG.md index 8873bf90..6ecd1021 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ All notable changes to this project will be documented in this file. This projec If you are upgrading from v0.16.x, replace the binary (or run `docker pull`). If you are upgrading from v0.15.x and below, please read the [upgrading documentation](https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING/v0_16.md) for more information on how to upgrade from previous versions. ## Added +- International Domain Names (IDN) support (#207). - OAuth: - OAuth Profile for Open Public Clients ([draft-ietf-mailmaint-oauth-public](https://datatracker.ietf.org/doc/draft-ietf-mailmaint-oauth-public/)) - Client secret verification for confidential clients. diff --git a/Cargo.lock b/Cargo.lock index 54b95579..88e21ebe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8877,6 +8877,7 @@ dependencies = [ "form_urlencoded", "futures", "http-body-util", + "idna", "mail-auth", "nohash-hasher", "parking_lot", diff --git a/crates/common/src/cache/principals.rs b/crates/common/src/cache/principals.rs index a39a2546..17f8ee81 100644 --- a/crates/common/src/cache/principals.rs +++ b/crates/common/src/cache/principals.rs @@ -41,9 +41,14 @@ use store::{ }; use trc::{AddContext, StoreEvent}; use types::id::Id; +use utils::DomainPart; impl Server { pub async fn domain(&self, domain: &str) -> trc::Result>> { + let Some(domain) = domain.to_ascii_domain() else { + return Ok(None); + }; + let domain = domain.as_ref(); let domain_names = &self.inner.cache.domain_names; if let Some(domain_id) = domain_names.get(domain) { diff --git a/crates/common/src/config/server/tls.rs b/crates/common/src/config/server/tls.rs index da48a34b..bce65f06 100644 --- a/crates/common/src/config/server/tls.rs +++ b/crates/common/src/config/server/tls.rs @@ -189,6 +189,17 @@ pub(crate) fn build_certified_key( pub(crate) fn build_self_signed_cert( domains: impl Into>, ) -> Result { + let domains = domains + .into() + .into_iter() + .map(|domain| { + if domain.is_ascii() { + domain + } else { + idna::domain_to_ascii(&domain).unwrap_or(domain) + } + }) + .collect::>(); let rcgen::CertifiedKey { cert, signing_key } = generate_simple_self_signed(domains) .map_err(|err| format!("Failed to generate self-signed certificate: {err}",))?; build_certified_key( diff --git a/crates/jmap/src/registry/mapping/bootstrap.rs b/crates/jmap/src/registry/mapping/bootstrap.rs index 351f81c9..6264522e 100644 --- a/crates/jmap/src/registry/mapping/bootstrap.rs +++ b/crates/jmap/src/registry/mapping/bootstrap.rs @@ -42,7 +42,7 @@ use store::{ write::{AnyKey, BatchBuilder}, }; use types::id::Id; -use utils::is_valid_domain; +use utils::{DomainPart, is_valid_domain}; pub(crate) async fn bootstrap_get( mut get: RegistryGetResponse<'_>, @@ -124,8 +124,20 @@ pub(crate) async fn bootstrap_set( } // Validate domain name and hostname - let server_hostname = bootstrap.server_hostname.trim().to_lowercase(); - let domain_name = bootstrap.default_domain.trim().to_lowercase(); + let server_hostname = bootstrap + .server_hostname + .trim() + .to_lowercase() + .to_ascii_domain() + .map(|hostname| hostname.into_owned()) + .unwrap_or_default(); + let domain_name = bootstrap + .default_domain + .trim() + .to_lowercase() + .to_ascii_domain() + .map(|domain| domain.into_owned()) + .unwrap_or_default(); if !is_valid_domain(&server_hostname) { set.response.not_updated.append( id, diff --git a/crates/smtp/src/inbound/data.rs b/crates/smtp/src/inbound/data.rs index c6a881f4..3f5896bb 100644 --- a/crates/smtp/src/inbound/data.rs +++ b/crates/smtp/src/inbound/data.rs @@ -769,7 +769,10 @@ impl Session { .map_or(0, |d| d.as_secs()); let mut message = Message { created, - return_path: mail_from.address.to_lowercase_domain().into_boxed_str(), + return_path: mail_from + .address + .to_lowercase_address(false) + .into_boxed_str(), recipients: Vec::with_capacity(rcpt_to.len()), flags: mail_from.flags, priority: self.data.priority, diff --git a/crates/smtp/src/inbound/mail.rs b/crates/smtp/src/inbound/mail.rs index d732e1e1..571c82f4 100644 --- a/crates/smtp/src/inbound/mail.rs +++ b/crates/smtp/src/inbound/mail.rs @@ -109,7 +109,7 @@ impl Session { } let (address, address_lcase, domain) = if !from.address.is_empty() { - let address_lcase = from.address.to_lowercase(); + let address_lcase = from.address.to_lowercase_address(true); let domain = address_lcase.domain_part().into(); (from.address.into_owned(), address_lcase, domain) } else { @@ -218,7 +218,7 @@ impl Session { ); if new_address.contains('@') { - mail_from.address_lcase = new_address.to_lowercase(); + mail_from.address_lcase = new_address.to_lowercase_address(true); mail_from.domain = mail_from.address_lcase.domain_part().into(); mail_from.address = new_address; } else if new_address.is_empty() { diff --git a/crates/smtp/src/inbound/rcpt.rs b/crates/smtp/src/inbound/rcpt.rs index 15a68919..15a22d91 100644 --- a/crates/smtp/src/inbound/rcpt.rs +++ b/crates/smtp/src/inbound/rcpt.rs @@ -70,7 +70,7 @@ impl Session { } // Build RCPT - let address_lcase = to.address.to_lowercase(); + let address_lcase = to.address.to_lowercase_address(true); let rcpt = SessionAddress { domain: address_lcase.domain_part().into(), address_lcase, @@ -167,7 +167,7 @@ impl Session { ); if new_address.contains('@') { - rcpt.address_lcase = new_address.to_lowercase(); + rcpt.address_lcase = new_address.to_lowercase_address(true); rcpt.domain = rcpt.address_lcase.domain_part().into(); rcpt.address = new_address; } diff --git a/crates/smtp/src/inbound/vrfy.rs b/crates/smtp/src/inbound/vrfy.rs index d3ce4c9e..29bbc134 100644 --- a/crates/smtp/src/inbound/vrfy.rs +++ b/crates/smtp/src/inbound/vrfy.rs @@ -8,13 +8,14 @@ use crate::core::Session; use common::network::{RcptResolution, SessionStream}; use std::{borrow::Cow, fmt::Write}; use trc::SmtpEvent; +use utils::DomainPart; impl Session { pub async fn handle_vrfy(&mut self, address: Cow<'_, str>) -> Result<(), ()> { if self.params.can_vrfy { match self .server - .rcpt_resolve(&address.to_lowercase(), self.data.session_id) + .rcpt_resolve(&address.to_lowercase_address(true), self.data.session_id) .await { Ok(RcptResolution::Accept | RcptResolution::Rewrite(_)) => { @@ -66,7 +67,7 @@ impl Session { if self.params.can_expn { match self .server - .rcpt_resolve(&address.to_lowercase(), self.data.session_id) + .rcpt_resolve(&address.to_lowercase_address(true), self.data.session_id) .await { Ok(RcptResolution::Expand(addresses)) => { diff --git a/crates/smtp/src/queue/mod.rs b/crates/smtp/src/queue/mod.rs index 2d0eaae3..23577916 100644 --- a/crates/smtp/src/queue/mod.rs +++ b/crates/smtp/src/queue/mod.rs @@ -410,7 +410,7 @@ pub fn instant_to_timestamp(now: Instant, time: Instant) -> u64 { impl Recipient { pub fn new(address: impl AsRef) -> Self { Recipient { - address: address.to_lowercase_domain().into_boxed_str(), + address: address.to_lowercase_address(false).into_boxed_str(), status: Status::Scheduled, flags: 0, orcpt: None, diff --git a/crates/smtp/src/queue/spool.rs b/crates/smtp/src/queue/spool.rs index f0c0f19c..5285b32a 100644 --- a/crates/smtp/src/queue/spool.rs +++ b/crates/smtp/src/queue/spool.rs @@ -93,7 +93,7 @@ impl SmtpSpool for Server { span_id, message: Message { created, - return_path: return_path.to_lowercase_domain().into_boxed_str(), + return_path: return_path.to_lowercase_address(false).into_boxed_str(), recipients: Vec::with_capacity(1), flags: 0, env_id: None, diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index 76b7831a..667ace6f 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -31,6 +31,7 @@ blake3 = "1.3.3" http-body-util = "0.1.0" form_urlencoded = "1.1.0" psl = "2" +idna = "1.0" quick_cache = "0.6.9" fast-float = "0.2.0" rkyv = { version = "0.8.10", features = ["little_endian"] } diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index 84484627..76ea7cd0 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -26,6 +26,7 @@ use futures::StreamExt; pub use reqwest::Client; use reqwest::Response; pub use reqwest::header::HeaderMap; +use std::borrow::Cow; use std::fmt::Write; pub trait HttpLimitResponse: Sync + Send { @@ -145,27 +146,42 @@ pub async fn wait_for_shutdown() { } pub trait DomainPart { - fn to_lowercase_domain(&self) -> String; + fn to_lowercase_address(&self, lower_local: bool) -> String; fn domain_part(&self) -> &str; fn try_domain_part(&self) -> Option<&str>; fn try_local_part(&self) -> Option<&str>; + fn to_ascii_domain(&self) -> Option>; } impl> DomainPart for T { - fn to_lowercase_domain(&self) -> String { + fn to_lowercase_address(&self, lower_local: bool) -> String { let address = self.as_ref(); if let Some((local, domain)) = address.rsplit_once('@') { let mut address = String::with_capacity(address.len()); - address.push_str(local); - address.push('@'); - for ch in domain.chars() { - for ch in ch.to_lowercase() { - address.push(ch); + if lower_local { + for ch in local.chars() { + for ch in ch.to_lowercase() { + address.push(ch); + } } + } else { + address.push_str(local); + } + address.push('@'); + if domain.is_ascii() { + for ch in domain.chars() { + for ch in ch.to_lowercase() { + address.push(ch); + } + } + } else { + let domain = + idna::domain_to_ascii(domain).unwrap_or_else(|_| domain.to_lowercase()); + address.push_str(&domain); } address } else { - address.to_string() + address.to_lowercase() } } @@ -186,6 +202,17 @@ impl> DomainPart for T { .map(|(_, d)| d) .unwrap_or_default() } + + #[inline(always)] + fn to_ascii_domain(&self) -> Option> { + let domain = self.as_ref(); + + if domain.is_ascii() { + Some(Cow::Borrowed(domain)) + } else { + idna::domain_to_ascii(domain).ok().map(Cow::Owned) + } + } } pub trait HexEncode { @@ -253,6 +280,8 @@ pub fn sanitize_email(email: &str) -> Option { } last_ch = NIL_CHAR; + let domain_start = result.len(); + let mut domain_is_ascii = true; for ch in chars { match ch { @@ -264,6 +293,9 @@ pub fn sanitize_email(email: &str) -> Option { } ' ' | '\x09'..='\x0d' => continue, _ => { + if !ch.is_ascii() { + domain_is_ascii = false; + } if ch.is_uppercase() { for ch in ch.to_lowercase() { result.push(ch); @@ -279,10 +311,20 @@ pub fn sanitize_email(email: &str) -> Option { last_ch = ch; } - if last_ch.is_alphanumeric() && is_valid_domain(&result) { - Some(result) + if !last_ch.is_alphanumeric() { + return None; + } + + if domain_is_ascii { + is_valid_domain(&result[domain_start..]).then_some(result) } else { - None + let domain = idna::domain_to_ascii(&result[domain_start..]).ok()?; + if !is_valid_domain(&domain) { + return None; + } + result.truncate(domain_start); + result.push_str(&domain); + Some(result) } } @@ -330,6 +372,7 @@ pub fn sanitize_domain(domain: &str) -> Option { let mut result = String::with_capacity(domain.len()); let mut found_dot = false; let mut last_ch = char::from(0); + let mut is_ascii = true; for ch in domain.chars() { if !ch.is_whitespace() { @@ -338,6 +381,8 @@ pub fn sanitize_domain(domain: &str) -> Option { if !(last_ch.is_alphanumeric() || last_ch == '-' || last_ch == '_') { return None; } + } else if !ch.is_ascii() { + is_ascii = false; } last_ch = ch; for ch in ch.to_lowercase() { @@ -346,10 +391,15 @@ pub fn sanitize_domain(domain: &str) -> Option { } } - if found_dot && last_ch != '.' && is_valid_domain(&result) { - Some(result) + if !(found_dot && last_ch != '.') { + return None; + } + + if is_ascii { + is_valid_domain(&result).then_some(result) } else { - None + let domain = idna::domain_to_ascii(&result).ok()?; + is_valid_domain(&domain).then_some(domain) } } @@ -372,3 +422,70 @@ pub fn is_valid_domain(domain: &str) -> bool { .rsplit_once('.') .is_some_and(|(_, tld)| RESERVED_TLDS.contains(&tld)) } + +#[cfg(test)] +mod tests { + use crate::DomainPart; + + use super::{sanitize_domain, sanitize_email}; + + #[test] + fn idn_domains_canonicalize_to_a_label() { + assert_eq!( + sanitize_domain("straß6.de").as_deref(), + Some("xn--stra6-oqa.de") + ); + assert_eq!( + sanitize_domain("STRASS.straß6.DE").as_deref(), + Some("strass.xn--stra6-oqa.de") + ); + assert_eq!( + sanitize_domain("münchen.de").as_deref(), + Some("xn--mnchen-3ya.de") + ); + } + + #[test] + fn a_label_and_ascii_domains_are_idempotent() { + assert_eq!( + sanitize_domain("xn--stra6-oqa.de").as_deref(), + Some("xn--stra6-oqa.de") + ); + assert_eq!( + sanitize_domain(&sanitize_domain("straß6.de").unwrap()).as_deref(), + Some("xn--stra6-oqa.de") + ); + assert_eq!( + sanitize_domain("Example.COM").as_deref(), + Some("example.com") + ); + } + + #[test] + fn email_domain_part_canonicalizes_local_part_preserved() { + assert_eq!( + sanitize_email("cornelius_strauss@straß6.de").as_deref(), + Some("cornelius_strauss@xn--stra6-oqa.de") + ); + assert_eq!( + sanitize_email("Foo.Bar@münchen.de").as_deref(), + Some("foo.bar@xn--mnchen-3ya.de") + ); + assert_eq!( + sanitize_email("user@example.com").as_deref(), + Some("user@example.com") + ); + } + + #[test] + fn to_ascii_domain_borrows_ascii_owns_idn() { + assert!(matches!( + "example.com".to_ascii_domain(), + Some(std::borrow::Cow::Borrowed(_)) + )); + assert!(matches!( + "straß6.de".to_ascii_domain(), + Some(std::borrow::Cow::Owned(_)) + )); + } +} diff --git a/tests/src/smtp/inbound/rcpt.rs b/tests/src/smtp/inbound/rcpt.rs index 33d71a73..d582cd25 100644 --- a/tests/src/smtp/inbound/rcpt.rs +++ b/tests/src/smtp/inbound/rcpt.rs @@ -48,6 +48,12 @@ async fn rcpt() { "Mike Foobar", &[], ), + ( + "cornelius@straß6.de", + "p4ssw0rd + extra safety", + "Cornelius Strauss", + &[], + ), ] { admin .create_user_account(name, secret, description, aliases, vec![]) @@ -189,4 +195,21 @@ async fn rcpt() { let rcpt = session.data.rcpt_to.last().unwrap(); assert!((rcpt.flags & (RCPT_NOTIFY_DELAY | RCPT_NOTIFY_SUCCESS | RCPT_NOTIFY_FAILURE)) != 0); assert_eq!(rcpt.dsn_info.as_ref().unwrap(), "Jane.Doe@Foobar.org"); + + let mut session = test.new_mta_session(); + session.data.remote_ip_str = "10.0.0.1".into(); + session.eval_session_params().await; + session.ehlo("mx1.foobar.org").await; + session.mail_from("idn@example.net", "250").await; + session.rcpt_to("cornelius@straß6.de", "250").await; + session.rcpt_to("cornelius@xn--stra6-oqa.de", "250").await; + assert_eq!(session.data.rcpt_to.len(), 2); + + let mut session = test.new_mta_session(); + session.data.remote_ip_str = "10.0.0.1".into(); + session.eval_session_params().await; + session.ehlo("mx1.foobar.org").await; + session.mail_from("idn2@example.net", "250").await; + session.rcpt_to("nobody@straß6.de", "550 5.1.2").await; + session.rcpt_to("nobody@xn--stra6-oqa.de", "550 5.1.2").await; }