From d8fa19d1056a94e53884922f6dd2fc6e1b8ec904 Mon Sep 17 00:00:00 2001 From: Maurus Decimus <11444311+mdecimus@users.noreply.github.com> Date: Wed, 1 Apr 2026 20:03:51 +0200 Subject: [PATCH] DNS, DKIM and ACME improvements - part 2 --- CHANGELOG.md | 7 +- Cargo.lock | 2 + crates/common/Cargo.toml | 1 + crates/common/src/cache/invalidate.rs | 20 +- crates/common/src/cache/principals.rs | 6 +- crates/common/src/config/network.rs | 183 ++++++- crates/common/src/config/server/tls.rs | 199 +------- crates/common/src/config/smtp/resolver.rs | 77 +-- crates/common/src/enterprise/config.rs | 2 +- crates/common/src/manager/defaults.rs | 4 +- crates/common/src/network/acme/account.rs | 12 +- crates/common/src/network/acme/directory.rs | 33 +- crates/common/src/network/acme/http.rs | 6 +- crates/common/src/network/acme/mod.rs | 29 +- crates/common/src/network/acme/order.rs | 136 ++--- crates/common/src/network/acme/renew.rs | 35 +- crates/common/src/network/acme/resolver.rs | 18 +- crates/common/src/network/autoconfig/mod.rs | 7 + crates/common/src/network/autoconfig/pacc.rs | 358 +++++++++++++ crates/common/src/network/dkim.rs | 80 ++- crates/common/src/network/dns/mod.rs | 9 + crates/common/src/network/dns/records.rs | 367 ++++++++++++++ .../src/network/{dns.rs => dns/resolve.rs} | 0 crates/common/src/network/dns/update.rs | 373 ++++++++++++++ crates/common/src/network/mod.rs | 1 + crates/common/src/network/tls.rs | 15 + crates/http/src/request.rs | 14 +- crates/jmap/src/registry/get.rs | 9 +- crates/jmap/src/registry/mapping/dkim.rs | 76 +-- crates/jmap/src/registry/mapping/domain.rs | 57 ++- crates/jmap/src/registry/mapping/task.rs | 2 +- crates/jmap/src/registry/set.rs | 22 +- crates/registry/src/utils/dkim.rs | 73 ++- crates/registry/src/utils/mod.rs | 6 +- crates/registry/src/utils/report.rs | 4 +- crates/registry/src/utils/task.rs | 6 +- crates/services/Cargo.toml | 1 + crates/services/src/task_manager/acme.rs | 74 +++ crates/services/src/task_manager/alarm.rs | 12 +- .../src/task_manager/destroy_account.rs | 2 +- crates/services/src/task_manager/dkim.rs | 476 ++++++++++++++++++ crates/services/src/task_manager/dns.rs | 33 ++ crates/services/src/task_manager/imip.rs | 2 +- crates/services/src/task_manager/index.rs | 12 +- .../services/src/task_manager/maintenance.rs | 7 +- crates/services/src/task_manager/manager.rs | 41 +- .../src/task_manager/merge_threads.rs | 4 +- crates/services/src/task_manager/mod.rs | 10 +- crates/services/src/task_manager/report.rs | 4 +- .../services/src/task_manager/restore_item.rs | 2 +- .../src/task_manager/spam_classifier.rs | 45 +- tests/src/directory/oidc.rs | 1 + tests/src/smtp/inbound/antispam.rs | 2 +- tests/src/smtp/inbound/sign.rs | 7 +- tests/src/system/tenant.rs | 10 +- 55 files changed, 2421 insertions(+), 573 deletions(-) create mode 100644 crates/common/src/network/autoconfig/mod.rs create mode 100644 crates/common/src/network/autoconfig/pacc.rs create mode 100644 crates/common/src/network/dns/mod.rs create mode 100644 crates/common/src/network/dns/records.rs rename crates/common/src/network/{dns.rs => dns/resolve.rs} (100%) create mode 100644 crates/common/src/network/dns/update.rs create mode 100644 crates/services/src/task_manager/acme.rs create mode 100644 crates/services/src/task_manager/dkim.rs create mode 100644 crates/services/src/task_manager/dns.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 13c0f8ac..e9be424b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,9 @@ This version includes **multiple breaking changes**. If you are upgrading from v - Store DKIM keys in the database (#1264) - DNS Management: - RFC2136 SIG0 support (#856) + - BunnyDNS provider support + - Porkbun provider support + - DNSimple provider support - Clustering: - Automatic cluster node ID generation and management. - Unified cluster management (#960) @@ -53,6 +56,7 @@ This version includes **multiple breaking changes**. If you are upgrading from v ## Changed - Replaced REST API with JMAP API (#2262 #959 #1480) +- Removed support for Authenticated Received Chain (ARC) sealing ([learn more](https://mailarchive.ietf.org/arch/msg/dmarc/KvX3-H1SL0Gh3IDl7FuR2hoR87M/)). - Directory: Removed `smtp`, `imap` and `memory` directory backends. ## Fixed @@ -63,7 +67,7 @@ This version includes **multiple breaking changes**. If you are upgrading from v - Recreated account cannot log in until server is restarted (#1469) - Subaddressing does not work for groups (#475) - New LDAP aliases are rejected (#1318). -- MTA +- MTA: - Relay to IP addresses (#838) - Duplicate delivery inverted check - SASL challenge responses include invalid `Go ahead` text @@ -80,7 +84,6 @@ This version includes **multiple breaking changes**. If you are upgrading from v - Spam filter: Skip invalid messages during training. - Calendar: Include minutes in localized invite templates (#2828) - ## [0.15.5] - 2026-02-14 If you are upgrading from v0.14.x and below, this version includes **multiple breaking changes**. Please read the [upgrading documentation](https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING/v0_15.md) for more information on how to upgrade from previous versions. diff --git a/Cargo.lock b/Cargo.lock index 87714271..2aabce1a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1117,6 +1117,7 @@ dependencies = [ "p384", "parking_lot", "pem", + "pkcs8", "privdrop", "prometheus", "proxy-header", @@ -7115,6 +7116,7 @@ dependencies = [ "common", "compact_str", "directory", + "dns-update", "email", "groupware", "hkdf", diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 180384d4..9e7cb168 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -84,6 +84,7 @@ rasn-cms = "0.10" rasn-pkix = "0.10" sequoia-openpgp = { version = "2.0", default-features = false, features = ["crypto-rust", "allow-experimental-crypto", "allow-variable-time-crypto"] } zxcvbn = "3.1.0" +pkcs8 = { version = "0.10.2", features = ["alloc", "std"] } [target.'cfg(unix)'.dependencies] privdrop = "0.5.3" diff --git a/crates/common/src/cache/invalidate.rs b/crates/common/src/cache/invalidate.rs index f062337c..4e980aa3 100644 --- a/crates/common/src/cache/invalidate.rs +++ b/crates/common/src/cache/invalidate.rs @@ -120,8 +120,13 @@ impl CacheInvalidationBuilder { } } - (ObjectInner::DkimSignature(_), ObjectInner::DkimSignature(_)) => { - self.invalidate(CacheInvalidation::DkimSignature(id)); + (ObjectInner::DkimSignature(current), ObjectInner::DkimSignature(new)) => { + let current_domain_id = current.domain_id().document_id(); + let new_domain_id = new.domain_id().document_id(); + self.invalidate(CacheInvalidation::DkimSignature(current_domain_id)); + if current_domain_id != new_domain_id { + self.invalidate(CacheInvalidation::DkimSignature(new_domain_id)); + } } (ObjectInner::Tenant(current), ObjectInner::Tenant(new)) => { @@ -172,8 +177,10 @@ impl CacheInvalidationBuilder { self.invalidate(CacheInvalidation::Domain(id)); self.invalidate(CacheInvalidation::DomainLogo(id)); } - ObjectInner::DkimSignature(_) => { - self.invalidate(CacheInvalidation::DkimSignature(id)); + ObjectInner::DkimSignature(object) => { + self.invalidate(CacheInvalidation::DkimSignature( + object.domain_id().document_id(), + )); } ObjectInner::Tenant(_) => { self.invalidate(CacheInvalidation::Tenant(id)); @@ -192,6 +199,11 @@ impl CacheInvalidationBuilder { pub fn invalidate(&mut self, change: CacheInvalidation) { self.changes.insert(change); } + + pub fn with_invalidation(mut self, change: CacheInvalidation) -> Self { + self.invalidate(change); + self + } } impl Server { diff --git a/crates/common/src/cache/principals.rs b/crates/common/src/cache/principals.rs index eba7abb6..2e4839f8 100644 --- a/crates/common/src/cache/principals.rs +++ b/crates/common/src/cache/principals.rs @@ -24,7 +24,7 @@ use crate::{ }; use registry::{ schema::{ - enums::{Locale, StorageQuota, TenantStorageQuota}, + enums::{DkimRotationStage, Locale, StorageQuota, TenantStorageQuota}, prelude::{ObjectType, Property}, structs::{ Account, DkimSignature, Domain, EncryptionAtRest, MailingList, MaskedEmail, @@ -804,7 +804,9 @@ impl Server { .await?; let mut signatures = Vec::with_capacity(ids.len()); for id in ids { - if let Some(signature) = self.registry().object::(id).await? { + if let Some(signature) = self.registry().object::(id).await? + && matches!(signature.stage(), DkimRotationStage::Active) + { match DkimSigner::new(domain.names[0].to_string(), signature).await { Ok(signer) => signatures.push(signer), Err(err) => { diff --git a/crates/common/src/config/network.rs b/crates/common/src/config/network.rs index 14efc137..e9aa4204 100644 --- a/crates/common/src/config/network.rs +++ b/crates/common/src/config/network.rs @@ -7,14 +7,24 @@ use super::*; use crate::{ expr::if_block::{BootstrapExprExt, IfBlock}, - network::security::Security, + network::{ + autoconfig::pacc::{ + Authentication, Configuration, HttpServer, Info, Logo, OAuthPublic, Protocols, + Provider, TextServer, + }, + security::Security, + }, }; use registry::schema::{ - enums::ClusterTaskType, + enums::{AcmeChallengeType, ClusterTaskType, ProviderInfo, ServiceProtocol}, prelude::ObjectType, - structs::{self, Asn, ClusterTaskGroup, HttpForm, Rate, SystemSettings, TaskManager}, + structs::{ + self, AcmeProvider, Asn, ClusterTaskGroup, HttpForm, MailExchanger, Rate, Service, + SystemSettings, TaskManager, + }, }; use std::{str::FromStr, time::Duration}; +use utils::map::vec_map::VecMap; #[derive(Clone)] pub struct Network { @@ -26,6 +36,16 @@ pub struct Network { pub contact_form: Option, pub asn_geo_lookup: AsnGeoLookupConfig, pub task_manager: TaskManager, + pub has_acme_tls_challenge: bool, + pub has_acme_http_challenge: bool, + pub info: NetworkInfo, +} + +#[derive(Clone)] +pub struct NetworkInfo { + pub pacc: String, + pub mxs: Vec, + pub services: VecMap, } #[derive(Clone)] @@ -131,6 +151,156 @@ impl ContactForm { impl Network { pub async fn parse(bp: &mut Bootstrap) -> Self { let system = bp.setting_infallible::().await; + let mut has_acme_tls_challenge = false; + let mut has_acme_http_challenge = false; + let mut has_acme_challenges = false; + + for provider in bp.list_infallible::().await { + match provider.object.challenge_type { + AcmeChallengeType::Http01 => has_acme_http_challenge = true, + AcmeChallengeType::TlsAlpn01 => has_acme_tls_challenge = true, + _ => {} + } + has_acme_challenges = true; + } + + if !has_acme_challenges { + // Assume this is an initial deployment and optimistically set both to true + // to avoid requiring a reload after ACME providers are added + has_acme_http_challenge = true; + has_acme_tls_challenge = true; + } + + let mut pacc = Configuration { + protocols: Protocols::default(), + authentication: Some(Authentication { + oauth_public: None, + password: true, + }), + info: Info { + provider: Provider { + name: "Stalwart".into(), + ..Default::default() + }, + ..Default::default() + }, + }; + + for (service, details) in &system.services { + let hostname = details + .hostname + .as_deref() + .unwrap_or(&system.default_hostname); + + match service { + ServiceProtocol::Jmap => { + pacc.authentication.as_mut().unwrap().oauth_public = OAuthPublic { + issuer: format!("https://{hostname}/",), + } + .into(); + pacc.protocols.jmap = HttpServer { + url: format!("https://{hostname}/jmap/session",), + } + .into(); + } + ServiceProtocol::Caldav => { + pacc.protocols.caldav = HttpServer { + url: format!("https://{hostname}/dav/cal/",), + } + .into(); + } + ServiceProtocol::Carddav => { + pacc.protocols.carddav = HttpServer { + url: format!("https://{hostname}/dav/card/",), + } + .into(); + } + ServiceProtocol::Webdav => { + pacc.protocols.webdav = HttpServer { + url: format!("https://{hostname}/dav/file/",), + } + .into(); + } + ServiceProtocol::Imap => { + pacc.protocols.imap = TextServer { + host: hostname.to_string(), + } + .into(); + } + ServiceProtocol::Pop3 => { + pacc.protocols.pop3 = TextServer { + host: hostname.to_string(), + } + .into(); + } + ServiceProtocol::Smtp => { + pacc.protocols.smtp = TextServer { + host: hostname.to_string(), + } + .into(); + } + ServiceProtocol::Managesieve => { + pacc.protocols.managesieve = TextServer { + host: hostname.to_string(), + } + .into(); + } + } + } + + for (tag, text) in system.provider_info { + match tag { + ProviderInfo::ProviderName => pacc.info.provider.name = text, + ProviderInfo::ProviderShortName => pacc.info.provider.short_name = Some(text), + ProviderInfo::UserDocumentation => { + pacc.info.help.get_or_insert_default().documentation = Some(text) + } + ProviderInfo::DeveloperDocumentation => { + pacc.info.help.get_or_insert_default().developer = Some(text) + } + ProviderInfo::ContactUri => { + pacc.info + .help + .get_or_insert_default() + .contact + .get_or_insert_default() + .push(text); + } + ProviderInfo::LogoUrl => { + let logo = pacc.info.provider.logo.get_or_insert_default(); + if logo.is_empty() { + logo.push(Logo { + url: text, + ..Default::default() + }); + } else { + logo[0].url = text; + } + } + ProviderInfo::LogoWidth => { + let logo = pacc.info.provider.logo.get_or_insert_default(); + if logo.is_empty() { + logo.push(Logo { + width: text.parse().ok(), + ..Default::default() + }); + } else { + logo[0].width = text.parse().ok(); + } + } + ProviderInfo::LogoHeight => { + let logo = pacc.info.provider.logo.get_or_insert_default(); + if logo.is_empty() { + logo.push(Logo { + height: text.parse().ok(), + ..Default::default() + }); + } else { + logo[0].height = text.parse().ok(); + } + } + } + } let mut network = Network { node_id: bp.node_id() as u64, @@ -141,6 +311,13 @@ impl Network { roles: ClusterRoles::default(), http: Http::parse(bp).await, task_manager: bp.setting_infallible::().await, + has_acme_tls_challenge, + has_acme_http_challenge, + info: NetworkInfo { + mxs: system.mail_exchangers.into_iter().collect(), + services: system.services, + pacc: serde_json::to_string(&pacc).unwrap_or_default(), + }, }; if let Some(role) = &bp.role { diff --git a/crates/common/src/config/server/tls.rs b/crates/common/src/config/server/tls.rs index 3ac8dd21..7c352e41 100644 --- a/crates/common/src/config/server/tls.rs +++ b/crates/common/src/config/server/tls.rs @@ -4,21 +4,11 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::Server; use ahash::{AHashMap, AHashSet}; -use dns_update::{ - Algorithm, DnsUpdater, TsigAlgorithm, - dnssec::{ - self, SigningKey, - crypto::{EcdsaSigningKey, Ed25519SigningKey}, - }, - providers::{ovh::OvhEndpoint, rfc2136::DnsAddress}, -}; use rcgen::generate_simple_self_signed; use registry::schema::{ - enums, prelude::Object, - structs::{Certificate, DnsServer, SystemSettings}, + structs::{Certificate, SystemSettings}, }; use rustls::{ SupportedProtocolVersion, @@ -27,199 +17,16 @@ use rustls::{ version::{TLS12, TLS13}, }; use rustls_pemfile::{Item, certs, read_all}; -use rustls_pki_types::{PrivateKeyDer, PrivatePkcs8KeyDer}; -use std::{io::Cursor, net::SocketAddr, sync::Arc}; +use rustls_pki_types::PrivateKeyDer; +use std::{io::Cursor, sync::Arc}; use store::{ registry::{bootstrap::Bootstrap, write::RegistryWrite}, write::now, }; -use trc::AddContext; pub static TLS13_VERSION: &[&SupportedProtocolVersion] = &[&TLS13]; pub static TLS12_VERSION: &[&SupportedProtocolVersion] = &[&TLS12]; -impl Server { - pub async fn build_dns_updater(&self, id: u64) -> trc::Result { - let Some(server) = self - .registry() - .object::(id.into()) - .await - .caused_by(trc::location!())? - else { - trc::bail!( - trc::DnsEvent::BuildError - .into_err() - .id(id.to_string()) - .details("DNS server settings not found") - ); - }; - - match server { - DnsServer::Tsig(server) => DnsUpdater::new_rfc2136_tsig( - match server.protocol { - enums::IpProtocol::Udp => DnsAddress::Tcp(SocketAddr::new( - server.host.into_inner(), - server.port as u16, - )), - enums::IpProtocol::Tcp => DnsAddress::Udp(SocketAddr::new( - server.host.into_inner(), - server.port as u16, - )), - }, - server.key_name, - server - .key - .secret() - .await - .map_err(|err| { - trc::DnsEvent::BuildError - .reason(err) - .details("Failed to obtain TSIG key secret") - .id(id.to_string()) - })? - .into_owned() - .into_bytes(), - match server.tsig_algorithm { - enums::TsigAlgorithm::HmacMd5 => TsigAlgorithm::HmacMd5, - enums::TsigAlgorithm::Gss => TsigAlgorithm::Gss, - enums::TsigAlgorithm::HmacSha1 => TsigAlgorithm::HmacSha1, - enums::TsigAlgorithm::HmacSha224 => TsigAlgorithm::HmacSha224, - enums::TsigAlgorithm::HmacSha256 => TsigAlgorithm::HmacSha256, - enums::TsigAlgorithm::HmacSha256128 => TsigAlgorithm::HmacSha256_128, - enums::TsigAlgorithm::HmacSha384 => TsigAlgorithm::HmacSha384, - enums::TsigAlgorithm::HmacSha384192 => TsigAlgorithm::HmacSha384_192, - enums::TsigAlgorithm::HmacSha512 => TsigAlgorithm::HmacSha512, - enums::TsigAlgorithm::HmacSha512256 => TsigAlgorithm::HmacSha512_256, - }, - ), - DnsServer::Sig0(server) => { - let key_bytes = server.key.secret().await.map_err(|err| { - trc::DnsEvent::BuildError - .reason(err) - .details("Failed to obtain key secret") - .id(id.to_string()) - })?; - - let pem_parsed = pem::parse(key_bytes.as_bytes()).map_err(|err| { - trc::DnsEvent::BuildError - .reason(err) - .details("Failed to parse PEM key") - .id(id.to_string()) - })?; - let pkcs8_der = PrivatePkcs8KeyDer::from(pem_parsed.contents()); - - let signing_key: Box = match server.sig0_algorithm { - enums::Sig0Algorithm::EcdsaP256Sha256 => Box::new( - EcdsaSigningKey::from_pkcs8(&pkcs8_der, dnssec::Algorithm::ECDSAP256SHA256) - .map_err(|err| { - trc::DnsEvent::BuildError - .reason(err) - .details("Failed to build ECDSA P-256 signing key") - .id(id.to_string()) - })?, - ), - enums::Sig0Algorithm::EcdsaP384Sha384 => Box::new( - EcdsaSigningKey::from_pkcs8(&pkcs8_der, dnssec::Algorithm::ECDSAP384SHA384) - .map_err(|err| { - trc::DnsEvent::BuildError - .reason(err) - .details("Failed to build ECDSA P-384 signing key") - .id(id.to_string()) - })?, - ), - enums::Sig0Algorithm::Ed25519 => { - Box::new(Ed25519SigningKey::from_pkcs8(&pkcs8_der).map_err(|err| { - trc::DnsEvent::BuildError - .reason(err) - .details("Failed to build Ed25519 signing key") - .id(id.to_string()) - })?) - } - }; - - DnsUpdater::new_rfc2136_sig0( - match server.protocol { - enums::IpProtocol::Udp => DnsAddress::Tcp(SocketAddr::new( - server.host.into_inner(), - server.port as u16, - )), - enums::IpProtocol::Tcp => DnsAddress::Udp(SocketAddr::new( - server.host.into_inner(), - server.port as u16, - )), - }, - server.signer_name, - signing_key, - server.public_key, - match server.sig0_algorithm { - enums::Sig0Algorithm::EcdsaP256Sha256 => Algorithm::ECDSAP256SHA256, - enums::Sig0Algorithm::EcdsaP384Sha384 => Algorithm::ECDSAP384SHA384, - enums::Sig0Algorithm::Ed25519 => Algorithm::ED25519, - }, - ) - } - DnsServer::Cloudflare(server) => DnsUpdater::new_cloudflare( - server.secret.secret().await.map_err(|err| { - trc::DnsEvent::BuildError - .reason(err) - .details("Failed to obtain key secret") - .id(id.to_string()) - })?, - server.email, - server.timeout.into_inner().into(), - ), - DnsServer::DigitalOcean(server) => DnsUpdater::new_digitalocean( - server.secret.secret().await.map_err(|err| { - trc::DnsEvent::BuildError - .reason(err) - .details("Failed to obtain key secret") - .id(id.to_string()) - })?, - server.timeout.into_inner().into(), - ), - DnsServer::DeSEC(server) => DnsUpdater::new_desec( - server.secret.secret().await.map_err(|err| { - trc::DnsEvent::BuildError - .reason(err) - .details("Failed to obtain key secret") - .id(id.to_string()) - })?, - server.timeout.into_inner().into(), - ), - DnsServer::Ovh(server) => DnsUpdater::new_ovh( - server.application_key, - server.application_secret.secret().await.map_err(|err| { - trc::DnsEvent::BuildError - .reason(err) - .details("Failed to obtain application secret") - .id(id.to_string()) - })?, - server.consumer_key.secret().await.map_err(|err| { - trc::DnsEvent::BuildError - .reason(err) - .details("Failed to obtain consumer key") - .id(id.to_string()) - })?, - match server.ovh_endpoint { - enums::OvhEndpoint::OvhEu => OvhEndpoint::OvhEu, - enums::OvhEndpoint::OvhCa => OvhEndpoint::OvhCa, - enums::OvhEndpoint::KimsufiEu => OvhEndpoint::KimsufiEu, - enums::OvhEndpoint::KimsufiCa => OvhEndpoint::KimsufiCa, - enums::OvhEndpoint::SoyoustartEu => OvhEndpoint::SoyoustartEu, - enums::OvhEndpoint::SoyoustartCa => OvhEndpoint::SoyoustartCa, - }, - server.timeout.into_inner().into(), - ), - } - .map_err(|err| { - trc::DnsEvent::BuildError - .reason(err) - .details("Failed to build DNS updater") - .id(id.to_string()) - }) - } -} - pub(crate) async fn parse_certificates( bp: &mut Bootstrap, certificates: &mut AHashMap, Arc>, diff --git a/crates/common/src/config/smtp/resolver.rs b/crates/common/src/config/smtp/resolver.rs index b5970ce0..fdf09751 100644 --- a/crates/common/src/config/smtp/resolver.rs +++ b/crates/common/src/config/smtp/resolver.rs @@ -4,7 +4,6 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::Server; use mail_auth::{ MessageAuthenticator, hickory_resolver::{ @@ -17,7 +16,7 @@ use mail_auth::{ use registry::schema::{ enums::{DnsResolverProtocol, PolicyEnforcement}, prelude::ObjectType, - structs::{DnsResolver, MtaSts}, + structs::{DnsResolver, MtaSts, SystemSettings}, }; use serde::{Deserialize, Serialize}; use std::{ @@ -214,7 +213,22 @@ impl Resolvers { impl Policy { pub async fn try_parse(bp: &mut Bootstrap) -> Option { let mta = bp.setting_infallible::().await; - if !mta.mx_hosts.is_empty() { + let mut mx_hosts = mta.mx_hosts.into_inner(); + + if mx_hosts.is_empty() { + let settings = bp.setting_infallible::().await; + let default_host = settings.default_hostname.as_str(); + mx_hosts = settings + .mail_exchangers + .iter() + .map(|mx| mx.hostname.as_deref().unwrap_or(default_host).to_string()) + .collect(); + } + + if !mx_hosts.is_empty() { + mx_hosts.sort_unstable(); + mx_hosts.dedup(); + let mut policy = Policy { id: Default::default(), mode: match mta.mode { @@ -222,8 +236,7 @@ impl Policy { PolicyEnforcement::Testing => Mode::Testing, PolicyEnforcement::Disable => Mode::None, }, - mx: mta - .mx_hosts + mx: mx_hosts .into_iter() .map(|mx| { if let Some(mx) = mx.strip_prefix("*.") { @@ -236,7 +249,6 @@ impl Policy { max_age: mta.max_age.into_inner().as_secs(), }; - policy.mx.sort_unstable(); policy.id = policy.hash().to_string(); Some(policy) @@ -245,35 +257,6 @@ impl Policy { } } - pub fn try_build(mut self, names: I) -> Option - where - I: IntoIterator, - T: AsRef, - { - if self.mx.is_empty() { - let mut mx = Vec::new(); - for name in names { - let name = name.as_ref(); - if let Some(domain) = name.strip_prefix('.') { - mx.push(MxPattern::StartsWith(domain.to_string())); - } else if name != "*" && !name.is_empty() { - mx.push(MxPattern::Equals(name.to_string())); - } - } - - if !mx.is_empty() { - mx.sort_unstable(); - self.id = self.hash().to_string(); - self.mx = mx.into_boxed_slice(); - Some(self) - } else { - None - } - } else { - Some(self) - } - } - fn hash(&self) -> u64 { let mut s = DefaultHasher::new(); self.mode.hash(&mut s); @@ -283,30 +266,6 @@ impl Policy { } } -impl Server { - pub fn build_mta_sts_policy(&self) -> Option { - self.core - .smtp - .session - .mta_sts_policy - .clone() - .and_then(|policy| { - policy.try_build( - self.inner - .data - .tls_certificates - .load() - .keys() - .filter(|key| { - !key.starts_with("mta-sts.") - && !key.starts_with("autoconfig.") - && !key.starts_with("autodiscover.") - }), - ) - }) - } -} - impl FromStr for Mode { type Err = String; fn from_str(value: &str) -> Result { diff --git a/crates/common/src/enterprise/config.rs b/crates/common/src/enterprise/config.rs index 30027b2f..83ed7d36 100644 --- a/crates/common/src/enterprise/config.rs +++ b/crates/common/src/enterprise/config.rs @@ -166,7 +166,7 @@ impl Enterprise { let api = api.object; let api = Arc::new(AiApiConfig { id: api.name, - api_type: match api.class { + api_type: match api.model_type { AiModelType::Chat => ApiType::ChatCompletion, AiModelType::Text => ApiType::TextCompletion, }, diff --git a/crates/common/src/manager/defaults.rs b/crates/common/src/manager/defaults.rs index 7b3343eb..7e81ed6e 100644 --- a/crates/common/src/manager/defaults.rs +++ b/crates/common/src/manager/defaults.rs @@ -398,6 +398,7 @@ async fn insert_safe_defaults(bp: &mut Bootstrap) -> trc::Result<()> { } } + let todo = "review"; #[cfg(not(feature = "test_mode"))] if let Some(domain_id) = default_domain_id { let now = store::write::now(); @@ -467,11 +468,8 @@ async fn insert_safe_defaults(bp: &mut Bootstrap) -> trc::Result<()> { { for (protocol, name, port, tls_implicit) in [ (NetworkListenerProtocol::Smtp, "smtp", 25, false), - (NetworkListenerProtocol::Smtp, "submission", 587, false), (NetworkListenerProtocol::Smtp, "submissions", 465, true), - (NetworkListenerProtocol::Imap, "imap", 143, false), (NetworkListenerProtocol::Imap, "imaps", 993, true), - (NetworkListenerProtocol::Pop3, "pop3", 110, false), (NetworkListenerProtocol::Pop3, "pop3s", 995, true), (NetworkListenerProtocol::ManageSieve, "sieve", 4190, false), (NetworkListenerProtocol::Http, "https", 443, true), diff --git a/crates/common/src/network/acme/account.rs b/crates/common/src/network/acme/account.rs index c0294b30..376e8d13 100644 --- a/crates/common/src/network/acme/account.rs +++ b/crates/common/src/network/acme/account.rs @@ -49,7 +49,7 @@ pub async fn acme_create_account( *contact = format!("mailto:{}", email); } - let directory = Directory::discover(&provider.directory).await?; + let directory = Directory::discover(&provider.directory, provider.max_retries as u32).await?; let account_key = EcdsaKeyPair::generate_pkcs8(ALG, &SystemRandom::new()).unwrap(); let key_pair = EcdsaKeyPair::from_pkcs8(ALG, account_key.as_ref(), &SystemRandom::new()) .map_err(|err| AcmeError::Crypto(format!("Failed to create ECDSA key pair: {}", err)))?; @@ -68,13 +68,19 @@ pub async fn acme_create_account( let body = sign( &key_pair, None, - directory.nonce().await?, + directory.nonce(provider.max_retries as u32).await?, &directory.new_account, &payload, )?; provider.account_uri = get_header( - &https(&directory.new_account, Method::POST, Some(body)).await?, + &https( + &directory.new_account, + Method::POST, + Some(body), + provider.max_retries as u32, + ) + .await?, "Location", )?; provider.account_key = URL_SAFE_NO_PAD.encode(account_key.as_ref()); diff --git a/crates/common/src/network/acme/directory.rs b/crates/common/src/network/acme/directory.rs index d78e43e8..af64fdfc 100644 --- a/crates/common/src/network/acme/directory.rs +++ b/crates/common/src/network/acme/directory.rs @@ -36,6 +36,7 @@ pub struct AcmeRequestBuilder { pub directory: Directory, pub kid: String, pub challenge: ChallengeType, + pub max_retries: u32, } pub struct AcmeResponse { @@ -48,7 +49,8 @@ static ALG: &EcdsaSigningAlgorithm = &ECDSA_P256_SHA256_FIXED_SIGNING; impl AcmeRequestBuilder { pub async fn new(provider: AcmeProvider) -> AcmeResult { - let directory = Directory::discover(&provider.directory).await?; + let directory = + Directory::discover(&provider.directory, provider.max_retries as u32).await?; let key_pair = EcdsaKeyPair::from_pkcs8( ALG, &URL_SAFE_NO_PAD @@ -64,7 +66,8 @@ impl AcmeRequestBuilder { key_pair, directory, kid: provider.account_uri, - challenge: provider.class.into(), + challenge: provider.challenge_type.into(), + max_retries: provider.max_retries as u32, }) } @@ -76,11 +79,11 @@ impl AcmeRequestBuilder { let body = sign( &self.key_pair, Some(&self.kid), - self.directory.nonce().await?, + self.directory.nonce(self.max_retries).await?, url.as_ref(), payload, )?; - let response = https(url.as_ref(), Method::POST, Some(body)).await?; + let response = https(url.as_ref(), Method::POST, Some(body), self.max_retries).await?; Ok(AcmeResponse { location: get_header(&response, "Location").ok(), @@ -173,14 +176,19 @@ impl AcmeRequestBuilder { } impl Directory { - pub async fn discover(url: impl AsRef) -> AcmeResult { - serde_json::from_str(&https(url, Method::GET, None).await?.text().await?) - .map_err(Into::into) + pub async fn discover(url: impl AsRef, max_retries: u32) -> AcmeResult { + serde_json::from_str( + &https(url, Method::GET, None, max_retries) + .await? + .text() + .await?, + ) + .map_err(Into::into) } - pub async fn nonce(&self) -> AcmeResult { + pub async fn nonce(&self, max_retries: u32) -> AcmeResult { get_header( - &https(&self.new_nonce.as_str(), Method::HEAD, None).await?, + &https(&self.new_nonce.as_str(), Method::HEAD, None, max_retries).await?, "replay-nonce", ) } @@ -199,12 +207,13 @@ impl AcmeResponse { } impl AcmeResponse { - pub fn assert_reasonable_retry_after(self) -> AcmeResult { + pub fn assert_reasonable_retry_after(self, max_retries: u32) -> AcmeResult { if let Some(retry_after) = self.retry_after && retry_after > Duration::from_secs(10 * 60) { - return Err(AcmeError::RetryAt { - time: Some(retry_after), + return Err(AcmeError::Backoff { + max_retries, + wait: retry_after.into(), }); } diff --git a/crates/common/src/network/acme/http.rs b/crates/common/src/network/acme/http.rs index 2a573714..f7fd2917 100644 --- a/crates/common/src/network/acme/http.rs +++ b/crates/common/src/network/acme/http.rs @@ -18,6 +18,7 @@ pub(crate) async fn https( url: impl AsRef, method: Method, body: Option, + max_retries: u32, ) -> AcmeResult { let url = url.as_ref(); let mut builder = reqwest::Client::builder() @@ -49,8 +50,9 @@ pub(crate) async fn https( response.status(), StatusCode::TOO_MANY_REQUESTS | StatusCode::SERVICE_UNAVAILABLE ) { - Err(AcmeError::RetryAt { - time: parse_retry_after(&response), + Err(AcmeError::Backoff { + wait: parse_retry_after(&response), + max_retries, }) } else { Err(AcmeError::HttpStatus(response.status())) diff --git a/crates/common/src/network/acme/mod.rs b/crates/common/src/network/acme/mod.rs index deb25abb..5c61fe2d 100644 --- a/crates/common/src/network/acme/mod.rs +++ b/crates/common/src/network/acme/mod.rs @@ -12,6 +12,7 @@ pub mod order; pub mod renew; pub mod resolver; +use crate::network::dns::update::DnsUpdater; use chrono::{DateTime, Utc}; use registry::schema::enums::AcmeChallengeType; use rustls::sign::CertifiedKey; @@ -31,18 +32,24 @@ pub enum AcmeError { Json(serde_json::Error), Crypto(String), Invalid(String), + Dns(String), AuthInvalid(AuthStatus), - OrderTimeout, OrderInvalid, - AuthTimeout, ChallengeNotSupported { requested: ChallengeType, supported: Vec, }, Internal(trc::Error), Registry(RegistryWriteResult), - RetryAt { - time: Option, + OrderTimeout { + max_retries: u32, + }, + AuthTimeout { + max_retries: u32, + }, + Backoff { + max_retries: u32, + wait: Option, }, } @@ -128,6 +135,11 @@ pub enum AuthStatus { Deactivated, } +pub struct AcmeDnsParameters { + pub updater: DnsUpdater, + pub origin: Option, +} + #[derive(Clone, Debug, serde::Serialize, Deserialize)] #[serde(tag = "type", content = "value", rename_all = "camelCase")] pub enum Identifier { @@ -196,12 +208,13 @@ impl Display for AcmeError { AcmeError::Http(err) => write!(f, "HTTP error: {}", err), AcmeError::HttpStatus(status) => write!(f, "HTTP error: status code {}", status), AcmeError::Json(err) => write!(f, "JSON error: {}", err), + AcmeError::Dns(err) => write!(f, "DNS error: {}", err), AcmeError::Crypto(err) => write!(f, "Cryptographic error: {}", err), AcmeError::Invalid(err) => write!(f, "Invalid request: {}", err), AcmeError::AuthInvalid(status) => write!(f, "Authentication failed: {:?}", status), - AcmeError::OrderTimeout => write!(f, "Order processing timed out"), + AcmeError::OrderTimeout { .. } => write!(f, "Order processing timed out"), AcmeError::OrderInvalid => write!(f, "Order is invalid"), - AcmeError::AuthTimeout => write!(f, "Authentication timed out"), + AcmeError::AuthTimeout { .. } => write!(f, "Authentication timed out"), AcmeError::ChallengeNotSupported { requested, supported, @@ -214,8 +227,8 @@ impl Display for AcmeError { } AcmeError::Internal(err) => write!(f, "Internal error: {}", err), AcmeError::Registry(err) => write!(f, "Registry error: {:?}", err), - AcmeError::RetryAt { time } => { - if let Some(time) = time { + AcmeError::Backoff { wait, .. } => { + if let Some(time) = wait { write!(f, "Rate limited. Retry after {} seconds", time.as_secs()) } else { write!(f, "Rate limited. Retry after some time") diff --git a/crates/common/src/network/acme/order.rs b/crates/common/src/network/acme/order.rs index fc1979a2..c05db28b 100644 --- a/crates/common/src/network/acme/order.rs +++ b/crates/common/src/network/acme/order.rs @@ -8,10 +8,12 @@ use crate::network::acme::directory::AcmeRequestBuilder; use crate::network::acme::{ - AcmeError, AcmeResult, AuthStatus, ChallengeType, Identifier, OrderStatus, ParsedCert, PemCert, + AcmeDnsParameters, AcmeError, AcmeResult, AuthStatus, ChallengeType, Identifier, OrderStatus, + ParsedCert, PemCert, }; use crate::{KV_ACME, Server}; use chrono::{TimeZone, Utc}; +use dns_update::DnsRecord; use futures::future::try_join_all; use rcgen::{CertificateParams, DistinguishedName, PKCS_ECDSA_P256_SHA256}; use std::collections::BTreeSet; @@ -30,6 +32,7 @@ impl AcmeRequestBuilder { server: &Server, domain: &str, hostnames: &[String], + dns_parameters: Option, ) -> AcmeResult { let domains = if hostnames.is_empty() { if matches!( @@ -76,7 +79,7 @@ impl AcmeRequestBuilder { let auth_futures = order .authorizations .iter() - .map(|url| self.authorize(server, url)); + .map(|url| self.authorize(server, url, dns_parameters.as_ref())); try_join_all(auth_futures).await?; trc::event!( Acme(AcmeEvent::AuthCompleted), @@ -103,7 +106,7 @@ impl AcmeRequestBuilder { let response = self .order(&order_url) .await? - .assert_reasonable_retry_after()?; + .assert_reasonable_retry_after(self.max_retries)?; order = response.body; retry_after = response.retry_after; if order.status != OrderStatus::Processing { @@ -111,7 +114,9 @@ impl AcmeRequestBuilder { } } if order.status == OrderStatus::Processing { - return Err(AcmeError::OrderTimeout); + return Err(AcmeError::OrderTimeout { + max_retries: self.max_retries, + }); } } OrderStatus::Ready => { @@ -147,8 +152,16 @@ impl AcmeRequestBuilder { } } - async fn authorize(&self, server: &Server, url: &String) -> AcmeResult<()> { - let response = self.auth(url).await?.assert_reasonable_retry_after()?; + async fn authorize( + &self, + server: &Server, + url: &String, + dns_parameters: Option<&AcmeDnsParameters>, + ) -> AcmeResult<()> { + let response = self + .auth(url) + .await? + .assert_reasonable_retry_after(self.max_retries)?; let mut retry_after = response.retry_after; let auth = response.body; @@ -200,105 +213,19 @@ impl AcmeRequestBuilder { .await?; } ChallengeType::Dns01 => { - let todo = "fix"; - todo!() - /*let dns_proof = self.dns_proof(challenge)?; + let dns_parameters = dns_parameters.unwrap(); let domain = domain.strip_prefix("*.").unwrap_or(&domain); - let name = format!("_acme-challenge.{}", domain); - let origin = origin - .as_deref() - .or_else(|| psl::domain_str(domain)) - .unwrap_or(domain) - .to_string(); - // First try deleting the record - if let Err(err) = updater.delete(&name, &origin, DnsRecordType::TXT).await { - // Errors are expected if the record does not exist - trc::event!( - Dns(DnsEvent::RecordDeletionFailed), - Hostname = name.to_string(), - Reason = err.to_string(), - Details = origin.to_string(), - Url = self.directory.new_order.to_string(), - ); - } - - // Create the record - if let Err(err) = updater + dns_parameters + .updater .create( - &name, - DnsRecord::TXT { - content: dns_proof.clone(), - }, - *ttl, - &origin, + dns_parameters.origin.as_deref().unwrap_or(domain), + &format!("_acme-challenge.{}", domain), + DnsRecord::TXT(self.dns_proof(challenge)?), + true, ) .await - { - return Err(EventType::Dns(DnsEvent::RecordCreationFailed) - .ctx(trc::Key::Id, self.id.to_string()) - .ctx(trc::Key::Hostname, name) - .ctx(trc::Key::Details, origin) - .reason(err)); - } - - trc::event!( - Dns(DnsEvent::RecordCreated), - Hostname = name.to_string(), - Details = origin.to_string(), - Url = self.directory.new_order.to_string(), - ); - - // Wait for changes to propagate - let wait_until = Instant::now() + *propagation_timeout; - let mut did_propagate = false; - while Instant::now() < wait_until { - match self.core.smtp.resolvers.dns.txt_raw_lookup(&name).await { - Ok(result) => { - let result = std::str::from_utf8(&result).unwrap_or_default(); - if result.contains(&dns_proof) { - did_propagate = true; - break; - } else { - trc::event!( - Dns(DnsEvent::RecordNotPropagated), - Url = self.directory.new_order.to_string(), - Hostname = name.to_string(), - Details = origin.to_string(), - Result = result.to_string(), - Value = dns_proof.to_string(), - ); - } - } - Err(err) => { - trc::event!( - Dns(DnsEvent::RecordLookupFailed), - Url = self.directory.new_order.to_string(), - Hostname = name.to_string(), - Details = origin.to_string(), - Reason = err.to_string(), - ); - } - } - - tokio::time::sleep(*polling_interval).await; - } - - if did_propagate { - trc::event!( - Dns(DnsEvent::RecordPropagated), - Url = self.directory.new_order.to_string(), - Hostname = name.to_string(), - Details = origin.to_string(), - ); - } else { - trc::event!( - Dns(DnsEvent::RecordPropagationTimeout), - Url = self.directory.new_order.to_string(), - Hostname = name.to_string(), - Details = origin.to_string(), - ); - }*/ + .map_err(AcmeError::Dns)?; } ChallengeType::DnsPersist01 => return Ok(()), ChallengeType::Unknown => unreachable!(), @@ -315,7 +242,10 @@ impl AcmeRequestBuilder { for i in 0u64..5 { tokio::time::sleep(retry_after.unwrap_or_else(|| Duration::from_secs(1u64 << i))).await; - let response = self.auth(url).await?.assert_reasonable_retry_after()?; + let response = self + .auth(url) + .await? + .assert_reasonable_retry_after(self.max_retries)?; retry_after = response.retry_after; match response.body.status { @@ -344,7 +274,9 @@ impl AcmeRequestBuilder { } } - Err(AcmeError::AuthTimeout) + Err(AcmeError::AuthTimeout { + max_retries: self.max_retries, + }) } } diff --git a/crates/common/src/network/acme/renew.rs b/crates/common/src/network/acme/renew.rs index 8eca2c89..48000cc2 100644 --- a/crates/common/src/network/acme/renew.rs +++ b/crates/common/src/network/acme/renew.rs @@ -7,11 +7,13 @@ use crate::{ Server, ipc::{BroadcastEvent, RegistryChange}, - network::acme::{AcmeError, AcmeResult, ParsedCert, directory::AcmeRequestBuilder}, + network::acme::{ + AcmeDnsParameters, AcmeError, AcmeResult, ParsedCert, directory::AcmeRequestBuilder, + }, }; use registry::{ schema::{ - enums::{AcmeRenewBefore, DnsRecordType}, + enums::{AcmeChallengeType, AcmeRenewBefore, DnsRecordType}, prelude::ObjectType, structs::{ AcmeProvider, Certificate, CertificateManagement, DnsManagement, Domain, PublicText, @@ -53,6 +55,31 @@ impl Server { cert.acme_provider_id ))); }; + let dns_parameters = match &domain.dns_management { + DnsManagement::Automatic(props) + if acme_provider.challenge_type == AcmeChallengeType::Dns01 => + { + match self.build_dns_updater(props.dns_server_id).await? { + Ok(updater) => Some(AcmeDnsParameters { + updater, + origin: props.origin.clone(), + }), + Err(err) => { + return Err(AcmeError::Invalid(format!( + "Failed to build DNS updater: {}", + err + ))); + } + } + } + _ => None, + }; + if acme_provider.challenge_type == AcmeChallengeType::Dns01 && dns_parameters.is_none() { + return Err(AcmeError::Invalid( + "ACME provider requires DNS challenge but a DNS provider was not configured" + .to_string(), + )); + } let renew_before = acme_provider.renew_before; let pem_cert = AcmeRequestBuilder::new(acme_provider) .await? @@ -60,10 +87,10 @@ impl Server { self, &domain.name, &cert.subject_alternative_names.into_inner(), + dns_parameters, ) .await?; let parsed_cert = ParsedCert::parse(&pem_cert.certificate)?; - let certificate = Certificate { private_key: SecretText::Text(SecretTextValue { secret: pem_cert.private_key, @@ -123,7 +150,7 @@ impl Server { // Update TLSA records if let DnsManagement::Automatic(props) = &domain.dns_management - && props.dns_publish_records.contains(&DnsRecordType::Tlsa) + && props.publish_records.contains(&DnsRecordType::Tlsa) { tasks.push(Task::DnsManagement(TaskDnsManagement { domain_id, diff --git a/crates/common/src/network/acme/resolver.rs b/crates/common/src/network/acme/resolver.rs index 0ed3032a..2f78bbf9 100644 --- a/crates/common/src/network/acme/resolver.rs +++ b/crates/common/src/network/acme/resolver.rs @@ -75,24 +75,14 @@ impl Server { } } + #[inline(always)] pub fn has_acme_tls_providers(&self) -> bool { - let todo = "fix"; - false - /*self.core - .acme - .providers - .values() - .any(|p| matches!(p.challenge, ChallengeSettings::TlsAlpn01))*/ + self.core.network.has_acme_tls_challenge } + #[inline(always)] pub fn has_acme_http_providers(&self) -> bool { - let todo = "fix"; - false - /*self.core - .acme - .providers - .values() - .any(|p| matches!(p.challenge, ChallengeSettings::Http01))*/ + self.core.network.has_acme_http_challenge } } diff --git a/crates/common/src/network/autoconfig/mod.rs b/crates/common/src/network/autoconfig/mod.rs new file mode 100644 index 00000000..22541416 --- /dev/null +++ b/crates/common/src/network/autoconfig/mod.rs @@ -0,0 +1,7 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub mod pacc; diff --git a/crates/common/src/network/autoconfig/pacc.rs b/crates/common/src/network/autoconfig/pacc.rs new file mode 100644 index 00000000..74b38997 --- /dev/null +++ b/crates/common/src/network/autoconfig/pacc.rs @@ -0,0 +1,358 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use serde::{Deserialize, Serialize}; + +/// Top-level configuration document. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct Configuration { + /// Supported protocols and their server endpoints. + pub protocols: Protocols, + + /// Authentication mechanisms the provider supports. + #[serde(skip_serializing_if = "Option::is_none")] + pub authentication: Option, + + /// Informational metadata about the provider. + pub info: Info, +} + +/// The `protocols` object listing available protocol endpoints. +/// +/// HTTP-based protocols (JMAP, CalDAV, CardDAV, WebDAV) use [`HttpServer`]. +/// Text-based protocols (IMAP, POP3, SMTP, ManageSieve) use [`TextServer`]. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] +pub struct Protocols { + #[serde(skip_serializing_if = "Option::is_none")] + pub jmap: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub imap: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub pop3: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub smtp: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub caldav: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub carddav: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub webdav: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub managesieve: Option, +} + +/// An HTTP-based protocol endpoint (JMAP, CalDAV, CardDAV, WebDAV). +/// +/// The `url` MUST use the `https` scheme and the default port 443. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct HttpServer { + /// HTTPS URL of the protocol endpoint. + pub url: String, +} + +/// A text-based protocol endpoint (IMAP, POP3, SMTP, ManageSieve). +/// +/// Connections use TLS on the protocol's default port +/// (993 IMAP, 995 POP3, 465 SMTP). +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct TextServer { + /// Hostname of the server. + pub host: String, +} + +/// Authentication mechanisms supported by the provider. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct Authentication { + /// OAuth Profile for Open Public Clients configuration. + #[serde(rename = "oauth-public", skip_serializing_if = "Option::is_none")] + pub oauth_public: Option, + + /// Whether the provider supports username/password authentication. + pub password: bool, +} + +/// OAuth Profile for Open Public Clients parameters. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct OAuthPublic { + /// The authorization server's issuer identifier (RFC 8414). + /// Must be an `https` URL with no query or fragment components. + pub issuer: String, +} + +/// Informational metadata presented to users and developers. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)] +pub struct Info { + /// Provider identity information (required). + pub provider: Provider, + + /// Help links for users and developers. + #[serde(skip_serializing_if = "Option::is_none")] + pub help: Option, +} + +/// Provider identity. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)] +pub struct Provider { + /// Display name of the provider (≤ 60 characters, SHOULD ≤ 30). + pub name: String, + + /// Short name (≤ 20 characters, SHOULD ≤ 12). + #[serde(rename = "shortName", skip_serializing_if = "Option::is_none")] + pub short_name: Option, + + /// Logo image variants. + #[serde(skip_serializing_if = "Option::is_none")] + pub logo: Option>, +} + +/// A single logo variant. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)] +pub struct Logo { + /// URL where the logo can be retrieved. + pub url: String, + + /// Media type of the logo image (e.g. `image/svg+xml`, `image/png`). + #[serde(rename = "content-type")] + pub content_type: String, + + /// Image width in pixels. Omitted for SVG. + #[serde(skip_serializing_if = "Option::is_none")] + pub width: Option, + + /// Image height in pixels. Omitted for SVG. + #[serde(skip_serializing_if = "Option::is_none")] + pub height: Option, +} + +/// Help links for users and developers. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)] +pub struct Help { + /// URL with user-facing documentation. + #[serde(skip_serializing_if = "Option::is_none")] + pub documentation: Option, + + /// URL with developer-facing documentation. + #[serde(skip_serializing_if = "Option::is_none")] + pub developer: Option, + + /// Contact URIs (e.g. `mailto:` URLs). NOT for end-user display. + #[serde(skip_serializing_if = "Option::is_none")] + pub contact: Option>, +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The full example from Section 4.1 of the draft. + const EXAMPLE_JSON: &str = r#"{ + "protocols": { + "jmap": { + "url": "https://jmap.example.com/session" + }, + "imap": { + "host": "imap.example.com" + }, + "pop3": { + "host": "pop3.example.com" + }, + "smtp": { + "host": "smtp.example.com" + }, + "caldav": { + "url": "https://sync.example.com/calendar/" + }, + "carddav": { + "url": "https://sync.example.com/contacts/" + } + }, + "authentication": { + "oauth-public": { + "issuer": "https://auth.example.com/" + }, + "password": true + }, + "info": { + "provider": { + "name": "Example Provider Name", + "shortName": "Example", + "logo": [ + { + "url": "https://www.example.net/logo.svg", + "content-type": "image/svg+xml" + } + ] + }, + "help": { + "documentation": "https://help.example.net/howto/set-up-your-mail-app.html", + "developer": "https://developer.example.net/client-apps/", + "contact": ["mailto:it@team.example.net"] + } + } +}"#; + + #[test] + fn deserialize_full_example() { + let config: Configuration = + serde_json::from_str(EXAMPLE_JSON).expect("failed to deserialize"); + + // Protocols + assert_eq!( + config.protocols.jmap.as_ref().unwrap().url, + "https://jmap.example.com/session" + ); + assert_eq!( + config.protocols.imap.as_ref().unwrap().host, + "imap.example.com" + ); + assert_eq!( + config.protocols.smtp.as_ref().unwrap().host, + "smtp.example.com" + ); + assert_eq!( + config.protocols.pop3.as_ref().unwrap().host, + "pop3.example.com" + ); + assert_eq!( + config.protocols.caldav.as_ref().unwrap().url, + "https://sync.example.com/calendar/" + ); + assert_eq!( + config.protocols.carddav.as_ref().unwrap().url, + "https://sync.example.com/contacts/" + ); + assert!(config.protocols.webdav.is_none()); + assert!(config.protocols.managesieve.is_none()); + + // Authentication + let auth = config.authentication.as_ref().unwrap(); + assert!(auth.password); + assert_eq!( + auth.oauth_public.as_ref().unwrap().issuer, + "https://auth.example.com/" + ); + + // Info + assert_eq!(config.info.provider.name, "Example Provider Name"); + assert_eq!(config.info.provider.short_name.as_deref(), Some("Example")); + + let logos = config.info.provider.logo.as_ref().unwrap(); + assert_eq!(logos.len(), 1); + assert_eq!(logos[0].content_type, "image/svg+xml"); + assert!(logos[0].width.is_none()); + + let help = config.info.help.as_ref().unwrap(); + assert_eq!( + help.documentation.as_deref(), + Some("https://help.example.net/howto/set-up-your-mail-app.html") + ); + assert_eq!( + help.contact.as_ref().unwrap(), + &["mailto:it@team.example.net"] + ); + } + + #[test] + fn roundtrip() { + let config: Configuration = + serde_json::from_str(EXAMPLE_JSON).expect("failed to deserialize"); + let serialized = serde_json::to_string_pretty(&config).expect("failed to serialize"); + let roundtripped: Configuration = + serde_json::from_str(&serialized).expect("failed to re-deserialize"); + assert_eq!(config, roundtripped); + } + + #[test] + fn minimal_config() { + let json = r#"{ + "protocols": {}, + "info": { + "provider": { + "name": "Minimal" + } + } + }"#; + let config: Configuration = serde_json::from_str(json).expect("failed to deserialize"); + assert_eq!(config.info.provider.name, "Minimal"); + assert!(config.authentication.is_none()); + assert!(config.protocols.jmap.is_none()); + } + + #[test] + fn ignores_unknown_properties() { + let json = r#"{ + "protocols": { + "imap": { "host": "imap.example.com" }, + "future-protocol": { "endpoint": "wss://example.com" } + }, + "info": { + "provider": { "name": "Test" } + }, + "futureField": 42 + }"#; + let config: Configuration = serde_json::from_str(json).expect("should ignore unknowns"); + assert_eq!( + config.protocols.imap.as_ref().unwrap().host, + "imap.example.com" + ); + } + + #[test] + fn logo_with_dimensions() { + let json = r#"{ + "protocols": {}, + "info": { + "provider": { + "name": "Test", + "logo": [ + { + "url": "https://example.com/logo.svg", + "content-type": "image/svg+xml" + }, + { + "url": "https://example.com/logo-128.png", + "content-type": "image/png", + "width": 128, + "height": 128 + }, + { + "url": "https://example.com/logo-512.png", + "content-type": "image/png", + "width": 512, + "height": 512 + } + ] + } + } + }"#; + let config: Configuration = serde_json::from_str(json).unwrap(); + let logos = config.info.provider.logo.as_ref().unwrap(); + assert_eq!(logos.len(), 3); + assert!(logos[0].width.is_none()); + assert_eq!(logos[1].width, Some(128)); + assert_eq!(logos[2].height, Some(512)); + } + + #[test] + fn password_only_auth() { + let json = r#"{ + "protocols": { "imap": { "host": "mail.example.com" } }, + "authentication": { "password": true }, + "info": { "provider": { "name": "PW Only" } } + }"#; + let config: Configuration = serde_json::from_str(json).unwrap(); + let auth = config.authentication.unwrap(); + assert!(auth.password); + assert!(auth.oauth_public.is_none()); + } +} diff --git a/crates/common/src/network/dkim.rs b/crates/common/src/network/dkim.rs index 08d1a57b..f0e2f2c4 100644 --- a/crates/common/src/network/dkim.rs +++ b/crates/common/src/network/dkim.rs @@ -4,10 +4,16 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::config::smtp::auth::{rsa_key_parse, simple_pem_parse}; use chrono::Utc; +use dns_update::{DnsRecord, NamedDnsRecord}; +use mail_auth::common::crypto::Ed25519Key; use mail_auth::dkim::generate::DkimKeyPair; use mail_builder::encoders::base64::base64_encode; +use pkcs8::Document; use registry::schema::enums::DkimSignatureType; +use registry::schema::structs::DkimSignature; +use rsa::pkcs1::DecodeRsaPublicKey; pub async fn generate_dkim_private_key( key_type: DkimSignatureType, @@ -49,6 +55,78 @@ pub async fn generate_dkim_private_key( .map_err(|err| err.to_string())) } +pub async fn generate_dkim_public_key(key: &DkimSignature) -> trc::Result { + match key { + DkimSignature::Dkim1RsaSha256(key) => key + .private_key + .secret() + .await + .map_err(|err| trc::DkimEvent::BuildError.reason(err)) + .and_then(|pem| rsa_key_parse(pem.as_bytes())) + .and_then(|pk| { + Document::from_pkcs1_der(&pk.public_key()).map_err(|err| { + trc::EventType::Dkim(trc::DkimEvent::BuildError) + .into_err() + .reason(err) + }) + }) + .map(|pk| { + String::from_utf8(base64_encode(pk.as_bytes()).unwrap_or_default()) + .unwrap_or_default() + }), + DkimSignature::Dkim1Ed25519Sha256(key) => key + .private_key + .secret() + .await + .map_err(|err| trc::DkimEvent::BuildError.reason(err)) + .and_then(|pem| { + simple_pem_parse(&pem).ok_or_else(|| { + trc::EventType::Dkim(trc::DkimEvent::BuildError) + .into_err() + .details("Failed to parse private key PEM") + }) + }) + .and_then(|der| { + Ed25519Key::from_pkcs8_maybe_unchecked_der(&der).map_err(|err| { + trc::EventType::Dkim(trc::DkimEvent::BuildError) + .into_err() + .reason(err) + }) + }) + .map(|pk| { + String::from_utf8(base64_encode(&pk.public_key()).unwrap_or_default()) + .unwrap_or_default() + }), + } +} + +pub async fn generate_dkim_dns_record( + key: &DkimSignature, + domain: &str, +) -> trc::Result { + let public_key = generate_dkim_public_key(key).await?; + + let (selector, record) = match key { + DkimSignature::Dkim1Ed25519Sha256(sign) => ( + &sign.selector, + format!("v=DKIM1; k=ed25519; h=sha256; p={public_key}"), + ), + DkimSignature::Dkim1RsaSha256(sign) => ( + &sign.selector, + format!("v=DKIM1; k=rsa; h=sha256; p={public_key}"), + ), + }; + + Ok(NamedDnsRecord { + name: format!("{selector}._domainkey.{domain}."), + record: DnsRecord::TXT(record), + }) +} + +pub fn generate_dkim_dns_record_name(key: &DkimSignature, domain: &str) -> String { + format!("{}._domainkey.{domain}.", key.selector()) +} + /// Generate a DKIM selector from a template string. /// /// Supported variables: @@ -58,8 +136,6 @@ pub async fn generate_dkim_private_key( /// - `{date-}` — current UTC date formatted with chrono strftime (e.g. `{date-%Y%m%d}`) /// - `{epoch}` — current UTC unix timestamp /// -/// The output is sanitized to only contain valid DKIM selector characters (`a-zA-Z0-9._-`). -/// Returns an error if the template contains unrecognized variables or produces an empty selector. pub fn generate_dkim_selector( template: &str, sig_type: DkimSignatureType, diff --git a/crates/common/src/network/dns/mod.rs b/crates/common/src/network/dns/mod.rs new file mode 100644 index 00000000..d2670fe0 --- /dev/null +++ b/crates/common/src/network/dns/mod.rs @@ -0,0 +1,9 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub mod records; +pub mod resolve; +pub mod update; diff --git a/crates/common/src/network/dns/records.rs b/crates/common/src/network/dns/records.rs new file mode 100644 index 00000000..8ea02e78 --- /dev/null +++ b/crates/common/src/network/dns/records.rs @@ -0,0 +1,367 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{Server, network::dkim::generate_dkim_dns_record}; +use ahash::{AHashMap, AHashSet}; +use base64::{Engine, engine::general_purpose}; +use dns_update::{ + CAARecord, DnsRecord, KeyValue, MXRecord, NamedDnsRecord, SRVRecord, TLSARecord, TlsaCertUsage, + TlsaMatching, TlsaSelector, bind::BindSerializer, +}; +use registry::schema::{ + enums::{DnsRecordType, ServiceProtocol}, + prelude::{ObjectType, Property}, + structs::{AcmeProvider, CertificateManagement, DkimSignature, DnsManagement, Domain}, +}; +use reqwest::Url; +use sha2::{Digest, Sha256}; +use store::registry::RegistryQuery; +use types::id::Id; +use x509_parser::parse_x509_certificate; + +impl Server { + pub async fn build_dns_records( + &self, + domain_id: Id, + domain: &Domain, + record_types: &[DnsRecordType], + ) -> trc::Result> { + let mut records = Vec::new(); + let network = &self.core.network; + let default_host = network.server_name.as_str(); + let domain_name = domain.name.as_str(); + let domain_name_suffix = format!(".{domain_name}"); + + for record_type in record_types { + match record_type { + DnsRecordType::Dkim => { + let signature_ids = self + .registry() + .query::>( + RegistryQuery::new(ObjectType::DkimSignature) + .equal(Property::DomainId, domain_id.document_id()), + ) + .await?; + + for id in signature_ids { + let Some(key) = self.registry().object::(id).await? else { + continue; + }; + records.push(generate_dkim_dns_record(&key, domain_name).await?); + } + } + DnsRecordType::Mx => { + for mx in &network.info.mxs { + records.push(NamedDnsRecord { + name: format!("{domain_name}."), + record: DnsRecord::MX(MXRecord { + exchange: mx + .hostname + .as_deref() + .unwrap_or(default_host) + .to_string(), + priority: mx.priority as u16, + }), + }); + } + } + DnsRecordType::Spf => { + let mxs = network + .info + .mxs + .iter() + .map(|mx| mx.hostname.as_deref().unwrap_or(default_host)) + .collect::>(); + + for mx in mxs { + if mx.ends_with(&domain_name_suffix) || mx == domain_name { + records.push(NamedDnsRecord { + name: format!("{mx}."), + record: DnsRecord::TXT("v=spf1 a -all".to_string()), + }); + } + } + + records.push(NamedDnsRecord { + name: format!("{domain_name}."), + record: DnsRecord::TXT("v=spf1 mx -all".to_string()), + }); + } + DnsRecordType::Dmarc => { + if let Some(uri) = &domain.report_address_uri { + let contents = if uri.starts_with("mailto:") && !uri.contains('@') { + format!("v=DMARC1; p=reject; rua={uri}@{domain_name}",) + } else { + format!("v=DMARC1; p=reject; rua={uri}",) + }; + + records.push(NamedDnsRecord { + name: format!("_dmarc.{domain_name}."), + record: DnsRecord::TXT(contents), + }); + } + } + DnsRecordType::TlsRpt => { + if let Some(uri) = &domain.report_address_uri { + let contents = if uri.starts_with("mailto:") && !uri.contains('@') { + format!("v=TLSRPTv1; rua={uri}@{domain_name}",) + } else { + format!("v=TLSRPTv1; rua={uri}",) + }; + + records.push(NamedDnsRecord { + name: format!("_smtp._tls.{domain_name}."), + record: DnsRecord::TXT(contents), + }); + } + } + DnsRecordType::MtaSts => { + if let Some(policy) = &self.core.smtp.session.mta_sts_policy { + records.push(NamedDnsRecord { + name: format!("mta-sts.{domain_name}."), + record: DnsRecord::CNAME(format!("{default_host}.")), + }); + + records.push(NamedDnsRecord { + name: format!("_mta-sts.{domain_name}."), + record: DnsRecord::TXT(format!("v=STSv1; id={}", policy.id)), + }); + } + } + DnsRecordType::AutoConfig => { + let pacc_digest = Sha256::digest(&network.info.pacc); + let pacc_digest_encoded = general_purpose::STANDARD.encode(pacc_digest); + + records.push(NamedDnsRecord { + name: format!("ua-auto-config.{domain_name}."), + record: DnsRecord::CNAME(format!("{default_host}.")), + }); + records.push(NamedDnsRecord { + name: format!("_ua-auto-config.{domain_name}."), + record: DnsRecord::TXT(format!( + "v=UAAC1; a=sha256; d={pacc_digest_encoded}" + )), + }); + } + DnsRecordType::AutoConfigLegacy => { + records.push(NamedDnsRecord { + name: format!("autoconfig.{domain_name}."), + record: DnsRecord::CNAME(format!("{default_host}.")), + }); + } + DnsRecordType::AutoDiscover => { + records.push(NamedDnsRecord { + name: format!("autodiscover.{domain_name}."), + record: DnsRecord::CNAME(format!("{default_host}.")), + }); + } + DnsRecordType::Srv => { + for (protocol, service) in &network.info.services { + let target = + format!("{}.", service.hostname.as_deref().unwrap_or(default_host)); + let services = match protocol { + ServiceProtocol::Jmap + | ServiceProtocol::Caldav + | ServiceProtocol::Carddav => { + let name = match protocol { + ServiceProtocol::Jmap => "jmap", + ServiceProtocol::Caldav => "caldavs", + ServiceProtocol::Carddav => "carddavs", + _ => unreachable!(), + }; + + records.push(NamedDnsRecord { + name: format!("_{name}._tcp.{domain_name}."), + record: DnsRecord::SRV(SRVRecord { + target: target.clone(), + priority: 0, + weight: 1, + port: 443, + }), + }); + continue; + } + ServiceProtocol::Webdav | ServiceProtocol::Managesieve => continue, + ServiceProtocol::Imap => [("imap", 143), ("imaps", 993)], + ServiceProtocol::Pop3 => [("pop3", 110), ("pop3s", 995)], + ServiceProtocol::Smtp => [("submission", 587), ("submissions", 465)], + }; + + for (is_tls, (service_name, port)) in services.into_iter().enumerate() { + if is_tls == 1 || service.cleartext { + records.push(NamedDnsRecord { + name: format!("_{service_name}._tcp.{domain_name}."), + record: DnsRecord::SRV(SRVRecord { + target: target.clone(), + priority: 0, + weight: 1, + port, + }), + }); + } + } + } + } + DnsRecordType::Caa => { + if let CertificateManagement::Automatic(props) = &domain.certificate_management + && let Some(provider) = self + .registry() + .object::(props.acme_provider_id) + .await? + && let Ok(provider_url) = Url::parse(&provider.directory) + && let Some(provider_name) = + provider_url.host_str().and_then(psl::domain_str) + { + records.push(NamedDnsRecord { + name: format!("{domain_name}."), + record: DnsRecord::CAA(CAARecord::Issue { + issuer_critical: false, + name: provider_name.to_string().into(), + options: vec![KeyValue { + key: "accounturi".to_string(), + value: provider.account_uri.clone(), + }], + }), + }); + + if let Some(uri) = &domain.report_address_uri + && uri.starts_with("mailto:") + { + let url = if !uri.contains('@') { + format!("{uri}@{domain_name}") + } else { + uri.to_string() + }; + records.push(NamedDnsRecord { + name: format!("{domain_name}."), + record: DnsRecord::CAA(CAARecord::Iodef { + issuer_critical: false, + url, + }), + }); + } + } + } + DnsRecordType::Tlsa => { + let mut hostnames: AHashMap> = AHashMap::new(); + + for mx in &network.info.mxs { + let hostname = mx.hostname.as_deref().unwrap_or(default_host); + if hostname.ends_with(&domain_name_suffix) || hostname == domain_name { + hostnames + .entry(hostname.to_string()) + .or_default() + .insert(25); + } + } + + for (protocol, service) in &network.info.services { + let hostname = service.hostname.as_deref().unwrap_or(default_host); + if hostname.ends_with(&domain_name_suffix) || hostname == domain_name { + let port = match protocol { + ServiceProtocol::Imap => 993, + ServiceProtocol::Pop3 => 995, + ServiceProtocol::Smtp => 465, + ServiceProtocol::Jmap + | ServiceProtocol::Caldav + | ServiceProtocol::Carddav + | ServiceProtocol::Webdav => 443, + ServiceProtocol::Managesieve => continue, + }; + hostnames + .entry(hostname.to_string()) + .or_default() + .insert(port); + } + } + + for (record_name, record_type) in [ + ("ua-auto-config", DnsRecordType::AutoConfig), + ("autoconfig", DnsRecordType::AutoConfigLegacy), + ("autodiscover", DnsRecordType::AutoDiscover), + ("mta-sts", DnsRecordType::MtaSts), + ] { + if matches!(&domain.dns_management, DnsManagement::Automatic(props) if props.publish_records.contains(&record_type)) + || matches!(domain.dns_management, DnsManagement::Manual) + { + hostnames + .entry(format!("{record_name}.{domain_name}")) + .or_default() + .insert(443); + } + } + + for (hostname, ports) in hostnames { + if let Some(key) = self.resolve_certificate(&hostname) { + for (cert_num, cert) in key.cert.iter().enumerate() { + let parsed_cert = match parse_x509_certificate(cert) { + Ok((_, parsed_cert)) => parsed_cert, + Err(err) => { + trc::error!( + trc::StoreEvent::UnexpectedError + .into_err() + .reason(err) + .caused_by(trc::location!()) + ); + continue; + } + }; + + let cert_usage = if cert_num == 0 { + TlsaCertUsage::DaneEe + } else { + TlsaCertUsage::DaneTa + }; + let cert_data = sha2::Sha256::digest(parsed_cert.subject_pki.raw); + + for port in &ports { + records.push(NamedDnsRecord { + name: format!("_{port}._tcp.{hostname}."), + record: DnsRecord::TLSA(TLSARecord { + cert_usage, + selector: TlsaSelector::Spki, + matching: TlsaMatching::Sha256, + cert_data: cert_data.to_vec(), + }), + }); + } + } + } + } + } + } + } + + Ok(records) + } + + pub async fn build_bind_dns_records( + &self, + domain_id: Id, + domain: &Domain, + ) -> trc::Result { + self.build_dns_records( + domain_id, + domain, + &[ + DnsRecordType::Dkim, + DnsRecordType::Tlsa, + DnsRecordType::Spf, + DnsRecordType::Mx, + DnsRecordType::Dmarc, + DnsRecordType::Srv, + DnsRecordType::MtaSts, + DnsRecordType::TlsRpt, + DnsRecordType::Caa, + DnsRecordType::AutoConfig, + DnsRecordType::AutoConfigLegacy, + DnsRecordType::AutoDiscover, + ], + ) + .await + .map(|records| BindSerializer::serialize(&records)) + } +} diff --git a/crates/common/src/network/dns.rs b/crates/common/src/network/dns/resolve.rs similarity index 100% rename from crates/common/src/network/dns.rs rename to crates/common/src/network/dns/resolve.rs diff --git a/crates/common/src/network/dns/update.rs b/crates/common/src/network/dns/update.rs new file mode 100644 index 00000000..ac6fec8d --- /dev/null +++ b/crates/common/src/network/dns/update.rs @@ -0,0 +1,373 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{Core, Server}; +use dns_update::{ + Algorithm, DnsRecord, DnsRecordType, TsigAlgorithm, + dnssec::{ + self, SigningKey, + crypto::{EcdsaSigningKey, Ed25519SigningKey}, + }, + providers::{ovh::OvhEndpoint, rfc2136::DnsAddress}, +}; +use registry::schema::{ + enums, + structs::{DnsManagement, DnsServer, Domain}, +}; +use rustls_pki_types::PrivatePkcs8KeyDer; +use std::{ + net::SocketAddr, + sync::Arc, + time::{Duration, Instant}, +}; +use trc::DnsEvent; +use types::id::Id; + +pub struct DnsUpdater { + pub updater: dns_update::DnsUpdater, + pub polling_interval: Duration, + pub propagation_timeout: Duration, + pub propagation_delay: Option, + pub ttl: Duration, + core: Arc, +} + +impl DnsUpdater { + pub async fn build(server: DnsServer, core: Arc) -> Result { + match server { + DnsServer::Tsig(server) => Ok(DnsUpdater { + polling_interval: server.polling_interval.into_inner(), + propagation_timeout: server.propagation_timeout.into_inner(), + propagation_delay: server.propagation_delay.map(|d| d.into_inner()), + ttl: server.ttl.into_inner(), + core, + updater: dns_update::DnsUpdater::new_rfc2136_tsig( + match server.protocol { + enums::IpProtocol::Udp => DnsAddress::Tcp(SocketAddr::new( + server.host.into_inner(), + server.port as u16, + )), + enums::IpProtocol::Tcp => DnsAddress::Udp(SocketAddr::new( + server.host.into_inner(), + server.port as u16, + )), + }, + server.key_name, + server.key.secret().await?.into_owned().into_bytes(), + match server.tsig_algorithm { + enums::TsigAlgorithm::HmacMd5 => TsigAlgorithm::HmacMd5, + enums::TsigAlgorithm::Gss => TsigAlgorithm::Gss, + enums::TsigAlgorithm::HmacSha1 => TsigAlgorithm::HmacSha1, + enums::TsigAlgorithm::HmacSha224 => TsigAlgorithm::HmacSha224, + enums::TsigAlgorithm::HmacSha256 => TsigAlgorithm::HmacSha256, + enums::TsigAlgorithm::HmacSha256128 => TsigAlgorithm::HmacSha256_128, + enums::TsigAlgorithm::HmacSha384 => TsigAlgorithm::HmacSha384, + enums::TsigAlgorithm::HmacSha384192 => TsigAlgorithm::HmacSha384_192, + enums::TsigAlgorithm::HmacSha512 => TsigAlgorithm::HmacSha512, + enums::TsigAlgorithm::HmacSha512256 => TsigAlgorithm::HmacSha512_256, + }, + ) + .map_err(|err| format!("Failed to build DNS updater: {}", err))?, + }), + DnsServer::Sig0(server) => { + let key_bytes = server.key.secret().await?; + let pem_parsed = pem::parse(key_bytes.as_bytes()) + .map_err(|err| format!("Failed to parse PEM key: {}", err))?; + let pkcs8_der = PrivatePkcs8KeyDer::from(pem_parsed.contents()); + let signing_key: Box = match server.sig0_algorithm { + enums::Sig0Algorithm::EcdsaP256Sha256 => Box::new( + EcdsaSigningKey::from_pkcs8(&pkcs8_der, dnssec::Algorithm::ECDSAP256SHA256) + .map_err(|err| { + format!("Failed to build ECDSA P-256 signing key: {}", err) + })?, + ), + enums::Sig0Algorithm::EcdsaP384Sha384 => Box::new( + EcdsaSigningKey::from_pkcs8(&pkcs8_der, dnssec::Algorithm::ECDSAP384SHA384) + .map_err(|err| { + format!("Failed to build ECDSA P-384 signing key: {}", err) + })?, + ), + enums::Sig0Algorithm::Ed25519 => { + Box::new(Ed25519SigningKey::from_pkcs8(&pkcs8_der).map_err(|err| { + format!("Failed to build Ed25519 signing key: {}", err) + })?) + } + }; + + Ok(DnsUpdater { + polling_interval: server.polling_interval.into_inner(), + propagation_timeout: server.propagation_timeout.into_inner(), + propagation_delay: server.propagation_delay.map(|d| d.into_inner()), + ttl: server.ttl.into_inner(), + core, + updater: dns_update::DnsUpdater::new_rfc2136_sig0( + match server.protocol { + enums::IpProtocol::Udp => DnsAddress::Tcp(SocketAddr::new( + server.host.into_inner(), + server.port as u16, + )), + enums::IpProtocol::Tcp => DnsAddress::Udp(SocketAddr::new( + server.host.into_inner(), + server.port as u16, + )), + }, + server.signer_name, + signing_key, + server.public_key, + match server.sig0_algorithm { + enums::Sig0Algorithm::EcdsaP256Sha256 => Algorithm::ECDSAP256SHA256, + enums::Sig0Algorithm::EcdsaP384Sha384 => Algorithm::ECDSAP384SHA384, + enums::Sig0Algorithm::Ed25519 => Algorithm::ED25519, + }, + ) + .map_err(|err| format!("Failed to build DNS updater: {}", err))?, + }) + } + DnsServer::Cloudflare(server) => Ok(DnsUpdater { + polling_interval: server.polling_interval.into_inner(), + propagation_timeout: server.propagation_timeout.into_inner(), + propagation_delay: server.propagation_delay.map(|d| d.into_inner()), + ttl: server.ttl.into_inner(), + core, + updater: dns_update::DnsUpdater::new_cloudflare( + server.secret.secret().await?, + server.email, + server.timeout.into_inner().into(), + ) + .map_err(|err| format!("Failed to build DNS updater: {}", err))?, + }), + DnsServer::DigitalOcean(server) => Ok(DnsUpdater { + polling_interval: server.polling_interval.into_inner(), + propagation_timeout: server.propagation_timeout.into_inner(), + propagation_delay: server.propagation_delay.map(|d| d.into_inner()), + ttl: server.ttl.into_inner(), + core, + updater: dns_update::DnsUpdater::new_digitalocean( + server.secret.secret().await?, + server.timeout.into_inner().into(), + ) + .map_err(|err| format!("Failed to build DNS updater: {}", err))?, + }), + DnsServer::DeSEC(server) => Ok(DnsUpdater { + polling_interval: server.polling_interval.into_inner(), + propagation_timeout: server.propagation_timeout.into_inner(), + propagation_delay: server.propagation_delay.map(|d| d.into_inner()), + ttl: server.ttl.into_inner(), + core, + updater: dns_update::DnsUpdater::new_desec( + server.secret.secret().await?, + server.timeout.into_inner().into(), + ) + .map_err(|err| format!("Failed to build DNS updater: {}", err))?, + }), + DnsServer::Ovh(server) => Ok(DnsUpdater { + polling_interval: server.polling_interval.into_inner(), + propagation_timeout: server.propagation_timeout.into_inner(), + propagation_delay: server.propagation_delay.map(|d| d.into_inner()), + ttl: server.ttl.into_inner(), + core, + updater: dns_update::DnsUpdater::new_ovh( + server.application_key, + server.application_secret.secret().await?, + server.consumer_key.secret().await?, + match server.ovh_endpoint { + enums::OvhEndpoint::OvhEu => OvhEndpoint::OvhEu, + enums::OvhEndpoint::OvhCa => OvhEndpoint::OvhCa, + enums::OvhEndpoint::KimsufiEu => OvhEndpoint::KimsufiEu, + enums::OvhEndpoint::KimsufiCa => OvhEndpoint::KimsufiCa, + enums::OvhEndpoint::SoyoustartEu => OvhEndpoint::SoyoustartEu, + enums::OvhEndpoint::SoyoustartCa => OvhEndpoint::SoyoustartCa, + }, + server.timeout.into_inner().into(), + ) + .map_err(|err| format!("Failed to build DNS updater: {}", err))?, + }), + DnsServer::Bunny(server) => Ok(DnsUpdater { + polling_interval: server.polling_interval.into_inner(), + propagation_timeout: server.propagation_timeout.into_inner(), + propagation_delay: server.propagation_delay.map(|d| d.into_inner()), + ttl: server.ttl.into_inner(), + core, + updater: dns_update::DnsUpdater::new_bunny( + server.secret.secret().await?, + server.timeout.into_inner().into(), + ) + .map_err(|err| format!("Failed to build DNS updater: {}", err))?, + }), + DnsServer::Porkbun(server) => Ok(DnsUpdater { + polling_interval: server.polling_interval.into_inner(), + propagation_timeout: server.propagation_timeout.into_inner(), + propagation_delay: server.propagation_delay.map(|d| d.into_inner()), + ttl: server.ttl.into_inner(), + core, + updater: dns_update::DnsUpdater::new_porkbun( + server.api_key.as_str(), + server.secret.secret().await?, + server.timeout.into_inner().into(), + ) + .map_err(|err| format!("Failed to build DNS updater: {}", err))?, + }), + DnsServer::Dnsimple(server) => Ok(DnsUpdater { + polling_interval: server.polling_interval.into_inner(), + propagation_timeout: server.propagation_timeout.into_inner(), + propagation_delay: server.propagation_delay.map(|d| d.into_inner()), + ttl: server.ttl.into_inner(), + core, + updater: dns_update::DnsUpdater::new_dnsimple( + server.secret.secret().await?, + server.account_identifier.as_str(), + server.timeout.into_inner().into(), + ) + .map_err(|err| format!("Failed to build DNS updater: {}", err))?, + }), + } + } + + pub async fn create( + &self, + origin: &str, + name: &str, + record: DnsRecord, + verify: bool, + ) -> Result { + // First try deleting the record + if let Err(err) = self.updater.delete(name, origin, record.as_type()).await { + // Errors are expected if the record does not exist + trc::event!( + Dns(DnsEvent::RecordDeletionFailed), + Hostname = name.to_string(), + Reason = err.to_string(), + Details = origin.to_string(), + ); + } + + // Create the record + if let Err(err) = self + .updater + .create(name, record.clone(), self.ttl.as_secs() as u32, origin) + .await + { + return Err(format!("Failed to create DNS record: {}", err)); + } + + trc::event!( + Dns(DnsEvent::RecordCreated), + Hostname = name.to_string(), + Details = origin.to_string(), + ); + + if verify && let DnsRecord::TXT(txt_record) = &record { + // Wait for changes to propagate + if let Some(initial_wait) = self.propagation_delay { + tokio::time::sleep(initial_wait).await; + } + let wait_until = Instant::now() + self.propagation_timeout; + let mut did_propagate = false; + while Instant::now() < wait_until { + match self.core.smtp.resolvers.dns.txt_raw_lookup(&name).await { + Ok(result) => { + let result = std::str::from_utf8(&result).unwrap_or_default(); + if result.contains(txt_record) { + did_propagate = true; + break; + } else { + trc::event!( + Dns(DnsEvent::RecordNotPropagated), + Hostname = name.to_string(), + Details = origin.to_string(), + Result = result.to_string(), + ); + } + } + Err(err) => { + trc::event!( + Dns(DnsEvent::RecordLookupFailed), + Hostname = name.to_string(), + Details = origin.to_string(), + Reason = err.to_string(), + ); + } + } + + tokio::time::sleep(self.polling_interval).await; + } + + if did_propagate { + trc::event!( + Dns(DnsEvent::RecordPropagated), + Hostname = name.to_string(), + Details = origin.to_string(), + ); + } else { + trc::event!( + Dns(DnsEvent::RecordPropagationTimeout), + Hostname = name.to_string(), + Details = origin.to_string(), + ); + } + + Ok(did_propagate) + } else { + Ok(true) + } + } + + pub async fn delete( + &self, + origin: &str, + name: &str, + record_type: DnsRecordType, + ) -> Result<(), String> { + // First try deleting the record + match self.updater.delete(name, origin, record_type).await { + Ok(_) => Ok(()), + Err(err) => { + trc::event!( + Dns(DnsEvent::RecordDeletionFailed), + Hostname = name.to_string(), + Reason = err.to_string(), + Details = origin.to_string(), + ); + Err(err.to_string()) + } + } + } +} + +impl Server { + pub async fn build_dns_updater_for_domain( + &self, + domain_id: Id, + ) -> trc::Result> { + if let Some(domain) = self.registry().object::(domain_id).await? { + match domain.dns_management { + DnsManagement::Automatic(props) => { + self.build_dns_updater(props.dns_server_id).await + } + DnsManagement::Manual => Ok(Err(format!( + "Domain with ID {} is set to manual DNS management", + domain_id + ))), + } + } else { + Ok(Err(format!("Domain with ID {} not found", domain_id))) + } + } + + pub async fn build_dns_updater( + &self, + dns_server_id: Id, + ) -> trc::Result> { + if let Some(settings) = self.registry().object::(dns_server_id).await? { + Ok(DnsUpdater::build(settings, self.core.clone()).await) + } else { + Ok(Err(format!( + "DNS server with ID {} not found", + dns_server_id + ))) + } + } +} diff --git a/crates/common/src/network/mod.rs b/crates/common/src/network/mod.rs index 67b4d082..0a9be5b6 100644 --- a/crates/common/src/network/mod.rs +++ b/crates/common/src/network/mod.rs @@ -25,6 +25,7 @@ use utils::snowflake::SnowflakeIdGenerator; pub mod acme; pub mod asn; +pub mod autoconfig; pub mod dkim; pub mod dns; pub mod limiter; diff --git a/crates/common/src/network/tls.rs b/crates/common/src/network/tls.rs index e040fde2..3e55a334 100644 --- a/crates/common/src/network/tls.rs +++ b/crates/common/src/network/tls.rs @@ -43,6 +43,21 @@ impl ResolvesServerCert for CertificateResolver { } } +impl Server { + pub fn resolve_certificate(&self, name: &str) -> Option> { + let certs = self.inner.data.tls_certificates.load(); + + certs + .get(name) + .or_else(|| { + // Try with a wildcard certificate + name.split_once('.') + .and_then(|(_, domain)| certs.get(domain)) + }) + .cloned() + } +} + impl CertificateResolver { pub(crate) fn resolve_certificate(&self, name: Option<&str>) -> Option> { let certs = self.inner.data.tls_certificates.load(); diff --git a/crates/http/src/request.rs b/crates/http/src/request.rs index 63e4be2d..979bbdd7 100644 --- a/crates/http/src/request.rs +++ b/crates/http/src/request.rs @@ -301,18 +301,30 @@ impl ParseHttp for Server { self.is_http_anonymous_request_allowed(session.remote_ip) .await?; - return if let Some(policy) = self.build_mta_sts_policy() { + return if let Some(policy) = &self.core.smtp.session.mta_sts_policy { Ok(Resource::new("text/plain", policy.to_string().into_bytes()) .into_http_response()) } else { Err(trc::ResourceEvent::NotFound.into_err()) }; } + ("user-agent-configuration.json", &Method::GET) => { + // Limit anonymous requests + self.is_http_anonymous_request_allowed(session.remote_ip) + .await?; + return Ok(Resource::new( + "application/json", + self.core.network.info.pacc.clone().into_bytes(), + ) + .into_http_response()); + } ("mail-v1.xml", &Method::GET) => { // Limit anonymous requests self.is_http_anonymous_request_allowed(session.remote_ip) .await?; + let todo = "fix autoconfig generation"; + return self.handle_autoconfig_request(&req).await; } ("autoconfig", &Method::GET) => { diff --git a/crates/jmap/src/registry/get.rs b/crates/jmap/src/registry/get.rs index f3fc7e19..a7ad3d3a 100644 --- a/crates/jmap/src/registry/get.rs +++ b/crates/jmap/src/registry/get.rs @@ -7,12 +7,12 @@ use crate::registry::{ EnterpriseRegistry, mapping::{ - RegistryGetResponse, account::account_get, dkim::generate_dkim_public_key, log::log_get, + RegistryGetResponse, account::account_get, log::log_get, queued_message::queued_message_get, report::report_get, spam_sample::spam_sample_get, task::task_get, }, }; -use common::{Server, auth::AccessToken}; +use common::{Server, auth::AccessToken, network::dkim::generate_dkim_public_key}; use jmap_proto::{ method::get::{GetRequest, GetResponse}, object::registry::Registry, @@ -257,7 +257,10 @@ impl RegistryGet for Server { if get.properties.is_empty() || get.properties.contains(&Property::DnsZoneFile) => { - let todo = "domain dns zone file"; + extra_properties.append( + Property::DnsZoneFile, + JmapValue::Str(self.build_bind_dns_records(id, obj).await?.into()), + ); } _ => {} } diff --git a/crates/jmap/src/registry/mapping/dkim.rs b/crates/jmap/src/registry/mapping/dkim.rs index 7c9f064e..77aefb62 100644 --- a/crates/jmap/src/registry/mapping/dkim.rs +++ b/crates/jmap/src/registry/mapping/dkim.rs @@ -7,13 +7,9 @@ use crate::registry::mapping::{ ObjectResponse, RegistrySetResponse, ValidationResult, principal::validate_tenant_quota, }; -use common::config::smtp::auth::{DkimSigner, rsa_key_parse, simple_pem_parse}; +use common::config::smtp::auth::DkimSigner; use jmap_proto::error::set::SetError; -use mail_auth::common::crypto::Ed25519Key; -use mail_builder::encoders::base64::base64_encode; -use pkcs8::Document; use registry::schema::{enums::TenantStorageQuota, structs::DkimSignature}; -use rsa::pkcs1::DecodeRsaPublicKey; pub(crate) async fn validate_dkim_signature( set: &RegistrySetResponse<'_>, @@ -31,31 +27,6 @@ pub(crate) async fn validate_dkim_signature( ObjectResponse::default() }; - // Generate private key if requested - /*let key_type = key.object_type(); - let pk = key.private_key_mut(); - if let Some(old_key) = old_key - && matches!(pk, DkimPrivateKey::Value(value) if value.secret == MASKED_PASSWORD) - { - *pk = old_key.private_key().clone(); - } - if pk == &DkimPrivateKey::Generate { - match generate_dkim_private_key(key_type).await? { - Ok(secret) => { - let pk_value = DkimPrivateKey::Value(SecretTextValue { secret }); - - response - .object - .insert(Property::PrivateKey, pk_value.clone().into_value()); - - *pk = pk_value; - } - Err(err) => { - return Ok(Err(SetError::forbidden().with_description(err.to_string()))); - } - } - }*/ - if old_key.is_none_or(|old_key| old_key.private_key() != key.private_key()) && let Err(err) = DkimSigner::new("example.com".to_string(), key.clone()).await { @@ -66,48 +37,3 @@ pub(crate) async fn validate_dkim_signature( Ok(Ok(response)) } - -pub async fn generate_dkim_public_key(key: &DkimSignature) -> trc::Result { - match key { - DkimSignature::Dkim1RsaSha256(key) => key - .private_key - .secret() - .await - .map_err(|err| trc::DkimEvent::BuildError.reason(err)) - .and_then(|pem| rsa_key_parse(pem.as_bytes())) - .and_then(|pk| { - Document::from_pkcs1_der(&pk.public_key()).map_err(|err| { - trc::EventType::Dkim(trc::DkimEvent::BuildError) - .into_err() - .reason(err) - }) - }) - .map(|pk| { - String::from_utf8(base64_encode(pk.as_bytes()).unwrap_or_default()) - .unwrap_or_default() - }), - DkimSignature::Dkim1Ed25519Sha256(key) => key - .private_key - .secret() - .await - .map_err(|err| trc::DkimEvent::BuildError.reason(err)) - .and_then(|pem| { - simple_pem_parse(&pem).ok_or_else(|| { - trc::EventType::Dkim(trc::DkimEvent::BuildError) - .into_err() - .details("Failed to parse private key PEM") - }) - }) - .and_then(|der| { - Ed25519Key::from_pkcs8_maybe_unchecked_der(&der).map_err(|err| { - trc::EventType::Dkim(trc::DkimEvent::BuildError) - .into_err() - .reason(err) - }) - }) - .map(|pk| { - String::from_utf8(base64_encode(&pk.public_key()).unwrap_or_default()) - .unwrap_or_default() - }), - } -} diff --git a/crates/jmap/src/registry/mapping/domain.rs b/crates/jmap/src/registry/mapping/domain.rs index f9811036..a9a0ab1f 100644 --- a/crates/jmap/src/registry/mapping/domain.rs +++ b/crates/jmap/src/registry/mapping/domain.rs @@ -7,15 +7,15 @@ use crate::registry::mapping::{ ObjectResponse, RegistrySetResponse, ValidationResult, principal::validate_tenant_quota, }; -use common::network::dkim::generate_dkim_selector; +use common::network::{dkim::generate_dkim_selector, dns::update::DnsUpdater}; use jmap_proto::error::set::SetError; use registry::{ schema::{ - enums::{AcmeChallengeType, DkimSignatureType, TaskDkimRotationStage, TenantStorageQuota}, + enums::{AcmeChallengeType, DkimSignatureType, DnsRecordType, TenantStorageQuota}, prelude::Property, structs::{ AcmeProvider, CertificateManagement, DkimManagement, DkimManagementProperties, - DnsManagement, Domain, Task, TaskDkimRotation, TaskDnsManagement, TaskDomainManagement, + DnsManagement, DnsServer, Domain, Task, TaskDnsManagement, TaskDomainManagement, TaskStatus, }, }, @@ -62,6 +62,9 @@ pub(crate) async fn validate_domain( } // Schedule DNS update task + let will_trigger_dkim = matches!(domain.dkim_management, DkimManagement::Automatic(_)) + && old_domain + .is_none_or(|old| !matches!(old.dkim_management, DkimManagement::Automatic(_))); let will_trigger_acme = if let DnsManagement::Automatic(details) = &domain.dns_management && old_domain.is_none_or(|old| !matches!(old.dns_management, DnsManagement::Automatic(_))) { @@ -72,7 +75,14 @@ pub(crate) async fn validate_domain( ); tasks.push(Task::DnsManagement(TaskDnsManagement { domain_id: Id::default(), - update_records: Map::new(details.dns_publish_records.as_slice().to_vec()), + update_records: Map::new( + details + .publish_records + .iter() + .filter(|&&r| r != DnsRecordType::Dkim || !will_trigger_dkim) + .copied() + .collect(), + ), on_success_renew_certificate, status: TaskStatus::now(), })); @@ -82,16 +92,9 @@ pub(crate) async fn validate_domain( }; // Schedule DKIM key rotation task - if matches!(domain.dkim_management, DkimManagement::Automatic(_)) - && old_domain.is_none_or(|old| !matches!(old.dkim_management, DkimManagement::Automatic(_))) - { - tasks.push(Task::DkimKeyRotation(TaskDkimRotation { + if will_trigger_dkim { + tasks.push(Task::DkimManagement(TaskDomainManagement { domain_id: Id::default(), - stage: if matches!(domain.dkim_management, DkimManagement::Automatic(_)) { - TaskDkimRotationStage::GenerateAndPublish - } else { - TaskDkimRotationStage::Generate - }, status: TaskStatus::now(), })); } @@ -117,7 +120,7 @@ pub(crate) async fn validate_domain( .with_description("ACME provider not found"))); }; - if matches!(provider.class, AcmeChallengeType::Dns01) + if matches!(provider.challenge_type, AcmeChallengeType::Dns01) && !matches!(domain.dns_management, DnsManagement::Automatic(_)) { return Ok(Err(SetError::invalid_properties() @@ -135,3 +138,29 @@ pub(crate) async fn validate_domain( Ok(Ok(response)) } + +pub(crate) async fn validate_dns_server( + set: &RegistrySetResponse<'_>, + dns: &mut DnsServer, + old_dns: Option<&DnsServer>, +) -> ValidationResult { + let response = if old_dns.is_none() { + match validate_tenant_quota(set, TenantStorageQuota::MaxDnsServers).await? { + Ok(response) => response, + Err(err) => { + return Ok(Err(err)); + } + } + } else { + ObjectResponse::default() + }; + + if old_dns.is_none_or(|old_dns| old_dns != dns) + && let Err(err) = DnsUpdater::build(dns.clone(), set.server.core.clone()).await + { + return Ok(Err(SetError::invalid_properties() + .with_description(format!("Failed to build DNS server: {err}")))); + } + + Ok(Ok(response)) +} diff --git a/crates/jmap/src/registry/mapping/task.rs b/crates/jmap/src/registry/mapping/task.rs index 60d38945..6b9e9996 100644 --- a/crates/jmap/src/registry/mapping/task.rs +++ b/crates/jmap/src/registry/mapping/task.rs @@ -91,7 +91,7 @@ pub(crate) async fn task_set( | TaskType::StoreMaintenance | TaskType::SpamFilterMaintenance | TaskType::AcmeRenewal - | TaskType::DkimKeyRotation + | TaskType::DkimManagement | TaskType::DnsManagement => { let mut index = IndexBuilder::default(); task.index(&mut index); diff --git a/crates/jmap/src/registry/set.rs b/crates/jmap/src/registry/set.rs index 2c4ede32..9f879e49 100644 --- a/crates/jmap/src/registry/set.rs +++ b/crates/jmap/src/registry/set.rs @@ -11,7 +11,7 @@ use crate::registry::{ account::account_set, action::action_set, dkim::validate_dkim_signature, - domain::validate_domain, + domain::{validate_dns_server, validate_domain}, map_bootstrap_error, principal::{ schedule_account_destruction, validate_account, validate_role, validate_tenant_quota, @@ -44,7 +44,7 @@ use registry::{ OBJ_FILTER_ACCOUNT, OBJ_FILTER_TENANT, OBJ_SINGLETON, Object, ObjectInner, ObjectType, Property, }, - structs::{Account, Certificate, DkimSignature, Domain, PublicKey, Role, Task}, + structs::{Account, Certificate, DkimSignature, DnsServer, Domain, PublicKey, Role, Task}, }, types::id::ObjectId, }; @@ -430,15 +430,15 @@ impl RegistrySet for Server { validate_domain(&set, domain, modification.as_domain(), &mut tasks) .await? } + ObjectInner::DnsServer(dns) => { + validate_dns_server(&set, dns, modification.as_dns_server()).await? + } ObjectInner::MailingList(_) if is_create => { validate_tenant_quota(&set, TenantStorageQuota::MaxMailingLists).await? } ObjectInner::OAuthClient(_) if is_create => { validate_tenant_quota(&set, TenantStorageQuota::MaxOauthClients).await? } - ObjectInner::DnsServer(_) if is_create => { - validate_tenant_quota(&set, TenantStorageQuota::MaxDnsServers).await? - } ObjectInner::Directory(_) if is_create => { validate_tenant_quota(&set, TenantStorageQuota::MaxDirectories).await? } @@ -563,7 +563,7 @@ impl RegistrySet for Server { for mut task in tasks.drain(..) { match &mut task { Task::AcmeRenewal(task) => task.domain_id = object_id, - Task::DkimKeyRotation(task) => task.domain_id = object_id, + Task::DkimManagement(task) => task.domain_id = object_id, Task::DnsManagement(task) => task.domain_id = object_id, _ => unreachable!(), } @@ -772,6 +772,16 @@ impl Modification { } } + fn as_dns_server(&self) -> Option<&DnsServer> { + match self { + Modification::Create { .. } => None, + Modification::Update { object, .. } => match &object.inner { + ObjectInner::DnsServer(dns) => Some(dns), + _ => None, + }, + } + } + fn as_certificate(&self) -> Option<&Certificate> { match self { Modification::Create { .. } => None, diff --git a/crates/registry/src/utils/dkim.rs b/crates/registry/src/utils/dkim.rs index 6f8eee4f..c8a61572 100644 --- a/crates/registry/src/utils/dkim.rs +++ b/crates/registry/src/utils/dkim.rs @@ -4,7 +4,78 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::schema::enums::DkimSignatureType; +use crate::schema::{ + enums::{DkimRotationStage, DkimSignatureType}, + prelude::{DkimSignature, UTCDateTime}, +}; +use types::id::Id; + +impl DkimSignature { + pub fn rotation_due(&self) -> Option { + let (stage, next_transition) = match self { + DkimSignature::Dkim1Ed25519Sha256(sign) => (sign.stage, sign.next_transition_at), + DkimSignature::Dkim1RsaSha256(sign) => (sign.stage, sign.next_transition_at), + }; + next_transition.and_then(|next_transition| { + if next_transition <= UTCDateTime::now() { + Some(stage) + } else { + None + } + }) + } + + pub fn next_transition(&self) -> Option { + match self { + DkimSignature::Dkim1Ed25519Sha256(sign) => sign.next_transition_at, + DkimSignature::Dkim1RsaSha256(sign) => sign.next_transition_at, + } + } + + pub fn set_next_transition(&mut self, next_transition: UTCDateTime) { + match self { + DkimSignature::Dkim1Ed25519Sha256(sign) => { + sign.next_transition_at = Some(next_transition) + } + DkimSignature::Dkim1RsaSha256(sign) => sign.next_transition_at = Some(next_transition), + } + } + + pub fn stage(&self) -> DkimRotationStage { + match self { + DkimSignature::Dkim1Ed25519Sha256(sign) => sign.stage, + DkimSignature::Dkim1RsaSha256(sign) => sign.stage, + } + } + + pub fn set_stage(&mut self, stage: DkimRotationStage) { + match self { + DkimSignature::Dkim1Ed25519Sha256(sign) => sign.stage = stage, + DkimSignature::Dkim1RsaSha256(sign) => sign.stage = stage, + } + } + + pub fn is_active(&self) -> bool { + match self { + DkimSignature::Dkim1Ed25519Sha256(sign) => sign.stage == DkimRotationStage::Active, + DkimSignature::Dkim1RsaSha256(sign) => sign.stage == DkimRotationStage::Active, + } + } + + pub fn selector(&self) -> &str { + match self { + DkimSignature::Dkim1Ed25519Sha256(sign) => &sign.selector, + DkimSignature::Dkim1RsaSha256(sign) => &sign.selector, + } + } + + pub fn domain_id(&self) -> Id { + match self { + DkimSignature::Dkim1Ed25519Sha256(sign) => sign.domain_id, + DkimSignature::Dkim1RsaSha256(sign) => sign.domain_id, + } + } +} impl DkimSignatureType { pub const fn algorithm(self) -> &'static str { diff --git a/crates/registry/src/utils/mod.rs b/crates/registry/src/utils/mod.rs index 349ea18e..2502d348 100644 --- a/crates/registry/src/utils/mod.rs +++ b/crates/registry/src/utils/mod.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::schema::prelude::{DkimSignature, Roles, SecretKey}; +use crate::schema::prelude::{DkimSignature, Roles, SecretText}; use types::id::Id; pub mod account; @@ -26,14 +26,14 @@ impl Roles { } impl DkimSignature { - pub fn private_key(&self) -> &SecretKey { + pub fn private_key(&self) -> &SecretText { match self { DkimSignature::Dkim1Ed25519Sha256(signature) => &signature.private_key, DkimSignature::Dkim1RsaSha256(signature) => &signature.private_key, } } - pub fn private_key_mut(&mut self) -> &mut SecretKey { + pub fn private_key_mut(&mut self) -> &mut SecretText { match self { DkimSignature::Dkim1Ed25519Sha256(signature) => &mut signature.private_key, DkimSignature::Dkim1RsaSha256(signature) => &mut signature.private_key, diff --git a/crates/registry/src/utils/report.rs b/crates/registry/src/utils/report.rs index 50bbc9f6..87c4de71 100644 --- a/crates/registry/src/utils/report.rs +++ b/crates/registry/src/utils/report.rs @@ -209,7 +209,7 @@ impl From for enums::SpfDomainScope { impl From for PolicyOverrideReason { fn from(value: structs::DmarcPolicyOverrideReason) -> Self { PolicyOverrideReason { - type_: value.class.into(), + type_: value.override_type.into(), comment: value.comment, } } @@ -218,7 +218,7 @@ impl From for PolicyOverrideReason { impl From for structs::DmarcPolicyOverrideReason { fn from(value: PolicyOverrideReason) -> Self { structs::DmarcPolicyOverrideReason { - class: value.type_.into(), + override_type: value.type_.into(), comment: value.comment, } } diff --git a/crates/registry/src/utils/task.rs b/crates/registry/src/utils/task.rs index db53da18..6b785e2c 100644 --- a/crates/registry/src/utils/task.rs +++ b/crates/registry/src/utils/task.rs @@ -27,7 +27,7 @@ impl Task { Task::StoreMaintenance(task) => task.status = status, Task::SpamFilterMaintenance(task) => task.status = status, Task::AcmeRenewal(task) => task.status = status, - Task::DkimKeyRotation(task) => task.status = status, + Task::DkimManagement(task) => task.status = status, Task::DnsManagement(task) => task.status = status, } } @@ -49,7 +49,7 @@ impl Task { Task::StoreMaintenance(task) => &task.status, Task::SpamFilterMaintenance(task) => &task.status, Task::AcmeRenewal(task) => &task.status, - Task::DkimKeyRotation(task) => &task.status, + Task::DkimManagement(task) => &task.status, Task::DnsManagement(task) => &task.status, } } @@ -87,7 +87,7 @@ impl Task { Task::StoreMaintenance(_) => Permission::TaskStoreMaintenance, Task::SpamFilterMaintenance(_) => Permission::TaskSpamFilterMaintenance, Task::AcmeRenewal(_) => Permission::TaskAcmeRenewal, - Task::DkimKeyRotation(_) => Permission::TaskDkimKeyRotation, + Task::DkimManagement(_) => Permission::TaskDkimManagement, Task::DnsManagement(_) => Permission::TaskDnsManagement, } } diff --git a/crates/services/Cargo.toml b/crates/services/Cargo.toml index bd45d194..f2708e5a 100644 --- a/crates/services/Cargo.toml +++ b/crates/services/Cargo.toml @@ -34,6 +34,7 @@ sha2 = "0.10" reqwest = { version = "0.12", default-features = false, features = ["rustls-tls-webpki-roots", "http2"]} base64 = "0.22" compact_str = "0.9.0" +dns-update = { path = "/Users/me/code/dns-update" } [dev-dependencies] diff --git a/crates/services/src/task_manager/acme.rs b/crates/services/src/task_manager/acme.rs new file mode 100644 index 00000000..3f78f232 --- /dev/null +++ b/crates/services/src/task_manager/acme.rs @@ -0,0 +1,74 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::task_manager::{TaskFailureType, TaskResult}; +use common::{Server, network::acme::AcmeError}; +use registry::schema::structs::TaskDomainManagement; +use store::write::now; + +pub(crate) trait AcmeTask: Sync + Send { + fn acme_management( + &self, + task: &TaskDomainManagement, + ) -> impl Future + Send; +} + +impl AcmeTask for Server { + async fn acme_management(&self, task: &TaskDomainManagement) -> TaskResult { + match acme_management(self, task).await { + Ok(result) => result, + Err(err) => { + let result = TaskResult::temporary(err.to_string()); + trc::error!( + err.caused_by(trc::location!()) + .details("Failed to run ACME task") + ); + result + } + } + } +} + +async fn acme_management(server: &Server, task: &TaskDomainManagement) -> trc::Result { + match server.acme_renew(task.domain_id).await { + Ok(tasks) => Ok(TaskResult::Success(tasks)), + Err(err) => match err { + AcmeError::Crypto(_) + | AcmeError::Invalid(_) + | AcmeError::ChallengeNotSupported { .. } + | AcmeError::OrderInvalid + | AcmeError::AuthInvalid(_) + | AcmeError::Json(_) + | AcmeError::Registry(_) => Ok(TaskResult::permanent(err.to_string())), + AcmeError::Http(_) | AcmeError::HttpStatus(_) | AcmeError::Dns(_) => { + Ok(TaskResult::temporary(err.to_string())) + } + AcmeError::OrderTimeout { max_retries } | AcmeError::AuthTimeout { max_retries } => { + Ok(TaskResult::Failure { + typ: TaskFailureType::Temporary, + message: err.to_string(), + max_attempts: (max_retries as u64).into(), + }) + } + AcmeError::Backoff { max_retries, wait } => { + if let Some(wait) = wait { + Ok(TaskResult::Failure { + typ: TaskFailureType::Retry(now() + wait.as_secs()), + message: err.to_string(), + max_attempts: (max_retries as u64).into(), + }) + } else { + Ok(TaskResult::Failure { + typ: TaskFailureType::Temporary, + message: err.to_string(), + max_attempts: (max_retries as u64).into(), + }) + } + } + AcmeError::Internal(error) => Err(error), + }, + } +} diff --git a/crates/services/src/task_manager/alarm.rs b/crates/services/src/task_manager/alarm.rs index c2040238..9a06e272 100644 --- a/crates/services/src/task_manager/alarm.rs +++ b/crates/services/src/task_manager/alarm.rs @@ -116,7 +116,7 @@ async fn send_email_alarm( AccountId = account_id, DocumentId = document_id, ); - return Ok(TaskResult::Success); + return Ok(TaskResult::Success(vec![])); } let account_info = server .account_info(account_id) @@ -130,7 +130,7 @@ async fn send_email_alarm( AccountId = account_id, DocumentId = document_id, ); - return Ok(TaskResult::Success); + return Ok(TaskResult::Success(vec![])); } // Fetch event @@ -151,7 +151,7 @@ async fn send_email_alarm( DocumentId = document_id, ); - return Ok(TaskResult::Success); + return Ok(TaskResult::Success(vec![])); }; // Unarchive event @@ -164,7 +164,7 @@ async fn send_email_alarm( let account_main_domain = account_main_email.rsplit('@').next().unwrap_or("localhost"); let logo_cid = format!("logo.{}@{account_main_domain}", now()); let Some(tpl) = build_template(server, &account_info, task, event, &logo_cid).await? else { - return Ok(TaskResult::Success); + return Ok(TaskResult::Success(vec![])); }; let txt_body = html_to_text(&tpl.body); @@ -335,7 +335,7 @@ async fn send_display_alarm( DocumentId = document_id, ); - return Ok(TaskResult::Success); + return Ok(TaskResult::Success(vec![])); }; // Unarchive event @@ -411,7 +411,7 @@ fn build_next_alarm( next_alarm.build_write_ops(account_id, document_id), )) } else { - Ok(TaskResult::Success) + Ok(TaskResult::Success(vec![])) } } diff --git a/crates/services/src/task_manager/destroy_account.rs b/crates/services/src/task_manager/destroy_account.rs index 2113dfed..4a33f323 100644 --- a/crates/services/src/task_manager/destroy_account.rs +++ b/crates/services/src/task_manager/destroy_account.rs @@ -156,7 +156,7 @@ async fn destroy_account(server: &Server, task: &TaskDestroyAccount) -> trc::Res .await .caused_by(trc::location!())?; - Ok(TaskResult::Success) + Ok(TaskResult::Success(vec![])) } pub async fn destroy_account_blobs(server: &Server, account_id: u32) -> trc::Result<()> { diff --git a/crates/services/src/task_manager/dkim.rs b/crates/services/src/task_manager/dkim.rs new file mode 100644 index 00000000..fb957bb8 --- /dev/null +++ b/crates/services/src/task_manager/dkim.rs @@ -0,0 +1,476 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::task_manager::TaskResult; +use common::{ + Server, + cache::invalidate::CacheInvalidationBuilder, + ipc::CacheInvalidation, + network::dkim::{ + generate_dkim_dns_record, generate_dkim_dns_record_name, generate_dkim_private_key, + generate_dkim_selector, + }, +}; +use registry::{ + schema::{ + enums::{DkimRotationStage, DkimSignatureType, DnsRecordType}, + prelude::{Object, ObjectType, Property}, + structs::{ + Dkim1Signature, DkimManagement, DkimSignature, DnsManagement, Domain, SecretText, + SecretTextValue, Task, TaskDomainManagement, TaskStatus, + }, + }, + types::{datetime::UTCDateTime, id::ObjectId}, +}; +use std::fmt::Write; +use store::{ + registry::{ + RegistryObject, RegistryQuery, + write::{RegistryWrite, RegistryWriteResult}, + }, + write::now, +}; +use types::id::Id; + +pub(crate) trait DkimManagementTask: Sync + Send { + fn dkim_management( + &self, + task: &TaskDomainManagement, + ) -> impl Future + Send; +} + +impl DkimManagementTask for Server { + async fn dkim_management(&self, task: &TaskDomainManagement) -> TaskResult { + match dkim_management(self, task).await { + Ok(result) => result, + Err(err) => { + let result = TaskResult::temporary(err.to_string()); + trc::error!( + err.caused_by(trc::location!()) + .details("Failed to run DKIM management task") + ); + result + } + } + } +} + +async fn dkim_management(server: &Server, task: &TaskDomainManagement) -> trc::Result { + let Some(domain) = server.registry().object::(task.domain_id).await? else { + return Ok(TaskResult::permanent("Domain not found".to_string())); + }; + let DkimManagement::Automatic(dkim) = domain.dkim_management else { + return Ok(TaskResult::permanent( + "Domain is not set to automatic DKIM management".to_string(), + )); + }; + let mut create_signatures = dkim.algorithms.into_inner(); + if create_signatures.is_empty() { + return Ok(TaskResult::permanent( + "No DKIM algorithms configured for domain".to_string(), + )); + } + + let dns_updater = match domain.dns_management { + DnsManagement::Automatic(props) if props.publish_records.contains(&DnsRecordType::Dkim) => { + match server.build_dns_updater(props.dns_server_id).await? { + Ok(updater) => Some((updater, props.origin.unwrap_or_else(|| domain.name.clone()))), + Err(err) => { + return Ok(TaskResult::permanent(format!( + "Failed to build DNS updater: {}", + err + ))); + } + } + } + _ => None, + }; + + // Fetch existing DKIM keys + let mut publish_signatures = Vec::new(); + let mut retire_signatures = Vec::new(); + let mut retiring_signatures = Vec::new(); + let mut delete_signatures = Vec::new(); + let mut next_transition = None; + + let signature_ids = server + .registry() + .query::>( + RegistryQuery::new(ObjectType::DkimSignature) + .equal(Property::DomainId, task.domain_id.document_id()), + ) + .await?; + + for id in signature_ids { + let id = ObjectId::new(ObjectType::DkimSignature, id); + let Some(key) = server.registry().get(id).await? else { + continue; + }; + let key: RegistryObject = RegistryObject { + id, + revision: key.revision, + object: key.into(), + }; + + let key_algo = key.object.object_type(); + if let Some(current_stage) = key.object.rotation_due() { + match current_stage { + DkimRotationStage::Pending => { + create_signatures.retain(|algo| algo != &key_algo); + publish_signatures.push(key) + } + DkimRotationStage::Active => retiring_signatures.push(key), + DkimRotationStage::Retiring => retire_signatures.push(key), + DkimRotationStage::Retired => delete_signatures.push(key), + } + } else { + if key.object.is_active() { + create_signatures.retain(|algo| algo != &key_algo); + } + + if let Some(transition) = key.object.next_transition() + && next_transition.is_none_or(|next| transition < next) + { + next_transition = Some(transition); + } + } + } + + let now = now(); + let mut do_refresh = false; + + for algorithm in create_signatures { + // Generate new key and selector + let secret = match generate_dkim_private_key(algorithm).await? { + Ok(secret) => secret, + Err(err) => { + return Ok(TaskResult::permanent(err.to_string())); + } + }; + let selector = match generate_dkim_selector(&dkim.selector_template, algorithm) { + Ok(selector) => selector, + Err(err) => { + return Ok(TaskResult::permanent(format!( + "Failed to generate DKIM selector: {}", + err + ))); + } + }; + + // Build key + let signature = Dkim1Signature { + stage: DkimRotationStage::Active, + domain_id: task.domain_id, + member_tenant_id: domain.member_tenant_id, + selector, + private_key: SecretText::Text(SecretTextValue { secret }), + ..Default::default() + }; + let mut signature = match algorithm { + DkimSignatureType::Dkim1Ed25519Sha256 => DkimSignature::Dkim1Ed25519Sha256(signature), + DkimSignatureType::Dkim1RsaSha256 => DkimSignature::Dkim1RsaSha256(signature), + }; + + // Publish key + if let Some((updater, origin)) = &dns_updater { + let record = generate_dkim_dns_record(&signature, &domain.name).await?; + let signature_transition = if updater + .create(origin, &record.name, record.record, true) + .await + .is_ok_and(|did_propagate| did_propagate) + { + do_refresh = true; + UTCDateTime::from_timestamp((now + dkim.rotate_after.as_secs()) as i64) + } else { + // Something went wrong, reschedule. + signature.set_stage(DkimRotationStage::Pending); + UTCDateTime::from_timestamp((now + 60) as i64) // Retry after 1 minute + }; + + if next_transition.is_none_or(|next| signature_transition < next) { + next_transition = Some(signature_transition); + } + + signature.set_next_transition(signature_transition); + } + + // Write key + match server + .registry() + .write(RegistryWrite::insert(&signature.into())) + .await? + { + RegistryWriteResult::Success(_) => (), + err => { + return Ok(TaskResult::permanent(format!( + "Failed to write DKIM signature: {err}" + ))); + } + } + } + + // Publish signatures + let mut temporary_errors = String::new(); + for signature in publish_signatures { + let record = generate_dkim_dns_record(&signature.object, &domain.name).await?; + if let Some((updater, origin)) = &dns_updater { + match updater + .create(origin, &record.name, record.record, true) + .await + { + Ok(true) => { + let signature_transition = + UTCDateTime::from_timestamp((now + dkim.rotate_after.as_secs()) as i64); + + if next_transition.is_none_or(|next| signature_transition < next) { + next_transition = Some(signature_transition); + } + + let mut new_signature = signature.object.clone(); + + new_signature.set_next_transition(signature_transition); + new_signature.set_stage(DkimRotationStage::Active); + + // Write key + if let Some(task_result) = update_signature( + server, + signature, + new_signature, + &record.name, + &mut temporary_errors, + ) + .await? + { + return Ok(task_result); + } + do_refresh = true; + } + Ok(false) => { + if !temporary_errors.is_empty() { + temporary_errors.push_str("; "); + } + let _ = write!( + &mut temporary_errors, + "DKIM record {} did not propagate, will retry.", + record.name + ); + } + Err(err) => { + if !temporary_errors.is_empty() { + temporary_errors.push_str("; "); + } + let _ = write!( + &mut temporary_errors, + "Failed to publish DKIM record {}: {err}.", + record.name + ); + } + } + } else { + if !temporary_errors.is_empty() { + temporary_errors.push_str("; "); + } + let _ = write!( + &mut temporary_errors, + "No DNS server configured, cannot publish DKIM record {}.", + record.name + ); + } + } + + // Retiring signatures + for signature in retiring_signatures { + let record = generate_dkim_dns_record_name(&signature.object, &domain.name); + let signature_transition = + UTCDateTime::from_timestamp((now + dkim.retire_after.as_secs()) as i64); + + if next_transition.is_none_or(|next| signature_transition < next) { + next_transition = Some(signature_transition); + } + + let mut new_signature = signature.object.clone(); + + new_signature.set_next_transition(signature_transition); + new_signature.set_stage(DkimRotationStage::Retiring); + + // Write key + if let Some(task_result) = update_signature( + server, + signature, + new_signature, + &record, + &mut temporary_errors, + ) + .await? + { + return Ok(task_result); + } + do_refresh = true; + } + + // Retire signatures + for signature in retire_signatures { + let record = generate_dkim_dns_record_name(&signature.object, &domain.name); + if let Some((updater, origin)) = &dns_updater { + match updater + .delete(origin, &record, dns_update::DnsRecordType::TXT) + .await + { + Ok(_) => { + let signature_transition = + UTCDateTime::from_timestamp((now + dkim.delete_after.as_secs()) as i64); + + if next_transition.is_none_or(|next| signature_transition < next) { + next_transition = Some(signature_transition); + } + + let mut new_signature = signature.object.clone(); + + new_signature.set_next_transition(signature_transition); + new_signature.set_stage(DkimRotationStage::Retired); + + // Write key + if let Some(task_result) = update_signature( + server, + signature, + new_signature, + &record, + &mut temporary_errors, + ) + .await? + { + return Ok(task_result); + } + + do_refresh = true; + } + Err(err) => { + if !temporary_errors.is_empty() { + temporary_errors.push_str("; "); + } + let _ = write!( + &mut temporary_errors, + "Failed to remove DKIM record {}: {err}.", + record + ); + } + } + } else { + if !temporary_errors.is_empty() { + temporary_errors.push_str("; "); + } + let _ = write!( + &mut temporary_errors, + "No DNS server configured, cannot retire DKIM record {}.", + record + ); + } + } + + // Delete signatures + for signature in delete_signatures { + let record = generate_dkim_dns_record_name(&signature.object, &domain.name); + match server + .registry() + .write(RegistryWrite::delete_object( + signature.id, + &Object { + inner: signature.object.into(), + revision: signature.revision, + }, + )) + .await + { + Ok(RegistryWriteResult::Success(_)) => {} + Ok(err) => { + return Ok(TaskResult::permanent(format!( + "Failed to delete DKIM signature for record {record}: {err}" + ))); + } + Err(err) => { + if err.is_assertion_failure() { + if !temporary_errors.is_empty() { + temporary_errors.push_str("; "); + } + let _ = write!( + temporary_errors, + "Failed to delete DKIM signature for record {record} due to concurrent modification, will retry.", + ); + } else { + return Err(err); + } + } + } + } + + if do_refresh + && let Err(err) = server + .invalidate_caches(CacheInvalidationBuilder::default().with_invalidation( + CacheInvalidation::DkimSignature(task.domain_id.document_id()), + )) + .await + { + trc::error!( + err.caused_by(trc::location!()) + .details("Failed to invalidate caches after DKIM management task") + ); + } + + if !temporary_errors.is_empty() { + Ok(TaskResult::temporary(temporary_errors)) + } else { + let tasks = if let Some(next_transition) = next_transition { + vec![Task::DkimManagement(TaskDomainManagement { + domain_id: task.domain_id, + status: TaskStatus::at(next_transition.timestamp()), + })] + } else { + vec![] + }; + + Ok(TaskResult::Success(tasks)) + } +} + +async fn update_signature( + server: &Server, + signature: RegistryObject, + new_signature: DkimSignature, + name: &str, + temporary_errors: &mut String, +) -> trc::Result> { + match server + .registry() + .write(RegistryWrite::update( + signature.id.id(), + &new_signature.into(), + &Object { + inner: signature.object.into(), + revision: signature.revision, + }, + )) + .await + { + Ok(RegistryWriteResult::Success(_)) => Ok(None), + Ok(err) => Ok(Some(TaskResult::permanent(format!( + "Failed to write DKIM signature for record {name}: {err}" + )))), + Err(err) => { + if err.is_assertion_failure() { + if !temporary_errors.is_empty() { + temporary_errors.push_str("; "); + } + let _ = write!( + temporary_errors, + "Failed to write DKIM signature for record {name} due to concurrent modification, will retry.", + ); + Ok(None) + } else { + Err(err) + } + } + } +} diff --git a/crates/services/src/task_manager/dns.rs b/crates/services/src/task_manager/dns.rs new file mode 100644 index 00000000..0bd4639b --- /dev/null +++ b/crates/services/src/task_manager/dns.rs @@ -0,0 +1,33 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::task_manager::TaskResult; +use common::Server; +use registry::schema::structs::TaskDnsManagement; + +pub(crate) trait DnsManagementTask: Sync + Send { + fn dns_management(&self, task: &TaskDnsManagement) -> impl Future + Send; +} + +impl DnsManagementTask for Server { + async fn dns_management(&self, task: &TaskDnsManagement) -> TaskResult { + match dns_management(self, task).await { + Ok(result) => result, + Err(err) => { + let result = TaskResult::temporary(err.to_string()); + trc::error!( + err.caused_by(trc::location!()) + .details("Failed to run DNS management task") + ); + result + } + } + } +} + +async fn dns_management(server: &Server, imip: &TaskDnsManagement) -> trc::Result { + todo!() +} diff --git a/crates/services/src/task_manager/imip.rs b/crates/services/src/task_manager/imip.rs index 30dd8084..05eb2387 100644 --- a/crates/services/src/task_manager/imip.rs +++ b/crates/services/src/task_manager/imip.rs @@ -271,7 +271,7 @@ async fn send_imip( } } - Ok(TaskResult::Success) + Ok(TaskResult::Success(vec![])) } pub struct Details { diff --git a/crates/services/src/task_manager/index.rs b/crates/services/src/task_manager/index.rs index cd78bf3e..16f7d27a 100644 --- a/crates/services/src/task_manager/index.rs +++ b/crates/services/src/task_manager/index.rs @@ -86,7 +86,7 @@ impl SearchIndexTask for Server { let result = match document { Ok(Some(doc)) if !doc.is_empty() => { document_insertions.push(doc); - TaskResult::Success + TaskResult::Success(vec![]) } Err(err) => { let result = TaskResult::temporary(err.to_string()); @@ -121,7 +121,7 @@ impl SearchIndexTask for Server { let result = match build_tracing_span_document(self, task.trace_id.id()).await { Ok(Some(doc)) if !doc.is_empty() => { document_insertions.push(doc); - TaskResult::Success + TaskResult::Success(vec![]) } Err(err) => { let result = TaskResult::temporary(err.to_string()); @@ -187,7 +187,7 @@ impl SearchIndexTask for Server { results.push(IndexTaskResult { task_type: TaskType::Delete, index: task.document_type, - result: TaskResult::Success, + result: TaskResult::Success(vec![]), }); } _ => unreachable!(), @@ -204,7 +204,7 @@ impl SearchIndexTask for Server { ); for r in results.iter_mut() { if r.task_type == TaskType::Delete - && r.result == TaskResult::Success + && r.result.is_success() && r.index == IndexDocumentType::Email { r.result = @@ -223,7 +223,7 @@ impl SearchIndexTask for Server { .details("Failed to index documents") ); for r in results.iter_mut() { - if r.task_type == TaskType::Insert && r.result == TaskResult::Success { + if r.task_type == TaskType::Insert && r.result.is_success() { r.result = TaskResult::temporary("Failed to index documents"); } } @@ -278,7 +278,7 @@ impl SearchIndexTask for Server { .ctx(trc::Key::Collection, index.name()) ); for r in results.iter_mut() { - if r.task_type == TaskType::Delete && r.result == TaskResult::Success { + if r.task_type == TaskType::Delete && r.result.is_success() { r.result = TaskResult::temporary("Failed to delete documents from index"); } } diff --git a/crates/services/src/task_manager/maintenance.rs b/crates/services/src/task_manager/maintenance.rs index 71b3e1bb..2ec4b92c 100644 --- a/crates/services/src/task_manager/maintenance.rs +++ b/crates/services/src/task_manager/maintenance.rs @@ -287,7 +287,7 @@ async fn store_maintenance( // Simulate success for testing purposes match test_var { 0 => { - return Ok(TaskResult::Success); + return Ok(TaskResult::Success(vec![])); } 1 => { return Ok(TaskResult::temporary( @@ -302,6 +302,7 @@ async fn store_maintenance( return Ok(TaskResult::Failure { typ: TaskFailureType::Retry(retry), message: "Simulated retry failure".to_string(), + max_attempts: None, }); } } @@ -338,7 +339,7 @@ async fn store_maintenance( } } - Ok(TaskResult::Success) + Ok(TaskResult::Success(vec![])) } async fn account_maintenance( @@ -360,7 +361,7 @@ async fn account_maintenance( } } - Ok(TaskResult::Success) + Ok(TaskResult::Success(vec![])) } async fn recalculate_quota(server: &Server, account_id: u32) -> trc::Result<()> { diff --git a/crates/services/src/task_manager/manager.rs b/crates/services/src/task_manager/manager.rs index 87eefeaf..aadc9e90 100644 --- a/crates/services/src/task_manager/manager.rs +++ b/crates/services/src/task_manager/manager.rs @@ -4,8 +4,11 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::task_manager::acme::AcmeTask; use crate::task_manager::alarm::SendAlarmTask; use crate::task_manager::destroy_account::DestroyAccountTask; +use crate::task_manager::dkim::DkimManagementTask; +use crate::task_manager::dns::DnsManagementTask; use crate::task_manager::imip::SendImipTask; use crate::task_manager::index::SearchIndexTask; use crate::task_manager::lock::TaskLockManager; @@ -100,7 +103,7 @@ pub fn spawn_task_manager(inner: Arc) { | TaskType::TlsReport | TaskType::RestoreArchivedItem | TaskType::AcmeRenewal - | TaskType::DkimKeyRotation + | TaskType::DkimManagement | TaskType::DnsManagement => TASK_QUEUE_BUFFER, }; @@ -237,12 +240,16 @@ pub fn spawn_task_manager(inner: Arc) { Task::SpamFilterMaintenance(task) => { server.spam_filter_maintenance(task).await } + Task::AcmeRenewal(task) => server.acme_management(task).await, + Task::DkimManagement(task_dkim_rotation) => { + server.dkim_management(task_dkim_rotation).await + } + Task::DnsManagement(task_dns_management) => { + server.dns_management(task_dns_management).await + } Task::IndexDocument(_) | Task::UnindexDocument(_) | Task::IndexTrace(_) => unreachable!(), - Task::AcmeRenewal(task_domain_management) => todo!(), - Task::DkimKeyRotation(task_dkim_rotation) => todo!(), - Task::DnsManagement(task_dns_management) => todo!(), }; refresh_queue = result.is_retry(); @@ -361,7 +368,7 @@ impl TaskQueueManager for Server { | TaskType::TlsReport | TaskType::RestoreArchivedItem | TaskType::AcmeRenewal - | TaskType::DkimKeyRotation + | TaskType::DkimManagement | TaskType::DnsManagement => true, }; @@ -488,7 +495,13 @@ async fn update_tasks( due: task.info.due, })); match result { - TaskResult::Success | TaskResult::Ignored => { + TaskResult::Success(tasks) => { + for task in tasks { + batch.schedule_task(task); + } + batch.clear(ValueClass::TaskQueue(TaskQueueClass::Task { id })); + } + TaskResult::Ignored => { batch.clear(ValueClass::TaskQueue(TaskQueueClass::Task { id })); } TaskResult::Update(ops) => { @@ -496,7 +509,11 @@ async fn update_tasks( batch.any_op(op); } } - TaskResult::Failure { typ, message } => { + TaskResult::Failure { + typ, + message, + max_attempts, + } => { let (attempt_number, created_at) = match task.task.status() { TaskStatus::Pending(status) => (0, status.created_at), TaskStatus::Retry(status) => (status.attempt_number, status.created_at), @@ -504,7 +521,7 @@ async fn update_tasks( }; let retry_at = match typ { TaskFailureType::Retry(retry_at) => (attempt_number - < server.core.network.task_manager.max_attempts + < max_attempts.unwrap_or(server.core.network.task_manager.max_attempts) && retry_at < retry_at.saturating_add( server.core.network.task_manager.total_deadline.as_secs(), @@ -512,6 +529,7 @@ async fn update_tasks( .then_some(retry_at), TaskFailureType::Temporary => next_retry_time( &server.core.network.task_manager, + max_attempts, created_at.timestamp() as u64, attempt_number, now(), @@ -576,11 +594,12 @@ async fn update_tasks( pub fn next_retry_time( manager: &TaskManager, + max_attempts_override: Option, create_time: u64, attempt: u64, now: u64, ) -> Option { - if attempt >= manager.max_attempts { + if attempt >= max_attempts_override.unwrap_or(manager.max_attempts) { return None; } @@ -610,6 +629,10 @@ pub fn next_retry_time( } impl TaskResult { + pub fn is_success(&self) -> bool { + matches!(self, TaskResult::Success(_)) + } + pub fn is_retry(&self) -> bool { matches!( self, diff --git a/crates/services/src/task_manager/merge_threads.rs b/crates/services/src/task_manager/merge_threads.rs index 319cb212..c0402d5b 100644 --- a/crates/services/src/task_manager/merge_threads.rs +++ b/crates/services/src/task_manager/merge_threads.rs @@ -129,7 +129,7 @@ async fn merge_threads( if thread_merge.num_thread_ids() < 2 { // Another process merged the threads already? - return Ok(TaskResult::Success); + return Ok(TaskResult::Success(vec![])); } // Add other messages with the same subject to the merge if they share a @@ -239,7 +239,7 @@ async fn merge_threads( } match server.commit_batch(batch).await { - Ok(_) => return Ok(TaskResult::Success), + Ok(_) => return Ok(TaskResult::Success(vec![])), Err(err) if err.is_assertion_failure() && try_count < MAX_RETRIES => { let backoff = store::rand::rng().random_range(50..=300); tokio::time::sleep(Duration::from_millis(backoff)).await; diff --git a/crates/services/src/task_manager/mod.rs b/crates/services/src/task_manager/mod.rs index dc5fb66b..17daa7e4 100644 --- a/crates/services/src/task_manager/mod.rs +++ b/crates/services/src/task_manager/mod.rs @@ -15,8 +15,11 @@ use store::{ahash::AHashMap, write::now}; use tokio::sync::mpsc; use trc::TaskManagerEvent; +pub mod acme; pub mod alarm; pub mod destroy_account; +pub mod dkim; +pub mod dns; pub mod imip; pub mod index; pub mod lock; @@ -59,11 +62,12 @@ pub(crate) struct TaskJob { #[derive(Debug, PartialEq, Eq)] pub(crate) enum TaskResult { - Success, + Success(Vec), Update([Operation; 2]), Failure { typ: TaskFailureType, message: String, + max_attempts: Option, }, Ignored, } @@ -98,7 +102,7 @@ impl TaskInfo for Task { Task::StoreMaintenance(_) => "StoreMaintenance", Task::SpamFilterMaintenance(_) => "SpamFilterMaintenance", Task::AcmeRenewal(_) => "AcmeRenewal", - Task::DkimKeyRotation(_) => "DkimKeyRotation", + Task::DkimManagement(_) => "DkimManagement", Task::DnsManagement(_) => "DnsManagement", } } @@ -109,6 +113,7 @@ impl TaskResult { TaskResult::Failure { typ: TaskFailureType::Permanent, message: message.into(), + max_attempts: None, } } @@ -116,6 +121,7 @@ impl TaskResult { TaskResult::Failure { typ: TaskFailureType::Temporary, message: message.into(), + max_attempts: None, } } } diff --git a/crates/services/src/task_manager/report.rs b/crates/services/src/task_manager/report.rs index 99638039..44bcaf60 100644 --- a/crates/services/src/task_manager/report.rs +++ b/crates/services/src/task_manager/report.rs @@ -35,10 +35,10 @@ async fn submit_report(server: &Server, report_id: ReportId) -> trc::Result server .send_dmarc_aggregate_report(item_id) .await - .map(|_| TaskResult::Success), + .map(|_| TaskResult::Success(vec![])), ReportId::Tls(item_id) => server .send_tls_aggregate_report(item_id) .await - .map(|_| TaskResult::Success), + .map(|_| TaskResult::Success(vec![])), } } diff --git a/crates/services/src/task_manager/restore_item.rs b/crates/services/src/task_manager/restore_item.rs index 7680324a..82dada2f 100644 --- a/crates/services/src/task_manager/restore_item.rs +++ b/crates/services/src/task_manager/restore_item.rs @@ -80,7 +80,7 @@ async fn restore_item(server: &Server, task: &TaskRestoreArchivedItem) -> trc::R }); server.store().write(batch.build_all()).await?; - Ok(TaskResult::Success) + Ok(TaskResult::Success(vec![])) } Err(mut err) if err.matches(trc::EventType::MessageIngest( diff --git a/crates/services/src/task_manager/spam_classifier.rs b/crates/services/src/task_manager/spam_classifier.rs index c6c71499..b0586e76 100644 --- a/crates/services/src/task_manager/spam_classifier.rs +++ b/crates/services/src/task_manager/spam_classifier.rs @@ -7,6 +7,7 @@ use crate::task_manager::{TaskFailureType, TaskResult}; use common::{ Server, + ipc::{BroadcastEvent, RegistryChange}, manager::{SPAM_CLASSIFIER_KEY, SPAM_TRAINER_KEY, fetch_resource}, }; use registry::{ @@ -78,7 +79,7 @@ async fn spam_filter_maintenance( } } - Ok(TaskResult::Success) + Ok(TaskResult::Success(vec![])) } struct RuleUpdateError { @@ -111,6 +112,7 @@ async fn update_spam_rules(server: &Server) -> trc::Result { return Ok(TaskResult::Failure { typ: err.typ, message: err.reason, + max_attempts: None, }); } }; @@ -118,10 +120,14 @@ async fn update_spam_rules(server: &Server) -> trc::Result { let registry = server.registry(); let mut stats: AHashMap = AHashMap::new(); + let mut reload_settings = false; + let mut reload_lookups = false; + for rule in rules.rules { match registry.write(RegistryWrite::insert(&rule.into())).await? { RegistryWriteResult::Success(_) => { stats.entry(ObjectType::SpamRule).or_default().success += 1; + reload_settings = true; } RegistryWriteResult::PrimaryKeyConflict { .. } => { stats @@ -142,6 +148,7 @@ async fn update_spam_rules(server: &Server) -> trc::Result { .entry(ObjectType::SpamDnsblServer) .or_default() .success += 1; + reload_settings = true; } RegistryWriteResult::PrimaryKeyConflict { .. } => { stats @@ -159,6 +166,7 @@ async fn update_spam_rules(server: &Server) -> trc::Result { match registry.write(RegistryWrite::insert(&tag.into())).await? { RegistryWriteResult::Success(_) => { stats.entry(ObjectType::SpamTag).or_default().success += 1; + reload_settings = true; } RegistryWriteResult::PrimaryKeyConflict { .. } => { stats.entry(ObjectType::SpamTag).or_default().already_exists += 1; @@ -176,6 +184,7 @@ async fn update_spam_rules(server: &Server) -> trc::Result { { RegistryWriteResult::Success(_) => { stats.entry(ObjectType::HttpLookup).or_default().success += 1; + reload_lookups = true; } RegistryWriteResult::PrimaryKeyConflict { .. } => { stats @@ -199,6 +208,7 @@ async fn update_spam_rules(server: &Server) -> trc::Result { .entry(ObjectType::MemoryLookupKey) .or_default() .success += 1; + reload_lookups = true; } RegistryWriteResult::PrimaryKeyConflict { .. } => { stats @@ -219,6 +229,7 @@ async fn update_spam_rules(server: &Server) -> trc::Result { .entry(ObjectType::SpamFileExtension) .or_default() .success += 1; + reload_settings = true; } RegistryWriteResult::PrimaryKeyConflict { .. } => { stats @@ -235,6 +246,34 @@ async fn update_spam_rules(server: &Server) -> trc::Result { } } + if reload_settings { + if let Err(err) = server + .reload_registry(RegistryChange::Reload(ObjectType::SpamRule)) + .await + { + trc::error!(err.details("Failed to reload registry after updating spam rules")); + } + server + .cluster_broadcast(BroadcastEvent::RegistryChange(RegistryChange::Reload( + ObjectType::SpamRule, + ))) + .await; + } + + if reload_lookups { + if let Err(err) = server + .reload_registry(RegistryChange::Reload(ObjectType::MemoryLookupKey)) + .await + { + trc::error!(err.details("Failed to reload registry after updating spam rules")); + } + server + .cluster_broadcast(BroadcastEvent::RegistryChange(RegistryChange::Reload( + ObjectType::MemoryLookupKey, + ))) + .await; + } + trc::event!( Spam(SpamEvent::RulesUpdated), Details = stats @@ -251,7 +290,7 @@ async fn update_spam_rules(server: &Server) -> trc::Result { Elapsed = started.elapsed(), ); - Ok(TaskResult::Success) + Ok(TaskResult::Success(vec![])) } async fn fetch_spam_rules(server: &Server) -> Result { @@ -275,8 +314,6 @@ async fn fetch_spam_rules(server: &Server) -> Result { }) })?; - let todo = "trigger task to reload settings and lookup stores"; - let mut rules = Rules::default(); for (object_type, values) in rules_json { let Some(object_type) = ObjectType::parse(&object_type) else { diff --git a/tests/src/directory/oidc.rs b/tests/src/directory/oidc.rs index b498025c..de11b3c2 100644 --- a/tests/src/directory/oidc.rs +++ b/tests/src/directory/oidc.rs @@ -24,6 +24,7 @@ pub async fn test() { "profile".to_string(), "openid".to_string(), ]), + member_tenant_id: None, }; let mut oidc = OpenIdDirectory::open(config.clone()).await.unwrap(); let token = get_token("john.doe@example.org", "this is an OIDC password").await; diff --git a/tests/src/smtp/inbound/antispam.rs b/tests/src/smtp/inbound/antispam.rs index c12fe6c7..b5300f2f 100644 --- a/tests/src/smtp/inbound/antispam.rs +++ b/tests/src/smtp/inbound/antispam.rs @@ -95,7 +95,7 @@ async fn antispam() { .await; let model_id = admin .registry_create_object(AiModel { - class: AiModelType::Chat, + model_type: AiModelType::Chat, allow_invalid_certs: true, model: "gpt-dummy".to_string(), name: "dummy".to_string(), diff --git a/tests/src/smtp/inbound/sign.rs b/tests/src/smtp/inbound/sign.rs index 6b19944e..0cfd7d09 100644 --- a/tests/src/smtp/inbound/sign.rs +++ b/tests/src/smtp/inbound/sign.rs @@ -18,8 +18,7 @@ use mail_auth::{ use registry::schema::{ enums::DkimCanonicalization, structs::{ - Dkim1Signature, DkimPrivateKey, DkimSignature, Domain, Expression, SecretTextValue, - SenderAuth, + Dkim1Signature, DkimSignature, Domain, Expression, SecretText, SecretTextValue, SenderAuth, }, }; use std::time::{Duration, Instant}; @@ -183,7 +182,7 @@ impl Account { selector: "rsa".to_string(), canonicalization: DkimCanonicalization::SimpleRelaxed, domain_id, - private_key: DkimPrivateKey::Value(SecretTextValue { + private_key: SecretText::Text(SecretTextValue { secret: RSA_KEY.to_string(), }), ..Default::default() @@ -196,7 +195,7 @@ impl Account { selector: "ed".to_string(), canonicalization: DkimCanonicalization::RelaxedSimple, domain_id, - private_key: DkimPrivateKey::Value(SecretTextValue { + private_key: SecretText::Text(SecretTextValue { secret: ED25519_KEY.to_string(), }), ..Default::default() diff --git a/tests/src/system/tenant.rs b/tests/src/system/tenant.rs index 8bc4f559..a56e1c17 100644 --- a/tests/src/system/tenant.rs +++ b/tests/src/system/tenant.rs @@ -18,10 +18,10 @@ use registry::{ enums::{AccountType, Permission, TenantStorageQuota}, prelude::{ObjectType, Property}, structs::{ - Account, Credential, Dkim1Signature, DkimPrivateKey, DkimSignature, DnsServer, - DnsServerCloudflare, Domain, GroupAccount, MailingList, OAuthClient, - PasswordCredential, Permissions, PermissionsList, Role, SecretKey, SecretKeyValue, - SecretTextValue, Tenant, UserAccount, UserRoles, + Account, Credential, Dkim1Signature, DkimSignature, DnsServer, DnsServerCloudflare, + Domain, GroupAccount, MailingList, OAuthClient, PasswordCredential, Permissions, + PermissionsList, Role, SecretKey, SecretKeyValue, SecretText, SecretTextValue, Tenant, + UserAccount, UserRoles, }, }, types::{EnumImpl, ObjectImpl, list::List, map::Map}, @@ -233,7 +233,7 @@ pub async fn test(test: &mut TestServer) { domain_id, member_tenant_id, selector: "ed-key".to_string(), - private_key: DkimPrivateKey::Value(SecretTextValue { + private_key: SecretText::Text(SecretTextValue { secret: DKIM_KEY.to_string(), }), ..Default::default()