From 4a4dcfd7e35115c495ac976483fd334022734756 Mon Sep 17 00:00:00 2001 From: Maurus Decimus <11444311+mdecimus@users.noreply.github.com> Date: Thu, 2 Apr 2026 19:32:51 +0200 Subject: [PATCH] DNS, DKIM and ACME improvements - part 4 --- Cargo.lock | 90 +++--- crates/common/Cargo.toml | 6 +- crates/common/src/network/acme/directory.rs | 22 +- crates/common/src/network/acme/http.rs | 7 +- crates/common/src/network/acme/mod.rs | 58 +++- crates/common/src/network/acme/order.rs | 68 +++-- crates/common/src/network/acme/renew.rs | 5 +- crates/common/src/network/dns/records.rs | 26 +- crates/common/src/network/dns/update.rs | 63 +++- crates/http-proto/src/context.rs | 10 +- crates/jmap/Cargo.toml | 2 +- crates/migration/Cargo.toml | 2 +- crates/services/src/task_manager/acme.rs | 83 +++--- crates/services/src/task_manager/dns.rs | 17 +- crates/services/src/task_manager/manager.rs | 2 +- crates/store/Cargo.toml | 2 +- tests/Cargo.toml | 3 +- tests/docker/INSTRUCTIONS.md | 63 +++- tests/docker/docker-compose.yml | 28 +- tests/docker/pebble/pebble-config.json | 4 +- tests/src/automation/acme.rs | 304 ++++++++++++++++++++ tests/src/automation/dkim.rs | 11 + tests/src/automation/dns.rs | 11 + tests/src/automation/mod.rs | 117 ++++++++ tests/src/cluster/broadcast.rs | 10 +- tests/src/lib.rs | 2 + tests/src/smtp/inbound/dmarc.rs | 7 +- tests/src/smtp/inbound/scripts.rs | 9 +- tests/src/smtp/inbound/sign.rs | 6 +- tests/src/store/registry.rs | 14 +- tests/src/system/directory.rs | 14 +- tests/src/system/tenant.rs | 14 +- tests/src/utils/account.rs | 8 +- tests/src/utils/registry.rs | 9 +- tests/src/utils/server.rs | 32 +-- tests/src/utils/storage.rs | 9 +- tests/src/utils/webdav.rs | 3 +- 37 files changed, 932 insertions(+), 209 deletions(-) create mode 100644 tests/src/automation/acme.rs create mode 100644 tests/src/automation/dkim.rs create mode 100644 tests/src/automation/dns.rs create mode 100644 tests/src/automation/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 04dace7c..b44b244b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1090,7 +1090,7 @@ dependencies = [ "dns-update", "futures", "hashify", - "hickory-proto 0.24.4", + "hickory-proto 0.25.2", "hostname", "hyper 1.8.1", "idna", @@ -1100,7 +1100,7 @@ dependencies = [ "infer 0.19.0", "jmap_proto", "libc", - "lz4_flex 0.12.0", + "lz4_flex 0.13.0", "mail-auth", "mail-builder", "mail-parser", @@ -1156,7 +1156,7 @@ dependencies = [ "whatlang", "x509-parser", "xxhash-rust", - "zip", + "zip 8.5.0", "zxcvbn", ] @@ -1329,21 +1329,6 @@ dependencies = [ "libc", ] -[[package]] -name = "crc" -version = "3.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" -dependencies = [ - "crc-catalog", -] - -[[package]] -name = "crc-catalog" -version = "2.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" - [[package]] name = "crc16" version = "0.4.0" @@ -2612,10 +2597,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "139ef39800118c7683f2fd3c98c1b23c09ae076556b435f8e9064ae108aaeeec" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi", "wasip2", "wasip3", + "wasm-bindgen", ] [[package]] @@ -2815,30 +2802,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "hickory-proto" -version = "0.24.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92652067c9ce6f66ce53cc38d1169daa36e6e7eb7dd3b63b5103bd9d97117248" -dependencies = [ - "async-trait", - "cfg-if", - "data-encoding", - "enum-as-inner", - "futures-channel", - "futures-io", - "futures-util", - "idna", - "ipnet", - "once_cell", - "rand 0.8.5", - "thiserror 1.0.69", - "tinyvec", - "tokio", - "tracing", - "url", -] - [[package]] name = "hickory-proto" version = "0.25.2" @@ -3687,7 +3650,7 @@ dependencies = [ "hyper-util", "jmap-tools", "jmap_proto", - "lz4_flex 0.12.0", + "lz4_flex 0.13.0", "mail-auth", "mail-builder", "mail-parser", @@ -4154,20 +4117,19 @@ dependencies = [ [[package]] name = "lz4_flex" -version = "0.12.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab6473172471198271ff72e9379150e9dfd70d8e533e0752a27e515b48dd375e" +checksum = "db9a0d582c2874f68138a16ce1867e0ffde6c0bb0a0df85e1f36d04146db488a" dependencies = [ "twox-hash", ] [[package]] name = "lzma-rust2" -version = "0.13.0" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c60a23ffb90d527e23192f1246b14746e2f7f071cb84476dd879071696c18a4a" +checksum = "47bb1e988e6fb779cf720ad431242d3f03167c1b3f2b1aae7f1a94b2495b36ae" dependencies = [ - "crc", "sha2 0.10.9", ] @@ -4188,7 +4150,7 @@ dependencies = [ "rustls-pki-types", "serde", "serde_json", - "zip", + "zip 6.0.0", ] [[package]] @@ -4354,7 +4316,7 @@ dependencies = [ "email", "futures", "groupware", - "lz4_flex 0.12.0", + "lz4_flex 0.13.0", "mail-auth", "mail-parser", "nlp", @@ -7585,7 +7547,7 @@ dependencies = [ "futures", "gethostname", "lru-cache", - "lz4_flex 0.12.0", + "lz4_flex 0.13.0", "memchr", "mysql_async", "nlp", @@ -7757,6 +7719,7 @@ dependencies = [ "dav", "dav-proto", "directory", + "dns-update", "ece", "email", "flate2", @@ -8389,6 +8352,12 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" +[[package]] +name = "typed-path" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" + [[package]] name = "typeid" version = "1.0.3" @@ -10072,14 +10041,26 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb2a05c7c36fde6c09b08576c9f7fb4cda705990f73b58fe011abf7dfb24168b" dependencies = [ - "aes", "arbitrary", + "crc32fast", + "flate2", + "indexmap 2.13.0", + "memchr", +] + +[[package]] +name = "zip" +version = "8.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2726508a48f38dceb22b35ecbbd2430efe34ff05c62bd3285f965d7911b33464" +dependencies = [ + "aes", "bzip2", - "constant_time_eq 0.3.1", + "constant_time_eq 0.4.2", "crc32fast", "deflate64", "flate2", - "getrandom 0.3.4", + "getrandom 0.4.1", "hmac 0.12.1", "indexmap 2.13.0", "lzma-rust2", @@ -10088,6 +10069,7 @@ dependencies = [ "ppmd-rust", "sha1", "time", + "typed-path", "zeroize", "zopfli", "zstd", diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index b794d6ec..6e1b75d1 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -60,7 +60,7 @@ unicode-security = "0.1.0" infer = "0.19" bincode = { version = "2.0", features = ["serde"] } hostname = "0.4.0" -zip = "6.0" +zip = "8.5" pwhash = "1.0.0" xxhash-rust = { version = "0.8.5", features = ["xxh3"] } psl = "2" @@ -75,8 +75,8 @@ rkyv = { version = "0.8.10", features = ["little_endian"] } indexmap = "2.7.1" tinyvec = { version = "1.10.0", features = ["alloc"] } compact_str = { version = "0.9.0", features = ["rkyv", "serde"] } -lz4_flex = { version = "0.12", features = ["frame"], default-features = false } -hickory-proto = "0.24" +lz4_flex = { version = "0.13", features = ["frame"], default-features = false } +hickory-proto = "0.25" nohash-hasher = "0.2.0" quick_cache = "0.6.9" rasn = "0.10" diff --git a/crates/common/src/network/acme/directory.rs b/crates/common/src/network/acme/directory.rs index af64fdfc..50b0dd55 100644 --- a/crates/common/src/network/acme/directory.rs +++ b/crates/common/src/network/acme/directory.rs @@ -141,16 +141,25 @@ impl AcmeRequestBuilder { } pub fn http_proof(&self, challenge: &Challenge) -> AcmeResult> { - key_authorization(&self.key_pair, &challenge.token).map(|key| key.into_bytes()) + let challenge_token = challenge.token.as_deref().ok_or_else(|| { + AcmeError::Invalid("Missing http-01 challenge token in response".to_string()) + })?; + key_authorization(&self.key_pair, challenge_token).map(|key| key.into_bytes()) } pub fn dns_proof(&self, challenge: &Challenge) -> AcmeResult { - key_authorization_sha256_base64(&self.key_pair, &challenge.token) + let challenge_token = challenge.token.as_deref().ok_or_else(|| { + AcmeError::Invalid("Missing dns-01 challenge token in response".to_string()) + })?; + key_authorization_sha256_base64(&self.key_pair, challenge_token) } pub fn tls_alpn_key(&self, challenge: &Challenge, domain: String) -> AcmeResult> { + let challenge_token = challenge.token.as_deref().ok_or_else(|| { + AcmeError::Invalid("Missing tls-alpn-01 challenge token in response".to_string()) + })?; let mut params = rcgen::CertificateParams::new(vec![domain]); - let key_auth = key_authorization_sha256(&self.key_pair, &challenge.token)?; + let key_auth = key_authorization_sha256(&self.key_pair, challenge_token)?; params.alg = &PKCS_ECDSA_P256_SHA256; params.custom_extensions = vec![CustomExtension::new_acme_identifier(key_auth.as_ref())]; let cert = Certificate::from_params(params).map_err(|err| { @@ -197,7 +206,12 @@ impl Directory { impl AcmeResponse { pub fn parse(input: AcmeResponse) -> AcmeResult> { serde_json::from_str(&input.body) - .map_err(Into::into) + .map_err(|err| { + AcmeError::Invalid(format!( + "ACME response parsing error: {}, body: {}", + err, input.body + )) + }) .map(|body| AcmeResponse { location: input.location, body, diff --git a/crates/common/src/network/acme/http.rs b/crates/common/src/network/acme/http.rs index f7fd2917..d9bac583 100644 --- a/crates/common/src/network/acme/http.rs +++ b/crates/common/src/network/acme/http.rs @@ -55,7 +55,12 @@ pub(crate) async fn https( max_retries, }) } else { - Err(AcmeError::HttpStatus(response.status())) + let status = response.status(); + let text = response.text().await.unwrap_or_default(); + Err(AcmeError::HttpStatus(format!( + "Unexpected status {}: {}", + status, text + ))) } } diff --git a/crates/common/src/network/acme/mod.rs b/crates/common/src/network/acme/mod.rs index 5c61fe2d..8c6fc002 100644 --- a/crates/common/src/network/acme/mod.rs +++ b/crates/common/src/network/acme/mod.rs @@ -28,13 +28,13 @@ pub type AcmeResult = Result; pub enum AcmeError { Http(reqwest::Error), - HttpStatus(reqwest::StatusCode), + HttpStatus(String), Json(serde_json::Error), Crypto(String), Invalid(String), Dns(String), - AuthInvalid(AuthStatus), - OrderInvalid, + AuthInvalid(String), + OrderInvalid(String), ChallengeNotSupported { requested: ChallengeType, supported: Vec, @@ -151,7 +151,7 @@ pub struct Challenge { #[serde(rename = "type")] pub typ: ChallengeType, pub url: String, - pub token: String, + pub token: Option, pub error: Option, } @@ -202,18 +202,64 @@ impl From for ChallengeType { } } +impl Display for AuthStatus { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + AuthStatus::Pending => write!(f, "pending"), + AuthStatus::Valid => write!(f, "valid"), + AuthStatus::Invalid => write!(f, "invalid"), + AuthStatus::Revoked => write!(f, "revoked"), + AuthStatus::Expired => write!(f, "expired"), + AuthStatus::Deactivated => write!(f, "deactivated"), + } + } +} + +impl Auth { + pub fn into_error(self) -> String { + let mut errors = format!("Status: {}", self.status); + for challenge in self.challenges { + if let Some(error) = challenge.error { + errors.push_str(&format!( + "; Challenge type: {}, error: {}", + challenge.typ.as_str(), + error + )); + } + } + + errors + } +} + +impl Display for Problem { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if let Some(typ) = &self.typ { + if let Some(detail) = &self.detail { + write!(f, "{}: {}", typ, detail) + } else { + write!(f, "{}", typ) + } + } else if let Some(detail) = &self.detail { + write!(f, "{}", detail) + } else { + write!(f, "Unknown error") + } + } +} + impl Display for AcmeError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { AcmeError::Http(err) => write!(f, "HTTP error: {}", err), - AcmeError::HttpStatus(status) => write!(f, "HTTP error: status code {}", status), + AcmeError::HttpStatus(status) => write!(f, "HTTP error: {}", status), AcmeError::Json(err) => write!(f, "JSON error: {}", err), AcmeError::Dns(err) => write!(f, "DNS error: {}", err), AcmeError::Crypto(err) => write!(f, "Cryptographic error: {}", err), AcmeError::Invalid(err) => write!(f, "Invalid request: {}", err), AcmeError::AuthInvalid(status) => write!(f, "Authentication failed: {:?}", status), AcmeError::OrderTimeout { .. } => write!(f, "Order processing timed out"), - AcmeError::OrderInvalid => write!(f, "Order is invalid"), + AcmeError::OrderInvalid(reason) => write!(f, "Order is invalid: {}", reason), AcmeError::AuthTimeout { .. } => write!(f, "Authentication timed out"), AcmeError::ChallengeNotSupported { requested, diff --git a/crates/common/src/network/acme/order.rs b/crates/common/src/network/acme/order.rs index c05db28b..68c9e3c1 100644 --- a/crates/common/src/network/acme/order.rs +++ b/crates/common/src/network/acme/order.rs @@ -41,20 +41,39 @@ impl AcmeRequestBuilder { ) { vec![format!("*.{domain}")] } else { - server - .core - .network - .server_name - .strip_suffix(domain) - .and_then(|host| host.strip_suffix(".")) - .map(|h| h.to_string()) - .into_iter() - .chain( - HOSTNAMES - .iter() - .map(|hostname| format!("{hostname}.{domain}")), - ) - .collect() + let server_name = server.core.network.server_name.as_str(); + let domain_suffix = format!(".{domain}"); + + // Add technical domains + let mut domains = HOSTNAMES + .iter() + .map(|hostname| format!("{hostname}.{domain}")) + .collect::>(); + + // Add server name if it matches the domain + if server_name.ends_with(&domain_suffix) { + domains.insert(server_name.to_string()); + } + + // Add mail exchangers + for exchanger in &server.core.network.info.mxs { + if let Some(exchanger) = &exchanger.hostname + && exchanger.ends_with(&domain_suffix) + { + domains.insert(exchanger.to_string()); + } + } + + // Add service hosts + for (_, service) in &server.core.network.info.services { + if let Some(service) = &service.hostname + && service.ends_with(&domain_suffix) + { + domains.insert(service.to_string()); + } + } + + domains.into_iter().collect() } } else { hostnames @@ -146,7 +165,13 @@ impl AcmeRequestBuilder { }); } OrderStatus::Invalid => { - return Err(AcmeError::OrderInvalid); + let reason = if let Some(reason) = order.error { + reason.to_string() + } else { + "Unknown reason".to_string() + }; + + return Err(AcmeError::OrderInvalid(reason)); } } } @@ -205,7 +230,12 @@ impl AcmeRequestBuilder { .key_set( KeyValue::with_prefix( KV_ACME, - &challenge.token, + challenge.token.as_deref().ok_or_else(|| { + AcmeError::Invalid( + "Missing http-01 challenge token in response" + .to_string(), + ) + })?, self.http_proof(challenge)?, ) .expires(3600), @@ -227,7 +257,7 @@ impl AcmeRequestBuilder { .await .map_err(AcmeError::Dns)?; } - ChallengeType::DnsPersist01 => return Ok(()), + ChallengeType::DnsPersist01 => {} ChallengeType::Unknown => unreachable!(), } @@ -236,7 +266,7 @@ impl AcmeRequestBuilder { } AuthStatus::Valid => return Ok(()), _ => { - return Err(AcmeError::AuthInvalid(auth.status)); + return Err(AcmeError::AuthInvalid(auth.into_error())); } }; @@ -269,7 +299,7 @@ impl AcmeRequestBuilder { return Ok(()); } _ => { - return Err(AcmeError::AuthInvalid(response.body.status)); + return Err(AcmeError::AuthInvalid(response.body.into_error())); } } } diff --git a/crates/common/src/network/acme/renew.rs b/crates/common/src/network/acme/renew.rs index 48000cc2..5fc0ad07 100644 --- a/crates/common/src/network/acme/renew.rs +++ b/crates/common/src/network/acme/renew.rs @@ -103,7 +103,8 @@ impl Server { not_valid_before: UTCDateTime::from_timestamp(parsed_cert.valid_not_before.timestamp()), subject_alternative_names: Map::new(parsed_cert.sans), }; - let expires_in = (parsed_cert.valid_not_after.timestamp() as u64).saturating_sub(now()); + let now = now(); + let expires_in = (parsed_cert.valid_not_after.timestamp() as u64).saturating_sub(now); if expires_in < 86400 { return Err(AcmeError::Invalid(format!( "Certificate expires in {} seconds, expected at least 86400 seconds", @@ -145,7 +146,7 @@ impl Server { }; tasks.push(Task::AcmeRenewal(TaskDomainManagement { domain_id, - status: TaskStatus::at(renew_in as i64), + status: TaskStatus::at((now + renew_in) as i64), })); // Update TLSA records diff --git a/crates/common/src/network/dns/records.rs b/crates/common/src/network/dns/records.rs index 21c6dda5..dcd5094b 100644 --- a/crates/common/src/network/dns/records.rs +++ b/crates/common/src/network/dns/records.rs @@ -212,8 +212,7 @@ impl Server { .object::(props.acme_provider_id) .await? && let Ok(provider_url) = Url::parse(&provider.directory) - && let Some(provider_name) = - provider_url.host_str().and_then(psl::domain_str) + && let Some(provider_name) = provider_domain(&provider_url) { records.push(NamedDnsRecord { name: format!("{domain_name}."), @@ -248,8 +247,13 @@ impl Server { records.push(NamedDnsRecord { name: format!("_validation-persist.{domain_name}."), record: DnsRecord::TXT(format!( - "{provider_name}; accounturi={}", - provider.account_uri + "{provider_name}; accounturi={}{}", + provider.account_uri, + if props.subject_alternative_names.is_empty() { + "; policy=wildcard" + } else { + "" + } )), }); } @@ -374,3 +378,17 @@ impl Server { .map(|records| BindSerializer::serialize(&records)) } } + +#[inline(always)] +#[allow(unused)] +fn provider_domain(url: &Url) -> Option<&str> { + #[cfg(feature = "test_mode")] + { + Some("pebble.letsencrypt.org") + } + + #[cfg(not(feature = "test_mode"))] + { + url.host_str().and_then(psl::domain_str) + } +} diff --git a/crates/common/src/network/dns/update.rs b/crates/common/src/network/dns/update.rs index ac6fec8d..241cbe2a 100644 --- a/crates/common/src/network/dns/update.rs +++ b/crates/common/src/network/dns/update.rs @@ -35,6 +35,11 @@ pub struct DnsUpdater { core: Arc, } +#[cfg(feature = "test_mode")] +pub static DNS_RECORDS: std::sync::LazyLock< + Arc>>, +> = std::sync::LazyLock::new(|| Arc::new(std::sync::Mutex::new(Vec::new()))); + impl DnsUpdater { pub async fn build(server: DnsServer, core: Arc) -> Result { match server { @@ -126,19 +131,43 @@ impl DnsUpdater { .map_err(|err| format!("Failed to build DNS updater: {}", err))?, }) } - DnsServer::Cloudflare(server) => Ok(DnsUpdater { - polling_interval: server.polling_interval.into_inner(), - propagation_timeout: server.propagation_timeout.into_inner(), - propagation_delay: server.propagation_delay.map(|d| d.into_inner()), - ttl: server.ttl.into_inner(), - core, - updater: dns_update::DnsUpdater::new_cloudflare( - server.secret.secret().await?, - server.email, - server.timeout.into_inner().into(), - ) - .map_err(|err| format!("Failed to build DNS updater: {}", err))?, - }), + DnsServer::Cloudflare(server) => { + let updater = { + #[cfg(feature = "test_mode")] + match server.email.as_deref() { + Some("test@pebble.org") => dns_update::DnsUpdater::new_pebble( + "http://localhost:8055", + server.timeout.into_inner().into(), + ), + Some("test@memory.org") => { + dns_update::DnsUpdater::new_in_memory(DNS_RECORDS.clone()) + } + _ => dns_update::DnsUpdater::new_cloudflare( + server.secret.secret().await?, + server.email, + server.timeout.into_inner().into(), + ) + .map_err(|err| format!("Failed to build DNS updater: {}", err))?, + } + + #[cfg(not(feature = "test_mode"))] + dns_update::DnsUpdater::new_cloudflare( + server.secret.secret().await?, + server.email, + server.timeout.into_inner().into(), + ) + .map_err(|err| format!("Failed to build DNS updater: {}", err))? + }; + + Ok(DnsUpdater { + polling_interval: server.polling_interval.into_inner(), + propagation_timeout: server.propagation_timeout.into_inner(), + propagation_delay: server.propagation_delay.map(|d| d.into_inner()), + ttl: server.ttl.into_inner(), + core, + updater, + }) + } DnsServer::DigitalOcean(server) => Ok(DnsUpdater { polling_interval: server.polling_interval.into_inner(), propagation_timeout: server.propagation_timeout.into_inner(), @@ -260,6 +289,14 @@ impl DnsUpdater { ); if verify && let DnsRecord::TXT(txt_record) = &record { + #[cfg(feature = "test_mode")] + if matches!( + self.updater, + dns_update::DnsUpdater::Pebble(_) | dns_update::DnsUpdater::InMemory(_) + ) { + return Ok(true); + } + // Wait for changes to propagate if let Some(initial_wait) = self.propagation_delay { tokio::time::sleep(initial_wait).await; diff --git a/crates/http-proto/src/context.rs b/crates/http-proto/src/context.rs index cb7d9a34..d358c93c 100644 --- a/crates/http-proto/src/context.rs +++ b/crates/http-proto/src/context.rs @@ -20,7 +20,15 @@ impl<'x> HttpContext<'x> { pub fn resolve_response_url(&self, server: &Server) -> String { if self.session.is_tls { - server.core.network.http.url_https.clone() + #[cfg(not(feature = "test_mode"))] + { + server.core.network.http.url_https.clone() + } + + #[cfg(feature = "test_mode")] + { + format!("https://127.0.0.1:{}", self.session.local_port) + } } else { format!( "{}:{}", diff --git a/crates/jmap/Cargo.toml b/crates/jmap/Cargo.toml index 62747706..cf31a71a 100644 --- a/crates/jmap/Cargo.toml +++ b/crates/jmap/Cargo.toml @@ -46,7 +46,7 @@ tungstenite = "0.28" chrono = "0.4" rand = "0.9.0" pkcs8 = { version = "0.10.2", features = ["alloc", "std"] } -lz4_flex = { version = "0.12", default-features = false } +lz4_flex = { version = "0.13", default-features = false } aes-gcm = "0.10.1" aes-gcm-siv = "0.11.1" rsa = "0.9.2" diff --git a/crates/migration/Cargo.toml b/crates/migration/Cargo.toml index a402e64f..74d367f6 100644 --- a/crates/migration/Cargo.toml +++ b/crates/migration/Cargo.toml @@ -28,7 +28,7 @@ serde_json = "1.0" rkyv = { version = "0.8.10", features = ["little_endian"] } compact_str = "0.9.0" bincode = "1.3.3" -lz4_flex = { version = "0.12", default-features = false } +lz4_flex = { version = "0.13", default-features = false } base64 = "0.22" futures = "0.3" num_cpus = "1.13.1" diff --git a/crates/services/src/task_manager/acme.rs b/crates/services/src/task_manager/acme.rs index 3f78f232..1de25090 100644 --- a/crates/services/src/task_manager/acme.rs +++ b/crates/services/src/task_manager/acme.rs @@ -7,6 +7,7 @@ use crate::task_manager::{TaskFailureType, TaskResult}; use common::{Server, network::acme::AcmeError}; use registry::schema::structs::TaskDomainManagement; +use std::time::Duration; use store::write::now; pub(crate) trait AcmeTask: Sync + Send { @@ -32,43 +33,59 @@ impl AcmeTask for Server { } } +#[cfg(not(feature = "test_mode"))] +const MAX_RETRIES: u32 = 3; + +#[cfg(feature = "test_mode")] +const MAX_RETRIES: u32 = 5; + +#[allow(unused_variables)] async fn acme_management(server: &Server, task: &TaskDomainManagement) -> trc::Result { - match server.acme_renew(task.domain_id).await { - Ok(tasks) => Ok(TaskResult::Success(tasks)), - Err(err) => match err { - AcmeError::Crypto(_) - | AcmeError::Invalid(_) - | AcmeError::ChallengeNotSupported { .. } - | AcmeError::OrderInvalid - | AcmeError::AuthInvalid(_) - | AcmeError::Json(_) - | AcmeError::Registry(_) => Ok(TaskResult::permanent(err.to_string())), - AcmeError::Http(_) | AcmeError::HttpStatus(_) | AcmeError::Dns(_) => { - Ok(TaskResult::temporary(err.to_string())) - } - AcmeError::OrderTimeout { max_retries } | AcmeError::AuthTimeout { max_retries } => { - Ok(TaskResult::Failure { + let mut last_temporary_error = Ok(TaskResult::temporary("")); + for retry in 0..MAX_RETRIES { + last_temporary_error = match server.acme_renew(task.domain_id).await { + Ok(tasks) => return Ok(TaskResult::Success(tasks)), + Err(err) => match err { + AcmeError::Crypto(_) + | AcmeError::Invalid(_) + | AcmeError::ChallengeNotSupported { .. } + | AcmeError::OrderInvalid(_) + | AcmeError::Json(_) + | AcmeError::Registry(_) => return Ok(TaskResult::permanent(err.to_string())), + AcmeError::Http(_) + | AcmeError::HttpStatus(_) + | AcmeError::Dns(_) + | AcmeError::AuthInvalid(_) => Ok(TaskResult::temporary(err.to_string())), + AcmeError::OrderTimeout { max_retries } + | AcmeError::AuthTimeout { max_retries } => Ok(TaskResult::Failure { typ: TaskFailureType::Temporary, message: err.to_string(), max_attempts: (max_retries as u64).into(), - }) - } - AcmeError::Backoff { max_retries, wait } => { - if let Some(wait) = wait { - Ok(TaskResult::Failure { - typ: TaskFailureType::Retry(now() + wait.as_secs()), - message: err.to_string(), - max_attempts: (max_retries as u64).into(), - }) - } else { - Ok(TaskResult::Failure { - typ: TaskFailureType::Temporary, - message: err.to_string(), - max_attempts: (max_retries as u64).into(), - }) + }), + AcmeError::Backoff { max_retries, wait } => { + return if let Some(wait) = wait { + Ok(TaskResult::Failure { + typ: TaskFailureType::Retry(now() + wait.as_secs()), + message: err.to_string(), + max_attempts: (max_retries as u64).into(), + }) + } else { + Ok(TaskResult::Failure { + typ: TaskFailureType::Temporary, + message: err.to_string(), + max_attempts: (max_retries as u64).into(), + }) + }; } - } - AcmeError::Internal(error) => Err(error), - }, + AcmeError::Internal(error) => return Err(error), + }, + }; + + #[cfg(not(feature = "test_mode"))] + tokio::time::sleep(Duration::from_secs(1 << (retry + 5))).await; + + #[cfg(feature = "test_mode")] + tokio::time::sleep(Duration::from_secs(1)).await; } + last_temporary_error } diff --git a/crates/services/src/task_manager/dns.rs b/crates/services/src/task_manager/dns.rs index 82cd26d5..5c872ff1 100644 --- a/crates/services/src/task_manager/dns.rs +++ b/crates/services/src/task_manager/dns.rs @@ -6,7 +6,9 @@ use crate::task_manager::TaskResult; use common::Server; -use registry::schema::structs::{DnsManagement, Domain, TaskDnsManagement}; +use registry::schema::structs::{ + DnsManagement, Domain, Task, TaskDnsManagement, TaskDomainManagement, TaskStatus, +}; use std::fmt::Write; use store::ahash::AHashSet; @@ -85,8 +87,17 @@ async fn dns_management(server: &Server, task: &TaskDnsManagement) -> trc::Resul } } - if !errors.is_empty() { - Ok(TaskResult::Success(vec![])) + if errors.is_empty() { + if task.on_success_renew_certificate { + Ok(TaskResult::Success(vec![Task::AcmeRenewal( + TaskDomainManagement { + domain_id: task.domain_id, + status: TaskStatus::now(), + }, + )])) + } else { + Ok(TaskResult::Success(vec![])) + } } else { Ok(TaskResult::permanent(errors)) } diff --git a/crates/services/src/task_manager/manager.rs b/crates/services/src/task_manager/manager.rs index aadc9e90..d62d810a 100644 --- a/crates/services/src/task_manager/manager.rs +++ b/crates/services/src/task_manager/manager.rs @@ -440,7 +440,7 @@ impl TaskQueueManager for Server { ); }); - if !tasks.is_empty() || !ipc.locked.is_empty() { + if !tasks.is_empty() { trc::event!( TaskManager(TaskManagerEvent::TaskAcquired), Total = tasks.len(), diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index 7ada5e86..98b85426 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -32,7 +32,7 @@ parking_lot = "0.12" lru-cache = { version = "0.1.2", optional = true } num_cpus = { version = "1.17", optional = true } blake3 = "1.8" -lz4_flex = { version = "0.12", default-features = false } +lz4_flex = { version = "0.13", default-features = false } deadpool-postgres = { version = "0.14", optional = true } tokio-postgres = { version = "0.7.10", features = ["with-serde_json-1"], optional = true } tokio-rustls = { version = "0.26", optional = true, default-features = false, features = ["ring", "tls12"] } diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 43358a91..e92b081d 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -32,7 +32,7 @@ dav-proto = { path = "../crates/dav-proto", features = ["test_mode"] } calcard = { version = "0.3", features = ["rkyv"] } groupware = { path = "../crates/groupware", features = ["test_mode"] } http = { path = "../crates/http", features = ["test_mode", "enterprise"] } -http_proto = { path = "../crates/http-proto" } +http_proto = { path = "../crates/http-proto", features = ["test_mode"] } services = { path = "../crates/services", features = ["test_mode", "enterprise"] } pop3 = { path = "../crates/pop3", features = ["test_mode"] } smtp = { path = "../crates/smtp", features = ["test_mode", "enterprise"] } @@ -81,6 +81,7 @@ rkyv = { version = "0.8.10", features = ["little_endian"] } compact_str = "0.9.0" quick-xml = "0.39" jmap-tools = { version = "0.1" } +dns-update = { path = "/Users/me/code/dns-update", features = ["test_provider"] } [target.'cfg(not(target_env = "msvc"))'.dependencies] jemallocator = "0.5.0" diff --git a/tests/docker/INSTRUCTIONS.md b/tests/docker/INSTRUCTIONS.md index 6f51a454..9595dafa 100644 --- a/tests/docker/INSTRUCTIONS.md +++ b/tests/docker/INSTRUCTIONS.md @@ -25,7 +25,8 @@ Wait ~30 seconds for all services to initialize (Keycloak takes the longest). | MinIO (S3) | localhost | 9000 / 9001 | `minioadmin` / `minioadmin`, bucket: `stalwart` | | Keycloak (OIDC)| localhost | 9080 | Admin: `admin` / `admin` | | OpenLDAP | localhost | 389 / 636 (TLS) | Admin DN: `cn=admin,dc=stalwart,dc=test`, pw: `admin` | -| Pebble (ACME) | localhost | 14000 / 15000 | Self-signed TLS, auto-valid challenges | +| Pebble (ACME) | localhost | 14000 / 15000 | Self-signed TLS, uses challtestsrv | +| Challtestsrv | localhost | 8055 | ACME challenge test server (management API)| | PowerDNS | localhost | 5300 / 8081 | API key: `stalwart-api-key` | | NATS | localhost | 4222 / 8222 | No auth | @@ -127,12 +128,64 @@ send EOF ``` -## ACME (Pebble) Details +## ACME (Pebble + Challenge Test Server) Details - **Directory URL**: `https://localhost:14000/dir` -- **Management URL**: `https://localhost:15000` -- **TLS**: Self-signed (use `PEBBLE_VA_ALWAYS_VALID=1` — all challenges auto-pass) -- Stalwart must be configured to trust the Pebble CA or skip TLS verification. +- **Pebble Management URL**: `https://localhost:15000` +- **Challenge Test Server API**: `http://localhost:8055` +- **TLS**: Self-signed — Stalwart must trust the Pebble CA or skip TLS verification +- Pebble uses the challenge test server (`pebble-challtestsrv`) as its DNS resolver, + so challenge validation goes through controllable DNS/HTTP/TLS-ALPN responders. + +### Challenge Test Server (challtestsrv) + +The challenge test server provides a management API on port 8055 to programmatically +control DNS records and challenge responses used during ACME validation. + +**Default behavior**: All A/AAAA queries resolve to `host.docker.internal` (the Docker +host), so Pebble can reach your test server on localhost automatically. Tests only need +to add challenge-specific records (TXT for DNS-01, HTTP tokens, etc.). + +### How Pebble Reaches Your Test Server + +When Pebble validates an HTTP-01 or TLS-ALPN-01 challenge, it: + +1. Resolves the domain via challtestsrv — by default all domains resolve to the Docker host +2. Connects to the resolved IP on port **5002** (HTTP-01) or **5001** (TLS-ALPN-01) + +These ports are configured in `pebble/pebble-config.json` (`httpPort` / `tlsPort`). +Change them to match whatever port your test Stalwart instance listens on. + +#### Management API Examples + +```bash +# Add a DNS-01 TXT challenge response +curl -s -X POST http://localhost:8055/add-dns \ + -d '{"host": "_acme-challenge.mail.stalwart.test.", "value": "dns-challenge-token"}' + +# Remove a DNS-01 TXT challenge response +curl -s -X POST http://localhost:8055/del-dns \ + -d '{"host": "_acme-challenge.mail.stalwart.test."}' + +# Add an HTTP-01 challenge response (served by challtestsrv itself) +curl -s -X POST http://localhost:8055/add-http \ + -d '{"token": "challenge-token", "content": "challenge-key-authorization"}' + +# Remove an HTTP-01 challenge response +curl -s -X POST http://localhost:8055/del-http \ + -d '{"token": "challenge-token"}' + +# Add a TLS-ALPN-01 challenge response (served by challtestsrv itself) +curl -s -X POST http://localhost:8055/add-tlsalpn \ + -d '{"host": "mail.stalwart.test", "content": "base64-encoded-key-authz"}' + +# Remove a TLS-ALPN-01 challenge response +curl -s -X POST http://localhost:8055/del-tlsalpn \ + -d '{"host": "mail.stalwart.test"}' + +# Clear all mock DNS/challenge data +curl -s -X POST http://localhost:8055/clear-request-count +``` ## Self-Signed TLS Certificate diff --git a/tests/docker/docker-compose.yml b/tests/docker/docker-compose.yml index ddcd31ac..e57fcf4d 100644 --- a/tests/docker/docker-compose.yml +++ b/tests/docker/docker-compose.yml @@ -186,20 +186,44 @@ services: - certs:/certs-shared:ro entrypoint: [ "/bin/bash", "-c", "mkdir -p /container/service/slapd/assets/config/bootstrap/ldif/custom && cp /seed/*.ldif /container/service/slapd/assets/config/bootstrap/ldif/custom/ && cp /certs-shared/* /container/service/slapd/assets/certs/ 2>/dev/null; exec /container/tool/run" ] + # --------------------------------------------------------------------------- + # Pebble Challenge Test Server – port 8055 (management API) + # Provides controllable DNS, HTTP, and TLS-ALPN challenge responders. + # Use the management API to add/remove challenge responses before requesting + # certificates from Pebble. + # --------------------------------------------------------------------------- + pebble-challtestsrv: + image: ghcr.io/letsencrypt/pebble-challtestsrv:latest + ports: + - "127.0.0.1:8055:8055" + command: [ "-defaultIPv6", "", "-defaultIPv4", "0.0.0.0" ] + + pebble-challtestsrv-init: + image: alpine:latest + depends_on: + - pebble-challtestsrv + network_mode: "service:pebble-challtestsrv" + entrypoint: [ "/bin/sh", "-c", "HOSTIP=$$(getent hosts host.docker.internal | awk '{print $$1}') && wget -qO- --post-data='{\"ip\":\"'$$HOSTIP'\"}' http://localhost:8055/set-default-ipv4" ] + # --------------------------------------------------------------------------- # Pebble (ACME server) – ports 14000 (directory) + 15000 (management) # --------------------------------------------------------------------------- pebble: image: ghcr.io/letsencrypt/pebble:latest + depends_on: + pebble-challtestsrv-init: + condition: service_completed_successfully environment: PEBBLE_VA_NOSLEEP: "1" - PEBBLE_VA_ALWAYS_VALID: "1" + PEBBLE_WFE_NONCEREJECT: "0" ports: - "127.0.0.1:14000:14000" - "127.0.0.1:15000:15000" + extra_hosts: + - "host.docker.internal:host-gateway" volumes: - ./pebble/pebble-config.json:/test/config/pebble-config.json:ro - command: -config /test/config/pebble-config.json + command: -config /test/config/pebble-config.json -dnsserver pebble-challtestsrv:8053 # --------------------------------------------------------------------------- # PowerDNS (DNS with TLSA + RFC2136) – port 5300 (moved from 53) diff --git a/tests/docker/pebble/pebble-config.json b/tests/docker/pebble/pebble-config.json index 8de8b18d..1f73a46d 100644 --- a/tests/docker/pebble/pebble-config.json +++ b/tests/docker/pebble/pebble-config.json @@ -4,8 +4,8 @@ "managementListenAddress": "0.0.0.0:15000", "certificate": "/test/certs/localhost/cert.pem", "privateKey": "/test/certs/localhost/key.pem", - "httpPort": 5002, - "tlsPort": 5001, + "httpPort": 8898, + "tlsPort": 8899, "ocspResponderURL": "", "externalAccountBindingRequired": false, "domainBlocklist": [], diff --git a/tests/src/automation/acme.rs b/tests/src/automation/acme.rs new file mode 100644 index 00000000..8a653ffc --- /dev/null +++ b/tests/src/automation/acme.rs @@ -0,0 +1,304 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::utils::server::TestServer; +use common::network::dns::update::DNS_RECORDS; +use dns_update::DnsRecord; +use jmap_proto::error::set::SetErrorType; +use registry::{ + schema::{ + enums::{AcmeChallengeType, AcmeRenewBefore, DnsRecordType}, + prelude::{ObjectType, Property}, + structs::{ + AcmeProvider, Certificate, CertificateManagement, CertificateManagementProperties, + DkimManagement, DnsManagement, DnsManagementProperties, DnsServer, DnsServerCloudflare, + Domain, SecretKey, SecretKeyValue, Task, TaskDomainManagement, + }, + }, + types::map::Map, +}; +use serde_json::json; + +pub async fn test(test: &TestServer) { + println!("Running ACME tests..."); + let account = test.account("admin@example.org"); + + // Create test Pebble and In Memory DNS servers + let pebble_dns_id = account + .registry_create_object(DnsServer::Cloudflare(DnsServerCloudflare { + email: "test@pebble.org".to_string().into(), + secret: SecretKey::Value(SecretKeyValue { + secret: "secret".into(), + }), + ..Default::default() + })) + .await; + let in_memory_dns_id = account + .registry_create_object(DnsServer::Cloudflare(DnsServerCloudflare { + email: "test@memory.org".to_string().into(), + secret: SecretKey::Value(SecretKeyValue { + secret: "secret".into(), + }), + ..Default::default() + })) + .await; + + // ACME provider creation should fail without a contact email + account + .registry_create_object_expect_err(AcmeProvider { + directory: "https://localhost:14000/dir".to_string(), + ..Default::default() + }) + .await + .assert_type(SetErrorType::InvalidProperties) + .assert_description_contains("At least one contact email is required"); + + // Create an ACME provider using TLS-ALPN-01 challenge + let tls_acme_id = account + .registry_create_object(AcmeProvider { + directory: "https://localhost:14000/dir".to_string(), + contact: Map::new(vec!["mailto:hello@tls.org".to_string()]), + challenge_type: AcmeChallengeType::TlsAlpn01, + renew_before: AcmeRenewBefore::R12, + ..Default::default() + }) + .await; + let provider = account.registry_get::(tls_acme_id).await; + assert_eq!(provider.directory, "https://localhost:14000/dir"); + assert_eq!( + provider.contact, + Map::new(vec!["mailto:hello@tls.org".to_string()]) + ); + assert!( + provider.account_uri.starts_with("https://localhost:14000"), + "Provider {:?} has invalid account URI", + provider + ); + + // Create a domain and trigger TLS-ALPN-01 ACME renewal + let domain_id = account + .registry_create_object(Domain { + name: "tls.org".to_string(), + certificate_management: CertificateManagement::Automatic( + CertificateManagementProperties { + acme_provider_id: tls_acme_id, + subject_alternative_names: Default::default(), + }, + ), + dkim_management: DkimManagement::Manual, + dns_management: DnsManagement::Automatic(DnsManagementProperties { + dns_server_id: in_memory_dns_id, + publish_records: Map::new(vec![ + DnsRecordType::Tlsa, + DnsRecordType::AutoConfig, + DnsRecordType::AutoConfigLegacy, + DnsRecordType::AutoDiscover, + DnsRecordType::MtaSts, + ]), + ..Default::default() + }), + ..Default::default() + }) + .await; + test.wait_for_tasks_skip_not_due().await; + let certificate = account + .registry_get_all::() + .await + .into_iter() + .next() + .unwrap() + .1; + let mut sans = certificate.subject_alternative_names.into_inner(); + sans.sort(); + assert_eq!( + sans, + vec![ + "autoconfig.tls.org".to_string(), + "autodiscover.tls.org".to_string(), + "mta-sts.tls.org".to_string(), + "ua-auto-config.tls.org".to_string() + ] + ); + + // Make sure the TLSA records were added to the in-memory DNS server + let records = DNS_RECORDS.lock().unwrap().clone(); + for record in [ + "_443._tcp.mta-sts.tls.org.", + "_443._tcp.autoconfig.tls.org.", + "_443._tcp.ua-auto-config.tls.org.", + "_443._tcp.autodiscover.tls.org.", + ] { + if records + .iter() + .find(|r| r.name == record && matches!(r.record, DnsRecord::TLSA(_))) + .is_none() + { + panic!( + "Expected TLSA record for {} not found in DNS records: {:?}", + record, records + ); + } + } + + // Make sure a task was created to renew the certificate before it expires + let tasks = account.registry_get_all::().await; + assert_eq!( + tasks.len(), + 1, + "Expected 1 task, found {}: {:?}", + tasks.len(), + tasks + ); + let task = tasks.into_iter().next().unwrap().1; + if let Task::AcmeRenewal(TaskDomainManagement { + domain_id: task_domain_id, + .. + }) = task + { + assert_eq!( + task_domain_id, domain_id, + "ACME renewal task has incorrect domain ID" + ); + } else { + panic!("Expected ACME renewal task, found: {:?}", task); + } + let not_valid_after = certificate.not_valid_after.timestamp(); + let not_valid_before = certificate.not_valid_before.timestamp(); + let length = not_valid_after - not_valid_before; + assert_eq!( + not_valid_after - length / 2, + task.due_timestamp() as i64, + "ACME renewal task has incorrect due timestamp, expected around {} but found {}", + not_valid_after - length / 2, + task.due_timestamp() as i64 + ); + account.registry_destroy_all(ObjectType::Certificate).await; + account.registry_destroy_all(ObjectType::Task).await; + + // Test ACME using HTTP-01 challenge and the server domain "mail.example.org" + let http_acme_id = account + .registry_create_object(AcmeProvider { + directory: "https://localhost:14000/dir".to_string(), + contact: Map::new(vec!["mailto:hello@example.org".to_string()]), + challenge_type: AcmeChallengeType::Http01, + ..Default::default() + }) + .await; + let domain_id = account.find_or_create_domain("example.org").await; + account.registry_update_object(ObjectType::Domain, domain_id, json!({ + Property::CertificateManagement: CertificateManagement::Automatic(CertificateManagementProperties { + acme_provider_id: http_acme_id, + subject_alternative_names: Default::default(), + }), + })).await; + test.wait_for_tasks_skip_not_due().await; + let certificate = account + .registry_get_all::() + .await + .into_iter() + .next() + .unwrap() + .1; + let mut sans = certificate.subject_alternative_names.into_inner(); + sans.sort(); + assert_eq!( + sans, + vec![ + "autoconfig.example.org".to_string(), + "autodiscover.example.org".to_string(), + "imap.example.org".to_string(), + "mail.example.org".to_string(), + "mta-sts.example.org".to_string(), + "mx1.example.org".to_string(), + "mx2.example.org".to_string(), + "pop3.example.org".to_string(), + "smtp.example.org".to_string(), + "ua-auto-config.example.org".to_string() + ] + ); + account.registry_destroy_all(ObjectType::Certificate).await; + account.registry_destroy_all(ObjectType::Task).await; + + // Test ACME using DNS-01 challenge + let dns_acme_id = account + .registry_create_object(AcmeProvider { + directory: "https://localhost:14000/dir".to_string(), + contact: Map::new(vec!["mailto:hello@dns.org".to_string()]), + challenge_type: AcmeChallengeType::Dns01, + ..Default::default() + }) + .await; + account + .registry_create_object(Domain { + name: "dns.org".to_string(), + certificate_management: CertificateManagement::Automatic( + CertificateManagementProperties { + acme_provider_id: dns_acme_id, + subject_alternative_names: Default::default(), + }, + ), + dkim_management: DkimManagement::Manual, + dns_management: DnsManagement::Automatic(DnsManagementProperties { + dns_server_id: pebble_dns_id, + publish_records: Map::new(vec![DnsRecordType::Caa]), + ..Default::default() + }), + ..Default::default() + }) + .await; + test.wait_for_tasks_skip_not_due().await; + let certificate = account + .registry_get_all::() + .await + .into_iter() + .next() + .unwrap() + .1; + let mut sans = certificate.subject_alternative_names.into_inner(); + sans.sort(); + assert_eq!(sans, vec!["*.dns.org".to_string()]); + account.registry_destroy_all(ObjectType::Certificate).await; + account.registry_destroy_all(ObjectType::Task).await; + + // Test ACME using DNS-01 challenge + let dns_acme_id = account + .registry_create_object(AcmeProvider { + directory: "https://localhost:14000/dir".to_string(), + contact: Map::new(vec!["mailto:hello@persist.org".to_string()]), + challenge_type: AcmeChallengeType::DnsPersist01, + ..Default::default() + }) + .await; + account + .registry_create_object(Domain { + name: "persist.org".to_string(), + certificate_management: CertificateManagement::Automatic( + CertificateManagementProperties { + acme_provider_id: dns_acme_id, + subject_alternative_names: Default::default(), + }, + ), + dkim_management: DkimManagement::Manual, + dns_management: DnsManagement::Automatic(DnsManagementProperties { + dns_server_id: pebble_dns_id, + publish_records: Map::new(vec![DnsRecordType::Caa]), + ..Default::default() + }), + ..Default::default() + }) + .await; + test.wait_for_tasks_skip_not_due().await; + let certificate = account + .registry_get_all::() + .await + .into_iter() + .next() + .unwrap() + .1; + let mut sans = certificate.subject_alternative_names.into_inner(); + sans.sort(); + assert_eq!(sans, vec!["*.persist.org".to_string()]); +} diff --git a/tests/src/automation/dkim.rs b/tests/src/automation/dkim.rs new file mode 100644 index 00000000..cb20bb8e --- /dev/null +++ b/tests/src/automation/dkim.rs @@ -0,0 +1,11 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::utils::server::TestServer; + +pub async fn test(test: &TestServer) { + println!("Running DKIM Management tests..."); +} diff --git a/tests/src/automation/dns.rs b/tests/src/automation/dns.rs new file mode 100644 index 00000000..0a83fce4 --- /dev/null +++ b/tests/src/automation/dns.rs @@ -0,0 +1,11 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::utils::server::TestServer; + +pub async fn test(test: &TestServer) { + println!("Running DNS Management tests..."); +} diff --git a/tests/src/automation/mod.rs b/tests/src/automation/mod.rs new file mode 100644 index 00000000..47e7b10e --- /dev/null +++ b/tests/src/automation/mod.rs @@ -0,0 +1,117 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub mod acme; +pub mod dkim; +pub mod dns; + +use registry::{ + schema::{ + enums::{NetworkListenerProtocol, ServiceProtocol}, + prelude::Property, + structs::{MailExchanger, Service, SystemSettings}, + }, + types::list::List, +}; +use utils::map::vec_map::VecMap; + +use crate::utils::server::TestServerBuilder; + +#[tokio::test(flavor = "multi_thread")] +async fn automation_tests() { + let mut test = TestServerBuilder::new("automation_tests") + .await + .with_listener(NetworkListenerProtocol::Http, "http", 8898, false) + .await + .with_default_listeners() + .await + .build() + .await; + + // Create admin account + let account = test.create_admin_account("admin@example.org").await; + account + .registry_update_setting( + SystemSettings { + mail_exchangers: List::from_iter([ + MailExchanger { + priority: 10u64, + hostname: "mx1.example.org".to_string().into(), + }, + MailExchanger { + priority: 20u64, + hostname: "mx2.example.org".to_string().into(), + }, + ]), + services: VecMap::from_iter([ + ( + ServiceProtocol::Caldav, + Service { + cleartext: false, + ..Default::default() + }, + ), + ( + ServiceProtocol::Carddav, + Service { + cleartext: false, + ..Default::default() + }, + ), + ( + ServiceProtocol::Imap, + Service { + cleartext: false, + hostname: "imap.example.org".to_string().into(), + }, + ), + ( + ServiceProtocol::Jmap, + Service { + cleartext: false, + ..Default::default() + }, + ), + ( + ServiceProtocol::Managesieve, + Service { + cleartext: false, + ..Default::default() + }, + ), + ( + ServiceProtocol::Pop3, + Service { + cleartext: false, + hostname: "pop3.example.org".to_string().into(), + }, + ), + ( + ServiceProtocol::Smtp, + Service { + cleartext: false, + hostname: "smtp.example.org".to_string().into(), + }, + ), + ( + ServiceProtocol::Webdav, + Service { + cleartext: false, + ..Default::default() + }, + ), + ]), + + ..Default::default() + }, + &[Property::MailExchangers, Property::Services], + ) + .await; + account.reload_settings().await; + test.insert_account(account); + + acme::test(&test).await; +} diff --git a/tests/src/cluster/broadcast.rs b/tests/src/cluster/broadcast.rs index ad860c2e..945753ca 100644 --- a/tests/src/cluster/broadcast.rs +++ b/tests/src/cluster/broadcast.rs @@ -18,7 +18,7 @@ use registry::{ prelude::{ObjectType, Property, SocketAddr}, structs::{ ClusterListenerGroup, ClusterListenerGroupProperties, ClusterRole, ClusterTaskGroup, - Coordinator, Expression, Http, NatsCoordinator, NetworkListener, RedisStore, + Coordinator, NatsCoordinator, NetworkListener, RedisStore, }, }, types::map::Map, @@ -60,14 +60,6 @@ fn cluster_tests() { // Create initial server let test = TestServerBuilder::new("cluster_test_0") - .await - .with_object(Http { - base_url: Expression { - else_: "'https://127.0.0.1:' + local_port".to_string(), - ..Default::default() - }, - ..Default::default() - }) .await .with_object(coordinator) .await diff --git a/tests/src/lib.rs b/tests/src/lib.rs index 2684ad5d..3e3c2546 100644 --- a/tests/src/lib.rs +++ b/tests/src/lib.rs @@ -13,6 +13,8 @@ use jemallocator::Jemalloc; #[global_allocator] static GLOBAL: Jemalloc = Jemalloc; +#[cfg(test)] +pub mod automation; #[cfg(test)] pub mod cluster; #[cfg(test)] diff --git a/tests/src/smtp/inbound/dmarc.rs b/tests/src/smtp/inbound/dmarc.rs index c1411237..05357cdd 100644 --- a/tests/src/smtp/inbound/dmarc.rs +++ b/tests/src/smtp/inbound/dmarc.rs @@ -21,8 +21,8 @@ use mail_auth::{ }; use registry::{ schema::structs::{ - DkimReportSettings, DmarcReportSettings, Domain, Expression, ExpressionMatch, SenderAuth, - SpfReportSettings, + CertificateManagement, DkimManagement, DkimReportSettings, DmarcReportSettings, + DnsManagement, Domain, Expression, ExpressionMatch, SenderAuth, SpfReportSettings, }, types::list::List, }; @@ -45,6 +45,9 @@ async fn dmarc() { let domain_id = admin .registry_create_object(Domain { name: "localdomain.org".into(), + certificate_management: CertificateManagement::Manual, + dns_management: DnsManagement::Manual, + dkim_management: DkimManagement::Manual, allow_relaying: true, ..Default::default() }) diff --git a/tests/src/smtp/inbound/scripts.rs b/tests/src/smtp/inbound/scripts.rs index b2783aa6..6da7d6bb 100644 --- a/tests/src/smtp/inbound/scripts.rs +++ b/tests/src/smtp/inbound/scripts.rs @@ -13,9 +13,9 @@ use crate::{ }; use core::panic; use registry::schema::structs::{ - Domain, Expression, LookupStore, MtaStageConnect, MtaStageData, MtaStageEhlo, - MtaStageMail, MtaStageRcpt, SieveSystemInterpreter, SieveSystemScript, SqliteStore, - StoreLookup, + CertificateManagement, DkimManagement, DnsManagement, Domain, Expression, LookupStore, + MtaStageConnect, MtaStageData, MtaStageEhlo, MtaStageMail, MtaStageRcpt, + SieveSystemInterpreter, SieveSystemScript, SqliteStore, StoreLookup, }; use smtp::scripts::{ScriptResult, event_loop::RunScript}; use std::{fs, path::PathBuf}; @@ -36,6 +36,9 @@ async fn sieve_scripts() { let domain_id = admin .registry_create_object(Domain { name: "foobar.org".into(), + certificate_management: CertificateManagement::Manual, + dns_management: DnsManagement::Manual, + dkim_management: DkimManagement::Manual, allow_relaying: true, ..Default::default() }) diff --git a/tests/src/smtp/inbound/sign.rs b/tests/src/smtp/inbound/sign.rs index 8b52bc3d..6272fd66 100644 --- a/tests/src/smtp/inbound/sign.rs +++ b/tests/src/smtp/inbound/sign.rs @@ -18,7 +18,8 @@ use mail_auth::{ use registry::schema::{ enums::{DkimCanonicalization, DkimRotationStage}, structs::{ - Dkim1Signature, DkimSignature, Domain, Expression, SecretText, SecretTextValue, SenderAuth, + CertificateManagement, Dkim1Signature, DkimManagement, DkimSignature, DnsManagement, + Domain, Expression, SecretText, SecretTextValue, SenderAuth, }, }; use std::time::{Duration, Instant}; @@ -40,6 +41,9 @@ async fn sign_and_seal() { let domain_id = admin .registry_create_object(Domain { name: "example.com".into(), + certificate_management: CertificateManagement::Manual, + dns_management: DnsManagement::Manual, + dkim_management: DkimManagement::Manual, allow_relaying: true, ..Default::default() }) diff --git a/tests/src/store/registry.rs b/tests/src/store/registry.rs index 1c025e4a..2b6a8801 100644 --- a/tests/src/store/registry.rs +++ b/tests/src/store/registry.rs @@ -12,8 +12,9 @@ use registry::{ enums::{AccountType, Locale, Permission, StorageQuota}, prelude::{Object, ObjectType, Property}, structs::{ - Account, Credential, CredentialPermissions, CredentialPermissionsList, CustomRoles, - Domain, EmailAlias, EncryptionAtRest, EncryptionSettings, GroupAccount, MailingList, + Account, CertificateManagement, Credential, CredentialPermissions, + CredentialPermissionsList, CustomRoles, DkimManagement, DnsManagement, Domain, + EmailAlias, EncryptionAtRest, EncryptionSettings, GroupAccount, MailingList, PasswordCredential, Permissions, PermissionsList, PublicKey, Roles, SecondaryCredential, UserAccount, UserRoles, }, @@ -114,6 +115,9 @@ pub async fn test(test: &TestServer) { .write(RegistryWrite::insert( &Domain { name: "test.org".into(), + certificate_management: CertificateManagement::Manual, + dns_management: DnsManagement::Manual, + dkim_management: DkimManagement::Manual, is_enabled: true, ..Default::default() } @@ -126,6 +130,9 @@ pub async fn test(test: &TestServer) { .write(RegistryWrite::insert( &Domain { name: "test.net".into(), + certificate_management: CertificateManagement::Manual, + dns_management: DnsManagement::Manual, + dkim_management: DkimManagement::Manual, is_enabled: true, ..Default::default() } @@ -243,6 +250,9 @@ pub async fn test(test: &TestServer) { Domain { name: "test.org".into(), is_enabled: true, + certificate_management: CertificateManagement::Manual, + dns_management: DnsManagement::Manual, + dkim_management: DkimManagement::Manual, ..Default::default() }, RegistryWriteResult::PrimaryKeyConflict { diff --git a/tests/src/system/directory.rs b/tests/src/system/directory.rs index f62adb15..adf8265b 100644 --- a/tests/src/system/directory.rs +++ b/tests/src/system/directory.rs @@ -15,8 +15,9 @@ use registry::{ enums::{AccountType, StorageQuota}, prelude::{ObjectType, Property}, structs::{ - Account, Credential, Domain, EmailAlias, Expression, ExpressionMatch, GroupAccount, - MailingList, PasswordCredential, SubAddressing, SubAddressingCustom, UserAccount, + Account, CertificateManagement, Credential, DkimManagement, DnsManagement, Domain, + EmailAlias, Expression, ExpressionMatch, GroupAccount, MailingList, PasswordCredential, + SubAddressing, SubAddressingCustom, UserAccount, }, }, types::{EnumImpl, list::List, map::Map}, @@ -33,6 +34,9 @@ pub async fn test(test: &TestServer) { let domain_id = account .registry_create_object(Domain { name: "example.com".to_string(), + certificate_management: CertificateManagement::Manual, + dns_management: DnsManagement::Manual, + dkim_management: DkimManagement::Manual, aliases: Map::new(vec!["beispiel.de".to_string()]), is_enabled: true, catch_all_address: Some("catchy@example.com".to_string()), @@ -60,6 +64,9 @@ pub async fn test(test: &TestServer) { account .registry_create_object_expect_err(Domain { name: "example.com".to_string(), + certificate_management: CertificateManagement::Manual, + dns_management: DnsManagement::Manual, + dkim_management: DkimManagement::Manual, ..Default::default() }) .await @@ -371,6 +378,9 @@ pub async fn test(test: &TestServer) { .registry_create_object(Domain { name: "another-example.com".to_string(), is_enabled: true, + certificate_management: CertificateManagement::Manual, + dns_management: DnsManagement::Manual, + dkim_management: DkimManagement::Manual, sub_addressing: SubAddressing::Custom(SubAddressingCustom { custom_rule: Expression { else_: "false".to_string(), diff --git a/tests/src/system/tenant.rs b/tests/src/system/tenant.rs index a56e1c17..73af402d 100644 --- a/tests/src/system/tenant.rs +++ b/tests/src/system/tenant.rs @@ -18,10 +18,10 @@ use registry::{ enums::{AccountType, Permission, TenantStorageQuota}, prelude::{ObjectType, Property}, structs::{ - Account, Credential, Dkim1Signature, DkimSignature, DnsServer, DnsServerCloudflare, - Domain, GroupAccount, MailingList, OAuthClient, PasswordCredential, Permissions, - PermissionsList, Role, SecretKey, SecretKeyValue, SecretText, SecretTextValue, Tenant, - UserAccount, UserRoles, + Account, CertificateManagement, Credential, Dkim1Signature, DkimManagement, + DkimSignature, DnsManagement, DnsServer, DnsServerCloudflare, Domain, GroupAccount, + MailingList, OAuthClient, PasswordCredential, Permissions, PermissionsList, Role, + SecretKey, SecretKeyValue, SecretText, SecretTextValue, Tenant, UserAccount, UserRoles, }, }, types::{EnumImpl, ObjectImpl, list::List, map::Map}, @@ -60,6 +60,9 @@ pub async fn test(test: &mut TestServer) { .registry_create_object(Domain { name: format!("tenant{name}.org"), member_tenant_id: tenant_id.into(), + certificate_management: CertificateManagement::Manual, + dns_management: DnsManagement::Manual, + dkim_management: DkimManagement::Manual, ..Default::default() }) .await; @@ -176,6 +179,9 @@ pub async fn test(test: &mut TestServer) { Domain { name: format!("tenant{tenant_id_pos}.org"), member_tenant_id, + certificate_management: CertificateManagement::Manual, + dns_management: DnsManagement::Manual, + dkim_management: DkimManagement::Manual, ..Default::default() }, ) diff --git a/tests/src/utils/account.rs b/tests/src/utils/account.rs index 515e8246..3665403d 100644 --- a/tests/src/utils/account.rs +++ b/tests/src/utils/account.rs @@ -12,8 +12,9 @@ use registry::{ enums::Permission, prelude::{ObjectType, Property}, structs::{ - self, Credential, CustomRoles, Domain, EmailAlias, GroupAccount, PasswordCredential, - Permissions, PermissionsList, Roles, UserAccount, + self, CertificateManagement, Credential, CustomRoles, DkimManagement, DnsManagement, + Domain, EmailAlias, GroupAccount, PasswordCredential, Permissions, PermissionsList, + Roles, UserAccount, }, }, types::{list::List, map::Map}, @@ -244,6 +245,9 @@ impl Account { self.registry_create_object(Domain { is_enabled: true, name: name.to_string(), + certificate_management: CertificateManagement::Manual, + dns_management: DnsManagement::Manual, + dkim_management: DkimManagement::Manual, ..Default::default() }) .await diff --git a/tests/src/utils/registry.rs b/tests/src/utils/registry.rs index c9cc6ef0..4f8e93ad 100644 --- a/tests/src/utils/registry.rs +++ b/tests/src/utils/registry.rs @@ -407,7 +407,14 @@ fn remove_server_set_props(value: &mut serde_json::Value) { .and_then(|v| v.as_str()) .is_some_and(|t| ["AppPassword", "ApiKey"].contains(&t)); obj.retain(|k, v| { - !(["createdAt", "credentialId", "retireAt"].contains(&k.as_str()) + !([ + "createdAt", + "credentialId", + "retireAt", + "accountKey", + "accountUri", + ] + .contains(&k.as_str()) || (is_app_pass && k == "secret") || (k == "memberTenantId" && v.is_null())) }); diff --git a/tests/src/utils/server.rs b/tests/src/utils/server.rs index 442a308f..9df7a95c 100644 --- a/tests/src/utils/server.rs +++ b/tests/src/utils/server.rs @@ -42,8 +42,8 @@ use registry::{ enums::{DataStoreType, EventPolicy, NetworkListenerProtocol, TracingLevel}, prelude::{Object, ObjectType, SocketAddr}, structs::{ - Authentication, Certificate, Expression, Http, NetworkListener, PublicText, - SecretKeyFile, SecretText, Tracer, TracerStdout, + Authentication, Certificate, NetworkListener, PublicText, SecretKeyFile, SecretText, + Tracer, TracerStdout, }, }, types::{EnumImpl, map::Map}, @@ -149,28 +149,12 @@ impl TestServerBuilder { ] { this = this.with_listener(protocol, name, port, use_tls).await; } - this.with_object(Http { - base_url: Expression { - else_: "'https://127.0.0.1:8899'".to_string(), - ..Default::default() - }, - ..Default::default() - }) - .await + this } pub async fn with_http_listener(self, port: u16) -> Self { self.with_listener(NetworkListenerProtocol::Http, "jmap", port, true) .await - .with_object(Http { - base_url: Expression { - else_: format!("'https://127.0.0.1:{}'", port), - - ..Default::default() - }, - ..Default::default() - }) - .await } pub async fn with_smtp_listener(self, port: u16) -> Self { @@ -216,7 +200,7 @@ impl TestServerBuilder { } self.insert_object(NetworkListener { bind: Map::new(vec![ - SocketAddr::from_str(&format!("127.0.0.1:{port}")).unwrap(), + SocketAddr::from_str(&format!("0.0.0.0:{port}")).unwrap(), ]), name: name.to_string(), protocol, @@ -469,11 +453,15 @@ impl TestServer { } pub async fn wait_for_tasks(&self) { - wait_for_tasks(&self.server, false).await; + wait_for_tasks(&self.server, false, false).await; } pub async fn wait_for_tasks_skip_failures(&self) { - wait_for_tasks(&self.server, true).await; + wait_for_tasks(&self.server, false, true).await; + } + + pub async fn wait_for_tasks_skip_not_due(&self) { + wait_for_tasks(&self.server, true, false).await; } pub async fn blob_expire_all(&self) { diff --git a/tests/src/utils/storage.rs b/tests/src/utils/storage.rs index 3f7e483e..9a12b268 100644 --- a/tests/src/utils/storage.rs +++ b/tests/src/utils/storage.rs @@ -21,6 +21,7 @@ use registry::{ }, types::{EnumImpl, duration::Duration}, }; +use store::write::now; use store::{ Deserialize, IterateParams, ValueKey, write::{TaskQueueClass, ValueClass}, @@ -153,7 +154,7 @@ fn build_search_store(typ: SearchStoreType, _path: &str) -> SearchStore { } } -pub async fn wait_for_tasks(server: &Server, skip_permanent_failures: bool) { +pub async fn wait_for_tasks(server: &Server, skip_not_due: bool, skip_permanent_failures: bool) { let mut count = 0; loop { let mut has_index_tasks = None; @@ -169,7 +170,9 @@ pub async fn wait_for_tasks(server: &Server, skip_permanent_failures: bool) { .ascending(), |_, value| { let task = Task::deserialize(value)?; - if skip_permanent_failures && matches!(task.status(), TaskStatus::Failed(_)) { + if (skip_permanent_failures && matches!(task.status(), TaskStatus::Failed(_))) + || (skip_not_due && task.due_timestamp() > now()) + { Ok(true) } else { has_index_tasks = Some(task); @@ -195,7 +198,7 @@ pub async fn wait_for_tasks(server: &Server, skip_permanent_failures: bool) { pub async fn assert_is_empty(server: &Server, include_registry: bool) { // Wait for pending index tasks - wait_for_tasks(server, false).await; + wait_for_tasks(server, false, false).await; // Assert is empty store_assert_is_empty( diff --git a/tests/src/utils/webdav.rs b/tests/src/utils/webdav.rs index 7961f743..399fa2e3 100644 --- a/tests/src/utils/webdav.rs +++ b/tests/src/utils/webdav.rs @@ -1174,7 +1174,8 @@ fn flatten_xml(xml: &str) -> Vec<(String, String)> { } } Event::GeneralRef(entity) => { - let value: Cow = match entity.as_ref() { + let entity_slice: &[u8] = entity.as_ref(); + let value: Cow = match entity_slice { b"lt" => "<".into(), b"gt" => ">".into(), b"amp" => "&".into(),