From 1b5334b65c3abd948af5c130741d58fb010320a1 Mon Sep 17 00:00:00 2001 From: Maurus Decimus <11444311+mdecimus@users.noreply.github.com> Date: Tue, 24 Mar 2026 18:43:08 +0100 Subject: [PATCH] Registry testing - part 14 --- crates/common/src/config/server/tls.rs | 12 +- crates/registry/src/schema/prelude.rs | 1 + crates/registry/src/types/map.rs | 4 +- crates/registry/src/types/socketaddr.rs | 9 +- crates/registry/src/utils/report.rs | 34 ++- crates/registry/src/utils/secret.rs | 14 +- crates/smtp/src/reporting/dmarc.rs | 2 +- crates/smtp/src/reporting/tls.rs | 5 +- tests/src/imap/mod.rs | 4 +- tests/src/smtp/inbound/data.rs | 19 +- tests/src/smtp/inbound/dmarc.rs | 48 +-- tests/src/smtp/inbound/ehlo.rs | 13 +- tests/src/smtp/inbound/mail.rs | 12 +- tests/src/smtp/inbound/milter.rs | 32 +- tests/src/smtp/inbound/mod.rs | 29 +- tests/src/smtp/inbound/rcpt.rs | 15 +- tests/src/smtp/inbound/rewrite.rs | 12 +- tests/src/smtp/inbound/scripts.rs | 12 +- tests/src/smtp/inbound/sign.rs | 48 +-- tests/src/smtp/inbound/throttle.rs | 13 +- tests/src/smtp/inbound/vrfy.rs | 12 +- tests/src/smtp/lookup/utils.rs | 12 +- tests/src/smtp/mod.rs | 6 +- tests/src/smtp/outbound/dane.rs | 207 ++++++------- tests/src/smtp/outbound/extensions.rs | 164 ++++++----- tests/src/smtp/outbound/fallback_relay.rs | 159 +++++----- tests/src/smtp/outbound/ip_lookup.rs | 118 +++++--- tests/src/smtp/outbound/lmtp.rs | 268 +++++++++++------ tests/src/smtp/outbound/mta_sts.rs | 182 ++++++------ tests/src/smtp/outbound/smtp.rs | 333 +++++++++++++-------- tests/src/smtp/outbound/throttle.rs | 341 ++++++++++++---------- tests/src/smtp/outbound/tls.rs | 172 +++++++---- tests/src/smtp/queue/concurrent.rs | 172 +++++++---- tests/src/smtp/queue/dsn.rs | 117 +++++--- tests/src/smtp/queue/manager.rs | 81 ++--- tests/src/smtp/queue/mod.rs | 36 ++- tests/src/smtp/queue/retry.rs | 225 +++++++++----- tests/src/smtp/queue/virtualq.rs | 220 +++++++++----- tests/src/smtp/reporting/analyze.rs | 166 ++++++----- tests/src/smtp/reporting/dmarc.rs | 168 ++++++----- tests/src/smtp/reporting/scheduler.rs | 289 +++++++++--------- tests/src/smtp/reporting/tls.rs | 200 +++++++------ tests/src/utils/registry.rs | 131 ++++++++- tests/src/utils/server.rs | 33 ++- 44 files changed, 2449 insertions(+), 1701 deletions(-) diff --git a/crates/common/src/config/server/tls.rs b/crates/common/src/config/server/tls.rs index a01d2137..f33e126d 100644 --- a/crates/common/src/config/server/tls.rs +++ b/crates/common/src/config/server/tls.rs @@ -282,8 +282,18 @@ pub(crate) async fn parse_certificates( continue; } }; + let public = match cert_obj.object.certificate.value().await { + Ok(value) => value.into_owned().into_bytes(), + Err(err) => { + bp.build_error( + cert_obj.id, + format!("Failed to obtain certificate value: {err}"), + ); + continue; + } + }; - match build_certified_key(cert_obj.object.certificate.into_bytes(), secret) { + match build_certified_key(public, secret) { Ok(cert) => { match cert .end_entity_cert() diff --git a/crates/registry/src/schema/prelude.rs b/crates/registry/src/schema/prelude.rs index 611ed495..15671964 100644 --- a/crates/registry/src/schema/prelude.rs +++ b/crates/registry/src/schema/prelude.rs @@ -29,6 +29,7 @@ pub use crate::types::map::Map; pub use crate::types::socketaddr::SocketAddr; pub use crate::types::string::StringValidator; pub use serde::{Deserialize, Serialize}; +pub use std::borrow::Cow; pub use std::str::FromStr; pub use types::blob::BlobId; pub use types::id::Id; diff --git a/crates/registry/src/types/map.rs b/crates/registry/src/types/map.rs index 19fbf5f7..654f58b5 100644 --- a/crates/registry/src/types/map.rs +++ b/crates/registry/src/types/map.rs @@ -207,11 +207,11 @@ impl<'de, T: MapItem> Deserialize<'de> for Map { { let mut items = Vec::with_capacity(map.size_hint().unwrap_or(0)); - while let Some(key) = map.next_key::<&str>()? { + while let Some(key) = map.next_key::>()? { let value: Option = map.next_value()?; if value == Some(true) { - let item = T::try_from_string(key) + let item = T::try_from_string(&key) .ok_or_else(|| de::Error::custom(format!("invalid map key: {key}")))?; if !items.contains(&item) { items.push(item); diff --git a/crates/registry/src/types/socketaddr.rs b/crates/registry/src/types/socketaddr.rs index 873fe8f7..ead5b9a0 100644 --- a/crates/registry/src/types/socketaddr.rs +++ b/crates/registry/src/types/socketaddr.rs @@ -13,6 +13,11 @@ use crate::{ }; use std::{fmt::Display, str::FromStr}; +const UNSET_SOCKET_ADDR: std::net::SocketAddr = std::net::SocketAddr::new( + std::net::IpAddr::V4(std::net::Ipv4Addr::from_octets([255, 255, 255, 255])), + u16::MAX, +); + #[derive(Debug, Clone, PartialEq, Eq)] pub struct SocketAddr(pub std::net::SocketAddr); @@ -22,7 +27,7 @@ impl SocketAddr { } pub fn is_valid(&self) -> bool { - !self.0.ip().is_unspecified() + self.0 != UNSET_SOCKET_ADDR } } @@ -63,7 +68,7 @@ impl<'de> serde::Deserialize<'de> for SocketAddr { impl Default for SocketAddr { fn default() -> Self { - SocketAddr(std::net::SocketAddr::from(([0, 0, 0, 0], 0))) + SocketAddr(UNSET_SOCKET_ADDR) } } diff --git a/crates/registry/src/utils/report.rs b/crates/registry/src/utils/report.rs index 253f142e..50bbc9f6 100644 --- a/crates/registry/src/utils/report.rs +++ b/crates/registry/src/utils/report.rs @@ -5,7 +5,11 @@ */ use crate::{ - schema::{enums, prelude::UTCDateTime, structs}, + schema::{ + enums, + prelude::UTCDateTime, + structs::{self, DmarcReportRecord, TlsFailureDetails}, + }, types::{ipaddr::IpAddr, list::List}, }; use mail_auth::{ @@ -927,3 +931,31 @@ impl From<&mail_auth::dmarc::Policy> for enums::DmarcDisposition { } } } + +impl DmarcReportRecord { + pub fn eq_except_count(&self, other: &Self) -> bool { + self.dkim_results == other.dkim_results + && self.envelope_from == other.envelope_from + && self.envelope_to == other.envelope_to + && self.evaluated_disposition == other.evaluated_disposition + && self.evaluated_dkim == other.evaluated_dkim + && self.evaluated_spf == other.evaluated_spf + && self.extensions == other.extensions + && self.header_from == other.header_from + && self.policy_override_reasons == other.policy_override_reasons + && self.source_ip == other.source_ip + && self.spf_results == other.spf_results + } +} + +impl TlsFailureDetails { + pub fn eq_except_count(&self, other: &Self) -> bool { + self.additional_information == other.additional_information + && self.failure_reason_code == other.failure_reason_code + && self.receiving_ip == other.receiving_ip + && self.receiving_mx_helo == other.receiving_mx_helo + && self.receiving_mx_hostname == other.receiving_mx_hostname + && self.result_type == other.result_type + && self.sending_mta_ip == other.sending_mta_ip + } +} diff --git a/crates/registry/src/utils/secret.rs b/crates/registry/src/utils/secret.rs index ce200bc8..42c691ca 100644 --- a/crates/registry/src/utils/secret.rs +++ b/crates/registry/src/utils/secret.rs @@ -5,8 +5,8 @@ */ use crate::schema::prelude::{ - SecretKey, SecretKeyEnvironmentVariable, SecretKeyFile, SecretKeyOptional, SecretKeyValue, - SecretText, SecretTextOptional, SecretTextValue, + PublicText, SecretKey, SecretKeyEnvironmentVariable, SecretKeyFile, SecretKeyOptional, + SecretKeyValue, SecretText, SecretTextOptional, SecretTextValue, }; use std::borrow::Cow; @@ -30,6 +30,16 @@ impl SecretText { } } +impl PublicText { + pub async fn value(&self) -> Result, String> { + match self { + PublicText::Text(value) => Ok(Cow::Borrowed(value.value.as_str())), + PublicText::File(file) => file.secret().await.map(Cow::Owned), + PublicText::EnvironmentVariable(env_var) => env_var.secret().map(Cow::Owned), + } + } +} + impl SecretKeyOptional { pub async fn secret(&self) -> Result>, String> { match self { diff --git a/crates/smtp/src/reporting/dmarc.rs b/crates/smtp/src/reporting/dmarc.rs index c20f3160..6775dbfb 100644 --- a/crates/smtp/src/reporting/dmarc.rs +++ b/crates/smtp/src/reporting/dmarc.rs @@ -605,7 +605,7 @@ impl DmarcReporting for Server { .0 .inner .iter() - .position(|d| d.value == record) + .position(|d| d.value.eq_except_count(&record)) { report.report.records.0.inner[idx].value.count += 1; } else { diff --git a/crates/smtp/src/reporting/tls.rs b/crates/smtp/src/reporting/tls.rs index 3898f03c..6334feba 100644 --- a/crates/smtp/src/reporting/tls.rs +++ b/crates/smtp/src/reporting/tls.rs @@ -421,18 +421,19 @@ impl TlsReporting for Server { }; // Add failure details - if let Some(failure) = event.failure.clone().map(TlsFailureDetails::from) { + if let Some(mut failure) = event.failure.clone().map(TlsFailureDetails::from) { if let Some(idx) = policy .failure_details .0 .inner .iter() - .position(|d| d.value == failure) + .position(|d| d.value.eq_except_count(&failure)) { policy.failure_details.0.inner[idx] .value .failed_session_count += 1; } else { + failure.failed_session_count = 1; policy.failure_details.push(failure); } diff --git a/tests/src/imap/mod.rs b/tests/src/imap/mod.rs index fd5891cd..2c390f1f 100644 --- a/tests/src/imap/mod.rs +++ b/tests/src/imap/mod.rs @@ -31,8 +31,8 @@ use registry::{ enums::{Permission, SpecialUse}, prelude::ObjectType, structs::{ - Action, Email, EmailFolder, Expression, Imap, MemoryLookupKey, MtaStageAuth, - MtaStageData, SpamClassifier, SpamTag, SpamTagScore, + Email, EmailFolder, Expression, Imap, MemoryLookupKey, MtaStageAuth, MtaStageData, + SpamClassifier, SpamTag, SpamTagScore, }, }, types::float::Float, diff --git a/tests/src/smtp/inbound/data.rs b/tests/src/smtp/inbound/data.rs index 4f92d4ac..ca008a62 100644 --- a/tests/src/smtp/inbound/data.rs +++ b/tests/src/smtp/inbound/data.rs @@ -16,8 +16,7 @@ use registry::{ enums::MtaQueueQuotaKey, prelude::ObjectType, structs::{ - Expression, ExpressionMatch, MtaQueueQuota, MtaStageAuth, MtaStageData, SenderAuth, - SpamSettings, + Expression, ExpressionMatch, MtaQueueQuota, MtaStageData, SenderAuth, SpamSettings, }, }, types::{list::List, map::Map}, @@ -58,15 +57,7 @@ async fn data() { } // Add test settings - admin - .registry_create_object(MtaStageAuth { - require: Expression { - else_: "false".into(), - ..Default::default() - }, - ..Default::default() - }) - .await; + admin.mta_no_auth().await; admin .registry_create_object(SpamSettings { enable: false, @@ -313,8 +304,10 @@ async fn data() { // Make sure store is empty test.clear_queue().await; - test.account("admin") - .registry_destroy_all(ObjectType::MtaQueueQuota) + let admin = test.account("admin"); + admin.registry_destroy_all(ObjectType::MtaQueueQuota).await; + admin + .registry_destroy_all(ObjectType::MtaInboundThrottle) .await; test.assert_is_empty().await; } diff --git a/tests/src/smtp/inbound/dmarc.rs b/tests/src/smtp/inbound/dmarc.rs index 49942ae5..c1411237 100644 --- a/tests/src/smtp/inbound/dmarc.rs +++ b/tests/src/smtp/inbound/dmarc.rs @@ -21,8 +21,8 @@ use mail_auth::{ }; use registry::{ schema::structs::{ - DkimReportSettings, DmarcReportSettings, Domain, Expression, ExpressionMatch, MtaStageAuth, - MtaStageData, SenderAuth, SpfReportSettings, + DkimReportSettings, DmarcReportSettings, Domain, Expression, ExpressionMatch, SenderAuth, + SpfReportSettings, }, types::list::List, }; @@ -50,48 +50,8 @@ async fn dmarc() { }) .await; admin.create_dkim_signatures(domain_id).await; - admin - .registry_create_object(MtaStageAuth { - require: Expression { - else_: "false".into(), - ..Default::default() - }, - ..Default::default() - }) - .await; - admin - .registry_create_object(MtaStageData { - add_date_header: Expression { - else_: "true".into(), - ..Default::default() - }, - add_message_id_header: Expression { - else_: "true".into(), - ..Default::default() - }, - add_received_header: Expression { - else_: "true".into(), - ..Default::default() - }, - add_received_spf_header: Expression { - else_: "true".into(), - ..Default::default() - }, - add_auth_results_header: Expression { - else_: "true".into(), - ..Default::default() - }, - add_return_path_header: Expression { - else_: "false".into(), - ..Default::default() - }, - enable_spam_filter: Expression { - else_: "false".into(), - ..Default::default() - }, - ..Default::default() - }) - .await; + admin.mta_no_auth().await; + admin.mta_add_all_headers().await; admin .registry_create_object(SenderAuth { dmarc_verify: Expression { diff --git a/tests/src/smtp/inbound/ehlo.rs b/tests/src/smtp/inbound/ehlo.rs index f5ab20ce..9e2db6cb 100644 --- a/tests/src/smtp/inbound/ehlo.rs +++ b/tests/src/smtp/inbound/ehlo.rs @@ -11,8 +11,7 @@ use crate::{ use mail_auth::{SpfResult, common::parse::TxtRecordParser, spf::Spf}; use registry::{ schema::structs::{ - Expression, ExpressionMatch, MtaExtensions, MtaStageAuth, MtaStageData, MtaStageEhlo, - SenderAuth, + Expression, ExpressionMatch, MtaExtensions, MtaStageData, MtaStageEhlo, SenderAuth, }, types::list::List, }; @@ -30,15 +29,7 @@ async fn ehlo() { // Add test settings let admin = test.account("admin"); - admin - .registry_create_object(MtaStageAuth { - require: Expression { - else_: "false".into(), - ..Default::default() - }, - ..Default::default() - }) - .await; + admin.mta_no_auth().await; admin .registry_create_object(MtaExtensions { future_release: Expression { diff --git a/tests/src/smtp/inbound/mail.rs b/tests/src/smtp/inbound/mail.rs index 394f9dab..a80f685f 100644 --- a/tests/src/smtp/inbound/mail.rs +++ b/tests/src/smtp/inbound/mail.rs @@ -13,7 +13,7 @@ use registry::{ schema::{ enums::MtaInboundThrottleKey, structs::{ - Expression, ExpressionMatch, MtaExtensions, MtaInboundThrottle, MtaStageAuth, + Expression, ExpressionMatch, MtaExtensions, MtaInboundThrottle, MtaStageData, MtaStageEhlo, MtaStageMail, Rate, SenderAuth, }, }, @@ -43,15 +43,7 @@ async fn mail() { ..Default::default() }) .await; - admin - .registry_create_object(MtaStageAuth { - require: Expression { - else_: "false".into(), - ..Default::default() - }, - ..Default::default() - }) - .await; + admin.mta_no_auth().await; admin .registry_create_object(SenderAuth { reverse_ip_verify: Expression { diff --git a/tests/src/smtp/inbound/milter.rs b/tests/src/smtp/inbound/milter.rs index 5fd2a2ec..045c9b4f 100644 --- a/tests/src/smtp/inbound/milter.rs +++ b/tests/src/smtp/inbound/milter.rs @@ -26,7 +26,7 @@ use registry::{ schema::{ enums::{self, MtaStage}, prelude::{ObjectType, Property}, - structs::{Expression, MtaHook, MtaMilter, MtaStageAuth, MtaStageRcpt}, + structs::{Expression, MtaHook, MtaMilter, MtaStageRcpt}, }, types::map::Map, }; @@ -67,24 +67,8 @@ async fn milter_session() { // Add test settings let admin = test.account("admin"); - admin - .registry_create_object(MtaStageAuth { - require: Expression { - else_: "false".into(), - ..Default::default() - }, - ..Default::default() - }) - .await; - admin - .registry_create_object(MtaStageRcpt { - allow_relaying: Expression { - else_: "true".into(), - ..Default::default() - }, - ..Default::default() - }) - .await; + admin.mta_no_auth().await; + admin.mta_allow_relaying().await; admin .registry_create_object(MtaMilter { enable: Expression { @@ -231,15 +215,7 @@ async fn mta_hook_session() { // Add test settings let admin = test.account("admin"); - admin - .registry_create_object(MtaStageAuth { - require: Expression { - else_: "false".into(), - ..Default::default() - }, - ..Default::default() - }) - .await; + admin.mta_no_auth().await; admin .registry_create_object(MtaStageRcpt { allow_relaying: Expression { diff --git a/tests/src/smtp/inbound/mod.rs b/tests/src/smtp/inbound/mod.rs index 84e1dd73..d4b1e3e6 100644 --- a/tests/src/smtp/inbound/mod.rs +++ b/tests/src/smtp/inbound/mod.rs @@ -7,6 +7,7 @@ use crate::utils::server::TestServer; use common::{ Server, + config::smtp::queue::QueueName, ipc::{DmarcEvent, QueueEvent, QueueEventStatus, ReportingEvent, TlsEvent}, }; use registry::{schema::prelude::ObjectType, types::ObjectImpl}; @@ -83,12 +84,12 @@ impl TestServer { self.last_queued_message().await } - pub async fn consume_message(&mut self, server: &Server) -> MessageWrapper { + pub async fn consume_message(&mut self) -> MessageWrapper { self.read_event().await.assert_refresh(); let message = self.last_queued_message().await; message .clone() - .remove(server, self.last_queued_due().await.into()) + .remove(&self.server, self.last_queued_due().await.into()) .await; message } @@ -103,7 +104,29 @@ impl TestServer { QueuedMessage { due: self.message_due(queue_id).await, queue_id, - queue_name: Default::default(), + queue_name: QueueName::new("remote").unwrap(), + } + } + + pub async fn expect_message_for_queue_then_deliver( + &mut self, + queue_name: &str, + ) -> QueuedMessage { + let message = self.expect_message().await; + + self.delivery_attempt_for_queue(message.queue_id, queue_name) + .await + } + + pub async fn delivery_attempt_for_queue( + &mut self, + queue_id: u64, + queue_name: &str, + ) -> QueuedMessage { + QueuedMessage { + due: self.message_due(queue_id).await, + queue_id, + queue_name: QueueName::new(queue_name).unwrap(), } } diff --git a/tests/src/smtp/inbound/rcpt.rs b/tests/src/smtp/inbound/rcpt.rs index 3ae5560f..cd0d0c63 100644 --- a/tests/src/smtp/inbound/rcpt.rs +++ b/tests/src/smtp/inbound/rcpt.rs @@ -12,8 +12,7 @@ use registry::{ schema::{ enums::MtaInboundThrottleKey, structs::{ - Expression, ExpressionMatch, MtaExtensions, MtaInboundThrottle, MtaStageAuth, - MtaStageRcpt, Rate, + Expression, ExpressionMatch, MtaExtensions, MtaInboundThrottle, MtaStageRcpt, Rate, }, }, types::{list::List, map::Map}, @@ -26,7 +25,7 @@ use std::time::Duration; async fn rcpt() { let mut test = TestServerBuilder::new("smtp_rcpt_test") .await - .with_http_listener(19004) + .with_http_listener(18999) .await .disable_services() .build() @@ -56,15 +55,7 @@ async fn rcpt() { } // Add test settings - admin - .registry_create_object(MtaStageAuth { - require: Expression { - else_: "false".into(), - ..Default::default() - }, - ..Default::default() - }) - .await; + admin.mta_no_auth().await; admin .registry_create_object(MtaStageRcpt { allow_relaying: Expression { diff --git a/tests/src/smtp/inbound/rewrite.rs b/tests/src/smtp/inbound/rewrite.rs index d8d19711..3cce8b7d 100644 --- a/tests/src/smtp/inbound/rewrite.rs +++ b/tests/src/smtp/inbound/rewrite.rs @@ -7,7 +7,7 @@ use crate::{smtp::session::TestSession, utils::server::TestServerBuilder}; use registry::{ schema::structs::{ - Expression, ExpressionMatch, MtaStageAuth, MtaStageMail, MtaStageRcpt, + Expression, ExpressionMatch, MtaStageMail, MtaStageRcpt, SieveSystemInterpreter, SieveSystemScript, }, types::list::List, @@ -39,15 +39,7 @@ async fn address_rewrite() { // Add test settings let admin = test.account("admin"); - admin - .registry_create_object(MtaStageAuth { - require: Expression { - else_: "false".into(), - ..Default::default() - }, - ..Default::default() - }) - .await; + admin.mta_no_auth().await; admin .registry_create_object(MtaStageMail { rewrite: Expression { diff --git a/tests/src/smtp/inbound/scripts.rs b/tests/src/smtp/inbound/scripts.rs index 0c0f690e..b2783aa6 100644 --- a/tests/src/smtp/inbound/scripts.rs +++ b/tests/src/smtp/inbound/scripts.rs @@ -13,7 +13,7 @@ use crate::{ }; use core::panic; use registry::schema::structs::{ - Domain, Expression, LookupStore, MtaStageAuth, MtaStageConnect, MtaStageData, MtaStageEhlo, + Domain, Expression, LookupStore, MtaStageConnect, MtaStageData, MtaStageEhlo, MtaStageMail, MtaStageRcpt, SieveSystemInterpreter, SieveSystemScript, SqliteStore, StoreLookup, }; @@ -41,15 +41,7 @@ async fn sieve_scripts() { }) .await; admin.create_dkim_signatures(domain_id).await; - admin - .registry_create_object(MtaStageAuth { - require: Expression { - else_: "false".into(), - ..Default::default() - }, - ..Default::default() - }) - .await; + admin.mta_no_auth().await; admin .registry_create_object(SieveSystemInterpreter { default_from_address: Expression { diff --git a/tests/src/smtp/inbound/sign.rs b/tests/src/smtp/inbound/sign.rs index 405373d3..6b19944e 100644 --- a/tests/src/smtp/inbound/sign.rs +++ b/tests/src/smtp/inbound/sign.rs @@ -18,8 +18,8 @@ use mail_auth::{ use registry::schema::{ enums::DkimCanonicalization, structs::{ - Dkim1Signature, DkimPrivateKey, DkimSignature, Domain, Expression, MtaStageAuth, - MtaStageData, SecretTextValue, SenderAuth, + Dkim1Signature, DkimPrivateKey, DkimSignature, Domain, Expression, SecretTextValue, + SenderAuth, }, }; use std::time::{Duration, Instant}; @@ -46,48 +46,8 @@ async fn sign_and_seal() { }) .await; admin.create_dkim_signatures(domain_id).await; - admin - .registry_create_object(MtaStageAuth { - require: Expression { - else_: "false".into(), - ..Default::default() - }, - ..Default::default() - }) - .await; - admin - .registry_create_object(MtaStageData { - add_date_header: Expression { - else_: "true".into(), - ..Default::default() - }, - add_message_id_header: Expression { - else_: "true".into(), - ..Default::default() - }, - add_received_header: Expression { - else_: "true".into(), - ..Default::default() - }, - add_received_spf_header: Expression { - else_: "true".into(), - ..Default::default() - }, - add_auth_results_header: Expression { - else_: "true".into(), - ..Default::default() - }, - add_return_path_header: Expression { - else_: "false".into(), - ..Default::default() - }, - enable_spam_filter: Expression { - else_: "false".into(), - ..Default::default() - }, - ..Default::default() - }) - .await; + admin.mta_no_auth().await; + admin.mta_add_all_headers().await; admin .registry_create_object(SenderAuth { dmarc_verify: Expression { diff --git a/tests/src/smtp/inbound/throttle.rs b/tests/src/smtp/inbound/throttle.rs index 55b0a583..57671388 100644 --- a/tests/src/smtp/inbound/throttle.rs +++ b/tests/src/smtp/inbound/throttle.rs @@ -8,7 +8,7 @@ use crate::utils::server::TestServerBuilder; use registry::{ schema::{ enums::MtaInboundThrottleKey, - structs::{Expression, MtaInboundThrottle, MtaStageAuth, Rate}, + structs::{Expression, MtaInboundThrottle, Rate}, }, types::map::Map, }; @@ -27,16 +27,7 @@ async fn throttle_inbound() { // Add test settings let admin = test.account("admin"); - admin - .registry_create_object(MtaStageAuth { - require: Expression { - else_: "false".into(), - ..Default::default() - }, - ..Default::default() - }) - .await; - + admin.mta_no_auth().await; admin .registry_create_object(MtaInboundThrottle { description: None, diff --git a/tests/src/smtp/inbound/vrfy.rs b/tests/src/smtp/inbound/vrfy.rs index 05b862d8..13262eff 100644 --- a/tests/src/smtp/inbound/vrfy.rs +++ b/tests/src/smtp/inbound/vrfy.rs @@ -9,7 +9,7 @@ use crate::{ utils::server::TestServerBuilder, }; use registry::{ - schema::structs::{Expression, ExpressionMatch, MailingList, MtaExtensions, MtaStageAuth}, + schema::structs::{Expression, ExpressionMatch, MailingList, MtaExtensions}, types::{list::List, map::Map}, }; @@ -54,15 +54,7 @@ async fn vrfy_expn() { .await; // Add test settings - admin - .registry_create_object(MtaStageAuth { - require: Expression { - else_: "false".into(), - ..Default::default() - }, - ..Default::default() - }) - .await; + admin.mta_no_auth().await; admin .registry_create_object(MtaExtensions { vrfy: Expression { diff --git a/tests/src/smtp/lookup/utils.rs b/tests/src/smtp/lookup/utils.rs index e20f0bf2..778ef7ba 100644 --- a/tests/src/smtp/lookup/utils.rs +++ b/tests/src/smtp/lookup/utils.rs @@ -18,7 +18,7 @@ use registry::{ enums::MtaIpStrategy, structs::{ Expression, MtaConnectionIpHost, MtaConnectionStrategy, MtaOutboundStrategy, MtaRoute, - MtaRouteMx, MtaStageAuth, + MtaRouteMx, }, }, types::{ipaddr::IpAddr, list::List}, @@ -75,15 +75,7 @@ async fn strategies() { // Add test settings let admin = test.account("admin"); - admin - .registry_create_object(MtaStageAuth { - require: Expression { - else_: "false".into(), - ..Default::default() - }, - ..Default::default() - }) - .await; + admin.mta_no_auth().await; admin .registry_create_object(MtaConnectionStrategy { name: "test".into(), diff --git a/tests/src/smtp/mod.rs b/tests/src/smtp/mod.rs index ff2efcb7..7fd6dc63 100644 --- a/tests/src/smtp/mod.rs +++ b/tests/src/smtp/mod.rs @@ -6,10 +6,10 @@ pub mod inbound; pub mod lookup; -pub mod session; -/* -pub mod management; pub mod outbound; pub mod queue; pub mod reporting; +pub mod session; +/* +pub mod management; */ diff --git a/tests/src/smtp/outbound/dane.rs b/tests/src/smtp/outbound/dane.rs index 9b7c8deb..04cbebca 100644 --- a/tests/src/smtp/outbound/dane.rs +++ b/tests/src/smtp/outbound/dane.rs @@ -4,30 +4,28 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::smtp::{ - DnsCache, - inbound::{TestMessage, TestQueueEvent, TestReportingEvent}, - session::{TestSession, VerifyResponse}, +use crate::{ + smtp::{ + inbound::{TestMessage, TestQueueEvent, TestReportingEvent}, + session::{TestSession, VerifyResponse}, + }, + utils::{dns::DnsCache, server::TestServerBuilder}, }; use common::{ - Core, - config::{ - server::ServerProtocol, - smtp::resolver::{DnssecResolver, Resolvers, Tlsa, TlsaEntry}, - }, + config::smtp::resolver::{Tlsa, TlsaEntry}, ipc::PolicyType, }; use mail_auth::{ - MX, MessageAuthenticator, + MX, common::parse::TxtRecordParser, - hickory_resolver::{ - TokioResolver, - config::{ResolverConfig, ResolverOpts}, - name_server::TokioConnectionProvider, - }, mta_sts::{ReportUri, TlsRpt}, report::tlsrpt::ResultType, }; +use registry::schema::{ + enums::MtaRequiredOrOptional, + prelude::ObjectType, + structs::{Expression, MtaTlsStrategy, TlsReportSettings}, +}; use rustls_pki_types::CertificateDer; use smtp::outbound::dane::{dnssec::TlsaLookup, verify::TlsaVerify}; use smtp::queue::{Error, ErrorDetails, Status}; @@ -41,71 +39,93 @@ use std::{ time::{Duration, Instant}, }; -const LOCAL: &str = r#" -[session.rcpt] -relay = true - -[report.tls.aggregate] -send = "weekly" - -[queue.tls.default] -dane = "require" -starttls = "require" - -"#; - -const REMOTE: &str = " -[session.ehlo] -reject-non-fqdn = false - -[session.rcpt] -relay = true - -[session.data.add-headers] -received = true -received-spf = true -auth-results = true -message-id = true -date = true -return-path = false - -"; - #[tokio::test] #[serial_test::serial] async fn dane_verify() { - - + let mut local = TestServerBuilder::new("smtp_dane_local") + .await + .with_http_listener(19018) + .await + .disable_services() + .capture_queue() + .capture_reporting() + .build() + .await; + let mut remote = TestServerBuilder::new("smtp_dane_remote") + .await + .with_dummy_tls_cert() + .await + .with_http_listener(19019) + .await + .with_smtp_listener(9925) + .await + .disable_services() + .capture_queue() + .build() + .await; - // Start test server - let mut remote = TestSMTP::new("smtp_dane_remote", REMOTE).await; - let _rx = remote.start(&[ServerProtocol::Smtp]).await; + let local_admin = local.account("admin"); + local_admin.mta_allow_relaying().await; + local_admin.mta_no_auth().await; + local_admin + .registry_create_object(TlsReportSettings { + send_frequency: Expression { + else_: "weekly".into(), + ..Default::default() + }, + ..Default::default() + }) + .await; + let (tls_strategy_id, mut tls_strategy) = local_admin + .registry_get_all::() + .await + .into_iter() + .find(|(_, s)| s.name == "default") + .unwrap(); + tls_strategy.dane = MtaRequiredOrOptional::Require; + tls_strategy.start_tls = MtaRequiredOrOptional::Require; + let mut tls_strategy = serde_json::to_value(tls_strategy).unwrap(); + tls_strategy + .as_object_mut() + .unwrap() + .retain(|k, _| k != "name"); + local_admin + .registry_update_object(ObjectType::MtaTlsStrategy, tls_strategy_id, tls_strategy) + .await; + local_admin.reload_settings().await; + local.reload_core(); + local.expect_reload_settings().await; - // Fail on missing TLSA record - let mut local = TestSMTP::new("smtp_dane_local", LOCAL).await; + let remote_admin = remote.account("admin"); + remote_admin.mta_no_auth().await; + remote_admin.mta_allow_relaying().await; + remote_admin.mta_add_all_headers().await; + remote_admin.mta_allow_non_fqdn().await; + remote_admin.reload_settings().await; + remote.reload_core(); + remote.expect_reload_settings().await; // Add mock DNS entries - let core = local.build_smtp(); - core.mx_add( + local.server.mx_add( "foobar.org", vec![MX { - exchanges: vec!["mx.foobar.org".to_string()], + exchanges: vec!["mx.foobar.org".into()].into_boxed_slice(), preference: 10, }], Instant::now() + Duration::from_secs(10), ); - core.ipv4_add( + local.server.ipv4_add( "mx.foobar.org", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(10), ); - core.txt_add( + local.server.txt_add( "_smtp._tls.foobar.org", TlsRpt::parse(b"v=TLSRPTv1; rua=mailto:reports@foobar.org").unwrap(), Instant::now() + Duration::from_secs(10), ); - let mut session = local.new_session(); + let mut session = local.new_mta_session(); session.data.remote_ip_str = "10.0.0.1".into(); session.eval_session_params().await; session.ehlo("mx.test.org").await; @@ -113,24 +133,22 @@ async fn dane_verify() { .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; local - .queue_receiver .expect_message_then_deliver() .await - .try_deliver(core.clone()); + .try_deliver(local.server.clone()); local - .queue_receiver .expect_message() .await - .read_lines(&local.queue_receiver) + .read_lines(&local) .await .assert_contains(" (DANE failed to authenticate") .assert_contains("No TLSA reco=") .assert_contains("rds found"); - local.queue_receiver.read_event().await.assert_done(); - local.queue_receiver.assert_no_events(); + local.read_event().await.assert_done(); + local.assert_no_events(); // Expect TLS failure report - let report = local.report_receiver.read_report().await.unwrap_tls(); + let report = local.read_report().await.unwrap_tls(); assert_eq!(report.domain, "foobar.org"); assert_eq!(report.policy, PolicyType::Tlsa(None)); assert_eq!( @@ -157,7 +175,7 @@ async fn dane_verify() { has_end_entities: true, has_intermediates: false, }); - core.tlsa_add( + local.server.tlsa_add( "_25._tcp.mx.foobar.org", tlsa.clone(), Instant::now() + Duration::from_secs(10), @@ -166,30 +184,28 @@ async fn dane_verify() { .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; local - .queue_receiver .expect_message_then_deliver() .await - .try_deliver(core.clone()); + .try_deliver(local.server.clone()); local - .queue_receiver .expect_message() .await - .read_lines(&local.queue_receiver) + .read_lines(&local) .await .assert_contains(" (DANE failed to authenticate") .assert_contains("No matching ") .assert_contains("certificates found"); - local.queue_receiver.read_event().await.assert_done(); - local.queue_receiver.assert_no_events(); + local.read_event().await.assert_done(); + local.assert_no_events(); // Expect TLS failure report - let report = local.report_receiver.read_report().await.unwrap_tls(); + let report = local.read_report().await.unwrap_tls(); assert_eq!(report.policy, PolicyType::Tlsa(tlsa.into())); assert_eq!( report.failure.as_ref().unwrap().result_type, ResultType::ValidationFailure ); - remote.queue_receiver.assert_no_events(); + remote.assert_no_events(); // DANE successful delivery let tlsa = Arc::new(Tlsa { @@ -205,7 +221,7 @@ async fn dane_verify() { has_end_entities: true, has_intermediates: false, }); - core.tlsa_add( + local.server.tlsa_add( "_25._tcp.mx.foobar.org", tlsa.clone(), Instant::now() + Duration::from_secs(10), @@ -214,43 +230,33 @@ async fn dane_verify() { .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; local - .queue_receiver .expect_message_then_deliver() .await - .try_deliver(core.clone()); - local.queue_receiver.read_event().await.assert_done(); - local.queue_receiver.assert_no_events(); + .try_deliver(local.server.clone()); + local.read_event().await.assert_done(); + local.assert_no_events(); remote - .queue_receiver .expect_message() .await - .read_lines(&remote.queue_receiver) + .read_lines(&remote) .await .assert_contains("using TLSv1.3 with cipher"); // Expect TLS success report - let report = local.report_receiver.read_report().await.unwrap_tls(); + let report = local.read_report().await.unwrap_tls(); assert_eq!(report.policy, PolicyType::Tlsa(tlsa.into())); assert!(report.failure.is_none()); } #[tokio::test] async fn dane_test() { - let conf = ResolverConfig::cloudflare_tls(); - let mut opts = ResolverOpts::default(); - opts.validate = true; - opts.try_tcp_on_error = true; - - let mut core = Core::default(); - core.smtp.resolvers = Resolvers { - dns: MessageAuthenticator::new_cloudflare().unwrap(), - dnssec: DnssecResolver { - resolver: TokioResolver::builder_with_config(conf, TokioConnectionProvider::default()) - .with_options(opts) - .build(), - }, - }; - let r = TestSMTP::from_core(core).build_smtp(); + let test = TestServerBuilder::new("smtp_dane_remote") + .await + .with_http_listener(19036) + .await + .disable_services() + .build() + .await; // Add dns entries let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); @@ -275,7 +281,7 @@ async fn dane_test() { match pos { 0 => { if hostname != item && !hostname.is_empty() { - r.tlsa_add( + test.server.tlsa_add( hostname, tlsa.into(), Instant::now() + Duration::from_secs(30), @@ -309,7 +315,7 @@ async fn dane_test() { } } } - r.tlsa_add( + test.server.tlsa_add( hostname, tlsa.into(), Instant::now() + Duration::from_secs(30), @@ -331,7 +337,8 @@ async fn dane_test() { } // Successful DANE verification - let tlsa = r + let tlsa = test + .server .tlsa_lookup(format!("_25._tcp.{host}.")) .await .unwrap() diff --git a/tests/src/smtp/outbound/extensions.rs b/tests/src/smtp/outbound/extensions.rs index 92f85317..f14be93e 100644 --- a/tests/src/smtp/outbound/extensions.rs +++ b/tests/src/smtp/outbound/extensions.rs @@ -4,79 +4,109 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::time::{Duration, Instant}; - -use common::config::server::ServerProtocol; -use mail_auth::MX; -use smtp_proto::{MAIL_REQUIRETLS, MAIL_RET_HDRS, MAIL_SMTPUTF8, RCPT_NOTIFY_NEVER}; - -use crate::smtp::{ - DnsCache, - inbound::{TestMessage, TestQueueEvent}, - session::{TestSession, VerifyResponse}, +use crate::{ + smtp::{ + inbound::{TestMessage, TestQueueEvent}, + session::{TestSession, VerifyResponse}, + }, + utils::{dns::DnsCache, server::TestServerBuilder}, }; - -const LOCAL: &str = r#" -[session.rcpt] -relay = true - -[session.extensions] -dsn = true -"#; - -const REMOTE: &str = r#" -[session.ehlo] -reject-non-fqdn = false - -[session.rcpt] -relay = true - -[session.data.limits] -size = 1500 - -[session.extensions] -dsn = true -requiretls = true - -[session.data.add-headers] -received = true -received-spf = true -auth-results = true -message-id = true -date = true -return-path = false -"#; +use mail_auth::MX; +use registry::schema::structs::{Expression, MtaStageData}; +use smtp_proto::{MAIL_REQUIRETLS, MAIL_RET_HDRS, MAIL_SMTPUTF8, RCPT_NOTIFY_NEVER}; +use std::time::{Duration, Instant}; #[tokio::test] #[serial_test::serial] async fn extensions() { - - + let mut local = TestServerBuilder::new("smtp_ext_local") + .await + .with_http_listener(19020) + .await + .disable_services() + .capture_queue() + .build() + .await; + let mut remote = TestServerBuilder::new("smtp_ext_remote") + .await + .with_http_listener(19021) + .await + .with_smtp_listener(9925) + .await + .disable_services() + .capture_queue() + .build() + .await; - // Start test server - let mut remote = TestSMTP::new("smtp_ext_remote", REMOTE).await; - let _rx = remote.start(&[ServerProtocol::Smtp]).await; + let local_admin = local.account("admin"); + local_admin.mta_allow_relaying().await; + local_admin.mta_no_auth().await; + local_admin.mta_all_extensions().await; + local_admin.reload_settings().await; + local.reload_core(); + local.expect_reload_settings().await; - // Successful delivery with DSN - let mut local = TestSMTP::new("smtp_ext_local", LOCAL).await; + let remote_admin = remote.account("admin"); + remote_admin.mta_all_extensions().await; + remote_admin.mta_allow_relaying().await; + remote_admin.mta_no_auth().await; + remote_admin + .registry_create_object(MtaStageData { + max_message_size: Expression { + else_: "1500".into(), + ..Default::default() + }, + add_date_header: Expression { + else_: "true".into(), + ..Default::default() + }, + add_message_id_header: Expression { + else_: "true".into(), + ..Default::default() + }, + add_received_header: Expression { + else_: "true".into(), + ..Default::default() + }, + add_received_spf_header: Expression { + else_: "true".into(), + ..Default::default() + }, + add_auth_results_header: Expression { + else_: "true".into(), + ..Default::default() + }, + add_return_path_header: Expression { + else_: "false".into(), + ..Default::default() + }, + enable_spam_filter: Expression { + else_: "false".into(), + ..Default::default() + }, + ..Default::default() + }) + .await; + remote_admin.reload_settings().await; + remote.reload_core(); + remote.expect_reload_settings().await; // Add mock DNS entries - let core = local.build_smtp(); - core.mx_add( + local.server.mx_add( "foobar.org", vec![MX { - exchanges: vec!["mx.foobar.org".to_string()], + exchanges: vec!["mx.foobar.org".into()].into_boxed_slice(), preference: 10, }], Instant::now() + Duration::from_secs(10), ); - core.ipv4_add( + local.server.ipv4_add( "mx.foobar.org", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(10), ); - let mut session = local.new_session(); + let mut session = local.new_mta_session(); session.data.remote_ip_str = "10.0.0.1".into(); session.eval_session_params().await; session.ehlo("mx.test.org").await; @@ -89,26 +119,23 @@ async fn extensions() { ) .await; local - .queue_receiver .expect_message_then_deliver() .await - .try_deliver(core.clone()); + .try_deliver(local.server.clone()); local - .queue_receiver .expect_message() .await - .read_lines(&local.queue_receiver) + .read_lines(&local) .await .assert_contains(" (delivered to") .assert_contains("Final-Recipient: rfc822;bill@foobar.org") .assert_contains("Action: delivered"); - local.queue_receiver.read_event().await.assert_done(); + local.read_event().await.assert_done(); remote - .queue_receiver .expect_message() .await - .read_lines(&remote.queue_receiver) + .read_lines(&remote) .await .assert_contains("using TLSv1.3 with cipher"); @@ -117,22 +144,20 @@ async fn extensions() { .send_message("john@test.org", &["bill@foobar.org"], "test:arc", "250") .await; local - .queue_receiver .expect_message_then_deliver() .await - .try_deliver(core.clone()); + .try_deliver(local.server.clone()); local - .queue_receiver .expect_message() .await - .read_lines(&local.queue_receiver) + .read_lines(&local) .await .assert_contains(" (host 'mx.foobar.org' rejected command 'MAIL FROM:") .assert_contains("Action: failed") .assert_contains("Diagnostic-Code: smtp;552") .assert_contains("Status: 5.3.4"); - local.queue_receiver.read_event().await.assert_done(); - remote.queue_receiver.assert_no_events(); + local.read_event().await.assert_done(); + remote.assert_no_events(); // Test DSN, SMTPUTF8 and REQUIRETLS extensions session @@ -144,12 +169,11 @@ async fn extensions() { ) .await; local - .queue_receiver .expect_message_then_deliver() .await - .try_deliver(core.clone()); - local.queue_receiver.read_event().await.assert_done(); - let message = remote.queue_receiver.expect_message().await; + .try_deliver(local.server.clone()); + local.read_event().await.assert_done(); + let message = remote.expect_message().await; assert_eq!(message.message.env_id, Some("abc123".into())); assert!((message.message.flags & MAIL_RET_HDRS) != 0); assert!((message.message.flags & MAIL_REQUIRETLS) != 0); diff --git a/tests/src/smtp/outbound/fallback_relay.rs b/tests/src/smtp/outbound/fallback_relay.rs index 5da05e39..fb5bd975 100644 --- a/tests/src/smtp/outbound/fallback_relay.rs +++ b/tests/src/smtp/outbound/fallback_relay.rs @@ -4,84 +4,115 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::time::{Duration, Instant}; - -use common::config::server::ServerProtocol; +use crate::{ + smtp::session::TestSession, + utils::{dns::DnsCache, server::TestServerBuilder}, +}; use mail_auth::MX; +use registry::{ + schema::{ + enums::MtaProtocol, + structs::{ + Expression, ExpressionMatch, MtaOutboundStrategy, MtaRoute, MtaRouteRelay, MtaStageRcpt, + }, + }, + types::list::List, +}; +use std::time::{Duration, Instant}; use store::write::now; -use crate::smtp::{DnsCache, session::TestSession}; - -const LOCAL: &str = r#" -[queue.strategy] -route = [{if = "retry_num > 0", then = "'fallback'"}, - {else = "'mx'"}] - -[session.rcpt] -relay = true -max-recipients = 100 - -[session.extensions] -dsn = true - -[queue.route.fallback] -type = "relay" -address = fallback.foobar.org -port = 9925 -protocol = 'smtp' -concurrency = 5 - -[queue.route.fallback.tls] -implicit = false -allow-invalid-certs = true - -"#; - -const REMOTE: &str = r#" -[session.rcpt] -relay = true - -[session.ehlo] -reject-non-fqdn = false - -[session.extensions] -dsn = true -chunking = false -"#; - #[tokio::test] #[serial_test::serial] async fn fallback_relay() { - - + let mut local = TestServerBuilder::new("smtp_fallback_local") + .await + .with_http_listener(19022) + .await + .disable_services() + .capture_queue() + .build() + .await; + let mut remote = TestServerBuilder::new("smtp_fallback_remote") + .await + .with_http_listener(19023) + .await + .with_smtp_listener(9925) + .await + .disable_services() + .capture_queue() + .build() + .await; - // Start test server - let mut remote = TestSMTP::new("smtp_fallback_remote", REMOTE).await; - let _rx = remote.start(&[ServerProtocol::Smtp]).await; - let mut local = TestSMTP::new("smtp_fallback_local", LOCAL).await; + let local_admin = local.account("admin"); + local_admin + .registry_create_object(MtaStageRcpt { + max_recipients: Expression { + else_: "100".into(), + + ..Default::default() + }, + allow_relaying: Expression { + else_: "true".into(), + + ..Default::default() + }, + ..Default::default() + }) + .await; + local_admin + .registry_create_object(MtaOutboundStrategy { + route: Expression { + match_: List::from_iter([ExpressionMatch { + if_: "retry_num > 0".into(), + then: "'fallback'".into(), + }]), + else_: "'mx'".into(), + }, + ..Default::default() + }) + .await; + local_admin + .registry_create_object(MtaRoute::Relay(MtaRouteRelay { + address: "fallback.foobar.org".into(), + implicit_tls: false, + allow_invalid_certs: true, + name: "fallback".into(), + port: 9925, + protocol: MtaProtocol::Smtp, + ..Default::default() + })) + .await; + local_admin.mta_no_auth().await; + local_admin.mta_all_extensions().await; + local_admin.reload_settings().await; + local.reload_core(); + local.expect_reload_settings().await; + + let remote_admin = remote.account("admin"); + remote_admin.mta_allow_relaying().await; + remote_admin.mta_no_auth().await; + remote_admin.mta_all_extensions().await; + remote_admin.mta_allow_non_fqdn().await; + remote_admin.reload_settings().await; + remote.reload_core(); + remote.expect_reload_settings().await; // Add mock DNS entries - let core = local.build_smtp(); - core.mx_add( + local.server.mx_add( "foobar.org", vec![MX { - exchanges: vec!["_dns_error.foobar.org".to_string()], + exchanges: vec!["_dns_error.foobar.org".into()].into_boxed_slice(), preference: 10, }], Instant::now() + Duration::from_secs(10), ); - /*core.ipv4_add( - "unreachable.foobar.org", - vec!["127.0.0.2".parse().unwrap()], - Instant::now() + Duration::from_secs(10), - );*/ - core.ipv4_add( + local.server.ipv4_add( "fallback.foobar.org", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(10), ); - let mut session = local.new_session(); + let mut session = local.new_mta_session(); session.data.remote_ip_str = "10.0.0.1".into(); session.eval_session_params().await; session.ehlo("mx.test.org").await; @@ -89,21 +120,19 @@ async fn fallback_relay() { .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; local - .queue_receiver .expect_message_then_deliver() .await - .try_deliver(core.clone()); - let mut retry = local.queue_receiver.expect_message().await; + .try_deliver(local.server.clone()); + let mut retry = local.expect_message().await; let prev_due = retry.message.recipients[0].retry.due; let next_due = now(); let queue_id = retry.queue_id; retry.message.recipients[0].retry.due = next_due; - retry.save_changes(&core, prev_due.into()).await; + retry.save_changes(&local.server, prev_due.into()).await; local - .queue_receiver .delivery_attempt(queue_id) .await - .try_deliver(core.clone()); + .try_deliver(local.server.clone()); tokio::time::sleep(Duration::from_millis(100)).await; - remote.queue_receiver.expect_message().await; + remote.expect_message().await; } diff --git a/tests/src/smtp/outbound/ip_lookup.rs b/tests/src/smtp/outbound/ip_lookup.rs index 1f813dde..4beaf26f 100644 --- a/tests/src/smtp/outbound/ip_lookup.rs +++ b/tests/src/smtp/outbound/ip_lookup.rs @@ -4,67 +4,103 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::{ + smtp::{inbound::TestQueueEvent, session::TestSession}, + utils::{dns::DnsCache, server::TestServerBuilder}, +}; +use mail_auth::MX; +use registry::{ + schema::{ + enums::MtaIpStrategy, + prelude::{ObjectType, Property}, + structs::MtaRoute, + }, + types::EnumImpl, +}; +use serde_json::json; use std::time::{Duration, Instant}; -use common::config::server::ServerProtocol; -use mail_auth::{IpLookupStrategy, MX}; - -use crate::smtp::{DnsCache, session::TestSession}; - -const LOCAL: &str = r#" -[session.rcpt] -relay = true - -[queue.route.mx] -ip-lookup = "ipv6_then_ipv4" -"#; - -const REMOTE: &str = r#" -[session.ehlo] -reject-non-fqdn = false - -[session.rcpt] -relay = true -"#; - #[tokio::test] #[serial_test::serial] async fn ip_lookup_strategy() { - - + let mut local = TestServerBuilder::new("smtp_iplookup_local") + .await + .with_http_listener(19024) + .await + .disable_services() + .capture_queue() + .build() + .await; + let mut remote = TestServerBuilder::new("smtp_iplookup_remote") + .await + .with_http_listener(19025) + .await + .with_smtp_listener(9925) + .await + .disable_services() + .capture_queue() + .build() + .await; - // Start test server - let mut remote = TestSMTP::new("smtp_iplookup_remote", REMOTE).await; - let _rx = remote.start(&[ServerProtocol::Smtp]).await; + let local_admin = local.account("admin"); + local_admin.mta_allow_relaying().await; + local_admin.mta_no_auth().await; + local_admin.reload_settings().await; + let (mx_route_id, _) = local_admin + .registry_get_all::() + .await + .into_iter() + .find(|(_, s)| matches!(s, MtaRoute::Mx(_))) + .unwrap(); + local.reload_core(); + local.expect_reload_settings().await; - for strategy in [IpLookupStrategy::Ipv6Only, IpLookupStrategy::Ipv6thenIpv4] { - //println!("-> Strategy: {:?}", strategy); + let remote_admin = remote.account("admin"); + remote_admin.mta_allow_relaying().await; + remote_admin.mta_no_auth().await; + remote_admin.reload_settings().await; + remote.reload_core(); + remote.expect_reload_settings().await; + + for strategy in [MtaIpStrategy::V6Only, MtaIpStrategy::V6ThenV4] { + local + .account("admin") + .registry_update_object( + ObjectType::MtaRoute, + mx_route_id, + json!({ + Property::IpLookupStrategy: strategy.as_str(), + }), + ) + .await; + local.account("admin").reload_settings().await; + local.expect_reload_settings().await; + + println!("-> Strategy: {:?}", strategy); // Add mock DNS entries - let mut local = TestSMTP::new("smtp_iplookup_local", LOCAL).await; - let core = local.build_smtp(); - core.mx_add( + local.server.mx_add( "foobar.org", vec![MX { - exchanges: vec!["mx.foobar.org".to_string()], + exchanges: vec!["mx.foobar.org".into()].into_boxed_slice(), preference: 10, }], Instant::now() + Duration::from_secs(10), ); - if matches!(strategy, IpLookupStrategy::Ipv6thenIpv4) { - core.ipv4_add( + if matches!(strategy, MtaIpStrategy::V6ThenV4) { + local.server.ipv4_add( "mx.foobar.org", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(10), ); } - core.ipv6_add( + local.server.ipv6_add( "mx.foobar.org", vec!["::1".parse().unwrap()], Instant::now() + Duration::from_secs(10), ); // Retry on failed STARTTLS - let mut session = local.new_session(); + let mut session = local.new_mta_session(); session.data.remote_ip_str = "10.0.0.1".into(); session.eval_session_params().await; session.ehlo("mx.test.org").await; @@ -72,21 +108,21 @@ async fn ip_lookup_strategy() { .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; local - .queue_receiver .expect_message_then_deliver() .await - .try_deliver(core.clone()); + .try_deliver(local.server.clone()); tokio::time::sleep(Duration::from_millis(100)).await; - if matches!(strategy, IpLookupStrategy::Ipv6thenIpv4) { - remote.queue_receiver.expect_message().await; + if matches!(strategy, MtaIpStrategy::V6ThenV4) { + remote.expect_message().await; } else { - let message = local.queue_receiver.last_queued_message().await; + let message = local.last_queued_message().await; let status = message.message.recipients[0].status.to_string(); assert!( status.contains("Connection refused"), "Message: {:?}", message ); + local.read_event().await.assert_refresh(); } } } diff --git a/tests/src/smtp/outbound/lmtp.rs b/tests/src/smtp/outbound/lmtp.rs index 5cd92996..e56e5ac5 100644 --- a/tests/src/smtp/outbound/lmtp.rs +++ b/tests/src/smtp/outbound/lmtp.rs @@ -4,96 +4,184 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::time::{Duration, Instant}; - -use crate::smtp::{ - DnsCache, - inbound::TestMessage, - queue::QueuedEvents, - session::{TestSession, VerifyResponse}, +use crate::{ + smtp::{ + inbound::TestMessage, + session::{TestSession, VerifyResponse}, + }, + utils::{dns::DnsCache, server::TestServerBuilder}, }; -use common::{ - config::{server::ServerProtocol, smtp::queue::QueueName}, - ipc::QueueEvent, +use common::{config::smtp::queue::QueueName, ipc::QueueEvent}; +use registry::{ + schema::{ + enums::{MtaProtocol, NetworkListenerProtocol}, + structs::{ + Expression, ExpressionMatch, MtaConnectionStrategy, MtaDeliveryExpiration, + MtaDeliveryExpirationTtl, MtaDeliverySchedule, MtaDeliveryScheduleInterval, + MtaDeliveryScheduleIntervals, MtaDeliveryScheduleIntervalsOrDefault, + MtaOutboundStrategy, MtaRoute, MtaRouteRelay, MtaStageRcpt, MtaVirtualQueue, + }, + }, + types::list::List, }; use smtp::queue::spool::{QUEUE_REFRESH, SmtpSpool}; +use std::time::{Duration, Instant}; use store::write::now; -const REMOTE: &str = " -[session.ehlo] -reject-non-fqdn = false - -[session.rcpt] -relay = true - -[session.extensions] -dsn = true -"; - -const LOCAL: &str = r#" -[queue.strategy] -route = [{if = "rcpt_domain = 'foobar.org'", then = "'lmtp'"}, - {else = "'mx'"}] -schedule = [{if = "rcpt_domain = 'foobar.org'", then = "'foobar'"}, - {else = "'default'"}] - -[session.rcpt] -relay = true -max-recipients = 100 - -[session.extensions] -dsn = true - -[queue.schedule.default] -retry = "1s" -notify = "1s" -expire = "5s" -queue-name = "default" - -[queue.schedule.foobar] -retry = "1s" -notify = ["1s", "2s"] -expire = "4s" -queue-name = "default" - -[queue.connection.default.timeout] -connect = "1s" -data = "50ms" - -[queue.route.lmtp] -type = "relay" -address = lmtp.foobar.org -port = 9924 -protocol = 'lmtp' -concurrency = 5 - -[queue.route.lmtp.tls] -implicit = true -allow-invalid-certs = true -"#; - #[tokio::test] #[serial_test::serial] async fn lmtp_delivery() { - - + let mut local = TestServerBuilder::new("lmtp_delivery_local") + .await + .with_http_listener(19026) + .await + .disable_services() + .capture_queue() + .build() + .await; + let mut remote = TestServerBuilder::new("lmtp_delivery_remote") + .await + .with_http_listener(19027) + .await + .with_listener(NetworkListenerProtocol::Lmtp, "lmtp-debug", 9924, true) + .await + .disable_services() + .capture_queue() + .build() + .await; - // Start test server - let mut remote = TestSMTP::new("lmtp_delivery_remote", REMOTE).await; - let _rx = remote.start(&[ServerProtocol::Lmtp]).await; + let local_admin = local.account("admin"); + local_admin + .registry_create_object(MtaStageRcpt { + max_recipients: Expression { + else_: "100".into(), - // Multiple delivery attempts - let mut local = TestSMTP::new("lmtp_delivery_local", LOCAL).await; + ..Default::default() + }, + allow_relaying: Expression { + else_: "true".into(), + + ..Default::default() + }, + ..Default::default() + }) + .await; + local_admin + .registry_create_object(MtaOutboundStrategy { + route: Expression { + match_: List::from_iter([ExpressionMatch { + if_: "rcpt_domain = 'foobar.org'".into(), + then: "'lmtp'".into(), + }]), + else_: "'mx'".into(), + }, + schedule: Expression { + match_: List::from_iter([ExpressionMatch { + if_: "rcpt_domain = 'foobar.org'".into(), + then: "'foobar'".into(), + }]), + else_: "'default'".into(), + }, + connection: Expression { + else_: "'impatient'".into(), + ..Default::default() + }, + ..Default::default() + }) + .await; + local_admin + .registry_create_object(MtaRoute::Relay(MtaRouteRelay { + address: "lmtp.foobar.org".into(), + allow_invalid_certs: true, + implicit_tls: true, + name: "lmtp".into(), + port: 9924, + protocol: MtaProtocol::Lmtp, + ..Default::default() + })) + .await; + let queue_id = local_admin + .registry_create_object(MtaVirtualQueue { + name: "default".into(), + threads_per_node: 25, + description: None, + }) + .await; + local_admin + .registry_create_object(MtaDeliverySchedule { + name: "foobar".into(), + retry: MtaDeliveryScheduleIntervalsOrDefault::Custom(MtaDeliveryScheduleIntervals { + intervals: List::from_iter([MtaDeliveryScheduleInterval { + duration: 1_000u64.into(), + }]), + }), + notify: MtaDeliveryScheduleIntervalsOrDefault::Custom(MtaDeliveryScheduleIntervals { + intervals: List::from_iter([ + MtaDeliveryScheduleInterval { + duration: 1_000u64.into(), + }, + MtaDeliveryScheduleInterval { + duration: 2_000u64.into(), + }, + ]), + }), + expiry: MtaDeliveryExpiration::Ttl(MtaDeliveryExpirationTtl { + expire: 4_000u64.into(), + }), + queue_id, + description: None, + }) + .await; + local_admin + .registry_create_object(MtaDeliverySchedule { + name: "default".into(), + retry: MtaDeliveryScheduleIntervalsOrDefault::Custom(MtaDeliveryScheduleIntervals { + intervals: List::from_iter([MtaDeliveryScheduleInterval { + duration: 1_000u64.into(), + }]), + }), + notify: MtaDeliveryScheduleIntervalsOrDefault::Custom(MtaDeliveryScheduleIntervals { + intervals: List::from_iter([MtaDeliveryScheduleInterval { + duration: 1_000u64.into(), + }]), + }), + expiry: MtaDeliveryExpiration::Ttl(MtaDeliveryExpirationTtl { + expire: 5_000u64.into(), + }), + queue_id, + description: None, + }) + .await; + local_admin + .registry_create_object(MtaConnectionStrategy { + name: "impatient".into(), + connect_timeout: 1_000u64.into(), + data_timeout: 50u64.into(), + ..Default::default() + }) + .await; + local_admin.mta_no_auth().await; + local_admin.mta_all_extensions().await; + local_admin.reload_settings().await; + local.reload_core(); + local.expect_reload_settings().await; + + let remote_admin = remote.account("admin"); + remote_admin.mta_allow_relaying().await; + remote_admin.mta_no_auth().await; + remote_admin.mta_all_extensions().await; + remote_admin.reload_settings().await; + remote.reload_core(); + remote.expect_reload_settings().await; // Add mock DNS entries - let core = local.build_smtp(); - core.ipv4_add( + local.server.ipv4_add( "lmtp.foobar.org", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(10), ); - let mut session = local.new_session(); + let mut session = local.new_mta_session(); session.data.remote_ip_str = "10.0.0.1".into(); session.eval_session_params().await; session.ehlo("mx.test.org").await; @@ -113,50 +201,53 @@ async fn lmtp_delivery() { ) .await; local - .queue_receiver - .expect_message_then_deliver() + .expect_message_for_queue_then_deliver("default") .await - .try_deliver(core.clone()); + .try_deliver(local.server.clone()); let mut dsn = Vec::new(); loop { - match local.queue_receiver.try_read_event().await { + match local.try_read_event().await { Some(QueueEvent::Refresh | QueueEvent::WorkerDone { .. }) => {} Some(QueueEvent::Paused(_)) | Some(QueueEvent::ReloadSettings) => unreachable!(), None | Some(QueueEvent::Stop) => break, } - let mut events = core.all_queued_messages().await; + let mut events = local.all_queued_messages().await; if events.messages.is_empty() { let now = now(); if events.next_refresh < now + QUEUE_REFRESH { tokio::time::sleep(Duration::from_secs(events.next_refresh - now)).await; - events = core.all_queued_messages().await; + events = local.all_queued_messages().await; } else { break; } } for event in events.messages { - let message = core + let message = local + .server .read_message(event.queue_id, QueueName::default()) .await .unwrap(); if message.message.return_path.is_empty() { - message.clone().remove(&core, event.due.into()).await; + message + .clone() + .remove(&local.server, event.due.into()) + .await; dsn.push(message); } else { - event.try_deliver(core.clone()); + event.try_deliver(local.server.clone()); tokio::time::sleep(Duration::from_millis(100)).await; } } } - local.queue_receiver.assert_queue_is_empty().await; + local.assert_queue_is_empty().await; assert_eq!(dsn.len(), 4); let mut dsn = dsn.into_iter(); dsn.next() .unwrap() - .read_lines(&local.queue_receiver) + .read_lines(&local) .await .assert_contains(" (delivered to") .assert_contains(" (delivered to") @@ -166,28 +257,27 @@ async fn lmtp_delivery() { dsn.next() .unwrap() - .read_lines(&local.queue_receiver) + .read_lines(&local) .await .assert_contains(" (host 'lmtp.foobar.org' rejected") .assert_contains("Action: delayed"); dsn.next() .unwrap() - .read_lines(&local.queue_receiver) + .read_lines(&local) .await .assert_contains(" (host 'lmtp.foobar.org' rejected") .assert_contains("Action: delayed"); dsn.next() .unwrap() - .read_lines(&local.queue_receiver) + .read_lines(&local) .await .assert_contains(" (host 'lmtp.foobar.org' rejected") .assert_contains("Action: failed"); assert_eq!( remote - .queue_receiver .expect_message() .await .message @@ -201,5 +291,5 @@ async fn lmtp_delivery() { "john@foobar.org".to_string() ] ); - remote.queue_receiver.assert_no_events(); + remote.assert_no_events(); } diff --git a/tests/src/smtp/outbound/mta_sts.rs b/tests/src/smtp/outbound/mta_sts.rs index b4a3fd87..455e0865 100644 --- a/tests/src/smtp/outbound/mta_sts.rs +++ b/tests/src/smtp/outbound/mta_sts.rs @@ -4,94 +4,118 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{ - sync::Arc, - time::{Duration, Instant}, -}; - -use common::{ - config::{server::ServerProtocol, smtp::resolver::Policy}, - ipc::PolicyType, +use crate::{ + smtp::{ + inbound::{TestMessage, TestQueueEvent, TestReportingEvent}, + session::{TestSession, VerifyResponse}, + }, + utils::{dns::DnsCache, server::TestServerBuilder}, }; +use common::{config::smtp::resolver::Policy, ipc::PolicyType}; use mail_auth::{ MX, common::parse::TxtRecordParser, mta_sts::{MtaSts, ReportUri, TlsRpt}, report::tlsrpt::ResultType, }; - -use crate::smtp::{ - DnsCache, - inbound::{TestMessage, TestQueueEvent, TestReportingEvent}, - session::{TestSession, VerifyResponse}, +use registry::schema::{ + enums::MtaRequiredOrOptional, + prelude::ObjectType, + structs::{Expression, MtaTlsStrategy, TlsReportSettings}, }; use smtp::outbound::mta_sts::{lookup::STS_TEST_POLICY, parse::ParsePolicy}; - -const LOCAL: &str = r#" -[session.rcpt] -relay = true - -[queue.tls.default] -mta-sts = "require" -allow-invalid-certs = false - -[report.tls.aggregate] -send = "weekly" - -"#; - -const REMOTE: &str = r#" -[session.ehlo] -reject-non-fqdn = false - -[session.rcpt] -relay = true - -[session.data.add-headers] -received = true -received-spf = true -auth-results = true -message-id = true -date = true -return-path = false - -"#; +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; #[tokio::test] #[serial_test::serial] async fn mta_sts_verify() { - - + let mut local = TestServerBuilder::new("smtp_mta_sts_local") + .await + .with_http_listener(19028) + .await + .disable_services() + .capture_queue() + .capture_reporting() + .build() + .await; + let mut remote = TestServerBuilder::new("smtp_mta_sts_remote") + .await + .with_http_listener(19029) + .await + .with_smtp_listener(9925) + .await + .with_dummy_tls_cert() + .await + .disable_services() + .capture_queue() + .build() + .await; - // Start test server - let mut remote = TestSMTP::new("smtp_mta_sts_remote", REMOTE).await; - let _rx = remote.start(&[ServerProtocol::Smtp]).await; + let local_admin = local.account("admin"); + local_admin.mta_allow_relaying().await; + local_admin.mta_no_auth().await; + local_admin + .registry_create_object(TlsReportSettings { + send_frequency: Expression { + else_: "weekly".into(), + ..Default::default() + }, + ..Default::default() + }) + .await; + let (tls_strategy_id, mut tls_strategy) = local_admin + .registry_get_all::() + .await + .into_iter() + .find(|(_, s)| s.name == "default") + .unwrap(); + tls_strategy.mta_sts = MtaRequiredOrOptional::Require; + tls_strategy.allow_invalid_certs = false; + let mut tls_strategy = serde_json::to_value(tls_strategy).unwrap(); + tls_strategy + .as_object_mut() + .unwrap() + .retain(|k, _| k != "name"); + local_admin + .registry_update_object(ObjectType::MtaTlsStrategy, tls_strategy_id, tls_strategy) + .await; + local_admin.reload_settings().await; + local.reload_core(); + local.expect_reload_settings().await; - // Fail on missing MTA-STS record - let mut local = TestSMTP::new("smtp_mta_sts_local", LOCAL).await; + let remote_admin = remote.account("admin"); + remote_admin.mta_no_auth().await; + remote_admin.mta_allow_relaying().await; + remote_admin.mta_allow_non_fqdn().await; + remote_admin.mta_add_all_headers().await; + remote_admin.reload_settings().await; + remote.reload_core(); + remote.expect_reload_settings().await; // Add mock DNS entries - let core = local.build_smtp(); - core.mx_add( + local.server.mx_add( "foobar.org", vec![MX { - exchanges: vec!["mx.foobar.org".to_string()], + exchanges: vec!["mx.foobar.org".into()].into_boxed_slice(), preference: 10, }], Instant::now() + Duration::from_secs(10), ); - core.ipv4_add( + local.server.ipv4_add( "mx.foobar.org", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(10), ); - core.txt_add( + local.server.txt_add( "_smtp._tls.foobar.org", TlsRpt::parse(b"v=TLSRPTv1; rua=mailto:reports@foobar.org").unwrap(), Instant::now() + Duration::from_secs(10), ); - let mut session = local.new_session(); + let mut session = local.new_mta_session(); session.data.remote_ip_str = "10.0.0.1".into(); session.eval_session_params().await; session.ehlo("mx.test.org").await; @@ -99,23 +123,21 @@ async fn mta_sts_verify() { .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; local - .queue_receiver .expect_message_then_deliver() .await - .try_deliver(core.clone()); + .try_deliver(local.server.clone()); local - .queue_receiver .expect_message() .await - .read_lines(&local.queue_receiver) + .read_lines(&local) .await .assert_contains(" (MTA-STS failed to authenticate") .assert_contains("Record not f=") .assert_contains("ound"); - local.queue_receiver.read_event().await.assert_done(); + local.read_event().await.assert_done(); // Expect TLS failure report - let report = local.report_receiver.read_report().await.unwrap_tls(); + let report = local.read_report().await.unwrap_tls(); assert_eq!(report.domain, "foobar.org"); assert_eq!(report.policy, PolicyType::Sts(None)); assert_eq!( @@ -128,7 +150,7 @@ async fn mta_sts_verify() { ); // MTA-STS policy fetch failure - core.txt_add( + local.server.txt_add( "_mta-sts.foobar.org", MtaSts::parse(b"v=STSv1; id=policy_will_fail;").unwrap(), Instant::now() + Duration::from_secs(10), @@ -137,22 +159,20 @@ async fn mta_sts_verify() { .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; local - .queue_receiver .expect_message_then_deliver() .await - .try_deliver(core.clone()); + .try_deliver(local.server.clone()); local - .queue_receiver .expect_message() .await - .read_lines(&local.queue_receiver) + .read_lines(&local) .await .assert_contains(" (MTA-STS failed to authenticate") .assert_contains("No 'mx' entries found"); - local.queue_receiver.read_event().await.assert_done(); + local.read_event().await.assert_done(); // Expect TLS failure report - let report = local.report_receiver.read_report().await.unwrap_tls(); + let report = local.read_report().await.unwrap_tls(); assert_eq!(report.policy, PolicyType::Sts(None)); assert_eq!( report.failure.as_ref().unwrap().result_type, @@ -171,22 +191,20 @@ async fn mta_sts_verify() { .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; local - .queue_receiver .expect_message_then_deliver() .await - .try_deliver(core.clone()); + .try_deliver(local.server.clone()); local - .queue_receiver .expect_message() .await - .read_lines(&local.queue_receiver) + .read_lines(&local) .await .assert_contains(" (MTA-STS failed to authenticate") .assert_contains("not authorized by policy"); - local.queue_receiver.read_event().await.assert_done(); + local.read_event().await.assert_done(); // Expect TLS failure report - let report = local.report_receiver.read_report().await.unwrap_tls(); + let report = local.read_report().await.unwrap_tls(); assert_eq!( report.policy, PolicyType::Sts( @@ -201,10 +219,10 @@ async fn mta_sts_verify() { report.failure.as_ref().unwrap().result_type, ResultType::ValidationFailure ); - remote.queue_receiver.assert_no_events(); + remote.assert_no_events(); // MTA-STS successful validation - core.txt_add( + local.server.txt_add( "_mta-sts.foobar.org", MtaSts::parse(b"v=STSv1; id=policy_will_work;").unwrap(), Instant::now() + Duration::from_secs(10), @@ -221,21 +239,19 @@ async fn mta_sts_verify() { .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; local - .queue_receiver .expect_message_then_deliver() .await - .try_deliver(core.clone()); - local.queue_receiver.read_event().await.assert_done(); + .try_deliver(local.server.clone()); + local.read_event().await.assert_done(); remote - .queue_receiver .expect_message() .await - .read_lines(&remote.queue_receiver) + .read_lines(&remote) .await .assert_contains("using TLSv1.3 with cipher"); // Expect TLS success report - let report = local.report_receiver.read_report().await.unwrap_tls(); + let report = local.read_report().await.unwrap_tls(); assert_eq!( report.policy, PolicyType::Sts( diff --git a/tests/src/smtp/outbound/smtp.rs b/tests/src/smtp/outbound/smtp.rs index 12eaa51d..4d384577 100644 --- a/tests/src/smtp/outbound/smtp.rs +++ b/tests/src/smtp/outbound/smtp.rs @@ -4,75 +4,30 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::time::{Duration, Instant}; - -use common::{ - config::{server::ServerProtocol, smtp::queue::QueueName}, - ipc::QueueEvent, +use crate::{ + smtp::{ + inbound::{TestMessage, TestQueueEvent}, + session::{TestSession, VerifyResponse}, + }, + utils::{dns::DnsCache, server::TestServerBuilder}, }; +use common::{config::smtp::queue::QueueName, ipc::QueueEvent}; use mail_auth::MX; -use store::write::now; - -use crate::smtp::{ - DnsCache, - inbound::{TestMessage, TestQueueEvent}, - queue::QueuedEvents, - session::{TestSession, VerifyResponse}, +use registry::{ + schema::{ + enums::NetworkListenerProtocol, + structs::{ + Expression, ExpressionMatch, MtaDeliveryExpiration, MtaDeliveryExpirationTtl, + MtaDeliverySchedule, MtaDeliveryScheduleInterval, MtaDeliveryScheduleIntervals, + MtaDeliveryScheduleIntervalsOrDefault, MtaExtensions, MtaOutboundStrategy, + MtaStageRcpt, MtaVirtualQueue, + }, + }, + types::list::List, }; use smtp::queue::spool::{QUEUE_REFRESH, SmtpSpool}; - -const LOCAL: &str = r#" -[session.rcpt] -relay = true -max-recipients = 100 - -[session.extensions] -dsn = true - -[queue.schedule.default] -retry = "1s" -notify = "1s" -expire = "7s" -queue-name = "default" - -[queue.schedule.foobar-org] -retry = "1s" -notify = ["1s", "2s"] -expire = "6s" -queue-name = "default" - -[queue.schedule.foobar-com] -retry = "1s" -notify = ["5s", "6s"] -expire = "7s" -queue-name = "default" - - -[queue.strategy] -schedule = [{if = "rcpt_domain == 'foobar.org'", then = "'foobar-org'"}, - {if = "rcpt_domain == 'foobar.com'", then = "'foobar-com'"}, - {else = "'default'"}] - -[spam-filter] -enable = false - -"#; - -const REMOTE: &str = r#" -[session.ehlo] -reject-non-fqdn = false - -[session.rcpt] -relay = true - -[session.extensions] -dsn = true -chunking = false - -[spam-filter] -enable = false - -"#; +use std::time::{Duration, Instant}; +use store::write::now; const SMUGGLER: &str = r#"From: Joe SixPack To: Suzie Q @@ -98,41 +53,192 @@ This is a smuggled message #[tokio::test] #[serial_test::serial] async fn smtp_delivery() { - - + let mut local = TestServerBuilder::new("smtp_delivery_local") + .await + .with_http_listener(19030) + .await + .disable_services() + .capture_queue() + .build() + .await; + let mut remote = TestServerBuilder::new("smtp_delivery_remote") + .await + .with_http_listener(19031) + .await + .with_listener(NetworkListenerProtocol::Smtp, "smtp-debug", 9925, false) + .await + .disable_services() + .capture_queue() + .build() + .await; - // Start test server - let mut remote = TestSMTP::new("smtp_delivery_remote", REMOTE).await; - let _rx = remote.start(&[ServerProtocol::Smtp]).await; - let remote_core = remote.build_smtp(); + let local_admin = local.account("admin"); + local_admin + .registry_create_object(MtaStageRcpt { + max_recipients: Expression { + else_: "100".into(), - // Multiple delivery attempts - let mut local = TestSMTP::new("smtp_delivery_local", LOCAL).await; + ..Default::default() + }, + allow_relaying: Expression { + else_: "true".into(), + + ..Default::default() + }, + ..Default::default() + }) + .await; + local_admin + .registry_create_object(MtaOutboundStrategy { + schedule: Expression { + match_: List::from_iter([ + ExpressionMatch { + if_: "rcpt_domain == 'foobar.org'".into(), + then: "'foobar-org'".into(), + }, + ExpressionMatch { + if_: "rcpt_domain == 'foobar.com'".into(), + then: "'foobar-com'".into(), + }, + ]), + else_: "'default'".into(), + }, + ..Default::default() + }) + .await; + let queue_id = local_admin + .registry_create_object(MtaVirtualQueue { + name: "default".into(), + threads_per_node: 25, + description: None, + }) + .await; + local_admin + .registry_create_object(MtaDeliverySchedule { + name: "default".into(), + retry: MtaDeliveryScheduleIntervalsOrDefault::Custom(MtaDeliveryScheduleIntervals { + intervals: List::from_iter([MtaDeliveryScheduleInterval { + duration: 1_000u64.into(), + }]), + }), + notify: MtaDeliveryScheduleIntervalsOrDefault::Custom(MtaDeliveryScheduleIntervals { + intervals: List::from_iter([MtaDeliveryScheduleInterval { + duration: 1_000u64.into(), + }]), + }), + expiry: MtaDeliveryExpiration::Ttl(MtaDeliveryExpirationTtl { + expire: 7_000u64.into(), + }), + queue_id, + description: None, + }) + .await; + local_admin + .registry_create_object(MtaDeliverySchedule { + name: "foobar-org".into(), + retry: MtaDeliveryScheduleIntervalsOrDefault::Custom(MtaDeliveryScheduleIntervals { + intervals: List::from_iter([MtaDeliveryScheduleInterval { + duration: 1_000u64.into(), + }]), + }), + notify: MtaDeliveryScheduleIntervalsOrDefault::Custom(MtaDeliveryScheduleIntervals { + intervals: List::from_iter([ + MtaDeliveryScheduleInterval { + duration: 1_000u64.into(), + }, + MtaDeliveryScheduleInterval { + duration: 2_000u64.into(), + }, + ]), + }), + expiry: MtaDeliveryExpiration::Ttl(MtaDeliveryExpirationTtl { + expire: 6_000u64.into(), + }), + queue_id, + description: None, + }) + .await; + local_admin + .registry_create_object(MtaDeliverySchedule { + name: "foobar-com".into(), + retry: MtaDeliveryScheduleIntervalsOrDefault::Custom(MtaDeliveryScheduleIntervals { + intervals: List::from_iter([MtaDeliveryScheduleInterval { + duration: 1_000u64.into(), + }]), + }), + notify: MtaDeliveryScheduleIntervalsOrDefault::Custom(MtaDeliveryScheduleIntervals { + intervals: List::from_iter([ + MtaDeliveryScheduleInterval { + duration: 5_000u64.into(), + }, + MtaDeliveryScheduleInterval { + duration: 6_000u64.into(), + }, + ]), + }), + expiry: MtaDeliveryExpiration::Ttl(MtaDeliveryExpirationTtl { + expire: 7_000u64.into(), + }), + queue_id, + description: None, + }) + .await; + local_admin.mta_no_auth().await; + local_admin.mta_all_extensions().await; + local_admin.mta_disable_spam_filter().await; + local_admin.reload_settings().await; + local.reload_core(); + local.expect_reload_settings().await; + + let remote_admin = remote.account("admin"); + remote_admin.mta_allow_relaying().await; + remote_admin.mta_no_auth().await; + remote_admin.mta_disable_spam_filter().await; + remote_admin.mta_allow_non_fqdn().await; + remote_admin + .registry_create_object(MtaExtensions { + chunking: Expression { + else_: "false".into(), + ..Default::default() + }, + dsn: Expression { + else_: "true".into(), + ..Default::default() + }, + ..Default::default() + }) + .await; + remote_admin.reload_settings().await; + remote.reload_core(); + remote.expect_reload_settings().await; // Add mock DNS entries - let core = local.build_smtp(); for domain in ["foobar.org", "foobar.net", "foobar.com"] { - core.mx_add( + local.server.mx_add( domain, vec![MX { - exchanges: vec![format!("mx1.{domain}"), format!("mx2.{domain}")], + exchanges: vec![ + format!("mx1.{domain}").into(), + format!("mx2.{domain}").into(), + ] + .into_boxed_slice(), preference: 10, }], Instant::now() + Duration::from_secs(10), ); - core.ipv4_add( + local.server.ipv4_add( format!("mx1.{domain}"), vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(30), ); - core.ipv4_add( + local.server.ipv4_add( format!("mx2.{domain}"), vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(30), ); } - let mut session = local.new_session(); + let mut session = local.new_mta_session(); session.data.remote_ip_str = "10.0.0.1".into(); session.eval_session_params().await; session.ehlo("mx.test.org").await; @@ -152,18 +258,17 @@ async fn smtp_delivery() { "250", ) .await; - let message = local.queue_receiver.expect_message().await; + let message = local.expect_message().await; let num_recipients = message.message.recipients.len(); assert_eq!(num_recipients, 7); local - .queue_receiver - .delivery_attempt(message.queue_id) + .delivery_attempt_for_queue(message.queue_id, "default") .await - .try_deliver(core.clone()); + .try_deliver(local.server.clone()); let mut dsn = Vec::new(); let mut rcpt_retries = vec![0; num_recipients]; loop { - match local.queue_receiver.try_read_event().await { + match local.try_read_event().await { Some(QueueEvent::Refresh | QueueEvent::WorkerDone { .. }) => {} Some(QueueEvent::Paused(_)) | Some(QueueEvent::ReloadSettings) => unreachable!(), None | Some(QueueEvent::Stop) => { @@ -171,29 +276,33 @@ async fn smtp_delivery() { } } - let mut events = core.all_queued_messages().await; + let mut events = local.all_queued_messages().await; if events.messages.is_empty() { let now = now(); if events.next_refresh < now + QUEUE_REFRESH { tokio::time::sleep(Duration::from_secs(events.next_refresh - now)).await; - events = core.all_queued_messages().await; + events = local.all_queued_messages().await; } else { break; } } for event in events.messages { - let message = core + let message = local + .server .read_message(event.queue_id, QueueName::default()) .await .unwrap(); if message.message.return_path.is_empty() { - message.clone().remove(&core, event.due.into()).await; + message + .clone() + .remove(&local.server, event.due.into()) + .await; dsn.push(message); } else { for (idx, rcpt) in message.message.recipients.iter().enumerate() { rcpt_retries[idx] = rcpt.retry.inner; } - event.try_deliver(core.clone()); + event.try_deliver(local.server.clone()); tokio::time::sleep(Duration::from_millis(100)).await; } } @@ -210,14 +319,14 @@ async fn smtp_delivery() { "retries {rcpt_retries:?}" ); - local.queue_receiver.assert_queue_is_empty().await; + local.assert_queue_is_empty().await; assert_eq!(dsn.len(), 5); let mut dsn = dsn.into_iter(); dsn.next() .unwrap() - .read_lines(&local.queue_receiver) + .read_lines(&local) .await .assert_contains(" (delivered to") .assert_contains(" (delivered to") @@ -227,7 +336,7 @@ async fn smtp_delivery() { dsn.next() .unwrap() - .read_lines(&local.queue_receiver) + .read_lines(&local) .await .assert_contains(" (host ") .assert_contains(" (host ") @@ -235,27 +344,26 @@ async fn smtp_delivery() { dsn.next() .unwrap() - .read_lines(&local.queue_receiver) + .read_lines(&local) .await .assert_contains(" (host ") .assert_contains("Action: delayed"); dsn.next() .unwrap() - .read_lines(&local.queue_receiver) + .read_lines(&local) .await .assert_contains(" (host "); dsn.next() .unwrap() - .read_lines(&local.queue_receiver) + .read_lines(&local) .await .assert_contains(" (host ") .assert_contains("Action: failed"); let mut recipients = remote - .queue_receiver - .consume_message(&remote_core) + .consume_message() .await .message .recipients @@ -264,8 +372,7 @@ async fn smtp_delivery() { .collect::>(); recipients.extend( remote - .queue_receiver - .consume_message(&remote_core) + .consume_message() .await .message .recipients @@ -278,7 +385,7 @@ async fn smtp_delivery() { vec!["ok@foobar.net".to_string(), "ok@foobar.org".to_string()] ); - remote.queue_receiver.assert_no_events(); + remote.assert_no_events(); // SMTP smuggling for separator in ["\n", "\r"].iter() { @@ -286,31 +393,21 @@ async fn smtp_delivery() { session.eval_session_params().await; session.ehlo("mx.test.org").await; - let message = SMUGGLER + let out_message = SMUGGLER .replace('\r', "") .replace('\n', "\r\n") .replace("", separator); session - .send_message("john@doe.org", &["bill@foobar.com"], &message, "250") + .send_message("john@doe.org", &["bill@foobar.com"], &out_message, "250") .await; local - .queue_receiver - .expect_message_then_deliver() + .expect_message_for_queue_then_deliver("default") .await - .try_deliver(core.clone()); - local - .queue_receiver - .read_event() - .await - .assert_refresh_or_done(); + .try_deliver(local.server.clone()); + local.read_event().await.assert_refresh_or_done(); - let message = remote - .queue_receiver - .consume_message(&remote_core) - .await - .read_message(&remote.queue_receiver) - .await; + let message = remote.consume_message().await.read_message(&remote).await; assert!( message.contains("This is a smuggled message"), @@ -324,8 +421,8 @@ async fn smtp_delivery() { ); assert!( message.contains(&format!("{separator}..\r\nMAIL FROM:<",)), - "message: {:?}", - message + "Message {message:?} does not contain separator {:?}", + format!("{separator}..\r\nMAIL FROM:<",) ); } } diff --git a/tests/src/smtp/outbound/throttle.rs b/tests/src/smtp/outbound/throttle.rs index d89eb57f..3e43cd1d 100644 --- a/tests/src/smtp/outbound/throttle.rs +++ b/tests/src/smtp/outbound/throttle.rs @@ -4,13 +4,27 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::smtp::{ - DnsCache, - inbound::TestQueueEvent, - queue::{build_rcpt, manager::new_message}, - session::TestSession, +use crate::{ + smtp::{ + inbound::TestQueueEvent, + queue::{build_rcpt, new_message}, + session::TestSession, + }, + utils::{dns::DnsCache, server::TestServerBuilder}, }; use mail_auth::MX; +use registry::{ + schema::{ + enums::MtaOutboundThrottleKey, + structs::{ + Expression, MtaDeliveryExpiration, MtaDeliveryExpirationTtl, MtaDeliverySchedule, + MtaDeliveryScheduleInterval, MtaDeliveryScheduleIntervals, + MtaDeliveryScheduleIntervalsOrDefault, MtaOutboundStrategy, MtaOutboundThrottle, + MtaVirtualQueue, Rate, + }, + }, + types::{list::List, map::Map}, +}; use smtp::queue::{Message, QueueEnvelope, Recipient, throttle::IsAllowed}; use std::{ net::{IpAddr, Ipv4Addr}, @@ -18,53 +32,97 @@ use std::{ }; use store::write::now; -const CONFIG: &str = r#" -[session.rcpt] -relay = true - -[queue.schedule.default] -retry = "1h" -notify = "1h" -expire = "1h" - -[[queue.limiter.outbound]] -match = "sender_domain = 'foobar.org'" -key = 'sender_domain' -enable = true - -[[queue.limiter.outbound]] -match = "sender_domain = 'foobar.net'" -key = 'sender_domain' -rate = '1/30m' -enable = true - -[[queue.limiter.outbound]] -match = "rcpt_domain = 'example.org'" -key = 'rcpt_domain' -enable = true - -[[queue.limiter.outbound]] -match = "rcpt_domain = 'example.net'" -key = 'rcpt_domain' -rate = '1/40m' -enable = true - -[[queue.limiter.outbound]] -match = "mx = 'mx.test.org'" -key = 'mx' -enable = true - -[[queue.limiter.outbound]] -match = "mx = 'mx.test.net'" -key = 'mx' -rate = '1/50m' -enable = true -"#; - #[tokio::test] async fn throttle_outbound() { - - + let mut local = TestServerBuilder::new("smtp_throttle_outbound") + .await + .with_http_listener(19032) + .await + .disable_services() + .capture_queue() + .build() + .await; + + let admin = local.account("admin"); + let queue_id = admin + .registry_create_object(MtaVirtualQueue { + name: "default".into(), + threads_per_node: 25, + description: None, + }) + .await; + admin + .registry_create_object(MtaDeliverySchedule { + name: "default".into(), + retry: MtaDeliveryScheduleIntervalsOrDefault::Custom(MtaDeliveryScheduleIntervals { + intervals: List::from_iter([MtaDeliveryScheduleInterval { + duration: 3_600_000u64.into(), + }]), + }), + notify: MtaDeliveryScheduleIntervalsOrDefault::Custom(MtaDeliveryScheduleIntervals { + intervals: List::from_iter([MtaDeliveryScheduleInterval { + duration: 3_600_000u64.into(), + }]), + }), + expiry: MtaDeliveryExpiration::Ttl(MtaDeliveryExpirationTtl { + expire: 3_600_000u64.into(), + }), + queue_id, + description: None, + }) + .await; + admin + .registry_create_object(MtaOutboundStrategy { + schedule: Expression { + else_: "'default'".into(), + ..Default::default() + }, + ..Default::default() + }) + .await; + + for (expr, key, rate_count, rate_duration) in [ + ( + "sender_domain = 'foobar.net'", + MtaOutboundThrottleKey::SenderDomain, + 1, + 30 * 60 * 1000, + ), + ( + "rcpt_domain = 'example.net'", + MtaOutboundThrottleKey::RcptDomain, + 1, + 40 * 60 * 1000, + ), + ("mx = 'mx.test.org'", MtaOutboundThrottleKey::Mx, 1, 99999), + ( + "mx = 'mx.test.net'", + MtaOutboundThrottleKey::Mx, + 1, + 50 * 60 * 1000, + ), + ] { + admin + .registry_create_object(MtaOutboundThrottle { + enable: true, + key: Map::new(vec![key]), + match_: Expression { + else_: expr.into(), + ..Default::default() + }, + rate: Rate { + count: rate_count, + period: rate_duration.into(), + }, + description: None, + }) + .await; + } + admin.mta_no_auth().await; + admin.mta_allow_relaying().await; + admin.reload_settings().await; + local.reload_core(); + local.expect_reload_settings().await; // Build test message let mut test_message = new_message(0).message; @@ -73,52 +131,42 @@ async fn throttle_outbound() { .recipients .push(build_rcpt("bill@test.org", 0, 0, 0)); - let mut local = TestSMTP::new("smtp_throttle_outbound", CONFIG).await; - - let core = local.build_smtp(); - let mut session = local.new_session(); + let mut session = local.new_mta_session(); session.data.remote_ip_str = "10.0.0.1".into(); session.eval_session_params().await; session.ehlo("mx.test.org").await; session .send_message("john@foobar.org", &["bill@test.org"], "test:no_dkim", "250") .await; - assert_eq!( - local.queue_receiver.last_queued_due().await as i64 - now() as i64, - 0 - ); + assert_eq!(local.last_queued_due().await as i64 - now() as i64, 0); // Throttle sender - let throttle = &core.core.smtp.queue.outbound_limiters; + let core = local.server.core.clone(); + let throttle = &core.smtp.queue.outbound_limiters; for t in &throttle.sender { - core.is_allowed( - t, - &QueueEnvelope::test(&test_message, &test_message.recipients[0], ""), - 0, - ) - .await - .unwrap(); + local + .server + .is_allowed( + t, + &QueueEnvelope::test(&test_message, &test_message.recipients[0], ""), + 0, + ) + .await + .unwrap(); } - // Expect concurrency throttle for sender domain 'foobar.org' - /*local - .queue_receiver - .expect_message_then_deliver() - .await - .try_deliver(core.clone()); - tokio::time::sleep(Duration::from_millis(100)).await; - local.queue_receiver.read_event().await.assert_on_hold();*/ - // Expect rate limit throttle for sender domain 'foobar.net' test_message.return_path = "test@foobar.net".into(); for t in &throttle.sender { - core.is_allowed( - t, - &QueueEnvelope::test(&test_message, &test_message.recipients[0], ""), - 0, - ) - .await - .unwrap(); + local + .server + .is_allowed( + t, + &QueueEnvelope::test(&test_message, &test_message.recipients[0], ""), + 0, + ) + .await + .unwrap(); } test_message.recipients.clear(); @@ -126,13 +174,12 @@ async fn throttle_outbound() { .send_message("john@foobar.net", &["bill@test.org"], "test:no_dkim", "250") .await; local - .queue_receiver - .expect_message_then_deliver() + .expect_message_for_queue_then_deliver("default") .await - .try_deliver(core.clone()); + .try_deliver(local.server.clone()); tokio::time::sleep(Duration::from_millis(100)).await; - local.queue_receiver.read_event().await.assert_refresh(); - let due = local.queue_receiver.last_queued_due().await - now(); + local.read_event().await.assert_refresh(); + let due = local.last_queued_due().await - now(); assert!(due > 0, "Due: {}", due); // Expect concurrency throttle for recipient domain 'example.org' @@ -141,43 +188,31 @@ async fn throttle_outbound() { .recipients .push(build_rcpt("test@example.org", 0, 0, 0)); for t in &throttle.rcpt { - core.is_allowed( - t, - &QueueEnvelope::test(&test_message, &test_message.recipients[0], ""), - 0, - ) - .await - .unwrap(); + local + .server + .is_allowed( + t, + &QueueEnvelope::test(&test_message, &test_message.recipients[0], ""), + 0, + ) + .await + .unwrap(); } - /*session - .send_message( - "john@test.net", - &["jane@example.org"], - "test:no_dkim", - "250", - ) - .await; - local - .queue_receiver - .expect_message_then_deliver() - .await - .try_deliver(core.clone()); - tokio::time::sleep(Duration::from_millis(100)).await; - local.queue_receiver.read_event().await.assert_on_hold();*/ - // Expect rate limit throttle for recipient domain 'example.net' test_message .recipients .push(build_rcpt("test@example.net", 0, 0, 0)); for t in &throttle.rcpt { - core.is_allowed( - t, - &QueueEnvelope::test(&test_message, &test_message.recipients[1], ""), - 0, - ) - .await - .unwrap(); + local + .server + .is_allowed( + t, + &QueueEnvelope::test(&test_message, &test_message.recipients[1], ""), + 0, + ) + .await + .unwrap(); } session @@ -189,25 +224,24 @@ async fn throttle_outbound() { ) .await; local - .queue_receiver - .expect_message_then_deliver() + .expect_message_for_queue_then_deliver("default") .await - .try_deliver(core.clone()); + .try_deliver(local.server.clone()); tokio::time::sleep(Duration::from_millis(100)).await; - local.queue_receiver.read_event().await.assert_refresh(); - let due = local.queue_receiver.last_queued_due().await - now(); + local.read_event().await.assert_refresh(); + let due = local.last_queued_due().await - now(); assert!(due > 0, "Due: {}", due); // Expect concurrency throttle for mx 'mx.test.org' - core.mx_add( + local.server.mx_add( "test.org", vec![MX { - exchanges: vec!["mx.test.org".into()], + exchanges: vec!["mx.test.org".into()].into_boxed_slice(), preference: 10, }], Instant::now() + Duration::from_secs(10), ); - core.ipv4_add( + local.server.ipv4_add( "mx.test.org", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(10), @@ -217,61 +251,54 @@ async fn throttle_outbound() { .push(build_rcpt("test@test.org", 0, 0, 0)); for t in &throttle.remote { - core.is_allowed( - t, - &QueueEnvelope::test(&test_message, &test_message.recipients[2], "mx.test.org"), - 0, - ) - .await - .unwrap(); + local + .server + .is_allowed( + t, + &QueueEnvelope::test(&test_message, &test_message.recipients[2], "mx.test.org"), + 0, + ) + .await + .unwrap(); } - /*session - .send_message("john@test.net", &["jane@test.org"], "test:no_dkim", "250") - .await; - local - .queue_receiver - .expect_message_then_deliver() - .await - .try_deliver(core.clone()); - local.queue_receiver.read_event().await.assert_on_hold();*/ - // Expect rate limit throttle for mx 'mx.test.net' - core.mx_add( + local.server.mx_add( "test.net", vec![MX { - exchanges: vec!["mx.test.net".into()], + exchanges: vec!["mx.test.net".into()].into_boxed_slice(), preference: 10, }], Instant::now() + Duration::from_secs(10), ); - core.ipv4_add( + local.server.ipv4_add( "mx.test.net", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(10), ); for t in &throttle.remote { - core.is_allowed( - t, - &QueueEnvelope::test(&test_message, &test_message.recipients[1], "mx.test.net"), - 0, - ) - .await - .unwrap(); + local + .server + .is_allowed( + t, + &QueueEnvelope::test(&test_message, &test_message.recipients[1], "mx.test.net"), + 0, + ) + .await + .unwrap(); } session .send_message("john@test.net", &["jane@test.net"], "test:no_dkim", "250") .await; local - .queue_receiver - .expect_message_then_deliver() + .expect_message_for_queue_then_deliver("default") .await - .try_deliver(core.clone()); + .try_deliver(local.server.clone()); tokio::time::sleep(Duration::from_millis(100)).await; - local.queue_receiver.read_event().await.assert_refresh(); - let due = local.queue_receiver.last_queued_due().await - now(); + local.read_event().await.assert_refresh(); + let due = local.last_queued_due().await - now(); assert!(due > 0, "Due: {}", due); } diff --git a/tests/src/smtp/outbound/tls.rs b/tests/src/smtp/outbound/tls.rs index e3aca014..da402c03 100644 --- a/tests/src/smtp/outbound/tls.rs +++ b/tests/src/smtp/outbound/tls.rs @@ -4,77 +4,130 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::time::{Duration, Instant}; - -use common::config::server::ServerProtocol; -use mail_auth::MX; -use store::write::now; - -use crate::smtp::{ - DnsCache, - inbound::TestMessage, - session::{TestSession, VerifyResponse}, +use crate::{ + smtp::{ + inbound::TestMessage, + session::{TestSession, VerifyResponse}, + }, + utils::{dns::DnsCache, server::TestServerBuilder}, }; - -const LOCAL: &str = r#" -[session.rcpt] -relay = true - -[queue.connection.default] -ehlo-hostname = "badtls.foobar.org" - -[queue.strategy] -tls = [ { if = "retry_num > 0 && last_error == 'tls'", then = "'no-tls'"}, - { else = "'default'" }] - -[queue.tls.no-tls] -starttls = false -allow-invalid-certs = true - -"#; - -const REMOTE: &str = r#" -[session.rcpt] -relay = true - -[session.ehlo] -reject-non-fqdn = false - -[session.extensions] -dsn = true -chunking = false -"#; +use mail_auth::MX; +use registry::{ + schema::{ + enums::MtaRequiredOrOptional, + structs::{ + Expression, ExpressionMatch, MtaConnectionStrategy, MtaDeliverySchedule, + MtaOutboundStrategy, MtaTlsStrategy, MtaVirtualQueue, + }, + }, + types::list::List, +}; +use std::time::{Duration, Instant}; +use store::write::now; #[tokio::test] #[serial_test::serial] async fn starttls_optional() { - - + let mut local = TestServerBuilder::new("smtp_starttls_local") + .await + .with_http_listener(19034) + .await + .disable_services() + .capture_queue() + .build() + .await; + let mut remote = TestServerBuilder::new("smtp_starttls_remote") + .await + .with_http_listener(19035) + .await + .with_smtp_listener(9925) + .await + .disable_services() + .capture_queue() + .build() + .await; - // Start test server - let mut remote = TestSMTP::new("smtp_starttls_remote", REMOTE).await; - let _rx = remote.start(&[ServerProtocol::Smtp]).await; + let local_admin = local.account("admin"); + local_admin.mta_no_auth().await; + local_admin.mta_allow_relaying().await; + local_admin + .registry_create_object(MtaOutboundStrategy { + tls: Expression { + match_: List::from_iter([ExpressionMatch { + if_: "retry_num > 0 && last_error == 'tls'".into(), + then: "'no-tls'".into(), + }]), + else_: "'default'".into(), + }, + connection: Expression { + else_: "'badtls'".into(), + ..Default::default() + }, + schedule: Expression { + else_: "'default'".into(), + ..Default::default() + }, + ..Default::default() + }) + .await; + let queue_id = local_admin + .registry_create_object(MtaVirtualQueue { + name: "default".into(), + threads_per_node: 25, + description: None, + }) + .await; + local_admin + .registry_create_object(MtaDeliverySchedule { + name: "default".into(), + queue_id, + ..Default::default() + }) + .await; + local_admin + .registry_create_object(MtaConnectionStrategy { + name: "badtls".into(), + ehlo_hostname: "badtls.foobar.org".to_string().into(), + ..Default::default() + }) + .await; + local_admin + .registry_create_object(MtaTlsStrategy { + name: "no-tls".into(), + allow_invalid_certs: true, + start_tls: MtaRequiredOrOptional::Disable, + ..Default::default() + }) + .await; + local_admin.reload_settings().await; + local.reload_core(); + local.expect_reload_settings().await; - // Retry on failed STARTTLS - let mut local = TestSMTP::new("smtp_starttls_local", LOCAL).await; + let remote_admin = remote.account("admin"); + remote_admin.mta_no_auth().await; + remote_admin.mta_allow_relaying().await; + remote_admin.mta_allow_non_fqdn().await; + remote_admin.mta_all_extensions().await; + remote_admin.reload_settings().await; + remote.reload_core(); + remote.expect_reload_settings().await; // Add mock DNS entries - let core = local.build_smtp(); - core.mx_add( + local.server.mx_add( "foobar.org", vec![MX { - exchanges: vec!["mx.foobar.org".to_string()], + exchanges: vec!["mx.foobar.org".into()].into_boxed_slice(), preference: 10, }], Instant::now() + Duration::from_secs(10), ); - core.ipv4_add( + local.server.ipv4_add( "mx.foobar.org", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(10), ); - let mut session = local.new_session(); + let mut session = local.new_mta_session(); session.data.remote_ip_str = "10.0.0.1".into(); session.eval_session_params().await; session.ehlo("mx.test.org").await; @@ -82,27 +135,24 @@ async fn starttls_optional() { .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; local - .queue_receiver - .expect_message_then_deliver() + .expect_message_for_queue_then_deliver("default") .await - .try_deliver(core.clone()); - let mut retry = local.queue_receiver.expect_message().await; + .try_deliver(local.server.clone()); + let mut retry = local.expect_message().await; let prev_due = retry.message.recipients[0].retry.due; let next_due = now(); let queue_id = retry.queue_id; retry.message.recipients[0].retry.due = next_due; - retry.save_changes(&core, prev_due.into()).await; + retry.save_changes(&local.server, prev_due.into()).await; local - .queue_receiver - .delivery_attempt(queue_id) + .delivery_attempt_for_queue(queue_id, "default") .await - .try_deliver(core.clone()); + .try_deliver(local.server.clone()); tokio::time::sleep(Duration::from_millis(100)).await; remote - .queue_receiver .expect_message() .await - .read_lines(&remote.queue_receiver) + .read_lines(&remote) .await .assert_not_contains("using TLSv1.3 with cipher"); } diff --git a/tests/src/smtp/queue/concurrent.rs b/tests/src/smtp/queue/concurrent.rs index d9093603..98e2b7b9 100644 --- a/tests/src/smtp/queue/concurrent.rs +++ b/tests/src/smtp/queue/concurrent.rs @@ -5,78 +5,135 @@ */ use crate::{ - smtp::{DnsCache, session::TestSession}, - store::cleanup::store_assert_is_empty, + smtp::session::TestSession, + utils::{dns::DnsCache, server::TestServerBuilder}, }; -use common::{config::server::ServerProtocol, ipc::QueueEvent}; +use common::{BuildServer, ipc::QueueEvent}; use mail_auth::MX; +use registry::{ + schema::{ + enums::NetworkListenerProtocol, + prelude::ObjectType, + structs::{ + Expression, MtaDeliveryExpiration, MtaDeliveryExpirationTtl, MtaDeliverySchedule, + MtaDeliveryScheduleInterval, MtaDeliveryScheduleIntervals, + MtaDeliveryScheduleIntervalsOrDefault, MtaOutboundStrategy, MtaStageData, + MtaVirtualQueue, + }, + }, + types::list::List, +}; use smtp::queue::manager::Queue; use std::time::{Duration, Instant}; -const LOCAL: &str = r#" -[spam-filter] -enable = false - -[session.rcpt] -relay = true - -[session.data.limits] -messages = 2000 - -[queue.virtual.default] -threads-per-node = 4 - -[queue.schedule.default] -retry = "1s" -notify = "1d" -expire = "1d" -queue-name = "default" -"#; - -const REMOTE: &str = r#" -[session.ehlo] -reject-non-fqdn = false - -[session.rcpt] -relay = true - -[spam-filter] -enable = false - -"#; - const NUM_MESSAGES: usize = 100; const NUM_QUEUES: usize = 10; #[tokio::test(flavor = "multi_thread", worker_threads = 18)] #[serial_test::serial] async fn concurrent_queue() { - - + let mut local = TestServerBuilder::new("smtp_concurrent_queue_local") + .await + .with_http_listener(19037) + .await + .disable_services() + .build() + .await; + let mut remote = TestServerBuilder::new("smtp_concurrent_queue_remote") + .await + .with_http_listener(19038) + .await + .with_listener(NetworkListenerProtocol::Smtp, "smtp-debug", 9925, false) + .await + .disable_services() + .capture_queue() + .build() + .await; - // Start test server - let remote = TestSMTP::new("smtp_concurrent_queue_remote", REMOTE).await; - let _rx = remote.start(&[ServerProtocol::Smtp]).await; + let local_admin = local.account("admin"); + local_admin + .registry_create_object(MtaStageData { + max_messages: Expression { + else_: "2000".into(), + ..Default::default() + }, + ..Default::default() + }) + .await; + local_admin + .registry_create_object(MtaOutboundStrategy { + schedule: Expression { + else_: "'default'".into(), + ..Default::default() + }, + ..Default::default() + }) + .await; + let queue_id = local_admin + .registry_create_object(MtaVirtualQueue { + name: "default".into(), + threads_per_node: 4, + description: None, + }) + .await; + local_admin + .registry_create_object(MtaDeliverySchedule { + name: "default".into(), + retry: MtaDeliveryScheduleIntervalsOrDefault::Custom(MtaDeliveryScheduleIntervals { + intervals: List::from_iter([MtaDeliveryScheduleInterval { + duration: 1_000u64.into(), + }]), + }), + notify: MtaDeliveryScheduleIntervalsOrDefault::Custom(MtaDeliveryScheduleIntervals { + intervals: List::from_iter([MtaDeliveryScheduleInterval { + duration: 86_400_000u64.into(), + }]), + }), + expiry: MtaDeliveryExpiration::Ttl(MtaDeliveryExpirationTtl { + expire: 86_400_000u64.into(), + }), + queue_id, + description: None, + }) + .await; + local_admin.mta_allow_relaying().await; + local_admin.mta_disable_spam_filter().await; + local_admin.mta_allow_non_fqdn().await; + local_admin.mta_no_auth().await; + local_admin + .registry_destroy_all(ObjectType::MtaInboundThrottle) + .await; + local_admin.reload_settings().await; + local.reload_core(); - let local = TestSMTP::with_database("smtp_concurrent_queue_local", LOCAL, "mysql").await; + let remote_admin = remote.account("admin"); + remote_admin.mta_allow_relaying().await; + remote_admin.mta_disable_spam_filter().await; + remote_admin.mta_allow_non_fqdn().await; + remote_admin.mta_no_auth().await; + remote_admin + .registry_destroy_all(ObjectType::MtaInboundThrottle) + .await; + remote_admin.reload_settings().await; + remote.reload_core(); + remote.expect_reload_settings().await; // Add mock DNS entries - let core = local.build_smtp(); - core.mx_add( + local.server.mx_add( "foobar.org", vec![MX { - exchanges: vec!["mx.foobar.org".to_string()], + exchanges: vec!["mx.foobar.org".into()].into_boxed_slice(), preference: 10, }], Instant::now() + Duration::from_secs(100), ); - core.ipv4_add( + local.server.ipv4_add( "mx.foobar.org", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(100), ); - let mut session = local.new_session(); + let mut session = local.new_mta_session(); session.data.remote_ip_str = "10.0.0.1".into(); session.eval_session_params().await; session.ehlo("mx.test.org").await; @@ -84,12 +141,12 @@ async fn concurrent_queue() { // Spawn concurrent queues let mut inners = vec![]; for _ in 0..NUM_QUEUES { - let (inner, rxs) = local.inner_with_rxs(); + let (inner, rxs) = local.inner_with_rxs().await; let server = inner.build_server(); server.mx_add( "foobar.org", vec![MX { - exchanges: vec!["mx.foobar.org".to_string()], + exchanges: vec!["mx.foobar.org".into()].into_boxed_slice(), preference: 10, }], Instant::now() + Duration::from_secs(100), @@ -132,8 +189,8 @@ async fn concurrent_queue() { loop { tokio::time::sleep(Duration::from_millis(1500)).await; - let m = local.queue_receiver.read_queued_messages().await.len(); - let e = local.queue_receiver.read_queued_events().await.len(); + let m = local.read_queued_messages().await.len(); + let e = local.read_queued_events().await.len(); if m + e != 0 { println!("Queue still has {} messages and {} events", m, e); @@ -145,15 +202,14 @@ async fn concurrent_queue() { } } - local.queue_receiver.assert_queue_is_empty().await; - let remote_messages = remote.queue_receiver.read_queued_messages().await; + local.assert_queue_is_empty().await; + let remote_messages = remote.read_queued_messages().await; assert_eq!(remote_messages.len(), NUM_MESSAGES); // Make sure local store is queue - store_assert_is_empty( - &core.core.storage.data, - core.core.storage.blob.clone(), - false, - ) - .await; + local + .account("admin") + .registry_destroy_all(ObjectType::MtaConnectionStrategy) + .await; + local.assert_is_empty().await; } diff --git a/tests/src/smtp/queue/dsn.rs b/tests/src/smtp/queue/dsn.rs index e78bcbac..dcc45b21 100644 --- a/tests/src/smtp/queue/dsn.rs +++ b/tests/src/smtp/queue/dsn.rs @@ -4,8 +4,12 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{smtp::inbound::sign::SIGNATURES, utils::server::TestServer}; +use crate::utils::server::{TestServer, TestServerBuilder}; use common::config::smtp::queue::{QueueExpiry, QueueName}; +use registry::schema::{ + enums::CompressionAlgo, + structs::{DsnReportSettings, Expression, ReportSettings}, +}; use smtp::queue::{ Error, ErrorDetails, HostResponse, Message, MessageWrapper, Recipient, Schedule, Status, UnexpectedResponse, dsn::SendDsn, @@ -20,28 +24,8 @@ use std::{ use store::write::now; use types::blob_hash::BlobHash; -const CONFIG: &str = r#" -[report] -submitter = "'mx.example.org'" - -[session.ehlo] -reject-non-fqdn = false - -[session.rcpt] -relay = true - -[report.dsn] -from-name = "'Mail Delivery Subsystem'" -from-address = "'MAILER-DAEMON@example.org'" -sign = "['rsa']" - -"#; - #[tokio::test] async fn generate_dsn() { - - - let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); path.push("resources"); path.push("smtp"); @@ -92,30 +76,71 @@ async fn generate_dsn() { }, }; - // Load config - let mut local = TestSMTP::new("smtp_dsn_test", CONFIG.to_string() + SIGNATURES).await; - let core = local.build_smtp(); - let qr = &mut local.queue_receiver; + let mut local = TestServerBuilder::new("smtp_queue_dsn") + .await + .with_http_listener(19039) + .await + .disable_services() + .capture_queue() + .build() + .await; + let local_admin = local.account("admin"); + local_admin + .registry_create_object(ReportSettings { + outbound_report_submitter: Expression { + else_: "'mx.example.org'".into(), + ..Default::default() + }, + ..Default::default() + }) + .await; + local_admin + .registry_create_object(DsnReportSettings { + dkim_sign_domain: Expression { + else_: "'example.org'".into(), + ..Default::default() + }, + from_address: Expression { + else_: "'MAILER-DAEMON@example.org'".into(), + ..Default::default() + }, + from_name: Expression { + else_: "'Mail Delivery Subsystem'".into(), + ..Default::default() + }, + }) + .await; + let domain_id = local_admin.find_or_create_domain("example.org").await; + local_admin.create_dkim_signatures(domain_id).await; + local_admin.mta_allow_non_fqdn().await; + local_admin.mta_allow_relaying().await; + local_admin.reload_settings().await; + local_admin.mta_allow_relaying().await; + local.reload_core(); + local.expect_reload_settings().await; // Create temp dir for queue - qr.blob_store + local + .server + .blob_store() .put_blob( message.message.blob_hash.as_slice(), dsn_original.as_bytes(), + CompressionAlgo::Lz4, ) .await .unwrap(); // Disabled DSN - core.send_dsn(&mut message).await; - qr.assert_no_events(); - qr.assert_queue_is_empty().await; + local.server.send_dsn(&mut message).await; + local.assert_no_events(); + local.assert_queue_is_empty().await; // Failure DSN message.message.recipients[0].flags = flags; - core.send_dsn(&mut message).await; - let dsn_message = qr.expect_message().await; - qr.compare_dsn(dsn_message.message, "failure.eml").await; + local.server.send_dsn(&mut message).await; + let dsn_message = local.expect_message().await; + local.compare_dsn(dsn_message.message, "failure.eml").await; // Success DSN message.message.recipients.push(Recipient { @@ -135,9 +160,9 @@ async fn generate_dsn() { expires: QueueExpiry::Ttl(10), queue: QueueName::default(), }); - core.send_dsn(&mut message).await; - let dsn_message = qr.expect_message().await; - qr.compare_dsn(dsn_message.message, "success.eml").await; + local.server.send_dsn(&mut message).await; + let dsn_message = local.expect_message().await; + local.compare_dsn(dsn_message.message, "success.eml").await; // Delay DSN message.message.recipients.push(Recipient { @@ -153,21 +178,21 @@ async fn generate_dsn() { expires: QueueExpiry::Ttl(10), queue: QueueName::default(), }); - core.send_dsn(&mut message).await; - let dsn_message = qr.expect_message().await; - qr.compare_dsn(dsn_message.message, "delay.eml").await; + local.server.send_dsn(&mut message).await; + let dsn_message = local.expect_message().await; + local.compare_dsn(dsn_message.message, "delay.eml").await; // Mixed DSN for rcpt in &mut message.message.recipients { rcpt.flags = flags; } message.message.recipients.last_mut().unwrap().notify.due = now(); - core.send_dsn(&mut message).await; - let dsn_message = qr.expect_message().await; - qr.compare_dsn(dsn_message.message, "mixed.eml").await; + local.server.send_dsn(&mut message).await; + let dsn_message = local.expect_message().await; + local.compare_dsn(dsn_message.message, "mixed.eml").await; // Load queue - let queue = qr.read_queued_messages().await; + let queue = local.read_queued_messages().await; assert_eq!(queue.len(), 4); } @@ -206,7 +231,7 @@ impl TestServer { fn remove_ids(message: Vec) -> String { let old_message = String::from_utf8(message).unwrap(); let mut message = String::with_capacity(old_message.len()); - let mut found_dkim = false; + let mut found_dkim = 0; let mut skip = false; let mut boundary = ""; @@ -220,8 +245,8 @@ fn remove_ids(message: Vec) -> String { } if line.starts_with("Date:") || line.starts_with("Message-ID:") { continue; - } else if !found_dkim && line.starts_with("DKIM-Signature:") { - found_dkim = true; + } else if found_dkim < 2 && line.starts_with("DKIM-Signature:") { + found_dkim += 1; skip = true; continue; } else if line.starts_with("--") { @@ -239,7 +264,7 @@ fn remove_ids(message: Vec) -> String { message.push_str("\r\n"); } - if !found_dkim { + if found_dkim == 0 { panic!("No DKIM signature found in: {old_message}"); } diff --git a/tests/src/smtp/queue/manager.rs b/tests/src/smtp/queue/manager.rs index ce356257..5efa8088 100644 --- a/tests/src/smtp/queue/manager.rs +++ b/tests/src/smtp/queue/manager.rs @@ -4,72 +4,69 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::smtp::{ - - queue::{QueuedEvents, build_rcpt}, +use crate::{ + smtp::queue::{build_rcpt, new_message}, + utils::server::TestServerBuilder, }; use common::config::smtp::queue::QueueName; -use smtp::queue::{ - Error, ErrorDetails, Message, MessageWrapper, Recipient, Status, spool::SmtpSpool, -}; -use std::{ - net::{IpAddr, Ipv4Addr}, - time::Duration, -}; +use smtp::queue::{Error, ErrorDetails, Message, Recipient, Status, spool::SmtpSpool}; +use std::time::Duration; use store::write::now; -const CONFIG: &str = r#" -[session.ehlo] -reject-non-fqdn = false - -[session.rcpt] -relay = true -"#; - #[tokio::test] async fn queue_due() { - - + let mut local = TestServerBuilder::new("smtp_queue_manager") + .await + .with_http_listener(19040) + .await + .disable_services() + .capture_queue() + .build() + .await; - let local = TestSMTP::new("smtp_queue_due_test", CONFIG).await; - let core = local.build_smtp(); - let qr = &local.queue_receiver; + let local_admin = local.account("admin"); + local_admin.mta_allow_relaying().await; + local_admin.mta_allow_non_fqdn().await; + local_admin.reload_settings().await; + local.reload_core(); + local.expect_reload_settings().await; let mut message = new_message(0); message.message.recipients.push(build_rcpt("c", 3, 8, 9)); - message.save_changes(&core, 0.into()).await; + message.save_changes(&local.server, 0.into()).await; let mut message = new_message(1); message.message.recipients.push(build_rcpt("b", 2, 6, 7)); - message.save_changes(&core, 0.into()).await; + message.save_changes(&local.server, 0.into()).await; let mut message = new_message(2); message.message.recipients.push(build_rcpt("a", 1, 4, 5)); - message.save_changes(&core, 0.into()).await; + message.save_changes(&local.server, 0.into()).await; for domain in vec!["a", "b", "c"].into_iter() { let now = now(); - let queued = core.all_queued_messages().await; + let queued = local.all_queued_messages().await; if queued.messages.is_empty() { let wake_up = queued.next_refresh - now; assert_eq!(wake_up, 1); std::thread::sleep(Duration::from_secs(wake_up)); } - for queue_event in core.all_queued_messages().await.messages { - if let Some(message) = core + for queue_event in local.all_queued_messages().await.messages { + if let Some(message) = local + .server .read_message(queue_event.queue_id, QueueName::default()) .await { message.message.rcpt(domain); - message.remove(&core, queue_event.due.into()).await; + message.remove(&local.server, queue_event.due.into()).await; } else { panic!("Message not found"); } } } - qr.assert_queue_is_empty().await; + local.assert_queue_is_empty().await; } #[test] @@ -161,28 +158,6 @@ fn delivery_events() { assert!(message.next_event(None).is_none()); } -pub fn new_message(queue_id: u64) -> MessageWrapper { - MessageWrapper { - queue_id, - span_id: 0, - queue_name: QueueName::default(), - is_multi_queue: false, - message: Message { - size: 0, - created: now(), - return_path: "sender@foobar.org".into(), - recipients: vec![], - flags: 0, - env_id: None, - priority: 0, - quota_keys: Default::default(), - blob_hash: Default::default(), - received_from_ip: IpAddr::V4(Ipv4Addr::LOCALHOST), - received_via_port: 0, - }, - } -} - fn next_event_after(message: &Message, queue: Option, instant: u64) -> Option { let mut next_event = None; diff --git a/tests/src/smtp/queue/mod.rs b/tests/src/smtp/queue/mod.rs index 36a1add6..d399181c 100644 --- a/tests/src/smtp/queue/mod.rs +++ b/tests/src/smtp/queue/mod.rs @@ -4,16 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{ - Server, - config::smtp::queue::{QueueExpiry, QueueName}, -}; -use smtp::queue::{ - Recipient, Schedule, Status, - manager::Queue, - spool::{QueuedMessages, SmtpSpool}, -}; -use tokio::sync::mpsc; +use common::config::smtp::queue::{QueueExpiry, QueueName}; +use smtp::queue::{Message, MessageWrapper, Recipient, Schedule, Status}; +use std::net::{IpAddr, Ipv4Addr}; +use store::write::now; pub mod concurrent; pub mod dsn; @@ -33,3 +27,25 @@ pub fn build_rcpt(address: &str, retry: u64, notify: u64, expires: u64) -> Recip queue: QueueName::default(), } } + +pub fn new_message(queue_id: u64) -> MessageWrapper { + MessageWrapper { + queue_id, + span_id: 0, + queue_name: QueueName::default(), + is_multi_queue: false, + message: Message { + size: 0, + created: now(), + return_path: "sender@foobar.org".into(), + recipients: vec![], + flags: 0, + env_id: None, + priority: 0, + quota_keys: Default::default(), + blob_hash: Default::default(), + received_from_ip: IpAddr::V4(Ipv4Addr::LOCALHOST), + received_via_port: 0, + }, + } +} diff --git a/tests/src/smtp/queue/retry.rs b/tests/src/smtp/queue/retry.rs index a7e1f5c3..0b1910e9 100644 --- a/tests/src/smtp/queue/retry.rs +++ b/tests/src/smtp/queue/retry.rs @@ -4,88 +4,167 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::time::Duration; - -use crate::smtp::{ - - inbound::{TestMessage, TestQueueEvent}, - queue::QueuedEvents, - session::{TestSession, VerifyResponse}, +use crate::{ + smtp::{ + inbound::{TestMessage, TestQueueEvent}, + session::{TestSession, VerifyResponse}, + }, + utils::server::TestServerBuilder, }; use ahash::AHashSet; use common::{ config::smtp::queue::QueueName, ipc::{QueueEvent, QueueEventStatus}, }; +use registry::{ + schema::structs::{ + Expression, ExpressionMatch, MtaDeliveryExpiration, MtaDeliveryExpirationTtl, + MtaDeliverySchedule, MtaDeliveryScheduleInterval, MtaDeliveryScheduleIntervals, + MtaDeliveryScheduleIntervalsOrDefault, MtaExtensions, MtaOutboundStrategy, MtaVirtualQueue, + }, + types::list::List, +}; use smtp::queue::spool::{QUEUE_REFRESH, SmtpSpool}; +use std::time::Duration; use store::write::now; -const CONFIG: &str = r#" -[session.ehlo] -reject-non-fqdn = false - -[session.rcpt] -relay = true - -[session.extensions] -deliver-by = "1h" -future-release = "1h" - -[queue.schedule.sender-default] -retry = ["1s", "2s", "3s"] -notify = ["15h", "22h"] -expire = "1d" -queue-name = "default" - -[queue.schedule.sender-test] -retry = ["1s", "2s", "3s"] -notify = ["1s", "2s"] -expire = "6s" -#max-attempts = 3 -queue-name = "default" - -[queue.strategy] -schedule = [{if = "sender_domain == 'test.org'", then = "'sender-test'"}, - {else = "'sender-default'"}] -"#; - #[tokio::test] async fn queue_retry() { - - + let mut local = TestServerBuilder::new("smtp_queue_retry") + .await + .with_http_listener(19041) + .await + .disable_services() + .capture_queue() + .build() + .await; - // Create temp dir for queue - let mut local = TestSMTP::new("smtp_queue_retry_test", CONFIG).await; - - // Create test message - let core = local.build_smtp(); - let mut session = local.new_session(); - let qr = &mut local.queue_receiver; + let local_admin = local.account("admin"); + local_admin.mta_allow_relaying().await; + local_admin.mta_allow_non_fqdn().await; + local_admin.mta_no_auth().await; + local_admin + .registry_create_object(MtaOutboundStrategy { + schedule: Expression { + match_: List::from_iter([ExpressionMatch { + if_: "sender_domain == 'test.org'".into(), + then: "'sender-test'".into(), + }]), + else_: "'sender-default'".into(), + }, + ..Default::default() + }) + .await; + local_admin + .registry_create_object(MtaExtensions { + deliver_by: Expression { + else_: "1h".into(), + ..Default::default() + }, + future_release: Expression { + else_: "1h".into(), + ..Default::default() + }, + ..Default::default() + }) + .await; + let queue_id = local_admin + .registry_create_object(MtaVirtualQueue { + name: "default".into(), + threads_per_node: 25, + description: None, + }) + .await; + local_admin + .registry_create_object(MtaDeliverySchedule { + name: "sender-default".into(), + retry: MtaDeliveryScheduleIntervalsOrDefault::Custom(MtaDeliveryScheduleIntervals { + intervals: List::from_iter([ + MtaDeliveryScheduleInterval { + duration: 1_000u64.into(), + }, + MtaDeliveryScheduleInterval { + duration: 2_000u64.into(), + }, + MtaDeliveryScheduleInterval { + duration: 3_000u64.into(), + }, + ]), + }), + notify: MtaDeliveryScheduleIntervalsOrDefault::Custom(MtaDeliveryScheduleIntervals { + intervals: List::from_iter([MtaDeliveryScheduleInterval { + duration: (15 * 60 * 60 * 1000u64).into(), + }]), + }), + expiry: MtaDeliveryExpiration::Ttl(MtaDeliveryExpirationTtl { + expire: 86_400_000u64.into(), + }), + queue_id, + description: None, + }) + .await; + local_admin + .registry_create_object(MtaDeliverySchedule { + name: "sender-test".into(), + retry: MtaDeliveryScheduleIntervalsOrDefault::Custom(MtaDeliveryScheduleIntervals { + intervals: List::from_iter([ + MtaDeliveryScheduleInterval { + duration: 1_000u64.into(), + }, + MtaDeliveryScheduleInterval { + duration: 2_000u64.into(), + }, + MtaDeliveryScheduleInterval { + duration: 3_000u64.into(), + }, + ]), + }), + notify: MtaDeliveryScheduleIntervalsOrDefault::Custom(MtaDeliveryScheduleIntervals { + intervals: List::from_iter([ + MtaDeliveryScheduleInterval { + duration: 1_000u64.into(), + }, + MtaDeliveryScheduleInterval { + duration: 2_000u64.into(), + }, + ]), + }), + expiry: MtaDeliveryExpiration::Ttl(MtaDeliveryExpirationTtl { + expire: 6_000u64.into(), + }), + queue_id, + description: None, + }) + .await; + local_admin.reload_settings().await; + local.reload_core(); + local.expect_reload_settings().await; + let mut session = local.new_mta_session(); session.data.remote_ip_str = "10.0.0.1".into(); session.eval_session_params().await; session.ehlo("mx.test.org").await; session .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; - let attempt = qr.expect_message_then_deliver().await; + let attempt = local.expect_message_for_queue_then_deliver("default").await; // Expect a failed DSN - attempt.try_deliver(core.clone()); - let message = qr.expect_message().await; + attempt.try_deliver(local.server.clone()); + let message = local.expect_message().await; assert_eq!(message.message.return_path.as_ref(), ""); assert_eq!( message.message.recipients.first().unwrap().address(), "john@test.org" ); message - .read_lines(qr) + .read_lines(&local) .await .assert_contains("Content-Type: multipart/report") .assert_contains("Final-Recipient: rfc822;bill@foobar.org") .assert_contains("Action: failed"); - qr.read_event().await.assert_done(); - qr.clear_queue(&core).await; + local.read_event().await.assert_done(); + local.clear_queue().await; // Expect a failed DSN for foobar.org, followed by two delayed DSN and // a final failed DSN for _dns_error.org. @@ -98,14 +177,14 @@ async fn queue_retry() { ) .await; let mut in_fight = AHashSet::new(); - let attempt = qr.expect_message_then_deliver().await; + let attempt = local.expect_message_for_queue_then_deliver("default").await; let mut dsn = Vec::new(); let mut retries = Vec::new(); in_fight.insert(attempt.queue_id); - attempt.try_deliver(core.clone()); + attempt.try_deliver(local.server.clone()); loop { - match qr.try_read_event().await { + match local.try_read_event().await { Some(QueueEvent::WorkerDone { queue_id, status, .. }) => { @@ -120,11 +199,11 @@ async fn queue_retry() { } let now = now(); - let mut events = core.all_queued_messages().await; + let mut events = local.all_queued_messages().await; if events.messages.is_empty() { if events.next_refresh < now + QUEUE_REFRESH { tokio::time::sleep(Duration::from_secs(events.next_refresh - now)).await; - events = core.all_queued_messages().await; + events = local.all_queued_messages().await; } else if in_fight.is_empty() { break; } @@ -135,29 +214,33 @@ async fn queue_retry() { continue; } - let message = core + let message = local + .server .read_message(event.queue_id, QueueName::default()) .await .unwrap(); if message.message.return_path.is_empty() { - message.clone().remove(&core, event.due.into()).await; + message + .clone() + .remove(&local.server, event.due.into()) + .await; dsn.push(message); } else { retries.push(event.due.saturating_sub(now)); in_fight.insert(event.queue_id); - event.try_deliver(core.clone()); + event.try_deliver(local.server.clone()); tokio::time::sleep(Duration::from_millis(100)).await; } } } - qr.assert_queue_is_empty().await; + local.assert_queue_is_empty().await; assert_eq!(retries, vec![1, 2, 3]); assert_eq!(dsn.len(), 4); let mut dsn = dsn.into_iter(); dsn.next() .unwrap() - .read_lines(qr) + .read_lines(&local) .await .assert_contains(" (failed to lookup 'foobar.org'") .assert_contains("Final-Recipient: rfc822;bill@foobar.org") @@ -165,7 +248,7 @@ async fn queue_retry() { dsn.next() .unwrap() - .read_lines(qr) + .read_lines(&local) .await .assert_contains(" (failed to lookup '_dns_error.org'") .assert_contains("Final-Recipient: rfc822;jane@_dns_error.org") @@ -173,7 +256,7 @@ async fn queue_retry() { dsn.next() .unwrap() - .read_lines(qr) + .read_lines(&local) .await .assert_contains(" (failed to lookup '_dns_error.org'") .assert_contains("Final-Recipient: rfc822;jane@_dns_error.org") @@ -181,7 +264,7 @@ async fn queue_retry() { dsn.next() .unwrap() - .read_lines(qr) + .read_lines(&local) .await .assert_contains(" (failed to lookup '_dns_error.org'") .assert_contains("Final-Recipient: rfc822;jane@_dns_error.org") @@ -199,8 +282,8 @@ async fn queue_retry() { ) .await; let now_ = now(); - let message = qr.expect_message().await; - assert!([59, 60].contains(&(qr.message_due(message.queue_id).await - now_))); + let message = local.expect_message().await; + assert!([59, 60].contains(&(local.message_due(message.queue_id).await - now_))); assert!([59, 60].contains(&(message.message.next_delivery_event(None).unwrap() - now_))); assert!( [3599, 3600].contains( @@ -215,7 +298,9 @@ async fn queue_retry() { ) ); assert!( - [54059, 54060].contains(&(message.message.recipients.first().unwrap().notify.due - now_)) + [54059, 54060].contains(&(message.message.recipients.first().unwrap().notify.due - now_)), + "diff: {}", + message.message.recipients.first().unwrap().notify.due - now_ ); // Test DELIVERBY (NOTIFY) @@ -227,8 +312,10 @@ async fn queue_retry() { "250", ) .await; - let schedule = qr.expect_message().await; + let schedule = local.expect_message().await; assert!( - [3599, 3600].contains(&(schedule.message.recipients.first().unwrap().notify.due - now())) + [3599, 3600].contains(&(schedule.message.recipients.first().unwrap().notify.due - now())), + "diff: {}", + schedule.message.recipients.first().unwrap().notify.due - now() ); } diff --git a/tests/src/smtp/queue/virtualq.rs b/tests/src/smtp/queue/virtualq.rs index 6d73af9e..4b06a7ba 100644 --- a/tests/src/smtp/queue/virtualq.rs +++ b/tests/src/smtp/queue/virtualq.rs @@ -5,76 +5,148 @@ */ use crate::{ - smtp::{DnsCache, session::TestSession}, - store::cleanup::store_assert_is_empty, -}; -use common::{ - config::{server::ServerProtocol, smtp::queue::QueueName}, - ipc::QueueEvent, + smtp::session::TestSession, + utils::{dns::DnsCache, server::TestServerBuilder}, }; +use common::{BuildServer, config::smtp::queue::QueueName, ipc::QueueEvent}; use mail_auth::MX; +use registry::{ + schema::{ + enums::NetworkListenerProtocol, + prelude::ObjectType, + structs::{ + Expression, ExpressionMatch, MtaDeliveryExpiration, MtaDeliveryExpirationTtl, + MtaDeliverySchedule, MtaDeliveryScheduleInterval, MtaDeliveryScheduleIntervals, + MtaDeliveryScheduleIntervalsOrDefault, MtaOutboundStrategy, MtaStageData, + MtaVirtualQueue, + }, + }, + types::list::List, +}; use smtp::queue::manager::Queue; use std::time::{Duration, Instant}; -const LOCAL: &str = r#" -[spam-filter] -enable = false - -[session.rcpt] -relay = true - -[session.data.limits] -messages = 2000 - -[queue.strategy] -schedule = [ { if = "rcpt == 'delay-random@foobar.org'", then = "'q2'" }, - { else = "'q1'"} ] - -[queue.virtual.q1] -threads-per-node = 5 - -[queue.virtual.q2] -threads-per-node = 4 - -[queue.schedule.q1] -retry = "1s" -notify = "1d" -expire = "1d" -queue-name = "q1" - -[queue.schedule.q2] -retry = "1s" -notify = "1d" -expire = "1d" -queue-name = "q2" - -"#; - -const REMOTE: &str = r#" -[session.ehlo] -reject-non-fqdn = false - -[session.rcpt] -relay = true - -[spam-filter] -enable = false - -"#; - const NUM_MESSAGES: usize = 100; const NUM_QUEUES: usize = 10; #[tokio::test(flavor = "multi_thread", worker_threads = 18)] #[serial_test::serial] async fn virtual_queue() { - - + let mut local = TestServerBuilder::new("smtp_virtual_queue_local") + .await + .with_http_listener(19042) + .await + .disable_services() + .build() + .await; + let mut remote = TestServerBuilder::new("smtp_virtual_queue_remote") + .await + .with_http_listener(19043) + .await + .with_listener(NetworkListenerProtocol::Smtp, "smtp-debug", 9925, false) + .await + .disable_services() + .capture_queue() + .build() + .await; - // Start test server - let remote = TestSMTP::new("smtp_virtual_queue_remote", REMOTE).await; - let _rx = remote.start(&[ServerProtocol::Smtp]).await; - let local = TestSMTP::with_database("smtp_virtual_queue_local", LOCAL, "mysql").await; + let local_admin = local.account("admin"); + local_admin + .registry_create_object(MtaOutboundStrategy { + schedule: Expression { + match_: List::from_iter([ExpressionMatch { + if_: "rcpt == 'delay-random@foobar.org'".into(), + then: "'q2'".into(), + }]), + else_: "'q1'".into(), + }, + ..Default::default() + }) + .await; + local_admin + .registry_create_object(MtaStageData { + max_messages: Expression { + else_: "2000".into(), + ..Default::default() + }, + ..Default::default() + }) + .await; + let queue1_id = local_admin + .registry_create_object(MtaVirtualQueue { + name: "q1".into(), + threads_per_node: 5, + description: None, + }) + .await; + let queue2_id = local_admin + .registry_create_object(MtaVirtualQueue { + name: "q2".into(), + threads_per_node: 4, + description: None, + }) + .await; + local_admin + .registry_create_object(MtaDeliverySchedule { + name: "q1".into(), + retry: MtaDeliveryScheduleIntervalsOrDefault::Custom(MtaDeliveryScheduleIntervals { + intervals: List::from_iter([MtaDeliveryScheduleInterval { + duration: 1_000u64.into(), + }]), + }), + notify: MtaDeliveryScheduleIntervalsOrDefault::Custom(MtaDeliveryScheduleIntervals { + intervals: List::from_iter([MtaDeliveryScheduleInterval { + duration: 86_400_000u64.into(), + }]), + }), + expiry: MtaDeliveryExpiration::Ttl(MtaDeliveryExpirationTtl { + expire: 86_400_000u64.into(), + }), + queue_id: queue1_id, + description: None, + }) + .await; + local_admin + .registry_create_object(MtaDeliverySchedule { + name: "q2".into(), + retry: MtaDeliveryScheduleIntervalsOrDefault::Custom(MtaDeliveryScheduleIntervals { + intervals: List::from_iter([MtaDeliveryScheduleInterval { + duration: 1_000u64.into(), + }]), + }), + notify: MtaDeliveryScheduleIntervalsOrDefault::Custom(MtaDeliveryScheduleIntervals { + intervals: List::from_iter([MtaDeliveryScheduleInterval { + duration: 86_400_000u64.into(), + }]), + }), + expiry: MtaDeliveryExpiration::Ttl(MtaDeliveryExpirationTtl { + expire: 86_400_000u64.into(), + }), + queue_id: queue2_id, + description: None, + }) + .await; + local_admin.mta_allow_relaying().await; + local_admin.mta_disable_spam_filter().await; + local_admin.mta_allow_non_fqdn().await; + local_admin.mta_no_auth().await; + local_admin + .registry_destroy_all(ObjectType::MtaInboundThrottle) + .await; + local_admin.reload_settings().await; + local.reload_core(); + + let remote_admin = remote.account("admin"); + remote_admin.mta_allow_relaying().await; + remote_admin.mta_disable_spam_filter().await; + remote_admin.mta_allow_non_fqdn().await; + remote_admin.mta_no_auth().await; + remote_admin + .registry_destroy_all(ObjectType::MtaInboundThrottle) + .await; + remote_admin.reload_settings().await; + remote.reload_core(); + remote.expect_reload_settings().await; // Validate parsing for value in ["a", "ab", "abcdefgh"] { @@ -107,22 +179,21 @@ async fn virtual_queue() { ); // Add mock DNS entries - let core = local.build_smtp(); - core.mx_add( + local.server.mx_add( "foobar.org", vec![MX { - exchanges: vec!["mx.foobar.org".to_string()], + exchanges: vec!["mx.foobar.org".into()].into_boxed_slice(), preference: 10, }], Instant::now() + Duration::from_secs(100), ); - core.ipv4_add( + local.server.ipv4_add( "mx.foobar.org", vec!["127.0.0.1".parse().unwrap()], Instant::now() + Duration::from_secs(100), ); - let mut session = local.new_session(); + let mut session = local.new_mta_session(); session.data.remote_ip_str = "10.0.0.1".into(); session.eval_session_params().await; session.ehlo("mx.test.org").await; @@ -130,12 +201,12 @@ async fn virtual_queue() { // Spawn concurrent queues let mut inners = vec![]; for _ in 0..NUM_QUEUES { - let (inner, rxs) = local.inner_with_rxs(); + let (inner, rxs) = local.inner_with_rxs().await; let server = inner.build_server(); server.mx_add( "foobar.org", vec![MX { - exchanges: vec!["mx.foobar.org".to_string()], + exchanges: vec!["mx.foobar.org".into()].into_boxed_slice(), preference: 10, }], Instant::now() + Duration::from_secs(100), @@ -183,8 +254,8 @@ async fn virtual_queue() { loop { tokio::time::sleep(Duration::from_millis(1500)).await; - let m = local.queue_receiver.read_queued_messages().await; - let e = local.queue_receiver.read_queued_events().await; + let m = local.read_queued_messages().await; + let e = local.read_queued_events().await; if m.len() + e.len() != 0 { println!( @@ -200,15 +271,14 @@ async fn virtual_queue() { } } - local.queue_receiver.assert_queue_is_empty().await; - let remote_messages = remote.queue_receiver.read_queued_messages().await; + local.assert_queue_is_empty().await; + let remote_messages = remote.read_queued_messages().await; assert_eq!(remote_messages.len(), NUM_MESSAGES * 2); // Make sure local store is queue - store_assert_is_empty( - &core.core.storage.data, - core.core.storage.blob.clone(), - false, - ) - .await; + local + .account("admin") + .registry_destroy_all(ObjectType::MtaConnectionStrategy) + .await; + local.assert_is_empty().await; } diff --git a/tests/src/smtp/reporting/analyze.rs b/tests/src/smtp/reporting/analyze.rs index d094b60f..96b9a8c9 100644 --- a/tests/src/smtp/reporting/analyze.rs +++ b/tests/src/smtp/reporting/analyze.rs @@ -4,34 +4,69 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::smtp::{ inbound::TestQueueEvent, session::TestSession}; +use crate::{ + smtp::{inbound::TestQueueEvent, session::TestSession}, + utils::server::TestServerBuilder, +}; +use ahash::AHashMap; +use registry::{ + schema::{ + enums::TaskStoreMaintenanceType, + structs::{ + ArfExternalReport, DataRetention, DmarcExternalReport, Expression, MtaStageData, + ReportSettings, Task, TaskStatus, TaskStoreMaintenance, TlsExternalReport, + }, + }, + types::map::Map, +}; use std::time::Duration; -use store::{IterateParams, ValueKey, write::ValueClass}; - -const CONFIG: &str = r#" -[session.rcpt] -relay = true - -[session.data.limits] -messages = 100 - -[report.analysis] -addresses = ["reports@*", "*@dmarc.foobar.org", "feedback@foobar.org"] -forward = false -store = "1s" -"#; #[tokio::test(flavor = "multi_thread")] async fn report_analyze() { - - + let mut test = TestServerBuilder::new("smtp_analyze_report_test") + .await + .with_http_listener(19044) + .await + .capture_queue() + .build() + .await; - // Create temp dir for queue - let mut local = TestSMTP::new("smtp_analyze_report_test", CONFIG).await; + let admin = test.account("admin"); + admin + .registry_create_object(MtaStageData { + max_messages: Expression { + else_: "100".into(), + ..Default::default() + }, + ..Default::default() + }) + .await; + admin + .registry_create_object(ReportSettings { + inbound_report_addresses: Map::new(vec![ + "reports@*".to_string(), + "*@dmarc.foobar.org".to_string(), + "feedback@foobar.org".to_string(), + ]), + inbound_report_forwarding: false, + ..Default::default() + }) + .await; + admin + .registry_create_object(DataRetention { + hold_mta_reports_for: Some(1u64.into()), + ..Default::default() + }) + .await; + admin.mta_no_auth().await; + admin.mta_allow_non_fqdn().await; + admin.mta_allow_relaying().await; + admin.reload_settings().await; + test.reload_core(); + test.expect_reload_settings().await; // Create test message - let mut session = local.new_session(); - let qr = &mut local.queue_receiver; + let mut session = test.new_mta_session(); session.data.remote_ip_str = "10.0.0.1".into(); session.eval_session_params().await; session.ehlo("mx.test.org").await; @@ -42,75 +77,70 @@ async fn report_analyze() { "feedback@foobar.org", ]; let mut ac = 0; - let mut total_reports_received = 0; - for (test, num_tests) in [("arf", 5), ("dmarc", 5), ("tls", 2)] { + let mut total_reports_received: AHashMap<&str, usize> = AHashMap::new(); + for (test_name, num_tests) in [("arf", 5), ("dmarc", 5), ("tls", 2)] { for num_test in 1..=num_tests { - total_reports_received += 1; + *total_reports_received.entry(test_name).or_insert(0) += 1; session .send_message( "john@test.org", &[addresses[ac % addresses.len()]], - &format!("report:{test}{num_test}"), + &format!("report:{test_name}{num_test}"), "250", ) .await; - qr.assert_no_events(); + test.assert_no_events(); ac += 1; } } tokio::time::sleep(Duration::from_millis(200)).await; - //let c = tokio::time::sleep(Duration::from_secs(86400)).await; - // Purging the database shouldn't remove the reports - qr.store.purge_store().await.unwrap(); + let admin = test.account("admin"); + admin + .registry_create_object(Task::StoreMaintenance(TaskStoreMaintenance { + maintenance_type: TaskStoreMaintenanceType::PurgeData, + shard_index: None, + status: TaskStatus::now(), + })) + .await; + test.wait_for_tasks().await; // Make sure the reports are in the store - let mut total_reports = 0; - qr.store - .iterate( - IterateParams::new( - ValueKey::from(ValueClass::Report(ReportClass::Tls { id: 0, expires: 0 })), - ValueKey::from(ValueClass::Report(ReportClass::Arf { - id: u64::MAX, - expires: u64::MAX, - })), - ), - |_, _| { - total_reports += 1; - Ok(true) - }, - ) - .await - .unwrap(); - assert_eq!(total_reports, total_reports_received); + assert_eq!( + admin.registry_get_all::().await.len(), + total_reports_received["dmarc"] + ); + assert_eq!( + admin.registry_get_all::().await.len(), + total_reports_received["tls"] + ); + assert_eq!( + admin.registry_get_all::().await.len(), + total_reports_received["arf"] + ); // Wait one second, purge, and make sure they are gone tokio::time::sleep(Duration::from_secs(1)).await; - qr.store.purge_store().await.unwrap(); - let mut total_reports = 0; - qr.store - .iterate( - IterateParams::new( - ValueKey::from(ValueClass::Report(ReportClass::Tls { id: 0, expires: 0 })), - ValueKey::from(ValueClass::Report(ReportClass::Arf { - id: u64::MAX, - expires: u64::MAX, - })), - ), - |_, _| { - total_reports += 1; - Ok(true) - }, - ) - .await - .unwrap(); - assert_eq!(total_reports, 0); + admin + .registry_create_object(Task::StoreMaintenance(TaskStoreMaintenance { + maintenance_type: TaskStoreMaintenanceType::PurgeData, + shard_index: None, + status: TaskStatus::now(), + })) + .await; + test.wait_for_tasks().await; + assert_eq!( + admin.registry_get_all::().await, + vec![] + ); + assert_eq!(admin.registry_get_all::().await, vec![]); + assert_eq!(admin.registry_get_all::().await, vec![]); // Test delivery to non-report addresses session .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; - qr.read_event().await.assert_refresh(); - qr.last_queued_message().await; + test.read_event().await.assert_refresh(); + test.last_queued_message().await; } diff --git a/tests/src/smtp/reporting/dmarc.rs b/tests/src/smtp/reporting/dmarc.rs index 32aff22d..65208c9a 100644 --- a/tests/src/smtp/reporting/dmarc.rs +++ b/tests/src/smtp/reporting/dmarc.rs @@ -4,9 +4,9 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::smtp::{ - inbound::{TestMessage, sign::SIGNATURES}, - session::VerifyResponse, +use crate::{ + smtp::{inbound::TestMessage, session::VerifyResponse}, + utils::{dns::DnsCache, server::TestServerBuilder}, }; use common::{config::smtp::report::AggregateFrequency, ipc::DmarcEvent}; use mail_auth::{ @@ -14,51 +14,85 @@ use mail_auth::{ dmarc::Dmarc, report::{ActionDisposition, Disposition, DmarcResult, Record, Report}, }; +use registry::schema::structs::{ + DmarcInternalReport, DmarcReportSettings, Expression, ReportSettings, +}; use smtp::reporting::dmarc::DmarcReporting; use std::{ net::IpAddr, sync::Arc, time::{Duration, Instant}, }; -use store::write::QueueClass; - -const CONFIG: &str = r#" -[session.rcpt] -relay = true - -[server] -hostname = "mx.example.org" - -[report] -submitter = "'mx.example.org'" - -[report.dmarc.aggregate] -from-name = "'DMARC Report'" -from-address = "'reports@' + config_get('report.domain')" -org-name = "'Foobar, Inc.'" -contact-info = "'https://foobar.org/contact'" -send = "daily" -max-size = 4096 -sign = "['rsa']" - -"#; #[tokio::test] async fn report_dmarc() { - - + let mut test = TestServerBuilder::new("smtp_report_dmarc_test") + .await + .with_http_listener(19045) + .await + .disable_services() + .capture_queue() + .build() + .await; - // Create scheduler - let mut local = TestSMTP::new("smtp_report_dmarc_test", CONFIG.to_string() + SIGNATURES).await; + let admin = test.account("admin"); + let domain_id = admin.find_or_create_domain("example.org").await; + admin.create_dkim_signatures(domain_id).await; + admin + .registry_create_object(ReportSettings { + outbound_report_submitter: Expression { + else_: "'mx.example.org'".into(), + ..Default::default() + }, + ..Default::default() + }) + .await; + admin + .registry_create_object(DmarcReportSettings { + aggregate_contact_info: Expression { + else_: "'https://foobar.org/contact'".into(), + ..Default::default() + }, + aggregate_dkim_sign_domain: Expression { + else_: "'example.org'".into(), + ..Default::default() + }, + aggregate_from_address: Expression { + else_: "'reports@' + system('domain')".into(), + ..Default::default() + }, + aggregate_from_name: Expression { + else_: "'DMARC Report'".into(), + ..Default::default() + }, + aggregate_max_report_size: Expression { + else_: "4096".into(), + ..Default::default() + }, + aggregate_org_name: Expression { + else_: "'Foobar, Inc.'".into(), + ..Default::default() + }, + aggregate_send_frequency: Expression { + else_: "daily".into(), + ..Default::default() + }, + ..Default::default() + }) + .await; + admin.mta_no_auth().await; + admin.mta_allow_non_fqdn().await; + admin.mta_allow_relaying().await; + admin.reload_settings().await; + test.reload_core(); + test.expect_reload_settings().await; // Authorize external report for foobar.org - let core = local.build_smtp(); - core.txt_add( + test.server.txt_add( "foobar.org._report._dmarc.foobar.net", Dmarc::parse(b"v=DMARC1;").unwrap(), Instant::now() + Duration::from_secs(10), ); - let qr = &mut local.queue_receiver; // Schedule two events with a same policy and another one with a different policy let dmarc_record = Arc::new( @@ -69,45 +103,47 @@ async fn report_dmarc() { ); assert_eq!(dmarc_record.rua().len(), 2); for _ in 0..2 { - core.schedule_dmarc(Box::new(DmarcEvent { + test.server + .schedule_dmarc(Box::new(DmarcEvent { + domain: "foobar.org".to_string(), + report_record: Record::new() + .with_source_ip("192.168.1.2".parse().unwrap()) + .with_action_disposition(ActionDisposition::Pass) + .with_dmarc_dkim_result(DmarcResult::Pass) + .with_dmarc_spf_result(DmarcResult::Fail) + .with_envelope_from("hello@example.org") + .with_envelope_to("other@example.org") + .with_header_from("bye@example.org"), + dmarc_record: dmarc_record.clone(), + interval: AggregateFrequency::Weekly, + span_id: 0, + })) + .await; + } + test.server + .schedule_dmarc(Box::new(DmarcEvent { domain: "foobar.org".to_string(), report_record: Record::new() - .with_source_ip("192.168.1.2".parse().unwrap()) - .with_action_disposition(ActionDisposition::Pass) - .with_dmarc_dkim_result(DmarcResult::Pass) - .with_dmarc_spf_result(DmarcResult::Fail) - .with_envelope_from("hello@example.org") - .with_envelope_to("other@example.org") - .with_header_from("bye@example.org"), + .with_source_ip("a:b:c::e:f".parse().unwrap()) + .with_action_disposition(ActionDisposition::Reject) + .with_dmarc_dkim_result(DmarcResult::Fail) + .with_dmarc_spf_result(DmarcResult::Pass), dmarc_record: dmarc_record.clone(), interval: AggregateFrequency::Weekly, + span_id: 0, })) .await; - } - core.schedule_dmarc(Box::new(DmarcEvent { - domain: "foobar.org".to_string(), - report_record: Record::new() - .with_source_ip("a:b:c::e:f".parse().unwrap()) - .with_action_disposition(ActionDisposition::Reject) - .with_dmarc_dkim_result(DmarcResult::Fail) - .with_dmarc_spf_result(DmarcResult::Pass), - dmarc_record: dmarc_record.clone(), - interval: AggregateFrequency::Weekly, - })) - .await; tokio::time::sleep(Duration::from_millis(200)).await; - let reports = qr.read_report_events().await; + let reports = test.read_report_events::().await; assert_eq!(reports.len(), 1); - match reports.into_iter().next().unwrap() { - QueueClass::DmarcReportHeader(event) => { - core.send_dmarc_aggregate_report(event).await; - } - _ => unreachable!(), - } + test.server + .send_dmarc_aggregate_report(reports.first().unwrap().0.id()) + .await + .unwrap(); // Expect report - let message = qr.expect_message().await; - qr.assert_no_events(); + let message = test.expect_message().await; + test.assert_no_events(); assert_eq!(message.message.recipients.len(), 1); assert_eq!( message.message.recipients.last().unwrap().address(), @@ -115,15 +151,15 @@ async fn report_dmarc() { ); assert_eq!(message.message.return_path.as_ref(), "reports@example.org"); message - .read_lines(qr) + .read_lines(&test) .await - .assert_contains("DKIM-Signature: v=1; a=rsa-sha256; s=rsa; d=example.com;") + .assert_contains("DKIM-Signature: v=1; a=rsa-sha256; s=rsa; d=example.org;") .assert_contains("To: ") .assert_contains("Report Domain: foobar.org") .assert_contains("Submitter: mx.example.org"); // Verify generated report - let report = Report::parse_rfc5322(message.read_message(qr).await.as_bytes()).unwrap(); + let report = Report::parse_rfc5322(message.read_message(&test).await.as_bytes()).unwrap(); assert_eq!(report.domain(), "foobar.org"); assert_eq!(report.email(), "reports@example.org"); assert_eq!(report.org_name(), "Foobar, Inc."); @@ -132,7 +168,7 @@ async fn report_dmarc() { "https://foobar.org/contact" ); assert_eq!(report.p(), Disposition::Quarantine); - assert_eq!(report.records().len(), 2); + assert_eq!(report.records().len(), 2, "records: {:?}", report.records()); for record in report.records() { let source_ip = record.source_ip().unwrap(); if source_ip == "192.168.1.2".parse::().unwrap() { @@ -148,5 +184,5 @@ async fn report_dmarc() { panic!("unexpected ip {source_ip}"); } } - qr.assert_report_is_empty().await; + test.assert_report_is_empty::().await; } diff --git a/tests/src/smtp/reporting/scheduler.rs b/tests/src/smtp/reporting/scheduler.rs index a66febaa..5c32d474 100644 --- a/tests/src/smtp/reporting/scheduler.rs +++ b/tests/src/smtp/reporting/scheduler.rs @@ -4,68 +4,73 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::sync::Arc; - +use crate::utils::server::TestServerBuilder; use common::{ config::smtp::report::AggregateFrequency, ipc::{DmarcEvent, PolicyType, TlsEvent}, }; use mail_auth::{ common::parse::TxtRecordParser, - dmarc::{Dmarc, URI}, + dmarc::Dmarc, mta_sts::TlsRpt, - report::{ActionDisposition, Alignment, Disposition, DmarcResult, PolicyPublished, Record}, + report::{ActionDisposition, DmarcResult, Record}, +}; +use registry::schema::structs::{ + DmarcInternalReport, DmarcReportSettings, Expression, TlsInternalReport, TlsReportSettings, }; -use store::write::QueueClass; - use smtp::reporting::{dmarc::DmarcReporting, tls::TlsReporting}; - -use crate::smtp::TestSMTP; - -const CONFIG: &str = r#" -[session.rcpt] -relay = true - -[report.dmarc.aggregate] -max-size = 500 -send = "daily" - -[report.tls.aggregate] -max-size = 550 -send = "daily" -"#; +use std::sync::Arc; #[tokio::test] async fn report_scheduler() { - - + let mut test = TestServerBuilder::new("smtp_report_queue_test") + .await + .with_http_listener(19046) + .await + .disable_services() + .capture_queue() + .build() + .await; - // Create scheduler - let local = TestSMTP::new("smtp_report_queue_test", CONFIG).await; - let core = local.build_smtp(); - let qr = &local.queue_receiver; + let admin = test.account("admin"); + admin + .registry_create_object(DmarcReportSettings { + aggregate_max_report_size: Expression { + else_: "500".into(), + ..Default::default() + }, + aggregate_send_frequency: Expression { + else_: "daily".into(), + ..Default::default() + }, + ..Default::default() + }) + .await; + admin + .registry_create_object(TlsReportSettings { + max_report_size: Expression { + else_: "550".into(), + ..Default::default() + }, + send_frequency: Expression { + else_: "daily".into(), + ..Default::default() + }, + ..Default::default() + }) + .await; + admin.mta_no_auth().await; + admin.mta_allow_non_fqdn().await; + admin.mta_allow_relaying().await; + admin.reload_settings().await; + test.reload_core(); + test.expect_reload_settings().await; // Schedule two events with a same policy and another one with a different policy let dmarc_record = Arc::new(Dmarc::parse(b"v=DMARC1; p=quarantine; rua=mailto:dmarc@foobar.org").unwrap()); - core.schedule_dmarc(Box::new(DmarcEvent { - domain: "foobar.org".to_string(), - report_record: Record::new() - .with_source_ip("192.168.1.2".parse().unwrap()) - .with_action_disposition(ActionDisposition::Pass) - .with_dmarc_dkim_result(DmarcResult::Pass) - .with_dmarc_spf_result(DmarcResult::Fail) - .with_envelope_from("hello@example.org") - .with_envelope_to("other@example.org") - .with_header_from("bye@example.org"), - dmarc_record: dmarc_record.clone(), - interval: AggregateFrequency::Weekly, - })) - .await; - - // No records should be added once the 550 bytes max size is reached - for _ in 0..10 { - core.schedule_dmarc(Box::new(DmarcEvent { + test.server + .schedule_dmarc(Box::new(DmarcEvent { domain: "foobar.org".to_string(), report_record: Record::new() .with_source_ip("192.168.1.2".parse().unwrap()) @@ -77,128 +82,108 @@ async fn report_scheduler() { .with_header_from("bye@example.org"), dmarc_record: dmarc_record.clone(), interval: AggregateFrequency::Weekly, + span_id: 0, })) .await; + + // No records should be added once the 550 bytes max size is reached + for _ in 0..10 { + test.server + .schedule_dmarc(Box::new(DmarcEvent { + domain: "foobar.org".to_string(), + report_record: Record::new() + .with_source_ip("192.168.1.2".parse().unwrap()) + .with_action_disposition(ActionDisposition::Pass) + .with_dmarc_dkim_result(DmarcResult::Pass) + .with_dmarc_spf_result(DmarcResult::Fail) + .with_envelope_from("hello@example.org") + .with_envelope_to("other@example.org") + .with_header_from("bye@example.org"), + dmarc_record: dmarc_record.clone(), + interval: AggregateFrequency::Weekly, + span_id: 0, + })) + .await; } let dmarc_record = Arc::new(Dmarc::parse(b"v=DMARC1; p=reject; rua=mailto:dmarc@foobar.org").unwrap()); - core.schedule_dmarc(Box::new(DmarcEvent { - domain: "foobar.org".to_string(), - report_record: Record::new() - .with_source_ip("a:b:c::e:f".parse().unwrap()) - .with_action_disposition(ActionDisposition::Reject) - .with_dmarc_dkim_result(DmarcResult::Fail) - .with_dmarc_spf_result(DmarcResult::Pass), - dmarc_record: dmarc_record.clone(), - interval: AggregateFrequency::Weekly, - })) - .await; + test.server + .schedule_dmarc(Box::new(DmarcEvent { + domain: "foobar.org".to_string(), + report_record: Record::new() + .with_source_ip("a:b:c::e:f".parse().unwrap()) + .with_action_disposition(ActionDisposition::Reject) + .with_dmarc_dkim_result(DmarcResult::Fail) + .with_dmarc_spf_result(DmarcResult::Pass), + dmarc_record: dmarc_record.clone(), + interval: AggregateFrequency::Weekly, + span_id: 0, + })) + .await; // Schedule TLS event let tls_record = Arc::new(TlsRpt::parse(b"v=TLSRPTv1;rua=mailto:reports@foobar.org").unwrap()); - core.schedule_tls(Box::new(TlsEvent { - domain: "foobar.org".to_string(), - policy: PolicyType::Tlsa(None), - failure: None, - tls_record: tls_record.clone(), - interval: AggregateFrequency::Daily, - })) - .await; - core.schedule_tls(Box::new(TlsEvent { - domain: "foobar.org".to_string(), - policy: PolicyType::Tlsa(None), - failure: None, - tls_record: tls_record.clone(), - interval: AggregateFrequency::Daily, - })) - .await; - core.schedule_tls(Box::new(TlsEvent { - domain: "foobar.org".to_string(), - policy: PolicyType::Sts(None), - failure: None, - tls_record: tls_record.clone(), - interval: AggregateFrequency::Daily, - })) - .await; - core.schedule_tls(Box::new(TlsEvent { - domain: "foobar.org".to_string(), - policy: PolicyType::None, - failure: None, - tls_record: tls_record.clone(), - interval: AggregateFrequency::Daily, - })) - .await; + test.server + .schedule_tls(Box::new(TlsEvent { + domain: "foobar.org".to_string(), + policy: PolicyType::Tlsa(None), + failure: None, + tls_record: tls_record.clone(), + interval: AggregateFrequency::Daily, + span_id: 0, + })) + .await; + test.server + .schedule_tls(Box::new(TlsEvent { + domain: "foobar.org".to_string(), + policy: PolicyType::Tlsa(None), + failure: None, + tls_record: tls_record.clone(), + interval: AggregateFrequency::Daily, + span_id: 0, + })) + .await; + test.server + .schedule_tls(Box::new(TlsEvent { + domain: "foobar.org".to_string(), + policy: PolicyType::Sts(None), + failure: None, + tls_record: tls_record.clone(), + interval: AggregateFrequency::Daily, + span_id: 0, + })) + .await; + test.server + .schedule_tls(Box::new(TlsEvent { + domain: "foobar.org".to_string(), + policy: PolicyType::None, + failure: None, + tls_record: tls_record.clone(), + interval: AggregateFrequency::Daily, + span_id: 0, + })) + .await; // Verify sizes and counts let mut total_tls = 0; let mut total_tls_policies = 0; let mut total_dmarc_policies = 0; - let mut last_domain = String::new(); - for report in qr.read_report_events().await { - match report { - QueueClass::DmarcReportHeader(event) => { - total_dmarc_policies += 1; - assert_eq!(event.due - event.seq_id, 7 * 86400); - } - QueueClass::TlsReportHeader(event) => { - if event.domain != last_domain { - last_domain.clone_from(&event.domain); - total_tls += 1; - } - total_tls_policies += 1; - assert_eq!(event.due - event.seq_id, 86400); - } - _ => unreachable!(), - } + for (_, report) in test.read_report_events::().await { + total_dmarc_policies += 1; + assert_eq!( + report.deliver_at.timestamp() - report.created_at.timestamp(), + 7 * 86400 + ); + } + for (_, report) in test.read_report_events::().await { + total_tls += 1; + total_tls_policies += report.report.policies.len(); + assert_eq!( + report.deliver_at.timestamp() - report.created_at.timestamp(), + 86400 + ); } assert_eq!(total_tls, 1); assert_eq!(total_tls_policies, 3); assert_eq!(total_dmarc_policies, 2); } - -#[test] -fn report_strip_json() { - let mut d = DmarcFormat { - rua: vec![ - URI { - uri: "hello".to_string(), - max_size: 0, - }, - URI { - uri: "world".to_string(), - max_size: 0, - }, - ], - policy: PolicyPublished { - domain: "example.org".to_string(), - version_published: None, - adkim: Alignment::Relaxed, - aspf: Alignment::Strict, - p: Disposition::Quarantine, - sp: Disposition::Reject, - testing: false, - fo: None, - }, - records: vec![ - Record::default() - .with_count(1) - .with_envelope_from("domain.net") - .with_envelope_to("other.org"), - ], - }; - let mut s = serde_json::to_string(&d).unwrap(); - s.truncate(s.len() - 2); - - let r = Record::default() - .with_count(2) - .with_envelope_from("otherdomain.net") - .with_envelope_to("otherother.org"); - let rs = serde_json::to_string(&r).unwrap(); - - d.records.push(r); - - assert_eq!( - serde_json::from_str::(&format!("{s},{rs}]}}")).unwrap(), - d - ); -} diff --git a/tests/src/smtp/reporting/tls.rs b/tests/src/smtp/reporting/tls.rs index 47805bb9..db97c37d 100644 --- a/tests/src/smtp/reporting/tls.rs +++ b/tests/src/smtp/reporting/tls.rs @@ -4,8 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{io::Read, sync::Arc, time::Duration}; - +use crate::{ + smtp::{inbound::TestMessage, session::VerifyResponse}, + utils::server::TestServerBuilder, +}; use common::{config::smtp::report::AggregateFrequency, ipc::TlsEvent}; use mail_auth::{ common::parse::TxtRecordParser, @@ -13,56 +15,84 @@ use mail_auth::{ mta_sts::TlsRpt, report::tlsrpt::{FailureDetails, PolicyType, ResultType, TlsReport}, }; -use store::write::QueueClass; - +use registry::schema::structs::{Expression, ReportSettings, TlsInternalReport, TlsReportSettings}; use smtp::reporting::tls::{TLS_HTTP_REPORT, TlsReporting}; - -use crate::smtp::{ - - inbound::{TestMessage, sign::SIGNATURES}, - session::VerifyResponse, -}; - -const CONFIG: &str = r#" -[session.rcpt] -relay = true - -[report] -submitter = "'mx.example.org'" - -[report.tls.aggregate] -from-name = "'Report Subsystem'" -from-address = "'reports@example.org'" -org-name = "'Foobar, Inc.'" -contact-info = "'https://foobar.org/contact'" -send = "daily" -max-size = 1532 -sign = "['rsa']" -"#; +use std::{io::Read, sync::Arc, time::Duration}; #[tokio::test] async fn report_tls() { - - + let mut test = TestServerBuilder::new("smtp_report_tls_test") + .await + .with_http_listener(19047) + .await + .disable_services() + .capture_queue() + .build() + .await; - // Create scheduler - let mut local = TestSMTP::new("smtp_report_tls_test", CONFIG.to_string() + SIGNATURES).await; - let core = local.build_smtp(); - let qr = &mut local.queue_receiver; + let admin = test.account("admin"); + let domain_id = admin.find_or_create_domain("example.org").await; + admin.create_dkim_signatures(domain_id).await; + admin + .registry_create_object(ReportSettings { + outbound_report_submitter: Expression { + else_: "'mx.example.org'".into(), + ..Default::default() + }, + ..Default::default() + }) + .await; + admin + .registry_create_object(TlsReportSettings { + contact_info: Expression { + else_: "'https://foobar.org/contact'".into(), + ..Default::default() + }, + dkim_sign_domain: Expression { + else_: "'example.org'".into(), + ..Default::default() + }, + from_address: Expression { + else_: "'reports@example.org'".into(), + ..Default::default() + }, + from_name: Expression { + else_: "'Report Subsystem'".into(), + ..Default::default() + }, + org_name: Expression { + else_: "'Foobar, Inc.'".into(), + ..Default::default() + }, + send_frequency: Expression { + else_: "daily".into(), + ..Default::default() + }, + ..Default::default() + }) + .await; + admin.mta_no_auth().await; + admin.mta_allow_non_fqdn().await; + admin.mta_allow_relaying().await; + admin.reload_settings().await; + test.reload_core(); + test.expect_reload_settings().await; // Schedule TLS reports to be delivered via email let tls_record = Arc::new(TlsRpt::parse(b"v=TLSRPTv1;rua=mailto:reports@foobar.org").unwrap()); for _ in 0..2 { // Add two successful records - core.schedule_tls(Box::new(TlsEvent { - domain: "foobar.org".to_string(), - policy: common::ipc::PolicyType::None, - failure: None, - tls_record: tls_record.clone(), - interval: AggregateFrequency::Daily, - })) - .await; + test.server + .schedule_tls(Box::new(TlsEvent { + domain: "foobar.org".to_string(), + policy: common::ipc::PolicyType::None, + failure: None, + tls_record: tls_record.clone(), + interval: AggregateFrequency::Daily, + span_id: 0, + })) + .await; } for (policy, rt) in [ @@ -84,48 +114,46 @@ async fn report_tls() { ResultType::StsWebpkiInvalid, ), ] { - core.schedule_tls(Box::new(TlsEvent { - domain: "foobar.org".to_string(), - policy, - failure: FailureDetails::new(rt).into(), - tls_record: tls_record.clone(), - interval: AggregateFrequency::Daily, - })) - .await; + test.server + .schedule_tls(Box::new(TlsEvent { + domain: "foobar.org".to_string(), + policy, + failure: FailureDetails::new(rt).into(), + tls_record: tls_record.clone(), + interval: AggregateFrequency::Daily, + span_id: 0, + })) + .await; } // Wait for flush tokio::time::sleep(Duration::from_millis(200)).await; - let reports = qr.read_report_events().await; - assert_eq!(reports.len(), 3); - let mut tls_reports = Vec::with_capacity(3); - for report in reports { - match report { - QueueClass::TlsReportHeader(event) => { - tls_reports.push(event); - } - _ => unreachable!(), - } - } - core.send_tls_aggregate_report(tls_reports).await; + let reports = test.read_report_events::().await; + assert_eq!(reports.len(), 1); + let (report_id, report) = reports.into_iter().next().unwrap(); + assert_eq!(report.report.policies.len(), 3); + test.server + .send_tls_aggregate_report(report_id.id()) + .await + .unwrap(); // Expect report - let message = qr.expect_message().await; + let message = test.expect_message().await; assert_eq!( message.message.recipients.last().unwrap().address(), "reports@foobar.org" ); assert_eq!(message.message.return_path.as_ref(), "reports@example.org"); message - .read_lines(qr) + .read_lines(&test) .await - .assert_contains("DKIM-Signature: v=1; a=rsa-sha256; s=rsa; d=example.com;") + .assert_contains("DKIM-Signature: v=1; a=rsa-sha256; s=rsa; d=example.org;") .assert_contains("To: ") .assert_contains("Report Domain: foobar.org") .assert_contains("Submitter: mx.example.org"); // Verify generated report - let report = TlsReport::parse_rfc5322(message.read_message(qr).await.as_bytes()).unwrap(); + let report = TlsReport::parse_rfc5322(message.read_message(&test).await.as_bytes()).unwrap(); assert_eq!(report.organization_name.unwrap(), "Foobar, Inc."); assert_eq!(report.contact_info.unwrap(), "https://foobar.org/contact"); assert_eq!(report.policies.len(), 3); @@ -145,10 +173,10 @@ async fn report_tls() { } PolicyType::Sts => { seen[1] = true; - assert_eq!(policy.summary.total_failure, 2); + assert_eq!(policy.summary.total_failure, 3); assert_eq!(policy.summary.total_success, 0); assert_eq!(policy.policy.policy_domain, "foobar.org"); - assert_eq!(policy.failure_details.len(), 2); + assert_eq!(policy.failure_details.len(), 3); assert!( policy .failure_details @@ -161,6 +189,12 @@ async fn report_tls() { .iter() .any(|d| d.result_type == ResultType::StsPolicyInvalid) ); + assert!( + policy + .failure_details + .iter() + .any(|d| d.result_type == ResultType::StsWebpkiInvalid) + ); } PolicyType::NoPolicyFound => { seen[2] = true; @@ -186,24 +220,24 @@ async fn report_tls() { for _ in 0..2 { // Add two successful records - core.schedule_tls(Box::new(TlsEvent { - domain: "foobar.org".to_string(), - policy: common::ipc::PolicyType::None, - failure: None, - tls_record: tls_record.clone(), - interval: AggregateFrequency::Daily, - })) - .await; + test.server + .schedule_tls(Box::new(TlsEvent { + domain: "foobar.org".to_string(), + policy: common::ipc::PolicyType::None, + failure: None, + tls_record: tls_record.clone(), + interval: AggregateFrequency::Daily, + span_id: 0, + })) + .await; } - let reports = qr.read_report_events().await; + let reports = test.read_report_events::().await; assert_eq!(reports.len(), 1); - match reports.into_iter().next().unwrap() { - QueueClass::TlsReportHeader(event) => { - core.send_tls_aggregate_report(vec![event]).await; - } - _ => unreachable!(), - } + test.server + .send_tls_aggregate_report(reports.first().unwrap().0.id()) + .await + .unwrap(); tokio::time::sleep(Duration::from_millis(200)).await; // Uncompress report @@ -217,5 +251,5 @@ async fn report_tls() { assert_eq!(report.contact_info.unwrap(), "https://foobar.org/contact"); assert_eq!(report.policies.len(), 1); } - qr.assert_report_is_empty().await; + test.assert_report_is_empty::().await; } diff --git a/tests/src/utils/registry.rs b/tests/src/utils/registry.rs index c4bbd535..c9cc6ef0 100644 --- a/tests/src/utils/registry.rs +++ b/tests/src/utils/registry.rs @@ -11,7 +11,10 @@ use crate::utils::{ use registry::{ schema::{ prelude::{ObjectType, Property}, - structs::Action, + structs::{ + Action, Expression, MtaExtensions, MtaStageAuth, MtaStageData, MtaStageEhlo, + MtaStageRcpt, SpamSettings, + }, }, types::{EnumImpl, ObjectImpl}, }; @@ -78,8 +81,8 @@ impl Account { let mut items = Vec::with_capacity(response.list().len()); for item in response.list() { let id = item.object_id(); - let item = serde_json::from_str(&item.to_string()).unwrap_or_else(|_| { - panic!("Failed to deserialize {item}"); + let item = serde_json::from_str(&item.to_string()).unwrap_or_else(|err| { + panic!("Failed to deserialize {item} : {err}"); }); items.push((id, item)); } @@ -252,6 +255,128 @@ impl Account { .await .assert_destroyed(&[account_id]); } + + pub async fn mta_allow_relaying(&self) { + self.registry_create_object(MtaStageRcpt { + allow_relaying: Expression { + else_: "true".into(), + ..Default::default() + }, + ..Default::default() + }) + .await; + } + + pub async fn mta_disable_spam_filter(&self) { + self.registry_create_object(SpamSettings { + enable: false, + ..Default::default() + }) + .await; + } + + pub async fn mta_no_auth(&self) { + self.registry_create_object(MtaStageAuth { + require: Expression { + else_: "false".into(), + ..Default::default() + }, + ..Default::default() + }) + .await; + } + + pub async fn mta_all_extensions(&self) { + self.registry_create_object(MtaExtensions { + chunking: Expression { + else_: "true".into(), + ..Default::default() + }, + deliver_by: Expression { + else_: "true".into(), + ..Default::default() + }, + dsn: Expression { + else_: "true".into(), + ..Default::default() + }, + expn: Expression { + else_: "true".into(), + ..Default::default() + }, + future_release: Expression { + else_: "true".into(), + ..Default::default() + }, + mt_priority: Expression { + else_: "true".into(), + ..Default::default() + }, + no_soliciting: Expression { + else_: "true".into(), + ..Default::default() + }, + pipelining: Expression { + else_: "true".into(), + ..Default::default() + }, + require_tls: Expression { + else_: "true".into(), + ..Default::default() + }, + vrfy: Expression { + else_: "true".into(), + ..Default::default() + }, + }) + .await; + } + + pub async fn mta_allow_non_fqdn(&self) { + self.registry_create_object(MtaStageEhlo { + reject_non_fqdn: Expression { + else_: "false".into(), + ..Default::default() + }, + ..Default::default() + }) + .await; + } + + pub async fn mta_add_all_headers(&self) { + self.registry_create_object(MtaStageData { + add_date_header: Expression { + else_: "true".into(), + ..Default::default() + }, + add_message_id_header: Expression { + else_: "true".into(), + ..Default::default() + }, + add_received_header: Expression { + else_: "true".into(), + ..Default::default() + }, + add_received_spf_header: Expression { + else_: "true".into(), + ..Default::default() + }, + add_auth_results_header: Expression { + else_: "true".into(), + ..Default::default() + }, + add_return_path_header: Expression { + else_: "false".into(), + ..Default::default() + }, + enable_spam_filter: Expression { + else_: "false".into(), + ..Default::default() + }, + ..Default::default() + }) + .await; + } } impl JmapResponse { diff --git a/tests/src/utils/server.rs b/tests/src/utils/server.rs index 4a8b08da..cad99810 100644 --- a/tests/src/utils/server.rs +++ b/tests/src/utils/server.rs @@ -41,7 +41,10 @@ use registry::{ schema::{ enums::{DataStoreType, EventPolicy, NetworkListenerProtocol, TracingLevel}, prelude::{Object, SocketAddr}, - structs::{Expression, Http, NetworkListener, Tracer, TracerStdout}, + structs::{ + Certificate, Expression, Http, NetworkListener, PublicText, SecretKeyFile, SecretText, + Tracer, TracerStdout, + }, }, types::{EnumImpl, map::Map}, }; @@ -55,7 +58,7 @@ use smtp::{ }, reporting::scheduler::SpawnReport, }; -use std::{str::FromStr, sync::Arc}; +use std::{path::PathBuf, str::FromStr, sync::Arc}; use store::{ RegistryStore, Store, ValueKey, registry::{bootstrap::Bootstrap, write::RegistryWrite}, @@ -161,6 +164,32 @@ impl TestServerBuilder { .await } + pub async fn with_smtp_listener(self, port: u16) -> Self { + self.with_listener(NetworkListenerProtocol::Smtp, "smtp", port, false) + .await + } + + pub async fn with_dummy_tls_cert(self) -> Self { + let mut cert_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + cert_path.push("resources"); + let mut cert = cert_path.clone(); + cert.push("tls_cert.pem"); + let mut pk = cert_path.clone(); + pk.push("tls_privatekey.pem"); + + self.with_object(Certificate { + private_key: SecretText::File(SecretKeyFile { + file_path: pk.to_string_lossy().to_string(), + }), + certificate: PublicText::File(SecretKeyFile { + file_path: cert.to_string_lossy().to_string(), + }), + + ..Default::default() + }) + .await + } + pub async fn with_listener( self, protocol: NetworkListenerProtocol,