filip/Stalwart
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Registry testing - part 15

Browse Source
This commit is contained in:
Maurus Decimus
2026-03-25 18:55:39 +01:00
parent 1b5334b65c
commit 0fde3e04fd
20 changed files with 1084 additions and 665 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tests/resources/smtp/antispam/rbl.test
View File

@@ -67,5 +67,5 @@ Content-Type: text/html; charset="utf-8"
<html>
<a href="https://bit.ly/abcde">test</a>
<img src="https://drive.google.com/path/to/file.exe">https://lnkiy.in/other/path?query=true</a<
<img src="https://drive.google.com/path/to/file.exe">https://cf-ipfs.com/other/path?query=true</a<
</html>

269
tests/src/smtp/inbound/antispam.rs
View File

@@ -4,24 +4,20 @@
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use crate::{
use crate::utils::{
dns::DnsCache,
http_server::{HttpMessage, spawn_mock_http_server},
jmap::server::enterprise::EnterpriseCore,
smtp::{DnsCache, session::TestSession},
server::TestServerBuilder,
};
use ahash::{AHashMap, AHashSet};
use ahash::AHashSet;
use common::{
Core, Server,
auth::AccessToken,
enterprise::{
SpamFilterLlmConfig,
llm::{
AiApiConfig, ChatCompletionChoice, ChatCompletionRequest, ChatCompletionResponse,
Message,
},
Server,
auth::{AccountCache, AccountInfo},
config::mailstore::spamfilter::SpamFilterAction,
enterprise::llm::{
ChatCompletionChoice, ChatCompletionRequest, ChatCompletionResponse, Message,
},
};
use email::message::ingest::EmailIngest;
use http_proto::{JsonResponse, ToHttpResponse};
use hyper::Method;
use mail_auth::{
@@ -29,7 +25,17 @@ use mail_auth::{
SpfResult, dkim::Signature, dmarc::Policy,
};
use mail_parser::MessageParser;
use smtp::core::{Session, SessionAddress};
use registry::{
schema::{
enums::{AiModelType, TaskSpamFilterMaintenanceType},
structs::{
self, AiModel, MemoryLookupKey, SpamLlm, SpamLlmProperties, SpamSettings,
SpamTrainingSample, Task, TaskSpamFilterMaintenance, TaskStatus,
},
},
types::{float::Float, map::Map},
};
use smtp::core::SessionAddress;
use smtp_proto::{MAIL_BODY_8BITMIME, MAIL_SMTPUTF8};
use spam_filter::{
SpamFilterInput,
@@ -171,46 +177,95 @@ allow-invalid-certs = true
#[tokio::test(flavor = "multi_thread")]
async fn antispam() {
// Prepare config
let tmp_dir = TempDir::new("smtp_antispam_test", true);
let mut config = CONFIG.replace("{PATH}", tmp_dir.temp_dir.as_path().to_str().unwrap());
let base_path = PathBuf::from(
std::env::var("SPAM_RULES_DIR")
.unwrap_or_else(|_| "/Users/me/code/spam-filter".to_string()),
);
for section in ["rules", "lists"] {
for entry in fs::read_dir(base_path.join(section)).unwrap() {
let entry = entry.unwrap();
let path = entry.path();
if path.is_file() {
let file_name = path.file_name().unwrap().to_str().unwrap();
if file_name.ends_with(".toml")
&& ((section == "rules" && file_name != "llm.toml")
|| (section == "lists" && file_name == "scores.toml"))
{
let contents = fs::read_to_string(&path).unwrap();
config.push_str("\n\n");
config.push_str(&contents);
}
}
}
}
// Parse config
let mut config = Config::new(&config).unwrap();
config.resolve_all_macros().await;
let stores = Stores::parse_all(&mut config, false).await;
let mut core = Core::parse(&mut config, stores, Default::default())
let mut test = TestServerBuilder::new("smtp_antispam_test")
.await
.enable_enterprise();
let ai_apis = AHashMap::from_iter([(
"dummy".to_string(),
AiApiConfig::parse(&mut config, "dummy").unwrap().into(),
)]);
core.enterprise.as_mut().unwrap().spam_filter_llm =
SpamFilterLlmConfig::parse(&mut config, &ai_apis);
crate::AssertConfig::assert_no_errors(config);
let server = TestSMTP::from_core(core).server;
.with_http_listener(19048)
.await
.build()
.await;
let admin = test.account("admin");
admin
.registry_create_object(SpamSettings {
score_spam: Float::new(5.0),
spam_filter_rules_url: std::env::var("SPAM_RULES_URL")
.unwrap_or_else(|_| {
"file:///Users/me/code/spam-filter/spam-filter-rules.json.gz".to_string()
})
.into(),
..Default::default()
})
.await;
admin
.registry_create_object(structs::SpamClassifier {
min_ham_samples: 10,
min_spam_samples: 10,
..Default::default()
})
.await;
let model_id = admin
.registry_create_object(AiModel {
class: AiModelType::Chat,
allow_invalid_certs: true,
model: "gpt-dummy".to_string(),
name: "dummy".to_string(),
url: "https://127.0.0.1:9090/v1/chat/completions".to_string(),
..Default::default()
})
.await;
admin
.registry_create_object(SpamLlm::Enable(SpamLlmProperties {
categories: Map::new(vec![
"Unsolicited".to_string(),
"Commercial".to_string(),
"Harmful".to_string(),
"Legitimate".to_string(),
]),
confidence: Map::new(vec![
"High".to_string(),
"Medium".to_string(),
"Low".to_string(),
]),
model_id,
prompt: "You are an AI assistant specialized in analyzing email content to detect spam"
.to_string(),
response_pos_category: 0,
response_pos_confidence: 1.into(),
response_pos_explanation: 2.into(),
separator: ",".to_string(),
..Default::default()
}))
.await;
admin
.registry_create_object(MemoryLookupKey {
is_glob_pattern: true,
key: "spamtrap@*".into(),
namespace: "spam-traps".into(),
})
.await;
admin
.registry_create_object(MemoryLookupKey {
is_glob_pattern: true,
key: "redirect.*".into(),
namespace: "url-redirectors".into(),
})
.await;
admin.mta_allow_relaying().await;
admin.mta_no_auth().await;
admin.mta_allow_non_fqdn().await;
admin.reload_settings().await;
// Fetch rules
admin
.registry_create_object(Task::SpamFilterMaintenance(TaskSpamFilterMaintenance {
maintenance_type: TaskSpamFilterMaintenanceType::UpdateRules,
status: TaskStatus::now(),
}))
.await;
test.wait_for_tasks().await;
admin.reload_settings().await;
test.reload_core();
let admin = test.account("admin");
// Add mock DNS entries
for (domain, ip) in [
@@ -245,12 +300,12 @@ async fn antispam() {
"127.0.0.8",
),
] {
server.ipv4_add(
test.server.ipv4_add(
domain,
vec![ip.parse().unwrap()],
Instant::now() + Duration::from_secs(100),
);
server.dnsbl_add(
test.server.dnsbl_add(
domain,
vec![ip.parse().unwrap()],
Instant::now() + Duration::from_secs(100),
@@ -262,10 +317,10 @@ async fn antispam() {
"gmail.com",
"custom.disposable.org",
] {
server.mx_add(
test.server.mx_add(
mx,
vec![MX {
exchanges: vec!["127.0.0.1".parse().unwrap()],
exchanges: vec!["127.0.0.1".into()].into_boxed_slice(),
preference: 10,
}],
Instant::now() + Duration::from_secs(100),
@@ -273,31 +328,35 @@ async fn antispam() {
}
// Spawn mock OpenAI server
let _tx = spawn_mock_http_server(Arc::new(|req: HttpMessage| {
assert_eq!(req.uri.path(), "/v1/chat/completions");
assert_eq!(req.method, Method::POST);
let req =
serde_json::from_slice::<ChatCompletionRequest>(req.body.as_ref().unwrap()).unwrap();
assert_eq!(req.model, "gpt-dummy");
let message = &req.messages[0].content;
assert!(message.contains("You are an AI assistant specialized in analyzing email"));
let _tx = spawn_mock_http_server(
&test,
Arc::new(|req: HttpMessage| {
assert_eq!(req.uri.path(), "/v1/chat/completions");
assert_eq!(req.method, Method::POST);
let req = serde_json::from_slice::<ChatCompletionRequest>(req.body.as_ref().unwrap())
.unwrap();
assert_eq!(req.model, "gpt-dummy");
let message = &req.messages[0].content;
assert!(message.contains("You are an AI assistant specialized in analyzing email"));
JsonResponse::new(&ChatCompletionResponse {
created: 0,
object: String::new(),
id: String::new(),
model: req.model,
choices: vec![ChatCompletionChoice {
index: 0,
finish_reason: "stop".to_string(),
message: Message {
role: "assistant".to_string(),
content: message.split_once("Subject: ").unwrap().1.to_string(),
},
}],
})
.into_http_response()
}))
JsonResponse::new(&ChatCompletionResponse {
created: 0,
object: String::new(),
id: String::new(),
model: req.model,
choices: vec![ChatCompletionChoice {
index: 0,
finish_reason: "stop".to_string(),
message: Message {
role: "assistant".to_string(),
content: message.split_once("Subject: ").unwrap().1.to_string(),
},
}],
})
.into_http_response()
}),
9090,
)
.await;
// Run tests
@@ -348,12 +407,10 @@ async fn antispam() {
continue;
}
"classifier_features" => {
classifier_features(&server, contents).await;
classifier_features(&test.server, contents).await;
continue;
}
"classifier" => {
let mut batch = BatchBuilder::new();
batch.with_account_id(u32::MAX);
for class in ["spam", "ham"] {
let contents =
fs::read_to_string(base_path.join(format!("classifier.{class}"))).unwrap();
@@ -363,17 +420,32 @@ async fn antispam() {
continue;
}
let (hash, blob_hold) = server
.put_temporary_blob(u32::MAX, sample.as_bytes(), 60)
let blob_id = test
.server
.put_jmap_blob(u32::MAX, sample.as_bytes())
.await
.unwrap();
server.add_spam_sample(&mut batch, hash, class == "spam", true, 0);
batch.clear(blob_hold);
admin
.registry_create_object(SpamTrainingSample {
blob_id,
from: "unknown".to_string(),
is_spam: class == "spam",
subject: "unknown".to_string(),
..Default::default()
})
.await;
}
}
assert!(!batch.is_empty());
server.store().write(batch.build_all()).await.unwrap();
server.spam_train(false).await.unwrap();
admin
.registry_create_object(Task::SpamFilterMaintenance(
TaskSpamFilterMaintenance {
maintenance_type: TaskSpamFilterMaintenanceType::Train,
status: TaskStatus::now(),
},
))
.await;
test.wait_for_tasks().await;
}
_ => {}
}
@@ -414,10 +486,14 @@ async fn antispam() {
session.data.helo_domain = value.to_string();
}
"authenticated_as" => {
session.data.authenticated_as = Some(Arc::new(AccessToken {
name: value.to_string(),
..Default::default()
}));
session.data.authenticated_as = Some(AccountInfo {
account_id: u32::MAX,
addresses: vec![value.to_string()],
account: Arc::new(AccountCache {
name: value.into(),
..Default::default()
}),
});
}
"spf.result" | "spf_ehlo.result" => {
session.data.spf_mail_from =
@@ -475,7 +551,7 @@ async fn antispam() {
result: IprevResult::None,
ptr: None,
})
.ptr = Some(Arc::new(vec![value.to_string()]));
.ptr = Some(Arc::from(vec![value.into()]));
}
"dmarc.result" => {
dmarc_result = DmarcResult::from_str(value).into();
@@ -586,6 +662,7 @@ async fn antispam() {
dmarc_policy.as_ref(),
);
spam_input.is_tls = is_tls;
let server = &test.server;
let mut spam_ctx = server.spam_filter_init(spam_input);
match test_name {
"html" => {

7
tests/src/smtp/inbound/mod.rs
View File

@@ -6,7 +6,6 @@
use crate::utils::server::TestServer;
use common::{
Server,
config::smtp::queue::QueueName,
ipc::{DmarcEvent, QueueEvent, QueueEventStatus, ReportingEvent, TlsEvent},
};
@@ -20,6 +19,7 @@ use store::{
use tokio::sync::mpsc::error::TryRecvError;
use types::id::Id;
pub mod antispam;
pub mod asn;
pub mod auth;
pub mod basic;
@@ -36,13 +36,8 @@ pub mod sign;
pub mod throttle;
pub mod vrfy;
/*
pub mod antispam;
*/
impl TestServer {
pub async fn read_event(&mut self) -> QueueEvent {
let todo = "fix antispam tests";
match tokio::time::timeout(Duration::from_millis(100), self.queue_rx.recv()).await {
Ok(Some(event)) => event,
Ok(None) => panic!("Channel closed."),

572
tests/src/smtp/management/queue.rs
View File

@@ -5,94 +5,143 @@
*/
use crate::{
jmap::ManagementApi,
smtp::{DnsCache, session::TestSession},
smtp::session::TestSession,
utils::{dns::DnsCache, server::TestServerBuilder},
};
use ahash::{AHashMap, HashMap, HashSet};
use common::config::server::ServerProtocol;
use mail_auth::MX;
use mail_parser::DateTime;
use reqwest::{Method, StatusCode, header::AUTHORIZATION};
use smtp::queue::{QueueId, Status, manager::SpawnQueue};
use registry::{
schema::{
enums::NetworkListenerProtocol,
prelude::{ObjectType, Property},
structs::{
Expression, MtaDeliveryExpiration, MtaDeliveryExpirationTtl, MtaDeliverySchedule,
MtaDeliveryScheduleInterval, MtaDeliveryScheduleIntervals,
MtaDeliveryScheduleIntervalsOrDefault, MtaExtensions, MtaOutboundStrategy,
MtaStageRcpt, MtaVirtualQueue, QueueExpiry, QueuedMessage, RecipientStatus,
},
},
types::{EnumImpl, datetime::UTCDateTime, list::List},
};
use serde_json::json;
use std::time::{Duration, Instant};
const LOCAL: &str = r#"
[storage]
directory = "local"
[directory."local"]
type = "memory"
[[directory."local".principals]]
name = "admin"
type = "admin"
description = "Superuser"
secret = "secret"
class = "admin"
[queue.schedule.default]
retry = "1000s"
notify = "2000s"
expire = "3000s"
queue-name = "default"
[session.rcpt]
relay = true
max-recipients = 100
[session.extensions]
dsn = true
future-release = "1h"
"#;
const REMOTE: &str = r#"
[session.ehlo]
reject-non-fqdn = false
[session.rcpt]
relay = true
"#;
#[derive(serde::Deserialize, Debug)]
#[allow(dead_code)]
pub(super) struct List<T> {
pub items: Vec<T>,
pub total: usize,
}
#[tokio::test]
#[serial_test::serial]
async fn manage_queue() {
let mut local = TestServerBuilder::new("smtp_manage_queue_local")
.await
.with_http_listener(19049)
.await
.disable_services()
.build()
.await;
let mut remote = TestServerBuilder::new("smtp_manage_queue_remote")
.await
.with_dummy_tls_cert()
.await
.with_http_listener(19050)
.await
.with_listener(NetworkListenerProtocol::Smtp, "smtp-debug", 9925, false)
.await
.disable_services()
.capture_queue()
.build()
.await;
// Start remote test server
let mut remote = TestSMTP::new("smtp_manage_queue_remote", REMOTE).await;
let _rx = remote.start(&[ServerProtocol::Smtp]).await;
let remote_core = remote.build_smtp();
let remote_admin = remote.account("admin");
remote_admin.mta_allow_relaying().await;
remote_admin.mta_no_auth().await;
remote_admin.mta_allow_non_fqdn().await;
remote_admin.reload_settings().await;
remote.reload_core();
remote.expect_reload_settings().await;
// Start local management interface
let local = TestSMTP::new("smtp_manage_queue_local", LOCAL).await;
let admin = local.account("admin");
admin
.registry_create_object(MtaExtensions {
dsn: Expression {
else_: "true".into(),
..Default::default()
},
future_release: Expression {
else_: "1h".into(),
..Default::default()
},
..Default::default()
})
.await;
admin
.registry_create_object(MtaStageRcpt {
max_recipients: Expression {
else_: "100".into(),
..Default::default()
},
allow_relaying: Expression {
else_: "true".into(),
..Default::default()
},
..Default::default()
})
.await;
let queue_id = admin
.registry_create_object(MtaVirtualQueue {
name: "default".into(),
threads_per_node: 25,
description: None,
})
.await;
admin
.registry_create_object(MtaDeliverySchedule {
name: "default".into(),
retry: MtaDeliveryScheduleIntervalsOrDefault::Custom(MtaDeliveryScheduleIntervals {
intervals: List::from_iter([MtaDeliveryScheduleInterval {
duration: 1_000_000u64.into(),
}]),
}),
notify: MtaDeliveryScheduleIntervalsOrDefault::Custom(MtaDeliveryScheduleIntervals {
intervals: List::from_iter([MtaDeliveryScheduleInterval {
duration: 2_000_000u64.into(),
}]),
}),
expiry: MtaDeliveryExpiration::Ttl(MtaDeliveryExpirationTtl {
expire: 3_000_000u64.into(),
}),
queue_id,
description: None,
})
.await;
admin
.registry_create_object(MtaOutboundStrategy {
schedule: Expression {
else_: "'default'".into(),
..Default::default()
},
..Default::default()
})
.await;
admin.mta_no_auth().await;
admin.mta_allow_non_fqdn().await;
admin.reload_settings().await;
local.reload_core();
let admin = local.account("admin");
// Add mock DNS entries
let core = local.build_smtp();
core.mx_add(
local.server.mx_add(
"foobar.org",
vec![MX {
exchanges: vec!["mx1.foobar.org".to_string()],
exchanges: vec!["mx1.foobar.org".into()].into_boxed_slice(),
preference: 10,
}],
Instant::now() + Duration::from_secs(10),
);
core.ipv4_add(
local.server.ipv4_add(
"mx1.foobar.org",
vec!["127.0.0.1".parse().unwrap()],
Instant::now() + Duration::from_secs(10),
);
let _rx_manage = local.start(&[ServerProtocol::Http]).await;
// Send test messages
let envelopes = HashMap::from_iter([
(
@@ -102,7 +151,7 @@ async fn manage_queue() {
vec![
"rcpt1@example1.org",
"rcpt1@example2.org",
"rcpt1@example2.org",
"rcpt2@example2.org",
],
),
),
@@ -130,11 +179,7 @@ async fn manage_queue() {
("e", ("bill5@foobar.net", vec!["john@foobar.org"])),
("f", ("", vec!["success@foobar.org", "delay@foobar.org"])),
]);
let mut session = local.new_session();
local
.queue_receiver
.queue_rx
.spawn(local.server.inner.clone());
let mut session = local.new_mta_session();
session.data.remote_ip_str = "10.0.0.1".into();
session.eval_session_params().await;
session.ehlo("foobar.net").await;
@@ -160,8 +205,7 @@ async fn manage_queue() {
tokio::time::sleep(Duration::from_millis(100)).await;
assert_eq!(
remote
.queue_receiver
.consume_message(&remote_core)
.consume_message()
.await
.message
.recipients
@@ -172,26 +216,30 @@ async fn manage_queue() {
);
// Fetch and validate messages
let api = ManagementApi::default();
let ids = api
.request::<List<QueueId>>(Method::GET, "/api/queue/messages")
.await
.unwrap()
.unwrap_data()
.items;
assert_eq!(ids.len(), 6);
assert_eq!(
admin
.registry_query_ids(
ObjectType::QueuedMessage,
Vec::<(&str, &str)>::new(),
Vec::<&str>::new()
)
.await
.len(),
6
);
let messages = admin.registry_get_all::<QueuedMessage>().await;
assert_eq!(messages.len(), 6);
let mut id_map = AHashMap::new();
let mut id_map_rev = AHashMap::new();
let mut test_search = String::new();
for (message, id) in api.get_messages(&ids).await.into_iter().zip(ids) {
let message = message.unwrap();
for (id, message) in messages {
let env_id = message.env_id.as_ref().unwrap().clone();
// Validate return path and recipients
let (sender, recipients) = envelopes.get(env_id.as_str()).unwrap();
assert_eq!(&message.return_path, sender);
'outer: for recipient in recipients {
for rcpt in &message.recipients {
for rcpt in message.recipients.iter() {
if &rcpt.address == recipient {
continue 'outer;
}
@@ -200,45 +248,43 @@ async fn manage_queue() {
}
// Validate status and datetimes
let created = message.created.to_timestamp();
let created = message.created_at.timestamp();
let hold_for = (env_id.as_bytes().first().unwrap() - b'a' + 1) as i64 * 100;
let next_retry = created + hold_for;
let next_notify = created + 2000 + hold_for;
let expires = created + 3000 + hold_for;
for rcpt in &message.recipients {
for rcpt in message.recipients.iter() {
if env_id == "c" {
let mut dt = *rcpt.next_retry.as_ref().unwrap();
dt.second -= 1;
test_search = dt.to_rfc3339();
let mut dt = rcpt.retry_due;
dt.add_seconds(-1);
test_search = dt.to_string();
}
if env_id != "f" {
// HOLDFOR messages
assert_eq!(rcpt.retry_num, 0);
assert_eq!(rcpt.retry_count, 0);
assert_timestamp(rcpt.retry_due.timestamp(), next_retry, "retry", &message);
assert_timestamp(rcpt.notify_due.timestamp(), next_notify, "notify", &message);
assert_timestamp(
rcpt.next_retry.as_ref().unwrap(),
next_retry,
"retry",
match &rcpt.expires {
QueueExpiry::Ttl(ttl) => ttl.expires_at.timestamp(),
QueueExpiry::Attempts(_) => unreachable!(),
},
expires,
"expires",
&message,
);
assert_timestamp(
rcpt.next_notify.as_ref().unwrap(),
next_notify,
"notify",
&message,
);
assert_timestamp(&rcpt.expires.unwrap(), expires, "expires", &message);
assert_eq!(&rcpt.status, &Status::Scheduled, "{message:#?}");
assert_eq!(&rcpt.status, &RecipientStatus::Scheduled, "{message:#?}");
} else if rcpt.address == "success@foobar.org" {
assert_eq!(rcpt.retry_num, 0);
assert_eq!(rcpt.retry_count, 0);
assert!(
matches!(&rcpt.status, Status::Completed(_)),
matches!(&rcpt.status, RecipientStatus::Completed(_)),
"{:?}",
rcpt.status
);
} else {
assert_eq!(rcpt.retry_num, 1);
assert_eq!(rcpt.retry_count, 1);
assert!(
matches!(&rcpt.status, Status::TemporaryFailure(_)),
matches!(&rcpt.status, RecipientStatus::TemporaryFailure(_)),
"{:?}",
rcpt.status
);
@@ -253,67 +299,66 @@ async fn manage_queue() {
// Test list search
for (query, expected_ids) in [
(
"/api/queue/messages?from=bill1@foobar.net".to_string(),
vec![(Property::ReturnPath.as_str(), "bill1@foobar.net")],
vec!["a"],
),
(
"/api/queue/messages?to=foobar.org".to_string(),
vec![(Property::To.as_str(), "foobar.org")],
vec!["d", "e", "f"],
),
(
"/api/queue/messages?from=bill3@foobar.net&to=rcpt5@example1.com".to_string(),
vec![
(Property::ReturnPath.as_str(), "bill3@foobar.net"),
(Property::To.as_str(), "rcpt5@example1.com"),
],
vec!["c"],
),
(
format!("/api/queue/messages?before={test_search}"),
vec![("dueIsLessThan", test_search.as_str())],
vec!["a", "b"],
),
(
format!("/api/queue/messages?after={test_search}"),
vec![("dueIsGreaterThan", test_search.as_str())],
vec!["d", "e", "f", "c"],
),
] {
let expected_ids = HashSet::from_iter(expected_ids.into_iter().map(|s| s.to_string()));
let ids = api
.request::<List<QueueId>>(Method::GET, &query)
.await
.unwrap()
.unwrap_data()
.items
.into_iter()
.map(|id| id_map_rev.get(&id).unwrap().clone())
.collect::<HashSet<_>>();
assert_eq!(ids, expected_ids, "failed for {query}");
let ids = admin
.registry_query_ids(ObjectType::QueuedMessage, query.clone(), Vec::<&str>::new())
.await;
assert_eq!(
HashSet::from_iter(ids.iter().map(|id| id_map_rev.get(id).unwrap().as_str())),
HashSet::from_iter(expected_ids.into_iter()),
"failed for query {query:?}"
);
}
// Retry delivery
for id in [id_map.get("e").unwrap(), id_map.get("f").unwrap()] {
assert!(
api.request::<bool>(Method::PATCH, &format!("/api/queue/messages/{id}",))
.await
.unwrap()
.unwrap_data(),
);
}
assert!(
api.request::<bool>(
Method::PATCH,
&format!(
"/api/queue/messages/{}?filter=example1.org&at=2200-01-01T00:00:00Z",
id_map.get("a").unwrap(),
for id in [id_map["e"], id_map["f"]] {
admin
.registry_update_object(
ObjectType::QueuedMessage,
id,
json!({
"recipients/0/retryDue": UTCDateTime::now()
}),
)
.await;
}
admin
.registry_update_object(
ObjectType::QueuedMessage,
id_map["a"],
json!({
"recipients/0/retryDue": "2200-01-01T00:00:00Z",
}),
)
.await
.unwrap()
.unwrap_data()
);
.await;
// Expect delivery to john@foobar.org
tokio::time::sleep(Duration::from_millis(100)).await;
assert_eq!(
remote
.queue_receiver
.consume_message(&remote_core)
.consume_message()
.await
.message
.recipients
@@ -323,30 +368,35 @@ async fn manage_queue() {
vec!["john@foobar.org".to_string()]
);
// Message 'e' should be gone, 'f' should have retry_num == 2
// while 'a' should have a retry time of 2200-01-01T00:00:00Z for example1.org
let mut messages = api
.get_messages(&[
*id_map.get("e").unwrap(),
*id_map.get("f").unwrap(),
*id_map.get("a").unwrap(),
])
.await
.into_iter();
assert_eq!(messages.next().unwrap(), None);
// Message 'e' should be gone, 'f' should have retry_count == 2
// while 'a' should have a retry time of 2200-01-01T00:00:00Z
assert_eq!(
messages
admin
.registry_get_many(ObjectType::QueuedMessage, [id_map["e"]])
.await
.not_found()
.next()
.unwrap(),
id_map["e"].to_string()
);
assert_eq!(
admin
.registry_get::<QueuedMessage>(id_map["f"])
.await
.recipients
.values()
.next()
.unwrap()
.unwrap()
.recipients
.first()
.unwrap()
.retry_num,
.retry_count,
2
);
for rcpt in messages.next().unwrap().unwrap().recipients {
let next_retry = rcpt.next_retry.as_ref().unwrap().to_rfc3339();
for rcpt in admin
.registry_get::<QueuedMessage>(id_map["a"])
.await
.recipients
.values()
{
let next_retry = rcpt.retry_due.to_string();
let matched =
["2200-01-01T00:00:00Z", "2199-12-31T23:59:59Z"].contains(&next_retry.as_str());
if rcpt.address.ends_with("example1.org") {
@@ -357,141 +407,96 @@ async fn manage_queue() {
}
// Cancel deliveries
for (id, filter) in [
("a", "example2.org"),
("b", "example1.net"),
("c", "rcpt6@example2.com"),
("d", ""),
] {
assert!(
api.request::<bool>(
Method::DELETE,
&format!(
"/api/queue/messages/{}{}{}",
id_map.get(id).unwrap(),
if !filter.is_empty() { "?filter=" } else { "" },
filter
)
for (id, filter) in [("a", &[1, 2][..]), ("b", &[0, 1][..]), ("c", &[1][..])] {
let mut map = serde_json::Map::new();
for i in filter {
map.insert(format!("recipients/{i}"), serde_json::Value::Null);
}
admin
.registry_update_object(
ObjectType::QueuedMessage,
id_map[id],
serde_json::Value::Object(map),
)
.await;
}
admin
.registry_destroy(ObjectType::QueuedMessage, [id_map["d"]])
.await
.assert_destroyed(&[id_map["d"]]);
tokio::time::sleep(Duration::from_millis(200)).await;
assert_eq!(admin.registry_get_all::<QueuedMessage>().await.len(), 3);
assert_eq!(
admin
.registry_query_ids(
ObjectType::QueuedMessage,
Vec::<(&str, &str)>::new(),
Vec::<&str>::new()
)
.await
.unwrap()
.unwrap_data(),
"failed for {id}: {filter}"
);
}
assert_eq!(
api.request::<List<QueueId>>(Method::GET, "/api/queue/messages")
.await
.unwrap()
.unwrap_data()
.items
.len(),
3
);
for (message, id) in api
.get_messages(&[
*id_map.get("a").unwrap(),
*id_map.get("b").unwrap(),
*id_map.get("c").unwrap(),
*id_map.get("d").unwrap(),
])
.await
.into_iter()
.zip(["a", "b", "c", "d"])
{
if ["b", "d"].contains(&id) {
assert_eq!(message, None);
} else {
let message = message.unwrap();
assert!(!message.recipients.is_empty());
for rcpt in message.recipients {
match id {
"a" => {
if rcpt.address.ends_with("example2.org") {
assert!(matches!(&rcpt.status, Status::PermanentFailure(_)));
} else {
assert!(matches!(&rcpt.status, Status::Scheduled));
}
for id in ["b", "d"] {
assert_eq!(
admin
.registry_get_many(ObjectType::QueuedMessage, [id_map[id]])
.await
.not_found()
.next()
.unwrap(),
id_map[id].to_string()
);
}
for id in ["a", "c"] {
let message = admin.registry_get::<QueuedMessage>(id_map[id]).await;
assert!(!message.recipients.is_empty());
for rcpt in message.recipients {
match id {
"a" => {
if rcpt.address.ends_with("example2.org") {
assert!(matches!(&rcpt.status, RecipientStatus::PermanentFailure(_)));
} else {
assert!(matches!(&rcpt.status, RecipientStatus::Scheduled));
}
"c" => {
if rcpt.address.ends_with("example2.com") {
if rcpt.address == "rcpt6@example2.com" {
assert!(matches!(&rcpt.status, Status::PermanentFailure(_)));
} else {
assert!(matches!(&rcpt.status, Status::Scheduled));
}
} else {
assert!(matches!(&rcpt.status, Status::Scheduled));
}
}
_ => unreachable!(),
}
"c" => {
if rcpt.address.ends_with("example2.com") {
if rcpt.address == "rcpt6@example2.com" {
assert!(matches!(&rcpt.status, RecipientStatus::PermanentFailure(_)));
} else {
assert!(matches!(&rcpt.status, RecipientStatus::Scheduled));
}
} else {
assert!(matches!(&rcpt.status, RecipientStatus::Scheduled));
}
}
_ => unreachable!(),
}
}
}
// Bulk cancel
admin.registry_destroy_all(ObjectType::QueuedMessage).await;
assert_eq!(
api.request::<List<Message>>(Method::GET, "/api/queue/messages?values=1")
admin
.registry_query_ids(
ObjectType::QueuedMessage,
Vec::<(&str, &str)>::new(),
Vec::<&str>::new()
)
.await
.unwrap()
.unwrap_data()
.items
.len(),
3
);
assert!(
api.request::<bool>(Method::DELETE, "/api/queue/messages?text=example2.com")
.await
.unwrap()
.unwrap_data()
);
tokio::time::sleep(Duration::from_millis(100)).await;
assert_eq!(
api.request::<List<QueueId>>(Method::GET, "/api/queue/messages")
.await
.unwrap()
.unwrap_data()
.items
.len(),
2
);
assert!(
api.request::<bool>(Method::DELETE, "/api/queue/messages")
.await
.unwrap()
.unwrap_data()
);
tokio::time::sleep(Duration::from_millis(100)).await;
assert_eq!(
api.request::<List<QueueId>>(Method::GET, "/api/queue/messages")
.await
.unwrap()
.unwrap_data()
.items
.len(),
0
);
// Test authentication error
assert_eq!(
reqwest::Client::builder()
.timeout(Duration::from_millis(500))
.danger_accept_invalid_certs(true)
.build()
.unwrap()
.get("https://127.0.0.1:9980/api/queue/messages")
.header(AUTHORIZATION, "Basic YWRtaW46aGVsbG93b3JsZA==")
.send()
.await
.unwrap()
.status(),
StatusCode::UNAUTHORIZED
);
}
fn assert_timestamp(timestamp: &DateTime, expected: i64, ctx: &str, message: &Message) {
let timestamp = timestamp.to_timestamp();
fn assert_timestamp(timestamp: i64, expected: i64, ctx: &str, message: &QueuedMessage) {
let diff = timestamp - expected;
if ![-2, -1, 0, 1, 2].contains(&diff) {
panic!(
@@ -499,20 +504,3 @@ fn assert_timestamp(timestamp: &DateTime, expected: i64, ctx: &str, message: &Me
);
}
}
impl ManagementApi {
async fn get_messages(&self, ids: &[QueueId]) -> Vec<Option<Message>> {
let mut results = Vec::with_capacity(ids.len());
for id in ids {
let message = self
.request::<Message>(Method::GET, &format!("/api/queue/messages/{id}",))
.await
.unwrap()
.try_unwrap_data();
results.push(message);
}
results
}
}

419
tests/src/smtp/management/report.rs
View File

@@ -4,13 +4,10 @@
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use crate::{
jmap::ManagementApi,
smtp::{ management::queue::List},
};
use ahash::{AHashMap, HashSet};
use crate::utils::server::TestServerBuilder;
use ahash::AHashMap;
use common::{
config::{server::ServerProtocol, smtp::report::AggregateFrequency},
config::smtp::report::AggregateFrequency,
ipc::{DmarcEvent, PolicyType, TlsEvent},
};
use mail_auth::{
@@ -22,235 +19,239 @@ use mail_auth::{
tlsrpt::{FailureDetails, ResultType},
},
};
use reqwest::Method;
use smtp::reporting::scheduler::SpawnReport;
use registry::schema::{
prelude::{ObjectType, Property},
structs::{
DmarcInternalReport, DmarcReportSettings, Expression, TlsInternalReport, TlsReportSettings,
},
};
use smtp::reporting::send::MtaReportSend;
use std::sync::Arc;
const CONFIG: &str = r#"
[storage]
directory = "local"
[directory."local"]
type = "memory"
[[directory."local".principals]]
name = "admin"
type = "admin"
description = "Superuser"
secret = "secret"
class = "admin"
[session.rcpt]
relay = true
[report.dmarc.aggregate]
max-size = 1024
[report.tls.aggregate]
max-size = 1024
"#;
#[tokio::test]
#[serial_test::serial]
async fn manage_reports() {
let mut test = TestServerBuilder::new("smtp_report_manage")
.await
.with_http_listener(19048)
.await
.disable_services()
.capture_queue()
.build()
.await;
// Start reporting service
let local = TestSMTP::new("smtp_manage_reports", CONFIG).await;
let _rx = local.start(&[ServerProtocol::Http]).await;
let core = local.build_smtp();
local
.report_receiver
.report_rx
.spawn(local.server.inner.clone());
let admin = test.account("admin");
admin
.registry_create_object(TlsReportSettings {
max_report_size: Expression {
else_: "1024".into(),
..Default::default()
},
..Default::default()
})
.await;
admin
.registry_create_object(DmarcReportSettings {
aggregate_max_report_size: Expression {
else_: "1024".into(),
..Default::default()
},
..Default::default()
})
.await;
admin.mta_allow_relaying().await;
admin.mta_no_auth().await;
admin.mta_allow_non_fqdn().await;
admin.reload_settings().await;
test.reload_core();
test.expect_reload_settings().await;
let admin = test.account("admin");
// Send test reporting events
core.schedule_report(DmarcEvent {
domain: "foobar.org".to_string(),
report_record: Record::new()
.with_source_ip("192.168.1.2".parse().unwrap())
.with_action_disposition(ActionDisposition::Pass)
.with_dmarc_dkim_result(DmarcResult::Pass)
.with_dmarc_spf_result(DmarcResult::Fail)
.with_envelope_from("hello@example.org")
.with_envelope_to("other@example.org")
.with_header_from("bye@example.org"),
dmarc_record: Arc::new(
Dmarc::parse(b"v=DMARC1; p=reject; rua=mailto:reports@foobar.org").unwrap(),
),
interval: AggregateFrequency::Daily,
})
.await;
core.schedule_report(DmarcEvent {
domain: "foobar.net".to_string(),
report_record: Record::new()
.with_source_ip("a:b:c::e:f".parse().unwrap())
.with_action_disposition(ActionDisposition::Reject)
.with_dmarc_dkim_result(DmarcResult::Fail)
.with_dmarc_spf_result(DmarcResult::Pass),
dmarc_record: Arc::new(
Dmarc::parse(
b"v=DMARC1; p=quarantine; rua=mailto:reports@foobar.net,mailto:reports@example.net",
)
.unwrap(),
),
interval: AggregateFrequency::Weekly,
})
.await;
core.schedule_report(TlsEvent {
domain: "foobar.org".to_string(),
policy: PolicyType::None,
failure: None,
tls_record: Arc::new(TlsRpt::parse(b"v=TLSRPTv1;rua=mailto:reports@foobar.org").unwrap()),
interval: AggregateFrequency::Daily,
})
.await;
core.schedule_report(TlsEvent {
domain: "foobar.net".to_string(),
policy: PolicyType::Sts(None),
failure: FailureDetails::new(ResultType::StsPolicyInvalid).into(),
tls_record: Arc::new(TlsRpt::parse(b"v=TLSRPTv1;rua=mailto:reports@foobar.net").unwrap()),
interval: AggregateFrequency::Weekly,
})
.await;
test.server
.schedule_report(DmarcEvent {
domain: "foobar.org".to_string(),
report_record: Record::new()
.with_source_ip("192.168.1.2".parse().unwrap())
.with_action_disposition(ActionDisposition::Pass)
.with_dmarc_dkim_result(DmarcResult::Pass)
.with_dmarc_spf_result(DmarcResult::Fail)
.with_envelope_from("hello@example.org")
.with_envelope_to("other@example.org")
.with_header_from("bye@example.org"),
dmarc_record: Arc::new(
Dmarc::parse(b"v=DMARC1; p=reject; rua=mailto:reports@foobar.org").unwrap(),
),
interval: AggregateFrequency::Daily,
span_id: 0,
})
.await;
test.server
.schedule_report(DmarcEvent {
domain: "foobar.net".to_string(),
report_record: Record::new()
.with_source_ip("a:b:c::e:f".parse().unwrap())
.with_action_disposition(ActionDisposition::Reject)
.with_dmarc_dkim_result(DmarcResult::Fail)
.with_dmarc_spf_result(DmarcResult::Pass),
dmarc_record: Arc::new(
Dmarc::parse(
concat!(
"v=DMARC1; p=quarantine; rua=mailto:reports",
"@foobar.net,mailto:reports@example.net"
)
.as_bytes(),
)
.unwrap(),
),
interval: AggregateFrequency::Weekly,
span_id: 0,
})
.await;
test.server
.schedule_report(TlsEvent {
domain: "foobar.org".to_string(),
policy: PolicyType::None,
failure: None,
tls_record: Arc::new(
TlsRpt::parse(b"v=TLSRPTv1;rua=mailto:reports@foobar.org").unwrap(),
),
interval: AggregateFrequency::Daily,
span_id: 0,
})
.await;
test.server
.schedule_report(TlsEvent {
domain: "foobar.net".to_string(),
policy: PolicyType::Sts(None),
failure: FailureDetails::new(ResultType::StsPolicyInvalid).into(),
tls_record: Arc::new(
TlsRpt::parse(b"v=TLSRPTv1;rua=mailto:reports@foobar.net").unwrap(),
),
interval: AggregateFrequency::Weekly,
span_id: 0,
})
.await;
// List reports
let api = ManagementApi::default();
let ids = api
.request::<List<String>>(Method::GET, "/api/queue/reports")
.await
.unwrap()
.unwrap_data()
.items;
assert_eq!(ids.len(), 4);
let mut id_map = AHashMap::new();
let mut id_map_rev = AHashMap::new();
for (report, id) in api.get_reports(&ids).await.into_iter().zip(ids) {
let mut parts = id.split('!');
let report = report.unwrap();
let mut id_num = if parts.next().unwrap() == "t" {
assert!(matches!(report, Report::Tls { .. }));
2
} else {
assert!(matches!(report, Report::Dmarc { .. }));
0
};
let (domain, range_to, range_from) = match report {
Report::Dmarc {
domain,
range_to,
range_from,
..
} => (domain, range_to, range_from),
Report::Tls {
domain,
range_to,
range_from,
..
} => (domain, range_to, range_from),
};
assert_eq!(parts.next().unwrap(), domain);
let diff = range_to.to_timestamp() - range_from.to_timestamp();
if domain == "foobar.org" {
// List DMARC reports
let mut dmarc_name_to_id = AHashMap::new();
let mut dmarc_id_to_name = AHashMap::new();
for (id, report) in admin.registry_get_all::<DmarcInternalReport>().await {
let diff =
report.report.date_range_end.timestamp() - report.report.date_range_begin.timestamp();
if report.domain == "foobar.org" {
assert_eq!(diff, 86400);
} else {
assert_eq!(diff, 7 * 86400);
id_num += 1;
}
id_map.insert(char::from(b'a' + id_num).to_string(), id.clone());
id_map_rev.insert(id, char::from(b'a' + id_num).to_string());
dmarc_name_to_id.insert(report.domain.clone(), id);
dmarc_id_to_name.insert(id, report.domain);
}
assert_eq!(dmarc_name_to_id.len(), 2);
// List TLS reports
let mut tls_name_to_id = AHashMap::new();
let mut tls_id_to_name = AHashMap::new();
for (id, report) in admin.registry_get_all::<TlsInternalReport>().await {
let diff =
report.report.date_range_end.timestamp() - report.report.date_range_start.timestamp();
if report.domain == "foobar.org" {
assert_eq!(diff, 86400);
} else {
assert_eq!(diff, 7 * 86400);
}
tls_name_to_id.insert(report.domain.clone(), id);
tls_id_to_name.insert(id, report.domain);
}
assert_eq!(tls_name_to_id.len(), 2);
// Test list search
for (query, expected_ids) in [
("/api/queue/reports?type=dmarc", vec!["a", "b"]),
("/api/queue/reports?type=tls", vec!["c", "d"]),
("/api/queue/reports?domain=foobar.org", vec!["a", "c"]),
("/api/queue/reports?domain=foobar.net", vec!["b", "d"]),
("/api/queue/reports?domain=foobar.org&type=dmarc", vec!["a"]),
("/api/queue/reports?domain=foobar.net&type=tls", vec!["d"]),
for (object, query, expected_ids) in [
(
ObjectType::DmarcInternalReport,
vec![],
vec![
dmarc_name_to_id["foobar.org"],
dmarc_name_to_id["foobar.net"],
],
),
(
ObjectType::TlsInternalReport,
vec![],
vec![tls_name_to_id["foobar.org"], tls_name_to_id["foobar.net"]],
),
(
ObjectType::DmarcInternalReport,
vec![(Property::Domain, "foobar.org".to_string())],
vec![dmarc_name_to_id["foobar.org"]],
),
(
ObjectType::DmarcInternalReport,
vec![(Property::Domain, "foobar.net".to_string())],
vec![dmarc_name_to_id["foobar.net"]],
),
(
ObjectType::TlsInternalReport,
vec![(Property::Domain, "foobar.org".to_string())],
vec![tls_name_to_id["foobar.org"]],
),
(
ObjectType::TlsInternalReport,
vec![(Property::Domain, "foobar.net".to_string())],
vec![tls_name_to_id["foobar.net"]],
),
] {
let expected_ids = HashSet::from_iter(expected_ids.into_iter().map(|s| s.to_string()));
let ids = api
.request::<List<String>>(Method::GET, query)
.await
.unwrap()
.unwrap_data()
.items
.into_iter()
.map(|id| id_map_rev.get(&id).unwrap().clone())
.collect::<HashSet<_>>();
assert_eq!(ids, expected_ids, "failed for {query}");
assert_eq!(
admin
.registry_query_ids(object, query.clone(), Vec::<&str>::new())
.await,
expected_ids,
"failed for {object:?} with query {query:?}"
);
}
// Cancel reports
for id in ["a", "b"] {
assert!(
api.request::<bool>(
Method::DELETE,
&format!("/api/queue/reports/{}", id_map.get(id).unwrap(),)
)
for (object, id) in [
(
ObjectType::DmarcInternalReport,
dmarc_name_to_id["foobar.org"],
),
(ObjectType::TlsInternalReport, tls_name_to_id["foobar.org"]),
] {
admin
.registry_destroy(object, vec![id])
.await
.unwrap()
.unwrap_data(),
"failed for {id}"
.assert_destroyed(&[id]);
}
for (object, id) in [
(
ObjectType::DmarcInternalReport,
dmarc_name_to_id["foobar.net"],
),
(ObjectType::TlsInternalReport, tls_name_to_id["foobar.net"]),
] {
assert_eq!(
admin
.registry_query_ids(object, Vec::<(&str, &str)>::new(), Vec::<&str>::new())
.await,
vec![id],
"failed for {object:?}"
);
}
assert_eq!(
api.request::<List<String>>(Method::GET, "/api/queue/reports")
.await
.unwrap()
.unwrap_data()
.items
.len(),
2
);
let mut ids = api
.get_reports(&[
id_map.get("a").unwrap().clone(),
id_map.get("b").unwrap().clone(),
id_map.get("c").unwrap().clone(),
id_map.get("d").unwrap().clone(),
])
.await
.into_iter();
assert!(ids.next().unwrap().is_none());
assert!(ids.next().unwrap().is_none());
assert!(ids.next().unwrap().is_some());
assert!(ids.next().unwrap().is_some());
// Cancel all reports
assert!(
api.request::<bool>(Method::DELETE, "/api/queue/reports")
.await
.unwrap()
.unwrap_data()
admin
.registry_destroy_all(ObjectType::DmarcInternalReport)
.await;
admin
.registry_destroy_all(ObjectType::TlsInternalReport)
.await;
assert_eq!(
admin.registry_get_all::<DmarcInternalReport>().await,
Vec::new()
);
assert_eq!(
api.request::<List<String>>(Method::GET, "/api/queue/reports")
.await
.unwrap()
.unwrap_data()
.items
.len(),
0
admin.registry_get_all::<TlsInternalReport>().await,
Vec::new()
);
}
impl ManagementApi {
async fn get_reports(&self, ids: &[String]) -> Vec<Option<Report>> {
let mut results = Vec::with_capacity(ids.len());
for id in ids {
let report = self
.request::<Report>(Method::GET, &format!("/api/queue/reports/{id}",))
.await
.unwrap()
.try_unwrap_data();
results.push(report);
}
results
}
}

4
tests/src/smtp/mod.rs
View File

@@ -6,10 +6,8 @@
pub mod inbound;
pub mod lookup;
pub mod management;
pub mod outbound;
pub mod queue;
pub mod reporting;
pub mod session;
/*
pub mod management;
*/

2
tests/src/smtp/outbound/fallback_relay.rs
View File

@@ -48,12 +48,10 @@ async fn fallback_relay() {
.registry_create_object(MtaStageRcpt {
max_recipients: Expression {
else_: "100".into(),
..Default::default()
},
allow_relaying: Expression {
else_: "true".into(),
..Default::default()
},
..Default::default()

3
tests/src/smtp/session.rs
View File

@@ -11,13 +11,12 @@ use common::{
network::{ServerInstance, SessionStream, TcpAcceptor, limiter::ConcurrencyLimiter},
};
use rustls::{ServerConfig, server::ResolvesServerCert};
use smtp::core::{Session, SessionAddress, SessionData, SessionParameters, State};
use std::{borrow::Cow, path::PathBuf, sync::Arc};
use tokio::{
io::{AsyncRead, AsyncWrite},
sync::watch,
};
use smtp::core::{Session, SessionAddress, SessionData, SessionParameters, State};
use tokio_rustls::TlsAcceptor;
use utils::snowflake::SnowflakeIdGenerator;