filip/Stalwart
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Store incoming reports in the data store

Browse Source
This commit is contained in:
mdecimus
2024-03-02 12:21:03 +01:00
parent 417bc38288
commit 875b1fa744
40 changed files with 1992 additions and 1541 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
tests/src/smtp/management/mod.rs
View File

@@ -23,7 +23,7 @@
use std::time::Duration;
use reqwest::header::AUTHORIZATION;
use reqwest::{header::AUTHORIZATION, Method};
use serde::{de::DeserializeOwned, Deserialize};
pub mod queue;
@@ -36,19 +36,22 @@ pub enum Response<T> {
Error { error: String, details: String },
}
pub async fn send_manage_request<T: DeserializeOwned>(query: &str) -> Result<Response<T>, String> {
send_manage_request_raw(query).await.map(|result| {
pub async fn send_manage_request<T: DeserializeOwned>(
method: Method,
query: &str,
) -> Result<Response<T>, String> {
send_manage_request_raw(method, query).await.map(|result| {
serde_json::from_str::<Response<T>>(&result).unwrap_or_else(|err| panic!("{err}: {result}"))
})
}
pub async fn send_manage_request_raw(query: &str) -> Result<String, String> {
pub async fn send_manage_request_raw(method: Method, query: &str) -> Result<String, String> {
reqwest::Client::builder()
.timeout(Duration::from_millis(500))
.danger_accept_invalid_certs(true)
.build()
.unwrap()
.get(format!("https://127.0.0.1:9980{query}"))
.request(method, format!("https://127.0.0.1:9980{query}"))
.header(AUTHORIZATION, "Basic YWRtaW46c2VjcmV0")
.send()
.await
@@ -69,6 +72,16 @@ impl<T> Response<T> {
}
}
pub fn try_unwrap_data(self) -> Option<T> {
match self {
Response::Data { data } => Some(data),
Response::Error { error, .. } if error == "not-found" => None,
Response::Error { error, details } => {
panic!("Expected data, found error {error:?}: {details:?}")
}
}
}
pub fn unwrap_error(self) -> (String, String) {
match self {
Response::Error { error, details } => (error, details),

101
tests/src/smtp/management/queue.rs
View File

@@ -30,7 +30,7 @@ use ahash::{AHashMap, HashMap, HashSet};
use directory::core::config::ConfigDirectory;
use mail_auth::MX;
use mail_parser::DateTime;
use reqwest::{header::AUTHORIZATION, StatusCode};
use reqwest::{header::AUTHORIZATION, Method, StatusCode};
use store::Store;
use utils::config::{if_block::IfBlock, Config, ServerProtocol};
@@ -61,9 +61,9 @@ member-of = ["superusers"]
#[derive(serde::Deserialize)]
#[allow(dead_code)]
struct List<T> {
items: Vec<T>,
total: usize,
pub(super) struct List<T> {
pub items: Vec<T>,
pub total: usize,
}
#[tokio::test]
@@ -192,7 +192,7 @@ async fn manage_queue() {
);
// Fetch and validate messages
let ids = send_manage_request::<List<QueueId>>("/api/queue/list")
let ids = send_manage_request::<List<QueueId>>(Method::GET, "/api/queue/messages")
.await
.unwrap()
.unwrap_data()
@@ -277,28 +277,28 @@ async fn manage_queue() {
// Test list search
for (query, expected_ids) in [
(
"/api/queue/list?from=bill1@foobar.net".to_string(),
"/api/queue/messages?from=bill1@foobar.net".to_string(),
vec!["a"],
),
(
"/api/queue/list?to=foobar.org".to_string(),
"/api/queue/messages?to=foobar.org".to_string(),
vec!["d", "e", "f"],
),
(
"/api/queue/list?from=bill3@foobar.net&to=rcpt5@example1.com".to_string(),
"/api/queue/messages?from=bill3@foobar.net&to=rcpt5@example1.com".to_string(),
vec!["c"],
),
(
format!("/api/queue/list?before={test_search}"),
format!("/api/queue/messages?before={test_search}"),
vec!["a", "b"],
),
(
format!("/api/queue/list?after={test_search}"),
format!("/api/queue/messages?after={test_search}"),
vec!["d", "e", "f", "c"],
),
] {
let expected_ids = HashSet::from_iter(expected_ids.into_iter().map(|s| s.to_string()));
let ids = send_manage_request::<List<QueueId>>(&query)
let ids = send_manage_request::<List<QueueId>>(Method::GET, &query)
.await
.unwrap()
.unwrap_data()
@@ -310,27 +310,24 @@ async fn manage_queue() {
}
// Retry delivery
assert_eq!(
send_manage_request::<Vec<bool>>(&format!(
"/api/queue/retry?id={},{}",
id_map.get("e").unwrap(),
id_map.get("f").unwrap()
))
.await
.unwrap()
.unwrap_data(),
vec![true, true]
);
assert_eq!(
send_manage_request::<Vec<bool>>(&format!(
"/api/queue/retry?id={}&filter=example1.org&at=2200-01-01T00:00:00Z",
for id in [id_map.get("e").unwrap(), id_map.get("f").unwrap()] {
assert!(
send_manage_request::<bool>(Method::PATCH, &format!("/api/queue/messages/{id}",))
.await
.unwrap()
.unwrap_data(),
);
}
assert!(send_manage_request::<bool>(
Method::PATCH,
&format!(
"/api/queue/messages/{}?filter=example1.org&at=2200-01-01T00:00:00Z",
id_map.get("a").unwrap(),
))
.await
.unwrap()
.unwrap_data(),
vec![true]
);
)
)
.await
.unwrap()
.unwrap_data());
// Expect delivery to john@foobar.org
tokio::time::sleep(Duration::from_millis(100)).await;
@@ -384,22 +381,24 @@ async fn manage_queue() {
("c", "rcpt6@example2.com"),
("d", ""),
] {
assert_eq!(
send_manage_request::<Vec<bool>>(&format!(
"/api/queue/cancel?id={}{}{}",
id_map.get(id).unwrap(),
if !filter.is_empty() { "&filter=" } else { "" },
filter
))
assert!(
send_manage_request::<bool>(
Method::DELETE,
&format!(
"/api/queue/messages/{}{}{}",
id_map.get(id).unwrap(),
if !filter.is_empty() { "?filter=" } else { "" },
filter
)
)
.await
.unwrap()
.unwrap_data(),
vec![true],
"failed for {id}: {filter}"
);
}
assert_eq!(
send_manage_request::<List<QueueId>>("/api/queue/list")
send_manage_request::<List<QueueId>>(Method::GET, "/api/queue/messages")
.await
.unwrap()
.unwrap_data()
@@ -485,14 +484,16 @@ fn assert_timestamp(timestamp: &DateTime, expected: i64, ctx: &str, message: &Me
}
async fn get_messages(ids: &[QueueId]) -> Vec<Option<Message>> {
send_manage_request(&format!(
"/api/queue/status?id={}",
ids.iter()
.map(|id| id.to_string())
.collect::<Vec<_>>()
.join(",")
))
.await
.unwrap()
.unwrap_data()
let mut results = Vec::with_capacity(ids.len());
for id in ids {
let message =
send_manage_request::<Message>(Method::GET, &format!("/api/queue/messages/{id}",))
.await
.unwrap()
.try_unwrap_data();
results.push(message);
}
results
}

56
tests/src/smtp/management/report.rs
View File

@@ -34,12 +34,16 @@ use mail_auth::{
ActionDisposition, DmarcResult, Record,
},
};
use reqwest::Method;
use store::Store;
use tokio::sync::mpsc;
use utils::config::{if_block::IfBlock, Config, ServerProtocol};
use crate::smtp::{
inbound::dummy_stores, management::send_manage_request, outbound::start_test_server, TestConfig,
inbound::dummy_stores,
management::{queue::List, send_manage_request},
outbound::start_test_server,
TestConfig,
};
use smtp::{
config::AggregateFrequency,
@@ -141,10 +145,11 @@ async fn manage_reports() {
.await;
// List reports
let ids = send_manage_request::<Vec<String>>("/admin/report/list")
let ids = send_manage_request::<List<String>>(Method::GET, "/api/queue/reports")
.await
.unwrap()
.unwrap_data();
.unwrap_data()
.items;
assert_eq!(ids.len(), 4);
let mut id_map = AHashMap::new();
let mut id_map_rev = AHashMap::new();
@@ -186,18 +191,19 @@ async fn manage_reports() {
// Test list search
for (query, expected_ids) in [
("/admin/report/list?type=dmarc", vec!["a", "b"]),
("/admin/report/list?type=tls", vec!["c", "d"]),
("/admin/report/list?domain=foobar.org", vec!["a", "c"]),
("/admin/report/list?domain=foobar.net", vec!["b", "d"]),
("/admin/report/list?domain=foobar.org&type=dmarc", vec!["a"]),
("/admin/report/list?domain=foobar.net&type=tls", vec!["d"]),
("/api/queue/reports?type=dmarc", vec!["a", "b"]),
("/api/queue/reports?type=tls", vec!["c", "d"]),
("/api/queue/reports?domain=foobar.org", vec!["a", "c"]),
("/api/queue/reports?domain=foobar.net", vec!["b", "d"]),
("/api/queue/reports?domain=foobar.org&type=dmarc", vec!["a"]),
("/api/queue/reports?domain=foobar.net&type=tls", vec!["d"]),
] {
let expected_ids = HashSet::from_iter(expected_ids.into_iter().map(|s| s.to_string()));
let ids = send_manage_request::<Vec<String>>(query)
let ids = send_manage_request::<List<String>>(Method::GET, query)
.await
.unwrap()
.unwrap_data()
.items
.into_iter()
.map(|id| id_map_rev.get(&id).unwrap().clone())
.collect::<HashSet<_>>();
@@ -206,23 +212,23 @@ async fn manage_reports() {
// Cancel reports
for id in ["a", "b"] {
assert_eq!(
send_manage_request::<Vec<bool>>(&format!(
"/admin/report/cancel?id={}",
id_map.get(id).unwrap(),
))
assert!(
send_manage_request::<bool>(
Method::DELETE,
&format!("/api/queue/reports/{}", id_map.get(id).unwrap(),)
)
.await
.unwrap()
.unwrap_data(),
vec![true],
"failed for {id}"
);
}
assert_eq!(
send_manage_request::<Vec<String>>("/admin/report/list")
send_manage_request::<List<String>>(Method::GET, "/api/queue/reports")
.await
.unwrap()
.unwrap_data()
.items
.len(),
2
);
@@ -241,8 +247,16 @@ async fn manage_reports() {
}
async fn get_reports(ids: &[String]) -> Vec<Option<Report>> {
send_manage_request(&format!("/admin/report/status?id={}", ids.join(",")))
.await
.unwrap()
.unwrap_data()
let mut results = Vec::with_capacity(ids.len());
for id in ids {
let report =
send_manage_request::<Report>(Method::GET, &format!("/api/queue/reports/{id}",))
.await
.unwrap()
.try_unwrap_data();
results.push(report);
}
results
}

2
tests/src/smtp/mod.rs
View File

@@ -408,7 +408,7 @@ impl TestConfig for ReportConfig {
addresses: vec![],
forward: true,
store: None,
report_id: 0.into(),
report_id: SnowflakeIdGenerator::new(),
},
dkim: Report::test(),
spf: Report::test(),

71
tests/src/smtp/reporting/analyze.rs
View File

@@ -21,25 +21,25 @@
* for more details.
*/
use std::{fs, sync::Arc, time::Duration};
use std::{sync::Arc, time::Duration};
use crate::smtp::{
inbound::TestQueueEvent, make_temp_dir, session::TestSession, TestConfig, TestSMTP,
};
use crate::smtp::{inbound::TestQueueEvent, session::TestSession, TestConfig, TestSMTP};
use smtp::{
config::AddressMatch,
core::{Session, SMTP},
};
use store::{
write::{ReportClass, ValueClass},
IterateParams, ValueKey,
};
use utils::config::if_block::IfBlock;
#[tokio::test]
#[tokio::test(flavor = "multi_thread")]
async fn report_analyze() {
let mut core = SMTP::test();
// Create temp dir for queue
let mut qr = core.init_test_queue("smtp_analyze_report_test");
let report_dir = make_temp_dir("smtp_report_incoming", true);
let config = &mut core.session.config.rcpt;
config.relay = IfBlock::new(true);
let config = &mut core.session.config.data;
@@ -51,10 +51,16 @@ async fn report_analyze() {
AddressMatch::Equals("feedback@foobar.org".to_string()),
];
config.forward = false;
config.store = report_dir.temp_dir.clone().into();
config.store = Duration::from_secs(1).into();
//config.store = Duration::from_secs(86400).into();
// Create test message
let core = Arc::new(core);
/*let rx_manage = crate::smtp::outbound::start_test_server(
core.clone(),
&[utils::config::ServerProtocol::Http],
);*/
let mut session = Session::test(core.clone());
session.data.remote_ip_str = "10.0.0.1".to_string();
session.eval_session_params().await;
@@ -84,14 +90,53 @@ async fn report_analyze() {
}
tokio::time::sleep(Duration::from_millis(200)).await;
//let c = tokio::time::sleep(Duration::from_secs(86400)).await;
// Purging the database shouldn't remove the reports
qr.store.purge_store().await.unwrap();
// Make sure the reports are in the store
let mut total_reports = 0;
for entry in fs::read_dir(&report_dir.temp_dir).unwrap() {
let path = entry.unwrap().path();
assert_ne!(fs::metadata(&path).unwrap().len(), 0);
total_reports += 1;
}
qr.store
.iterate(
IterateParams::new(
ValueKey::from(ValueClass::Report(ReportClass::Tls { id: 0, expires: 0 })),
ValueKey::from(ValueClass::Report(ReportClass::Arf {
id: u64::MAX,
expires: u64::MAX,
})),
),
|_, _| {
total_reports += 1;
Ok(true)
},
)
.await
.unwrap();
assert_eq!(total_reports, total_reports_received);
// Wait one second, purge, and make sure they are gone
tokio::time::sleep(Duration::from_secs(1)).await;
qr.store.purge_store().await.unwrap();
let mut total_reports = 0;
qr.store
.iterate(
IterateParams::new(
ValueKey::from(ValueClass::Report(ReportClass::Tls { id: 0, expires: 0 })),
ValueKey::from(ValueClass::Report(ReportClass::Arf {
id: u64::MAX,
expires: u64::MAX,
})),
),
|_, _| {
total_reports += 1;
Ok(true)
},
)
.await
.unwrap();
assert_eq!(total_reports, 0);
// Test delivery to non-report addresses
session
.send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250")

2
tests/src/smtp/reporting/dmarc.rs
View File

@@ -117,7 +117,7 @@ async fn report_dmarc() {
assert_eq!(reports.len(), 1);
match reports.into_iter().next().unwrap() {
QueueClass::DmarcReportHeader(event) => {
core.generate_dmarc_report(event).await;
core.send_dmarc_aggregate_report(event).await;
}
_ => unreachable!(),
}

22
tests/src/smtp/reporting/tls.rs
View File

@@ -85,7 +85,7 @@ async fn report_tls() {
for (policy, rt) in [
(
smtp::reporting::PolicyType::None,
smtp::reporting::PolicyType::None, // Quota limited at 1532 bytes, this should not be included in the report.
ResultType::CertificateExpired,
),
(
@@ -101,7 +101,7 @@ async fn report_tls() {
ResultType::StsPolicyInvalid,
),
(
smtp::reporting::PolicyType::Sts(None), // Quota limited at 1532 bytes, this should not be included in the report.
smtp::reporting::PolicyType::Sts(None),
ResultType::StsWebpkiInvalid,
),
] {
@@ -128,8 +128,7 @@ async fn report_tls() {
_ => unreachable!(),
}
}
core.generate_tls_report(tls_reports.first().unwrap().domain.clone(), tls_reports)
.await;
core.send_tls_aggregate_report(tls_reports).await;
// Expect report
let message = qr.expect_message().await;
@@ -167,10 +166,10 @@ async fn report_tls() {
}
PolicyType::Sts => {
seen[1] = true;
assert_eq!(policy.summary.total_failure, 2);
assert_eq!(policy.summary.total_failure, 3);
assert_eq!(policy.summary.total_success, 0);
assert_eq!(policy.policy.policy_domain, "foobar.org");
assert_eq!(policy.failure_details.len(), 2);
assert_eq!(policy.failure_details.len(), 3);
assert!(policy
.failure_details
.iter()
@@ -182,14 +181,14 @@ async fn report_tls() {
}
PolicyType::NoPolicyFound => {
seen[2] = true;
assert_eq!(policy.summary.total_failure, 1);
assert_eq!(policy.summary.total_failure, 0);
assert_eq!(policy.summary.total_success, 2);
assert_eq!(policy.policy.policy_domain, "foobar.org");
assert_eq!(policy.failure_details.len(), 1);
assert_eq!(
assert_eq!(policy.failure_details.len(), 0);
/*assert_eq!(
policy.failure_details.first().unwrap().result_type,
ResultType::CertificateExpired
);
);*/
}
PolicyType::Other => unreachable!(),
}
@@ -218,8 +217,7 @@ async fn report_tls() {
assert_eq!(reports.len(), 1);
match reports.into_iter().next().unwrap() {
QueueClass::TlsReportHeader(event) => {
core.generate_tls_report(event.domain.clone(), vec![event])
.await;
core.send_tls_aggregate_report(vec![event]).await;
}
_ => unreachable!(),
}

8
tests/src/store/lookup.rs
View File

@@ -57,7 +57,7 @@ pub async fn lookup_tests() {
.key_set(key.clone(), "world".to_string().into_bytes(), None)
.await
.unwrap();
store.purge_expired().await.unwrap();
store.purge_lookup_store().await.unwrap();
assert_eq!(
store.key_get::<String>(key.clone()).await.unwrap(),
Some("world".to_string())
@@ -75,7 +75,7 @@ pub async fn lookup_tests() {
tokio::time::sleep(tokio::time::Duration::from_secs(2)).await;
assert_eq!(None, store.key_get::<String>(key.clone()).await.unwrap());
store.purge_expired().await.unwrap();
store.purge_lookup_store().await.unwrap();
if let LookupStore::Store(store) = &store {
store.assert_is_empty(store.clone().into()).await;
}
@@ -106,7 +106,7 @@ pub async fn lookup_tests() {
.unwrap();
assert_eq!(1, store.counter_get(key.clone()).await.unwrap());
tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
store.purge_expired().await.unwrap();
store.purge_lookup_store().await.unwrap();
assert_eq!(0, store.counter_get(key.clone()).await.unwrap());
// Test rate limiter
@@ -127,7 +127,7 @@ pub async fn lookup_tests() {
.unwrap()
.is_none());
tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
store.purge_expired().await.unwrap();
store.purge_lookup_store().await.unwrap();
if let LookupStore::Store(store) = &store {
store.assert_is_empty(store.clone().into()).await;
}