From 6c4d28a877e94c2e43229704c373fa63c30d70f2 Mon Sep 17 00:00:00 2001 From: mdecimus <11444311+mdecimus@users.noreply.github.com> Date: Sat, 6 Dec 2025 16:59:43 +0100 Subject: [PATCH] Spam filter performance and accuracy improvements (part 7) --- crates/jmap/src/email/import.rs | 13 +- crates/spam-filter/src/analysis/score.rs | 10 +- crates/spam-filter/src/analysis/subject.rs | 4 + crates/store/src/write/blob.rs | 37 +++- crates/trc/src/event/description.rs | 4 +- crates/trc/src/event/level.rs | 5 +- crates/trc/src/lib.rs | 2 +- crates/trc/src/serializers/binary.rs | 4 +- tests/src/imap/{bayes.rs => antispam.rs} | 235 +++++++++++++++++---- tests/src/imap/mod.rs | 29 ++- tests/src/imap/pop.rs | 22 +- tests/src/jmap/auth/permissions.rs | 14 +- tests/src/jmap/mail/antispam.rs | 175 +++++++++++++++ tests/src/jmap/mail/delivery.rs | 215 +++++++++++++++---- tests/src/jmap/mail/mod.rs | 1 + tests/src/jmap/mod.rs | 11 +- tests/src/jmap/server/enterprise.rs | 21 +- tests/src/jmap/server/purge.rs | 3 +- tests/src/smtp/inbound/antispam.rs | 8 - tests/src/store/cleanup.rs | 40 ++-- 20 files changed, 701 insertions(+), 152 deletions(-) rename tests/src/imap/{bayes.rs => antispam.rs} (83%) create mode 100644 tests/src/jmap/mail/antispam.rs diff --git a/crates/jmap/src/email/import.rs b/crates/jmap/src/email/import.rs index 59ba5031..8717b195 100644 --- a/crates/jmap/src/email/import.rs +++ b/crates/jmap/src/email/import.rs @@ -10,6 +10,7 @@ use crate::{ use common::{Server, auth::AccessToken}; use email::{ cache::{MessageCacheFetch, mailbox::MailboxCacheAccess}, + mailbox::JUNK_ID, message::ingest::{EmailIngest, IngestEmail, IngestSource}, }; use http_proto::HttpSessionData; @@ -22,7 +23,7 @@ use jmap_proto::{ }; use mail_parser::MessageParser; use std::future::Future; -use types::{acl::Acl, id::Id}; +use types::{acl::Acl, id::Id, keyword::Keyword}; use utils::map::vec_map::VecMap; pub trait EmailImport: Sync + Send { @@ -149,12 +150,16 @@ impl EmailImport for Server { message: MessageParser::new().parse(&raw_message), blob_hash: Some(&blob_id.hash), access_token: import_access_token.as_deref().unwrap_or(access_token), + source: IngestSource::Jmap { + train_classifier: email + .keywords + .iter() + .any(|k| matches!(k, Keyword::Junk | Keyword::NotJunk)) + || mailbox_ids.contains(&JUNK_ID), + }, mailbox_ids, keywords: email.keywords, received_at: email.received_at.map(|r| r.into()), - source: IngestSource::Jmap { - train_classifier: true, - }, session_id: session.session_id, }) .await diff --git a/crates/spam-filter/src/analysis/score.rs b/crates/spam-filter/src/analysis/score.rs index e469e7e8..c21ff7fc 100644 --- a/crates/spam-filter/src/analysis/score.rs +++ b/crates/spam-filter/src/analysis/score.rs @@ -78,7 +78,10 @@ impl SpamFilterAnalyzeScore for Server { let mut final_score = ctx.result.score; let mut avg_confidence: f32 = 0.0; let mut total_results = 0; - let mut user_results = vec![false; ctx.result.classifier_confidence.len()]; + let mut user_results = vec![ + ctx.result.score >= self.core.spam.scores.spam_threshold; + ctx.input.env_rcpt_to.len() + ]; if !ctx.result.classifier_confidence.is_empty() { for (idx, &confidence) in ctx.result.classifier_confidence.iter().enumerate() { if let Some(confidence) = confidence { @@ -95,9 +98,8 @@ impl SpamFilterAnalyzeScore for Server { .copied() .unwrap_or_default(); - if ctx.result.score + user_score >= self.core.spam.scores.spam_threshold { - user_results[idx] = true; - } + user_results[idx] = + ctx.result.score + user_score >= self.core.spam.scores.spam_threshold; } } diff --git a/crates/spam-filter/src/analysis/subject.rs b/crates/spam-filter/src/analysis/subject.rs index c5bb6a3a..413bdb82 100644 --- a/crates/spam-filter/src/analysis/subject.rs +++ b/crates/spam-filter/src/analysis/subject.rs @@ -84,6 +84,10 @@ impl SpamFilterAnalyzeSubject for Server { } else if ctx.output.subject.ends_with(' ') { // Subject ends with whitespace ctx.result.add_tag("SUBJECT_ENDS_SPACES"); + } else if ctx.output.subject + == "XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X" + { + ctx.result.add_tag("GTUBE_TEST"); } if ctx.output.subject_thread.len() >= 10 diff --git a/crates/store/src/write/blob.rs b/crates/store/src/write/blob.rs index 94e8b7e4..68c28adf 100644 --- a/crates/store/src/write/blob.rs +++ b/crates/store/src/write/blob.rs @@ -4,12 +4,14 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use std::time::Instant; + use super::{BlobOp, Operation, ValueClass, ValueOp, key::DeserializeBigEndian, now}; use crate::{ BlobStore, IterateParams, Store, U32_LEN, U64_LEN, ValueKey, write::{BatchBuilder, BlobLink}, }; -use trc::AddContext; +use trc::{AddContext, PurgeEvent}; use types::{ blob::BlobClass, blob_hash::{BLOB_HASH_LEN, BlobHash}, @@ -117,6 +119,10 @@ impl Store { } pub async fn purge_blobs(&self, blob_store: BlobStore) -> trc::Result<()> { + let mut total_active = 0; + let mut total_deleted = 0; + let started = Instant::now(); + for byte in 0..=u8::MAX { // Validate linked blobs let mut from_hash = BlobHash::default(); @@ -194,8 +200,18 @@ impl Store { .await .caused_by(trc::location!())?; } + + total_active += state.total_active - 1; // Exclude default hash + total_deleted += state.total_deleted; } + trc::event!( + Purge(PurgeEvent::BlobCleanup), + Expires = total_deleted, + Total = total_active, + Elapsed = started.elapsed() + ); + Ok(()) } } @@ -206,6 +222,8 @@ struct BlobPurgeState { delete_keys: Vec<(Option, BlobOp)>, spam_train_samples: Vec<(u32, u64)>, now: u64, + total_deleted: u64, + total_active: u64, } impl BlobPurgeState { @@ -216,6 +234,8 @@ impl BlobPurgeState { delete_keys: Vec::new(), spam_train_samples: Vec::new(), now: now(), + total_deleted: 0, + total_active: 0, } } @@ -228,6 +248,7 @@ impl BlobPurgeState { pub fn finalize(&mut self, new_hash: BlobHash) { if !self.last_hash_is_linked { + self.total_deleted += 1; self.delete_keys.push(( None, BlobOp::Commit { @@ -235,6 +256,7 @@ impl BlobPurgeState { }, )); } else { + self.total_active += 1; if !self.spam_train_samples.is_empty() { if self.spam_train_samples.len() > 1 { // Sort by account_id ascending, then until descending @@ -243,27 +265,25 @@ impl BlobPurgeState { a_id.cmp(b_id).then_with(|| b_until.cmp(a_until)) }); let mut samples = self.spam_train_samples.iter().peekable(); - while let Some((account_id, until)) = samples.next() { + while let Some((account_id, _)) = samples.next() { // Keep only the latest sample per account - let mut last_until = *until; while let Some((next_account_id, next_until)) = samples.peek() { if next_account_id == account_id { self.delete_keys.push(( Some(*account_id), BlobOp::SpamSample { hash: self.last_hash.clone(), - until: last_until, + until: *next_until, }, )); self.delete_keys.push(( Some(*account_id), BlobOp::Link { hash: self.last_hash.clone(), - to: BlobLink::Temporary { until: last_until }, + to: BlobLink::Temporary { until: *next_until }, }, )); samples.next(); - last_until = *next_until; } else { break; } @@ -280,6 +300,7 @@ impl BlobPurgeState { pub fn process_key(&mut self, key: &[u8], value: &[u8]) -> trc::Result<()> { const TEMP_LINK: usize = BLOB_HASH_LEN + U32_LEN + U64_LEN; const DOC_LINK: usize = BLOB_HASH_LEN + U64_LEN + 1; + const ID_LINK: usize = BLOB_HASH_LEN + U64_LEN; match key.len() { BLOB_HASH_LEN => { @@ -339,8 +360,8 @@ impl BlobPurgeState { } Ok(()) } - DOC_LINK => { - // Document link + DOC_LINK | ID_LINK => { + // Document/Id link self.last_hash_is_linked = true; Ok(()) } diff --git a/crates/trc/src/event/description.rs b/crates/trc/src/event/description.rs index 3a6647f1..123e4c12 100644 --- a/crates/trc/src/event/description.rs +++ b/crates/trc/src/event/description.rs @@ -1279,7 +1279,7 @@ impl PurgeEvent { PurgeEvent::Error => "Purge error", PurgeEvent::InProgress => "Active purge in progress", PurgeEvent::AutoExpunge => "Auto-expunge executed", - PurgeEvent::TombstoneCleanup => "Tombstone cleanup executed", + PurgeEvent::BlobCleanup => "Blob storage cleanup completed", } } @@ -1291,7 +1291,7 @@ impl PurgeEvent { PurgeEvent::Error => "An error occurred with the purge", PurgeEvent::InProgress => "An active purge is in progress", PurgeEvent::AutoExpunge => "Auto-expunge has been executed", - PurgeEvent::TombstoneCleanup => "Tombstone cleanup has been executed", + PurgeEvent::BlobCleanup => "Blob storage cleanup has completed", } } } diff --git a/crates/trc/src/event/level.rs b/crates/trc/src/event/level.rs index 6fbb406e..e9493914 100644 --- a/crates/trc/src/event/level.rs +++ b/crates/trc/src/event/level.rs @@ -273,9 +273,8 @@ impl EventType { PurgeEvent::Finished => Level::Debug, PurgeEvent::Running => Level::Info, PurgeEvent::Error => Level::Error, - PurgeEvent::InProgress | PurgeEvent::AutoExpunge | PurgeEvent::TombstoneCleanup => { - Level::Debug - } + PurgeEvent::BlobCleanup => Level::Info, + PurgeEvent::InProgress | PurgeEvent::AutoExpunge => Level::Debug, }, EventType::Eval(event) => match event { EvalEvent::Error | EvalEvent::StoreNotFound => Level::Debug, diff --git a/crates/trc/src/lib.rs b/crates/trc/src/lib.rs index 85e72989..9aa0a8db 100644 --- a/crates/trc/src/lib.rs +++ b/crates/trc/src/lib.rs @@ -717,7 +717,7 @@ pub enum PurgeEvent { Error, InProgress, AutoExpunge, - TombstoneCleanup, + BlobCleanup, } #[event_type] diff --git a/crates/trc/src/serializers/binary.rs b/crates/trc/src/serializers/binary.rs index 41319240..acb76c0c 100644 --- a/crates/trc/src/serializers/binary.rs +++ b/crates/trc/src/serializers/binary.rs @@ -668,7 +668,7 @@ impl EventType { EventType::Purge(PurgeEvent::InProgress) => 367, EventType::Purge(PurgeEvent::Running) => 368, EventType::Purge(PurgeEvent::Started) => 369, - EventType::Purge(PurgeEvent::TombstoneCleanup) => 370, + EventType::Purge(PurgeEvent::BlobCleanup) => 370, EventType::PushSubscription(PushSubscriptionEvent::Error) => 371, EventType::PushSubscription(PushSubscriptionEvent::NotFound) => 372, EventType::PushSubscription(PushSubscriptionEvent::Success) => 373, @@ -1299,7 +1299,7 @@ impl EventType { 367 => Some(EventType::Purge(PurgeEvent::InProgress)), 368 => Some(EventType::Purge(PurgeEvent::Running)), 369 => Some(EventType::Purge(PurgeEvent::Started)), - 370 => Some(EventType::Purge(PurgeEvent::TombstoneCleanup)), + 370 => Some(EventType::Purge(PurgeEvent::BlobCleanup)), 371 => Some(EventType::PushSubscription(PushSubscriptionEvent::Error)), 372 => Some(EventType::PushSubscription(PushSubscriptionEvent::NotFound)), 373 => Some(EventType::PushSubscription(PushSubscriptionEvent::Success)), diff --git a/tests/src/imap/bayes.rs b/tests/src/imap/antispam.rs similarity index 83% rename from tests/src/imap/bayes.rs rename to tests/src/imap/antispam.rs index cc04de6b..db125e07 100644 --- a/tests/src/imap/bayes.rs +++ b/tests/src/imap/antispam.rs @@ -5,42 +5,40 @@ */ use super::{IMAPTest, ImapConnection}; -use crate::{ - imap::Type, - jmap::{mail::delivery::SmtpConnection, wait_for_index}, - smtp::session::VerifyResponse, -}; -use directory::backend::internal::manage::ManageDirectory; +use crate::{imap::Type, jmap::mail::delivery::SmtpConnection, smtp::session::VerifyResponse}; +use common::{Server, config::spamfilter::SpamClassifierModel}; use imap_proto::ResponseType; +use spam_filter::modules::classifier::SpamClassifier; +use store::{ + IterateParams, U32_LEN, U64_LEN, ValueKey, + write::{AlignedBytes, Archive, BlobOp, ValueClass, key::DeserializeBigEndian}, +}; +use types::{blob_hash::BlobHash, collection::Collection, field::PrincipalField}; pub async fn test(handle: &IMAPTest) { println!("Running Spam classifier tests..."); let mut imap = ImapConnection::connect(b"_x ").await; imap.assert_read(Type::Untagged, ResponseType::Ok).await; - imap.send("AUTHENTICATE PLAIN AGJheWVzQGV4YW1wbGUuY29tAHNlY3JldA==") - .await; - imap.assert_read(Type::Tagged, ResponseType::Ok).await; + imap.authenticate("sgd@example.com", "secret").await; - let todo = "fix + test jmap"; - - // Make sure the bayes classifier is empty - /*let account_id = handle + let account_id = handle .server - .store() - .get_principal_id("bayes@example.com") + .directory() + .email_to_id("sgd@example.com") .await .unwrap() .unwrap(); - let w = handle.spam_weights(account_id).await; - assert_eq!(w.ham, 0); - assert_eq!(w.spam, 0); + + // Make sure there are no training samples + spam_delete_samples(&handle.server).await; + assert_eq!(spam_training_samples(&handle.server).await.total_count, 0); // Train the classifier via APPEND imap.append("INBOX", HAM[0]).await; imap.append("Junk Mail", SPAM[0]).await; - let w = handle.spam_weights(account_id).await; - assert_eq!(w.ham, 1); - assert_eq!(w.spam, 1); + let samples = spam_training_samples(&handle.server).await; + assert_eq!(samples.ham_count, 1); + assert_eq!(samples.spam_count, 1); // Append two spam samples to "Drafts", then train the classifier via STORE and MOVE imap.append("Drafts", SPAM[1]).await; @@ -48,9 +46,9 @@ pub async fn test(handle: &IMAPTest) { imap.send_ok("SELECT Drafts").await; imap.send_ok("STORE 1 +FLAGS ($Junk)").await; imap.send_ok("MOVE 2 \"Junk Mail\"").await; - let w = handle.spam_weights(account_id).await; - assert_eq!(w.ham, 1); - assert_eq!(w.spam, 3); + let samples = spam_training_samples(&handle.server).await; + assert_eq!(samples.ham_count, 1); + assert_eq!(samples.spam_count, 3); // Add the remaining messages via APPEND for message in HAM.iter().skip(1) { @@ -59,14 +57,42 @@ pub async fn test(handle: &IMAPTest) { for message in SPAM.iter().skip(3) { imap.append("Junk Mail", message).await; } - let w = handle.spam_weights(account_id).await; - assert_eq!(w.ham, 10); - assert_eq!(w.spam, 10); + let samples = spam_training_samples(&handle.server).await; + assert_eq!(samples.ham_count, 10); + assert_eq!(samples.spam_count, 10); + assert_eq!(samples.samples.len(), 20); + assert!( + samples + .samples + .iter() + .all(|s| s.account_id == account_id && s.remove.is_none()) + ); + + // Train the classifier + handle.server.spam_train(false).await.unwrap(); + let model = spam_classifier_model(&handle.server).await; + assert_eq!(model.ham_count, 10); + assert_eq!(model.spam_count, 10); + assert_eq!( + model.last_sample_expiry, + samples.samples.iter().map(|s| s.until).max().unwrap() + ); + assert_eq!(spam_training_samples(&handle.server).await.total_count, 20); + assert!( + handle + .server + .inner + .data + .spam_classifier + .load() + .model + .is_active() + ); // Send 3 test emails for message in TEST { let mut lmtp = SmtpConnection::connect_port(11201).await; - lmtp.ingest("bill@example.com", &["bayes@example.com"], message) + lmtp.ingest("bill@example.com", &["sgd@example.com"], message) .await; } tokio::time::sleep(std::time::Duration::from_millis(200)).await; @@ -76,24 +102,159 @@ pub async fn test(handle: &IMAPTest) { .await .assert_not_contains("FLAGS ($Junk") .assert_contains("Subject: can someone explain") - .assert_contains("X-Spam-Bayes: No"); + .assert_contains("X-Spam-Status: No") + .assert_contains("PROB_HAM_HIGH"); imap.send("FETCH 12 (FLAGS RFC822.TEXT)").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_not_contains("FLAGS ($Junk") .assert_contains("Subject: classifier test") - .assert_not_contains("X-Spam-Bayes: "); + .assert_contains("X-Spam-Status: No") + .assert_contains("PROB_HAM_MEDIUM"); imap.send_ok("SELECT \"Junk Mail\"").await; imap.send("FETCH 10 (FLAGS RFC822.TEXT)").await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("FLAGS ($Junk") .assert_contains("Subject: save up to") - .assert_contains("X-Spam-Bayes: Yes"); + .assert_contains("X-Spam-Status: Yes") + .assert_contains("PROB_SPAM_HIGH"); imap.send_ok("MOVE 10 INBOX").await; - let w = handle.spam_weights(account_id).await; - assert_eq!(w.ham, 11); - assert_eq!(w.spam, 10);*/ + let samples = spam_training_samples(&handle.server).await; + assert_eq!(samples.ham_count, 11); + assert_eq!(samples.spam_count, 10); + + // Make sure spam traps trigger spam classification + let mut lmtp = SmtpConnection::connect_port(11201).await; + lmtp.ingest("bill@example.com", &["spamtrap@example.com"], SPAM[4]) + .await; + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + let samples = spam_training_samples(&handle.server).await; + assert_eq!(samples.ham_count, 11); + assert_eq!(samples.spam_count, 11); +} + +#[derive(Default, Debug)] +pub struct TrainingSamples { + pub samples: Vec, + pub spam_count: usize, + pub ham_count: usize, + pub total_count: usize, +} + +#[derive(Debug)] +#[allow(dead_code)] +pub struct TrainingSample { + pub hash: BlobHash, + pub account_id: u32, + pub is_spam: bool, + pub remove: Option, + pub until: u64, +} + +pub async fn spam_classifier_model(server: &Server) -> SpamClassifierModel { + server + .store() + .get_value::>(ValueKey::property( + u32::MAX, + Collection::Principal, + u32::MAX, + PrincipalField::SpamModel, + )) + .await + .and_then(|archive| match archive { + Some(archive) => archive.deserialize::().map(Some), + None => Ok(None), + }) + .unwrap() + .unwrap() +} + +pub async fn spam_delete_samples(server: &Server) { + let from_key = ValueKey { + account_id: 0, + collection: 0, + document_id: 0, + class: ValueClass::Blob(BlobOp::SpamSample { + hash: BlobHash::default(), + until: 0, + }), + }; + let to_key = ValueKey { + account_id: u32::MAX, + collection: u8::MAX, + document_id: u32::MAX, + class: ValueClass::Blob(BlobOp::SpamSample { + hash: BlobHash::new_max(), + until: u64::MAX, + }), + }; + server.store().delete_range(from_key, to_key).await.unwrap(); +} + +pub async fn spam_training_samples(server: &Server) -> TrainingSamples { + let mut samples = TrainingSamples::default(); + let from_key = ValueKey { + account_id: 0, + collection: 0, + document_id: 0, + class: ValueClass::Blob(BlobOp::SpamSample { + hash: BlobHash::default(), + until: 0, + }), + }; + let to_key = ValueKey { + account_id: u32::MAX, + collection: u8::MAX, + document_id: u32::MAX, + class: ValueClass::Blob(BlobOp::SpamSample { + hash: BlobHash::new_max(), + until: u64::MAX, + }), + }; + server + .store() + .iterate( + IterateParams::new(from_key, to_key).ascending(), + |key, value| { + let until = key.deserialize_be_u64(1)?; + let account_id = key.deserialize_be_u32(U64_LEN + 1)?; + let hash = + BlobHash::try_from_hash_slice(key.get(U64_LEN + U32_LEN + 1..).ok_or_else( + || trc::Error::corrupted_key(key, value.into(), trc::location!()), + )?) + .unwrap(); + let (Some(is_spam), Some(hold)) = (value.first(), value.get(1)) else { + return Err(trc::Error::corrupted_key( + key, + value.into(), + trc::location!(), + )); + }; + + let do_remove = *hold == 0; + let is_spam = *is_spam == 1; + samples.samples.push(TrainingSample { + hash, + account_id, + is_spam, + remove: do_remove.then_some(until), + until, + }); + if is_spam { + samples.spam_count += 1; + } else { + samples.ham_count += 1; + } + samples.total_count += 1; + + Ok(true) + }, + ) + .await + .unwrap(); + + samples } impl ImapConnection { @@ -113,7 +274,7 @@ impl ImapConnection { } } -const SPAM: [&str; 10] = [ +pub const SPAM: [&str; 10] = [ concat!( "Subject: save up to NUMBER on life insurance\r\n\r\n wh", "y spend more than you have to life quote savings e", @@ -426,7 +587,7 @@ const SPAM: [&str; 10] = [ ), ]; -const HAM: [&str; 10] = [ +pub const HAM: [&str; 10] = [ concat!( "Message-ID: \r\nSubject: i have been", " trying to research via sa mirrors and search engi", @@ -654,7 +815,7 @@ const TEST: [&str; 3] = [ ), concat!( "Subject: classifier test\r\n\r\nthis is a novel text tha", - "t the bayes classifier has never seen before, it s", + "t the sgd classifier has never seen before, it s", "hould be classified as ham or non-ham\r\n" ), ]; diff --git a/tests/src/imap/mod.rs b/tests/src/imap/mod.rs index 0fb0a801..76f6b669 100644 --- a/tests/src/imap/mod.rs +++ b/tests/src/imap/mod.rs @@ -5,9 +5,9 @@ */ pub mod acl; +pub mod antispam; pub mod append; pub mod basic; -pub mod bayes; pub mod body_structure; pub mod condstore; pub mod copy_move; @@ -109,8 +109,8 @@ pub async fn imap_tests() { imap.assert_read(Type::Untagged, ResponseType::Bye).await; } - // Bayes training - bayes::test(&handle).await; + // Antispam training + antispam::test(&handle).await; // Run ManageSieve tests managesieve::test().await; @@ -268,10 +268,18 @@ async fn init_imap_tests(delete_if_exists: bool) -> IMAPTest { .await; store .create_test_user( - "bayes@example.com", + "sgd@example.com", "secret", - "Thomas Bayes", - &["bayes@example.com"], + "Sigmund Gudmund Dudmundsson", + &["sgd@example.com"], + ) + .await; + store + .create_test_user( + "spamtrap@example.com", + "secret", + "Spam Trap", + &["spamtrap@example.com"], ) .await; store @@ -658,12 +666,11 @@ wait = "1ms" [spam-filter] enable = true -[spam-filter.bayes.account] -enable = true +[spam-filter.list] +scores = {"PROB_SPAM_HIGH" = "10.0"} -[spam-filter.bayes.classify] -balance = "0.0" -learns = 10 +[lookup] +"spam-traps" = {"spamtrap@*"} [queue] path = "{TMP}" diff --git a/tests/src/imap/pop.rs b/tests/src/imap/pop.rs index e55ea2b7..0424f7dd 100644 --- a/tests/src/imap/pop.rs +++ b/tests/src/imap/pop.rs @@ -79,7 +79,7 @@ pub async fn test() { pop3.send("STAT").await; pop3.assert_read(ResponseType::Ok) .await - .assert_contains("+OK 3 546"); + .assert_contains("+OK 3 603"); // UTF8 pop3.send("UTF8").await; @@ -90,13 +90,13 @@ pub async fn test() { pop3.assert_read(ResponseType::Multiline) .await .assert_contains("+OK 3 messages") - .assert_contains("1 182") - .assert_contains("2 182") - .assert_contains("3 182"); + .assert_contains("1 201") + .assert_contains("2 201") + .assert_contains("3 201"); pop3.send("LIST 2").await; pop3.assert_read(ResponseType::Ok) .await - .assert_contains("+OK 2 182"); + .assert_contains("+OK 2 201"); // UIDL pop3.send("UIDL").await; @@ -115,13 +115,13 @@ pub async fn test() { pop3.send("RETR 1").await; pop3.assert_read(ResponseType::Multiline) .await - .assert_contains("+OK 182 octets") + .assert_contains("+OK 201 octets") .assert_contains("I'm going to need those TPS 0 reports ASAP.") .assert_contains("So, if you could do that, that'd be great."); pop3.send("RETR 3").await; pop3.assert_read(ResponseType::Multiline) .await - .assert_contains("+OK 182 octets") + .assert_contains("+OK 201 octets") .assert_contains("I'm going to need those TPS 2 reports ASAP.") .assert_contains("So, if you could do that, that'd be great."); pop3.send("RETR 4").await; @@ -131,13 +131,13 @@ pub async fn test() { pop3.send("TOP 1 4").await; pop3.assert_read(ResponseType::Multiline) .await - .assert_contains("+OK 182 octets") + .assert_contains("+OK 201 octets") .assert_contains("Subject: TPS Report 0") .assert_not_contains("I'm going to need those TPS 0 reports ASAP."); pop3.send("TOP 3 4").await; pop3.assert_read(ResponseType::Multiline) .await - .assert_contains("+OK 182 octets") + .assert_contains("+OK 201 octets") .assert_contains("Subject: TPS Report 2") .assert_not_contains("I'm going to need those TPS 2 reports ASAP."); @@ -153,7 +153,7 @@ pub async fn test() { pop3.send("STAT").await; pop3.assert_read(ResponseType::Ok) .await - .assert_contains("+OK 3 546"); + .assert_contains("+OK 3 603"); // DELE + QUIT (should delete messages) pop3.send("DELE 2").await; @@ -164,7 +164,7 @@ pub async fn test() { pop3.send("STAT").await; pop3.assert_read(ResponseType::Ok) .await - .assert_contains("+OK 2 364"); + .assert_contains("+OK 2 402"); pop3.send("TOP 1 4").await; pop3.assert_read(ResponseType::Multiline) .await diff --git a/tests/src/jmap/auth/permissions.rs b/tests/src/jmap/auth/permissions.rs index dfbf3a67..e98f2859 100644 --- a/tests/src/jmap/auth/permissions.rs +++ b/tests/src/jmap/auth/permissions.rs @@ -21,6 +21,13 @@ pub async fn test(params: &JMAPTest) { println!("Running permissions tests..."); let server = params.server.clone(); + // Disable spam filtering to avoid adding extra headers + let old_core = params.server.core.clone(); + let mut new_core = old_core.as_ref().clone(); + new_core.spam.enabled = false; + new_core.smtp.session.data.add_delivered_to = false; + params.server.inner.shared_core.store(Arc::new(new_core)); + // Remove unlimited requests permission for &account in params.accounts.keys() { params @@ -675,13 +682,14 @@ pub async fn test(params: &JMAPTest) { ); // Quota for the tenant and user should be updated + const EXTRA_BYTES: i64 = 19; // Storage overhead assert_eq!( server.get_used_quota(tenant_id).await.unwrap(), - TEST_MESSAGE.len() as i64 + TEST_MESSAGE.len() as i64 + EXTRA_BYTES ); assert_eq!( server.get_used_quota(tenant_user_id).await.unwrap(), - TEST_MESSAGE.len() as i64 + TEST_MESSAGE.len() as i64 + EXTRA_BYTES ); // Next delivery should fail due to tenant quota @@ -720,7 +728,7 @@ pub async fn test(params: &JMAPTest) { assert_eq!(server.get_used_quota(tenant_id).await.unwrap(), 0); assert_eq!( server.get_used_quota(other_tenant_id).await.unwrap(), - TEST_MESSAGE.len() as i64 + TEST_MESSAGE.len() as i64 + EXTRA_BYTES ); // Deleting tenants with data should fail diff --git a/tests/src/jmap/mail/antispam.rs b/tests/src/jmap/mail/antispam.rs new file mode 100644 index 00000000..09248025 --- /dev/null +++ b/tests/src/jmap/mail/antispam.rs @@ -0,0 +1,175 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::sync::Arc; + +use email::mailbox::{DRAFTS_ID, INBOX_ID, JUNK_ID}; +use store::write::now; +use types::{id::Id, keyword::Keyword}; + +use crate::{imap::antispam::*, jmap::JMAPTest}; + +pub async fn test(params: &mut JMAPTest) { + println!("Running Email Spam classifier tests..."); + let account = params.account("jdoe@example.com"); + let client = account.client(); + let account_id = account.id().document_id(); + + // Make sure there are no training samples + spam_delete_samples(¶ms.server).await; + assert_eq!(spam_training_samples(¶ms.server).await.total_count, 0); + + // Import samples + let mut spam_ids = vec![]; + let mut ham_ids = vec![]; + for (idx, samples) in [&SPAM, &HAM].into_iter().enumerate() { + let is_spam = idx == 0; + for (num, sample) in samples.iter().enumerate() { + let mut mailbox_ids = vec![]; + let mut keywords = vec![]; + + if num == 0 { + if is_spam { + mailbox_ids.push(Id::from(JUNK_ID).to_string()); + keywords.push(Keyword::Junk.to_string()); + } else { + mailbox_ids.push(Id::from(INBOX_ID).to_string()); + keywords.push(Keyword::NotJunk.to_string()); + } + } else { + mailbox_ids.push(Id::from(DRAFTS_ID).to_string()); + } + + let mail_id = client + .email_import( + sample.as_bytes().to_vec(), + &mailbox_ids, + Some(&keywords), + None, + ) + .await + .unwrap() + .take_id(); + if is_spam { + spam_ids.push(mail_id); + } else { + ham_ids.push(mail_id); + } + } + } + let samples = spam_training_samples(¶ms.server).await; + assert_eq!(samples.ham_count, 1); + assert_eq!(samples.spam_count, 1); + + // Train the classifier via JMAP + for (ids, is_spam) in [(&spam_ids, true), (&ham_ids, false)] { + for (idx, id) in ids.iter().skip(1).enumerate() { + // Set keywords and mailboxes + let mut request = client.build(); + let req = request.set_email().update(id); + if idx < 5 || !is_spam { + // Update via keywords + let keyword = if is_spam { + Keyword::Junk + } else { + Keyword::NotJunk + } + .to_string(); + req.keywords([&keyword]); + } else { + // Update via mailbox + let mailbox_id = if is_spam { JUNK_ID } else { INBOX_ID }; + req.mailbox_ids([&Id::from(mailbox_id).to_string()]); + } + + request.send_set_email().await.unwrap().updated(id).unwrap(); + } + } + let samples = spam_training_samples(¶ms.server).await; + assert_eq!(samples.ham_count, 10); + assert_eq!(samples.spam_count, 10); + + // Reclassifying an email should not add a new sample + let mut request = client.build(); + request + .set_email() + .update(&ham_ids[0]) + .keywords([Keyword::Junk.to_string()]); + request + .send_set_email() + .await + .unwrap() + .updated(&ham_ids[0]) + .unwrap(); + let samples = spam_training_samples(¶ms.server).await; + assert_eq!(samples.ham_count, 9); + assert_eq!(samples.spam_count, 11); + assert_eq!(samples.samples.len(), 20); + let hold_for = params + .server + .core + .spam + .classifier + .as_ref() + .unwrap() + .hold_samples_for; + assert!(hold_for > 2 * 86400); + let hold_until = now() + hold_for; + let hold_range = (hold_until - 86400)..=hold_until; + assert!(samples.samples.iter().all(|s| s.account_id == account_id + && s.remove.is_none() + && hold_range.contains(&s.until))); + + // Purging blobs should not remove training samples + params + .server + .store() + .purge_blobs(params.server.blob_store().clone()) + .await + .unwrap(); + let samples = spam_training_samples(¶ms.server).await; + assert_eq!(samples.ham_count, 9); + assert_eq!(samples.spam_count, 11); + assert_eq!(samples.samples.len(), 20); + + // Extend hold period so a new training sample is generated + let old_core = params.server.core.clone(); + let mut new_core = old_core.as_ref().clone(); + new_core.spam.classifier.as_mut().unwrap().hold_samples_for += 2 * 86400; + params.server.inner.shared_core.store(Arc::new(new_core)); + + // Reclassifying an email will now add a new sample + let mut request = client.build(); + request + .set_email() + .update(&ham_ids[0]) + .keywords([Keyword::NotJunk.to_string()]); + request + .send_set_email() + .await + .unwrap() + .updated(&ham_ids[0]) + .unwrap(); + let samples = spam_training_samples(¶ms.server).await; + assert_eq!(samples.ham_count, 10); + assert_eq!(samples.spam_count, 11); + assert_eq!(samples.samples.len(), 21); + + // Blob purge should remove the duplicated sample + params + .server + .store() + .purge_blobs(params.server.blob_store().clone()) + .await + .unwrap(); + let samples = spam_training_samples(¶ms.server).await; + assert_eq!(samples.ham_count, 10); + assert_eq!(samples.spam_count, 10); + assert_eq!(samples.samples.len(), 20); + + params.destroy_all_mailboxes(account).await; + params.assert_is_empty().await; +} diff --git a/tests/src/jmap/mail/delivery.rs b/tests/src/jmap/mail/delivery.rs index 31b0b12c..996ffc9e 100644 --- a/tests/src/jmap/mail/delivery.rs +++ b/tests/src/jmap/mail/delivery.rs @@ -5,12 +5,16 @@ */ use crate::{ - directory::internal::TestInternalDirectory, jmap::JMAPTest, - store::cleanup::store_blob_expire_all, webdav::DummyWebDavClient, + directory::internal::TestInternalDirectory, + imap::antispam::{spam_delete_samples, spam_training_samples}, + jmap::JMAPTest, + store::cleanup::store_blob_expire_all, + webdav::DummyWebDavClient, }; +use common::Server; use email::{ cache::{MessageCacheFetch, email::MessageCacheAccess}, - mailbox::{INBOX_ID, JUNK_ID}, + mailbox::{INBOX_ID, JUNK_ID, SENT_ID}, message::metadata::MessageMetadata, }; use groupware::DavResourceName; @@ -18,6 +22,7 @@ use jmap::blob::download::BlobDownload; use std::{sync::Arc, time::Duration}; use store::{ ValueKey, + roaring::RoaringBitmap, write::{AlignedBytes, Archive}, }; use tokio::{ @@ -26,9 +31,9 @@ use tokio::{ }; use types::{ blob::{BlobClass, BlobId}, - blob_hash::BlobHash, collection::Collection, field::EmailField, + id::Id, }; use utils::chained_bytes::ChainedBytes; @@ -72,7 +77,6 @@ pub async fn test(params: &mut JMAPTest) { "From: bill@example.com\r\n", "To: jdoe@example.com\r\n", "Subject: TPS Report\r\n", - "X-Spam-Status: No\r\n", "\r\n", "I'm going to need those TPS reports ASAP. ", "So, if you could do that, that'd be great." @@ -89,15 +93,18 @@ pub async fn test(params: &mut JMAPTest) { assert_eq!(john_cache.in_mailbox(INBOX_ID).count(), 1); assert_eq!(john_cache.in_mailbox(JUNK_ID).count(), 0); - // Delivering to individuals' aliases + // Make sure there are no spam training samples + spam_delete_samples(¶ms.server).await; + assert_eq!(spam_training_samples(¶ms.server).await.total_count, 0); + + // Test spam filtering lmtp.ingest( "bill@example.com", &["john.doe@example.com"], concat!( "From: bill@example.com\r\n", "To: john.doe@example.com\r\n", - "Subject: Fwd: TPS Report\r\n", - "X-Spam-Status: Yes, score=13.9\r\n", + "Subject: XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X\r\n", "\r\n", "--- Forwarded Message ---\r\n\r\n ", "I'm going to need those TPS reports ASAP. ", @@ -109,10 +116,25 @@ pub async fn test(params: &mut JMAPTest) { .get_cached_messages(john.id().document_id()) .await .unwrap(); - + let inbox_ids = john_cache + .in_mailbox(INBOX_ID) + .map(|e| e.document_id) + .collect::(); + let junk_ids = john_cache + .in_mailbox(JUNK_ID) + .map(|e| e.document_id) + .collect::(); assert_eq!(john_cache.emails.items.len(), 2); - assert_eq!(john_cache.in_mailbox(INBOX_ID).count(), 1); - assert_eq!(john_cache.in_mailbox(JUNK_ID).count(), 1); + assert_eq!(inbox_ids.len(), 1); + assert_eq!(junk_ids.len(), 1); + assert_message_headers_contains( + &server, + john.id().document_id(), + junk_ids.min().unwrap(), + "X-Spam-Status: Yes", + ) + .await; + assert_eq!(spam_training_samples(¶ms.server).await.total_count, 0); // CardDAV spam override let dav_client = DummyWebDavClient::new(u32::MAX, john.name(), john.secret(), john.emails()[0]); @@ -139,8 +161,7 @@ END:VCARD concat!( "From: dmarc-bill@example.com\r\n", "To: john.doe@example.com\r\n", - "Subject: Fwd: TPS Report (CardDAV spam override)\r\n", - "X-Spam-Status: Yes, score=13.9\r\n", + "Subject: XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X\r\n", "\r\n", "--- Forwarded Message ---\r\n\r\n ", "I'm going to need those TPS reports ASAP. ", @@ -152,11 +173,100 @@ END:VCARD .get_cached_messages(john.id().document_id()) .await .unwrap(); - + let inbox_ids = john_cache + .in_mailbox(INBOX_ID) + .map(|e| e.document_id) + .collect::(); + let junk_ids = john_cache + .in_mailbox(JUNK_ID) + .map(|e| e.document_id) + .collect::(); assert_eq!(john_cache.emails.items.len(), 3); - assert_eq!(john_cache.in_mailbox(INBOX_ID).count(), 2); - assert_eq!(john_cache.in_mailbox(JUNK_ID).count(), 1); + assert_eq!(inbox_ids.len(), 2); + assert_eq!(junk_ids.len(), 1); dav_client.delete_default_containers().await; + assert_message_headers_contains( + &server, + john.id().document_id(), + inbox_ids.max().unwrap(), + "X-Spam-Status: No, reason=card-exists", + ) + .await; + let samples = spam_training_samples(¶ms.server).await; + assert_eq!(samples.ham_count, 1); + assert_eq!(samples.spam_count, 0); + + // Test trusted reply override + john.client() + .email_import( + concat!( + "From: john.doe@example.com\r\n", + "To: dmarc-bill@example.com\r\n", + "Message-ID: \r\n", + "Subject: XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X\r\n", + "\r\n", + "This is a trusted reply." + ) + .as_bytes() + .to_vec(), + vec![Id::from(SENT_ID).to_string()], + None::>, + None, + ) + .await + .unwrap() + .take_id(); + assert_eq!( + server + .get_cached_messages(john.id().document_id()) + .await + .unwrap() + .emails + .items + .len(), + 4 + ); + lmtp.ingest( + "dmarc-bill@example.com", + &["john.doe@example.com"], + concat!( + "From: dmarc-bill@example.com\r\n", + "To: john.doe@example.com\r\n", + "Message-ID: \r\n", + "References: \r\n", + "Subject: XJS*C4JDBQADN1.NSBN3*2IDNEN*GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X\r\n", + "\r\n", + "--- Forwarded Message ---\r\n\r\n ", + "I'm going to need those TPS reports ASAP. ", + "So, if you could do that, that'd be great." + ), + ) + .await; + let john_cache = server + .get_cached_messages(john.id().document_id()) + .await + .unwrap(); + let inbox_ids = john_cache + .in_mailbox(INBOX_ID) + .map(|e| e.document_id) + .collect::(); + let junk_ids = john_cache + .in_mailbox(JUNK_ID) + .map(|e| e.document_id) + .collect::(); + assert_eq!(john_cache.emails.items.len(), 5); + assert_eq!(inbox_ids.len(), 3); + assert_eq!(junk_ids.len(), 1); + assert_message_headers_contains( + &server, + john.id().document_id(), + inbox_ids.max().unwrap(), + "X-Spam-Status: No, reason=trusted-reply", + ) + .await; + let samples = spam_training_samples(¶ms.server).await; + assert_eq!(samples.ham_count, 2); + assert_eq!(samples.spam_count, 0); // EXPN and VRFY lmtp.expn("members@example.com", 2) @@ -185,7 +295,7 @@ END:VCARD ) .await; - for (account, num_messages) in [(john, 4), (jane, 1), (bill, 1)] { + for (account, num_messages) in [(john, 6), (jane, 1), (bill, 1)] { assert_eq!( server .get_cached_messages(account.id().document_id()) @@ -223,7 +333,7 @@ END:VCARD ) .await; - for (account, num_messages) in [(john, 4), (jane, 2), (bill, 2)] { + for (account, num_messages) in [(john, 6), (jane, 2), (bill, 2)] { assert_eq!( server .get_cached_messages(account.id().document_id()) @@ -262,7 +372,7 @@ END:VCARD // Make sure blobs are properly linked store_blob_expire_all(params.server.store()).await; - for (account, num_messages) in [(john, 5), (jane, 3), (bill, 3)] { + for (account, num_messages) in [(john, 7), (jane, 3), (bill, 3)] { let account_id = account.id().document_id(); let cache = server.get_cached_messages(account_id).await.unwrap(); assert_eq!( @@ -273,31 +383,20 @@ END:VCARD ); let access_token = server.get_access_token(account_id).await.unwrap(); - for document_id in cache.emails.items.iter().map(|e| e.document_id) { - let archive = server - .store() - .get_value::>(ValueKey::property( - account_id, - Collection::Email, - document_id, - EmailField::Metadata, - )) - .await - .unwrap() - .unwrap(); - let metadata = archive.to_unarchived::().unwrap(); + for document_id in cache.in_mailbox(INBOX_ID).map(|e| e.document_id) { + let metadata = message_metadata(&server, account_id, document_id).await; let partial_message = server .store() - .get_blob(metadata.inner.blob_hash.0.as_ref(), 0..usize::MAX) + .get_blob(metadata.blob_hash.0.as_ref(), 0..usize::MAX) .await .unwrap() .unwrap(); - assert_ne!(metadata.inner.blob_body_offset.to_native(), 0); + assert_ne!(metadata.blob_body_offset, 0); let expected_full_message = String::from_utf8( - ChainedBytes::new(metadata.inner.raw_headers.as_ref()) + ChainedBytes::new(metadata.raw_headers.as_ref()) .with_last( partial_message - .get(metadata.inner.blob_body_offset.to_native() as usize..) + .get(metadata.blob_body_offset as usize..) .unwrap_or_default(), ) .to_bytes(), @@ -312,7 +411,7 @@ END:VCARD server .blob_download( &BlobId { - hash: BlobHash::from(&metadata.inner.blob_hash), + hash: metadata.blob_hash, class: BlobClass::Linked { account_id, collection: Collection::Email.into(), @@ -349,6 +448,48 @@ END:VCARD ]); } +async fn assert_message_headers_contains( + server: &Server, + account_id: u32, + document_id: u32, + value: &str, +) { + let headers = message_headers(server, account_id, document_id).await; + assert!( + headers.contains(value), + "Expected message headers to contain {:?}, got {:?}", + value, + headers + ); +} + +async fn message_headers(server: &Server, account_id: u32, document_id: u32) -> String { + std::str::from_utf8( + message_metadata(server, account_id, document_id) + .await + .raw_headers + .as_ref(), + ) + .unwrap() + .to_string() +} + +async fn message_metadata(server: &Server, account_id: u32, document_id: u32) -> MessageMetadata { + server + .store() + .get_value::>(ValueKey::property( + account_id, + Collection::Email, + document_id, + EmailField::Metadata, + )) + .await + .unwrap() + .unwrap() + .deserialize::() + .unwrap() +} + pub struct SmtpConnection { reader: Lines>>, writer: WriteHalf, diff --git a/tests/src/jmap/mail/mod.rs b/tests/src/jmap/mail/mod.rs index 0a46503d..0351c552 100644 --- a/tests/src/jmap/mail/mod.rs +++ b/tests/src/jmap/mail/mod.rs @@ -5,6 +5,7 @@ */ pub mod acl; +pub mod antispam; pub mod changes; pub mod copy; pub mod crypto; diff --git a/tests/src/jmap/mod.rs b/tests/src/jmap/mod.rs index a8fbffe4..60cc7f44 100644 --- a/tests/src/jmap/mod.rs +++ b/tests/src/jmap/mod.rs @@ -78,7 +78,7 @@ async fn jmap_tests() { server::webhooks::test(&mut params).await; - /*mail::get::test(&mut params).await; + mail::get::test(&mut params).await; mail::set::test(&mut params).await; mail::parse::test(&mut params).await; mail::query::test(&mut params, delete).await; @@ -95,6 +95,7 @@ async fn jmap_tests() { mail::vacation_response::test(&mut params).await; mail::submission::test(&mut params).await; mail::crypto::test(&mut params).await; + mail::antispam::test(&mut params).await; core::event_source::test(&mut params).await; core::websocket::test(&mut params).await; @@ -122,7 +123,7 @@ async fn jmap_tests() { calendar::acl::test(&mut params).await; principal::get::test(&mut params).await; - principal::availability::test(&mut params).await;*/ + principal::availability::test(&mut params).await; server::purge::test(&mut params).await; server::enterprise::test(&mut params).await; @@ -1580,9 +1581,6 @@ WiYrLO4z8/kmkqvA7wGElBok9IqhRANCAAQxZK68FnQtHC0eyh8CA05xRIvxhVHn ''' signature-algorithm = "ES256" -[spam-filter.bayes.auto-learn] -card-is-ham = false - [session.extensions] expn = true vrfy = true @@ -1590,6 +1588,9 @@ vrfy = true [spam-filter] enable = true +[spam-filter.list] +scores = {"GTUBE_TEST" = "1000.0"} + [sharing] allow-directory-query = true diff --git a/tests/src/jmap/server/enterprise.rs b/tests/src/jmap/server/enterprise.rs index 999736a2..08484289 100644 --- a/tests/src/jmap/server/enterprise.rs +++ b/tests/src/jmap/server/enterprise.rs @@ -32,8 +32,12 @@ use common::{ tracers::store::TracingStore, }, }; -use http::management::enterprise::undelete::{ - DeletedBlobResponse, DeletedItemResponse, UndeleteRequest, UndeleteResponse, +use directory::{QueryBy, backend::internal::manage::ManageDirectory}; +use http::management::{ + enterprise::undelete::{ + DeletedBlobResponse, DeletedItemResponse, UndeleteRequest, UndeleteResponse, + }, + stores::destroy_account_data, }; use imap_proto::ResponseType; use nlp::language::Language; @@ -135,7 +139,7 @@ pub async fn test(params: &mut JMAPTest) { // Create test account let server = params.server.inner.build_server(); - server + let account_id = server .store() .create_test_user( "jdoe@example.com", @@ -150,6 +154,17 @@ pub async fn test(params: &mut JMAPTest) { tracing(params).await; metrics(params).await; + // Delete test account + server + .store() + .delete_principal(QueryBy::Id(account_id)) + .await + .unwrap(); + destroy_account_data(&server, account_id, true) + .await + .unwrap(); + params.assert_is_empty().await; + params.server.inner.shared_core.store( params .server diff --git a/tests/src/jmap/server/purge.rs b/tests/src/jmap/server/purge.rs index 461f5103..c68b4e55 100644 --- a/tests/src/jmap/server/purge.rs +++ b/tests/src/jmap/server/purge.rs @@ -6,7 +6,7 @@ use crate::{ imap::{AssertResult, ImapConnection, Type}, - jmap::JMAPTest, + jmap::{JMAPTest, wait_for_index}, }; use ahash::AHashSet; use common::Server; @@ -149,6 +149,7 @@ pub async fn test(params: &mut JMAPTest) { } // Delete account + wait_for_index(&server).await; server .store() .delete_principal(QueryBy::Id(account.id().document_id())) diff --git a/tests/src/smtp/inbound/antispam.rs b/tests/src/smtp/inbound/antispam.rs index d7087a76..32d0c3b7 100644 --- a/tests/src/smtp/inbound/antispam.rs +++ b/tests/src/smtp/inbound/antispam.rs @@ -61,14 +61,6 @@ use store::{Stores, write::BatchBuilder}; use utils::config::Config; const CONFIG: &str = r#" -[spam-filter.bayes.classify] -balance = "0.0" -learns = 10 - -[spam-filter.bayes.auto-learn.threshold] -ham = "-0.5" -spam = "6.0" - [spam-filter.score] spam = "5.0" diff --git a/tests/src/store/cleanup.rs b/tests/src/store/cleanup.rs index d3025c7e..37c12842 100644 --- a/tests/src/store/cleanup.rs +++ b/tests/src/store/cleanup.rs @@ -120,8 +120,8 @@ pub async fn store_blob_expire_all(store: &Store) { let mut last_account_id = u32::MAX; store .iterate( - IterateParams::new(from_key, to_key).ascending().no_values(), - |key, _| { + IterateParams::new(from_key, to_key).ascending(), + |key, value| { if key.len() == BLOB_HASH_LEN + U32_LEN + U64_LEN { let account_id = key .deserialize_be_u32(BLOB_HASH_LEN) @@ -136,16 +136,32 @@ pub async fn store_blob_expire_all(store: &Store) { .deserialize_be_u64(BLOB_HASH_LEN + U32_LEN) .caused_by(trc::location!())?; - batch - .clear(ValueClass::Blob(BlobOp::Link { - hash: hash.clone(), - to: BlobLink::Temporary { until }, - })) - .clear(ValueClass::Blob(BlobOp::Quota { - hash: hash.clone(), - until, - })) - .clear(ValueClass::Blob(BlobOp::Undelete { hash, until })); + match value.first().copied() { + Some(BlobLink::QUOTA_LINK) => { + batch.clear(ValueClass::Blob(BlobOp::Quota { + hash: hash.clone(), + until, + })); + } + Some(BlobLink::UNDELETE_LINK) => { + batch.clear(ValueClass::Blob(BlobOp::Undelete { + hash: hash.clone(), + until, + })); + } + Some(BlobLink::SPAM_SAMPLE_LINK) => { + batch.clear(ValueClass::Blob(BlobOp::SpamSample { + hash: hash.clone(), + until, + })); + } + _ => {} + } + + batch.clear(ValueClass::Blob(BlobOp::Link { + hash, + to: BlobLink::Temporary { until }, + })); } Ok(true)