From 1efaedcc276da5f5056766eb7f9763c1227b350e Mon Sep 17 00:00:00 2001 From: mdecimus Date: Sat, 24 May 2025 10:17:41 +0200 Subject: [PATCH] Train spam messages as ham when the sender is in the user's address book --- CHANGELOG.md | 1 + crates/common/src/config/spamfilter.rs | 8 +++++ crates/common/src/lib.rs | 3 ++ crates/dav/src/calendar/mod.rs | 2 +- crates/dav/src/card/mod.rs | 2 +- crates/email/src/message/ingest.rs | 48 +++++++++++++++++++++----- crates/groupware/src/calendar/index.rs | 2 +- crates/groupware/src/contact/index.rs | 47 +++++++++++++++++++++++-- crates/groupware/src/lib.rs | 2 -- 9 files changed, 100 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a3169d42..9a9f6c65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ This version includes **breaking changes** to the database layout and requires a - Peer-to-peer [cluster coordination](https://stalw.art/docs/cluster/coordination/overview) or with Apache Kafka, Redpanda, NATS or Redis. - Incremental caching of emails, calendars, contacts and file metadata. - Zero-copy deserialization. +- Train spam messages as ham when the sender is in the user's address book. - `XOAUTH2` SASL mechanism support (#1194 #1369). - Support for RFC9698, the `JMAPACCESS` Extension for IMAP. - Search index for accounts and other principals (#1368). diff --git a/crates/common/src/config/spamfilter.rs b/crates/common/src/config/spamfilter.rs index 4f387046..5d08934b 100644 --- a/crates/common/src/config/spamfilter.rs +++ b/crates/common/src/config/spamfilter.rs @@ -24,6 +24,7 @@ use super::{Variable, functions::ResolveVariable, if_block::IfBlock, tokenizer:: #[derive(Debug, Clone, Default)] pub struct SpamFilterConfig { pub enabled: bool, + pub card_is_ham: bool, pub dnsbl: DnsBlConfig, pub rules: SpamFilterRules, pub lists: SpamFilterLists, @@ -85,6 +86,7 @@ pub struct BayesConfig { pub auto_learn_reply_ham: bool, pub auto_learn_spam_threshold: f64, pub auto_learn_ham_threshold: f64, + pub auto_learn_card_is_ham: bool, pub score_spam: f64, pub score_ham: f64, pub account_score_spam: f64, @@ -178,6 +180,9 @@ impl SpamFilterConfig { enabled: config .property_or_default("spam-filter.enable", "true") .unwrap_or(true), + card_is_ham: config + .property_or_default("spam-filter.card-is-ham", "true") + .unwrap_or(true), dnsbl: DnsBlConfig::parse(config), rules: SpamFilterRules::parse(config), lists: SpamFilterLists::parse(config), @@ -569,6 +574,9 @@ impl BayesConfig { account_score_ham: config .property_or_default("spam-filter.bayes.account.score.ham", "0.5") .unwrap_or(0.5), + auto_learn_card_is_ham: config + .property_or_default("spam-filter.bayes.auto-learn.card-is-ham", "true") + .unwrap_or(true), } .into() } diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index ce4b41b8..9b25270b 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -106,6 +106,9 @@ pub const KV_LOCK_HOUSEKEEPER: u8 = 24; pub const KV_LOCK_DAV: u8 = 25; pub const KV_SIEVE_ID: u8 = 26; +pub const IDX_UID: u8 = 0; +pub const IDX_EMAIL: u8 = 1; + #[derive(Clone)] pub struct Server { pub inner: Arc, diff --git a/crates/dav/src/calendar/mod.rs b/crates/dav/src/calendar/mod.rs index c72d3adf..5edb6526 100644 --- a/crates/dav/src/calendar/mod.rs +++ b/crates/dav/src/calendar/mod.rs @@ -14,12 +14,12 @@ pub mod query; pub mod update; use crate::{DavError, DavErrorCondition}; +use common::IDX_UID; use common::{DavResources, Server}; use dav_proto::schema::{ property::{CalDavProperty, CalendarData, DavProperty, WebDavProperty}, response::CalCondition, }; -use groupware::IDX_UID; use hyper::StatusCode; use jmap_proto::types::collection::Collection; use store::query::Filter; diff --git a/crates/dav/src/card/mod.rs b/crates/dav/src/card/mod.rs index dc46cd18..d7160957 100644 --- a/crates/dav/src/card/mod.rs +++ b/crates/dav/src/card/mod.rs @@ -4,12 +4,12 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use common::IDX_UID; use common::{DavResources, Server}; use dav_proto::schema::{ property::{CardDavProperty, DavProperty, WebDavProperty}, response::CardCondition, }; -use groupware::IDX_UID; use hyper::StatusCode; use jmap_proto::types::collection::Collection; use store::query::Filter; diff --git a/crates/email/src/message/ingest.rs b/crates/email/src/message/ingest.rs index f622c6de..49ee4ad1 100644 --- a/crates/email/src/message/ingest.rs +++ b/crates/email/src/message/ingest.rs @@ -18,7 +18,7 @@ use crate::{ }, }; use common::{ - Server, + IDX_EMAIL, Server, auth::{AccessToken, ResourceToken}, storage::index::ObjectIndexBuilder, }; @@ -47,11 +47,13 @@ use std::{ use store::{ BlobClass, IndexKey, IndexKeyPrefix, IterateParams, U32_LEN, ahash::AHashMap, + query::Filter, roaring::RoaringBitmap, write::{BatchBuilder, TaskQueueClass, ValueClass, key::DeserializeBigEndian, now}, }; use store::{SerializeInfallible, rand::Rng}; use trc::{AddContext, MessageIngestEvent}; +use utils::sanitize_email; #[derive(Default)] pub struct IngestedEmail { @@ -194,14 +196,44 @@ impl EmailIngest for Server { .is_some_and(|v| v.contains("Yes")); } + // If the message is classified as spam, check whether the sender address is present in the user's address book + if is_spam && self.core.spam.card_is_ham { + if let Some(sender) = message + .from() + .and_then(|s| s.first()) + .and_then(|s| s.address()) + .and_then(sanitize_email) + { + if !self + .store() + .filter( + account_id, + Collection::ContactCard, + vec![Filter::eq(IDX_EMAIL, sender.into_bytes())], + ) + .await + .caused_by(trc::location!())? + .results + .is_empty() + { + is_spam = false; + if self + .core + .spam + .bayes + .as_ref() + .is_some_and(|config| config.auto_learn_card_is_ham) + { + train_spam = Some(false); + } + } + } + } + // Classify the message with user's model - if let Some(bayes_config) = self - .core - .spam - .bayes - .as_ref() - .filter(|config| config.account_classify && params.spam_train) - { + if let Some(bayes_config) = self.core.spam.bayes.as_ref().filter(|config| { + config.account_classify && params.spam_train && train_spam.is_none() + }) { // Initialize spam filter let ctx = self.spam_filter_init(SpamFilterInput::from_account_message( &message, diff --git a/crates/groupware/src/calendar/index.rs b/crates/groupware/src/calendar/index.rs index 452a70b5..8c6b735f 100644 --- a/crates/groupware/src/calendar/index.rs +++ b/crates/groupware/src/calendar/index.rs @@ -8,7 +8,7 @@ use super::{ ArchivedCalendar, ArchivedCalendarEvent, ArchivedCalendarPreferences, ArchivedDefaultAlert, ArchivedTimezone, Calendar, CalendarEvent, CalendarPreferences, DefaultAlert, Timezone, }; -use crate::IDX_UID; +use common::IDX_UID; use common::storage::index::{IndexValue, IndexableAndSerializableObject, IndexableObject}; use jmap_proto::types::{collection::SyncCollection, value::AclGrant}; diff --git a/crates/groupware/src/contact/index.rs b/crates/groupware/src/contact/index.rs index 104c8db8..db8cd8c0 100644 --- a/crates/groupware/src/contact/index.rs +++ b/crates/groupware/src/contact/index.rs @@ -5,9 +5,14 @@ */ use super::{AddressBook, ArchivedAddressBook, ArchivedContactCard, ContactCard}; -use crate::IDX_UID; -use common::storage::index::{IndexValue, IndexableAndSerializableObject, IndexableObject}; +use calcard::vcard::VCardProperty; +use common::storage::index::{ + IndexItem, IndexValue, IndexableAndSerializableObject, IndexableObject, +}; +use common::{IDX_EMAIL, IDX_UID}; use jmap_proto::types::{collection::SyncCollection, value::AclGrant}; +use std::collections::HashSet; +use utils::sanitize_email; impl IndexableObject for AddressBook { fn index_values(&self) -> impl Iterator> { @@ -67,6 +72,15 @@ impl IndexableObject for ContactCard { field: IDX_UID, value: self.card.uid().into(), }, + IndexValue::IndexList { + field: IDX_EMAIL, + value: self + .emails() + .map(Into::into) + .collect::>() + .into_iter() + .collect(), + }, IndexValue::Quota { used: self.dead_properties.size() as u32 + self.display_name.as_ref().map_or(0, |n| n.len() as u32) @@ -89,6 +103,15 @@ impl IndexableObject for &ArchivedContactCard { field: IDX_UID, value: self.card.uid().into(), }, + IndexValue::IndexList { + field: IDX_EMAIL, + value: self + .emails() + .map(Into::into) + .collect::>() + .into_iter() + .collect(), + }, IndexValue::Quota { used: self.dead_properties.size() as u32 + self.display_name.as_ref().map_or(0, |n| n.len() as u32) @@ -109,3 +132,23 @@ impl IndexableAndSerializableObject for ContactCard { true } } + +impl ContactCard { + pub fn emails(&self) -> impl Iterator { + self.card.properties(&VCardProperty::Email).flat_map(|e| { + e.values + .iter() + .filter_map(|v| v.as_text().and_then(sanitize_email)) + }) + } +} + +impl ArchivedContactCard { + pub fn emails(&self) -> impl Iterator { + self.card.properties(&VCardProperty::Email).flat_map(|e| { + e.values + .iter() + .filter_map(|v| v.as_text().and_then(sanitize_email)) + }) + } +} diff --git a/crates/groupware/src/lib.rs b/crates/groupware/src/lib.rs index e287b32f..50f77af8 100644 --- a/crates/groupware/src/lib.rs +++ b/crates/groupware/src/lib.rs @@ -13,8 +13,6 @@ pub mod calendar; pub mod contact; pub mod file; -pub const IDX_UID: u8 = 0; - #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum DavResourceName { Card,