filip/Stalwart
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Train spam messages as ham when the sender is in the user's address book

Browse Source
This commit is contained in:
mdecimus
2025-05-24 10:17:41 +02:00
parent c19d2ceb43
commit 1efaedcc27
9 changed files with 100 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
crates/common/src/config/spamfilter.rs
View File

@@ -24,6 +24,7 @@ use super::{Variable, functions::ResolveVariable, if_block::IfBlock, tokenizer::
#[derive(Debug, Clone, Default)]
pub struct SpamFilterConfig {
pub enabled: bool,
pub card_is_ham: bool,
pub dnsbl: DnsBlConfig,
pub rules: SpamFilterRules,
pub lists: SpamFilterLists,
@@ -85,6 +86,7 @@ pub struct BayesConfig {
pub auto_learn_reply_ham: bool,
pub auto_learn_spam_threshold: f64,
pub auto_learn_ham_threshold: f64,
pub auto_learn_card_is_ham: bool,
pub score_spam: f64,
pub score_ham: f64,
pub account_score_spam: f64,
@@ -178,6 +180,9 @@ impl SpamFilterConfig {
enabled: config
.property_or_default("spam-filter.enable", "true")
.unwrap_or(true),
card_is_ham: config
.property_or_default("spam-filter.card-is-ham", "true")
.unwrap_or(true),
dnsbl: DnsBlConfig::parse(config),
rules: SpamFilterRules::parse(config),
lists: SpamFilterLists::parse(config),
@@ -569,6 +574,9 @@ impl BayesConfig {
account_score_ham: config
.property_or_default("spam-filter.bayes.account.score.ham", "0.5")
.unwrap_or(0.5),
auto_learn_card_is_ham: config
.property_or_default("spam-filter.bayes.auto-learn.card-is-ham", "true")
.unwrap_or(true),
}
.into()
}

3
crates/common/src/lib.rs
View File

@@ -106,6 +106,9 @@ pub const KV_LOCK_HOUSEKEEPER: u8 = 24;
pub const KV_LOCK_DAV: u8 = 25;
pub const KV_SIEVE_ID: u8 = 26;
pub const IDX_UID: u8 = 0;
pub const IDX_EMAIL: u8 = 1;
#[derive(Clone)]
pub struct Server {
pub inner: Arc<Inner>,

2
crates/dav/src/calendar/mod.rs
View File

@@ -14,12 +14,12 @@ pub mod query;
pub mod update;
use crate::{DavError, DavErrorCondition};
use common::IDX_UID;
use common::{DavResources, Server};
use dav_proto::schema::{
property::{CalDavProperty, CalendarData, DavProperty, WebDavProperty},
response::CalCondition,
};
use groupware::IDX_UID;
use hyper::StatusCode;
use jmap_proto::types::collection::Collection;
use store::query::Filter;

2
crates/dav/src/card/mod.rs
View File

@@ -4,12 +4,12 @@
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use common::IDX_UID;
use common::{DavResources, Server};
use dav_proto::schema::{
property::{CardDavProperty, DavProperty, WebDavProperty},
response::CardCondition,
};
use groupware::IDX_UID;
use hyper::StatusCode;
use jmap_proto::types::collection::Collection;
use store::query::Filter;

48
crates/email/src/message/ingest.rs
View File

@@ -18,7 +18,7 @@ use crate::{
},
};
use common::{
Server,
IDX_EMAIL, Server,
auth::{AccessToken, ResourceToken},
storage::index::ObjectIndexBuilder,
};
@@ -47,11 +47,13 @@ use std::{
use store::{
BlobClass, IndexKey, IndexKeyPrefix, IterateParams, U32_LEN,
ahash::AHashMap,
query::Filter,
roaring::RoaringBitmap,
write::{BatchBuilder, TaskQueueClass, ValueClass, key::DeserializeBigEndian, now},
};
use store::{SerializeInfallible, rand::Rng};
use trc::{AddContext, MessageIngestEvent};
use utils::sanitize_email;
#[derive(Default)]
pub struct IngestedEmail {
@@ -194,14 +196,44 @@ impl EmailIngest for Server {
.is_some_and(|v| v.contains("Yes"));
}
// If the message is classified as spam, check whether the sender address is present in the user's address book
if is_spam && self.core.spam.card_is_ham {
if let Some(sender) = message
.from()
.and_then(|s| s.first())
.and_then(|s| s.address())
.and_then(sanitize_email)
{
if !self
.store()
.filter(
account_id,
Collection::ContactCard,
vec![Filter::eq(IDX_EMAIL, sender.into_bytes())],
)
.await
.caused_by(trc::location!())?
.results
.is_empty()
{
is_spam = false;
if self
.core
.spam
.bayes
.as_ref()
.is_some_and(|config| config.auto_learn_card_is_ham)
{
train_spam = Some(false);
}
}
}
}
// Classify the message with user's model
if let Some(bayes_config) = self
.core
.spam
.bayes
.as_ref()
.filter(|config| config.account_classify && params.spam_train)
{
if let Some(bayes_config) = self.core.spam.bayes.as_ref().filter(|config| {
config.account_classify && params.spam_train && train_spam.is_none()
}) {
// Initialize spam filter
let ctx = self.spam_filter_init(SpamFilterInput::from_account_message(
&message,

2
crates/groupware/src/calendar/index.rs
View File

@@ -8,7 +8,7 @@ use super::{
ArchivedCalendar, ArchivedCalendarEvent, ArchivedCalendarPreferences, ArchivedDefaultAlert,
ArchivedTimezone, Calendar, CalendarEvent, CalendarPreferences, DefaultAlert, Timezone,
};
use crate::IDX_UID;
use common::IDX_UID;
use common::storage::index::{IndexValue, IndexableAndSerializableObject, IndexableObject};
use jmap_proto::types::{collection::SyncCollection, value::AclGrant};

47
crates/groupware/src/contact/index.rs
View File

@@ -5,9 +5,14 @@
*/
use super::{AddressBook, ArchivedAddressBook, ArchivedContactCard, ContactCard};
use crate::IDX_UID;
use common::storage::index::{IndexValue, IndexableAndSerializableObject, IndexableObject};
use calcard::vcard::VCardProperty;
use common::storage::index::{
IndexItem, IndexValue, IndexableAndSerializableObject, IndexableObject,
};
use common::{IDX_EMAIL, IDX_UID};
use jmap_proto::types::{collection::SyncCollection, value::AclGrant};
use std::collections::HashSet;
use utils::sanitize_email;
impl IndexableObject for AddressBook {
fn index_values(&self) -> impl Iterator<Item = IndexValue<'_>> {
@@ -67,6 +72,15 @@ impl IndexableObject for ContactCard {
field: IDX_UID,
value: self.card.uid().into(),
},
IndexValue::IndexList {
field: IDX_EMAIL,
value: self
.emails()
.map(Into::into)
.collect::<HashSet<IndexItem>>()
.into_iter()
.collect(),
},
IndexValue::Quota {
used: self.dead_properties.size() as u32
+ self.display_name.as_ref().map_or(0, |n| n.len() as u32)
@@ -89,6 +103,15 @@ impl IndexableObject for &ArchivedContactCard {
field: IDX_UID,
value: self.card.uid().into(),
},
IndexValue::IndexList {
field: IDX_EMAIL,
value: self
.emails()
.map(Into::into)
.collect::<HashSet<IndexItem>>()
.into_iter()
.collect(),
},
IndexValue::Quota {
used: self.dead_properties.size() as u32
+ self.display_name.as_ref().map_or(0, |n| n.len() as u32)
@@ -109,3 +132,23 @@ impl IndexableAndSerializableObject for ContactCard {
true
}
}
impl ContactCard {
pub fn emails(&self) -> impl Iterator<Item = String> {
self.card.properties(&VCardProperty::Email).flat_map(|e| {
e.values
.iter()
.filter_map(|v| v.as_text().and_then(sanitize_email))
})
}
}
impl ArchivedContactCard {
pub fn emails(&self) -> impl Iterator<Item = String> {
self.card.properties(&VCardProperty::Email).flat_map(|e| {
e.values
.iter()
.filter_map(|v| v.as_text().and_then(sanitize_email))
})
}
}

2
crates/groupware/src/lib.rs
View File

@@ -13,8 +13,6 @@ pub mod calendar;
pub mod contact;
pub mod file;
pub const IDX_UID: u8 = 0;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum DavResourceName {
Card,