filip/Stalwart
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Spam filter performance and accuracy improvements (part 5)

Browse Source
This commit is contained in:
mdecimus
2025-12-04 17:46:38 +01:00
parent 50dce48a85
commit c467ce07f1
34 changed files with 967 additions and 484 deletions
Download Patch File Download Diff File Expand all files Collapse all files

114
crates/http/src/management/spam.rs
View File

@@ -4,20 +4,18 @@
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use std::net::IpAddr;
use common::{Server, auth::AccessToken, config::spamfilter::SpamFilterAction, psl};
use compact_str::CompactString;
use directory::{
Permission,
backend::internal::manage::{self, ManageDirectory},
};
use email::message::ingest::EmailIngest;
use http_proto::{request::decode_path_element, *};
use hyper::Method;
use mail_auth::{
AuthenticatedMessage, DmarcResult, dmarc::verify::DmarcParameters, spf::verify::SpfParameters,
};
use mail_parser::{Message, MessageParser};
use mail_parser::MessageParser;
use serde::{Deserialize, Serialize};
use serde_json::json;
use spam_filter::{
@@ -25,9 +23,8 @@ use spam_filter::{
analysis::{init::SpamFilterInit, score::SpamFilterAnalyzeScore},
};
use std::future::Future;
use store::ahash::AHashMap;
use http_proto::{request::decode_path_element, *};
use std::net::IpAddr;
use store::{ahash::AHashMap, write::BatchBuilder};
pub trait ManageSpamHandler: Sync + Send {
fn handle_manage_spam(
@@ -65,8 +62,8 @@ pub struct SpamClassifyRequest {
#[derive(Debug, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct SpamClassifyResponse {
pub score: f64,
pub tags: AHashMap<String, SpamFilterDisposition<f64>>,
pub score: f32,
pub tags: AHashMap<String, SpamFilterDisposition<f32>>,
pub disposition: SpamFilterDisposition<String>,
}
@@ -88,25 +85,63 @@ impl ManageSpamHandler for Server {
session: &HttpSessionData,
access_token: &AccessToken,
) -> trc::Result<HttpResponse> {
// Validate the access token
//access_token.assert_has_permission(Permission::SpamFilterTrain)?;
match (path.get(1).copied(), path.get(2).copied(), req.method()) {
(Some("train"), Some(class @ ("ham" | "spam")), &Method::POST) => {
let todo = "fix";
/*let message = parse_message_or_err(body.as_deref().unwrap_or_default())?;
let input = if let Some(account) = path.get(3).copied().filter(|a| !a.is_empty()) {
let account_id = self
(Some("sample"), Some(class @ ("ham" | "spam")), &Method::POST) => {
// Validate the access token
access_token.assert_has_permission(Permission::SpamFilterTrain)?;
let message =
body.ok_or_else(|| manage::error("Failed to parse message.", None::<u64>))?;
let account_id = if let Some(account) =
path.get(3).copied().filter(|a| !a.is_empty())
{
let principal = self
.store()
.get_principal_id(decode_path_element(account).as_ref())
.get_principal_info(decode_path_element(account).as_ref())
.await?
.ok_or_else(|| manage::not_found(account.to_string()))?;
SpamFilterInput::from_account_message(&message, account_id, session.session_id)
if access_token.tenant.is_some() && principal.tenant != access_token.tenant_id()
{
return Err(manage::error(
"Account does not belong to this tenant.",
None::<u64>,
));
}
principal.id
} else if access_token.tenant.is_none() {
u32::MAX
} else {
SpamFilterInput::from_message(&message, session.session_id)
return Err(manage::error(
"Account ID is required for tenants.",
None::<u64>,
));
};
self.bayes_train(&self.spam_filter_init(input), class == "spam", true)
.await?; */
// Write sample
let (blob_hash, blob_hold) =
self.put_temporary_blob(account_id, &message, 60).await?;
let mut batch = BatchBuilder::new();
batch.with_account_id(account_id).clear(blob_hold);
self.add_spam_sample(
&mut batch,
blob_hash,
class == "spam",
true,
session.session_id,
);
self.store().write(batch.build_all()).await?;
Ok(JsonResponse::new(json!({
"data": (),
}))
.into_http_response())
}
(Some("train"), _, &Method::GET) => {
// Validate the access token
access_token.assert_has_permission(Permission::SpamFilterTrain)?;
let todo = "implement";
Ok(JsonResponse::new(json!({
"data": (),
@@ -114,6 +149,9 @@ impl ManageSpamHandler for Server {
.into_http_response())
}
(Some("classify"), _, &Method::POST) => {
// Validate the access token
access_token.assert_has_permission(Permission::SpamFilterTest)?;
// Parse request
let request = serde_json::from_slice::<SpamClassifyRequest>(
body.as_deref().unwrap_or_default(),
@@ -123,7 +161,10 @@ impl ManageSpamHandler for Server {
})?;
// Built spam filter input
let message = parse_message_or_err(request.message.as_bytes())?;
let message = MessageParser::new()
.parse(request.message.as_bytes())
.filter(|m| m.root_part().headers().iter().any(|h| !h.name.is_other()))
.ok_or_else(|| manage::error("Failed to parse message.", None::<u64>))?;
let remote_ip = request.remote_ip;
let ehlo_domain = request.ehlo_domain.to_lowercase();
@@ -243,21 +284,26 @@ impl ManageSpamHandler for Server {
env_from_flags: request.env_from_flags,
env_rcpt_to: request.env_rcpt_to.iter().map(String::as_str).collect(),
is_test: true,
is_train: false,
};
// Classify
let mut ctx = self.spam_filter_init(input);
let result = self.spam_filter_classify(&mut ctx).await;
let todo = "fix";
// Build response
/* let mut response = SpamClassifyResponse {
let mut response = SpamClassifyResponse {
score: ctx.result.score,
tags: AHashMap::with_capacity(ctx.result.tags.len()),
disposition: match result {
SpamFilterAction::Allow(value) => SpamFilterDisposition::Allow { value },
SpamFilterAction::Allow(value) => SpamFilterDisposition::Allow {
value: value.headers,
},
SpamFilterAction::Discard => SpamFilterDisposition::Discard,
SpamFilterAction::Reject => SpamFilterDisposition::Reject,
SpamFilterAction::Disabled => SpamFilterDisposition::Allow {
value: String::new(),
},
},
};
for tag in ctx.result.tags {
@@ -267,7 +313,9 @@ impl ManageSpamHandler for Server {
}
Some(SpamFilterAction::Discard) => SpamFilterDisposition::Discard,
Some(SpamFilterAction::Reject) => SpamFilterDisposition::Reject,
None => SpamFilterDisposition::Allow { value: 0.0 },
Some(SpamFilterAction::Disabled) | None => {
SpamFilterDisposition::Allow { value: 0.0 }
}
};
response.tags.insert(tag, disposition);
}
@@ -275,17 +323,9 @@ impl ManageSpamHandler for Server {
Ok(JsonResponse::new(json!({
"data": response,
}))
.into_http_response())*/
todo!()
.into_http_response())
}
_ => Err(trc::ResourceEvent::NotFound.into_err()),
}
}
}
fn parse_message_or_err(bytes: &[u8]) -> trc::Result<Message<'_>> {
MessageParser::new()
.parse(bytes)
.filter(|m| m.root_part().headers().iter().any(|h| !h.name.is_other()))
.ok_or_else(|| manage::error("Failed to parse message.", None::<u64>))
}

1
crates/http/src/request.rs
View File

@@ -85,7 +85,6 @@ impl ParseHttp for Server {
}
}
let todo = "hashify";
match path.next().unwrap_or_default() {
"jmap" => {
match (path.next().unwrap_or_default(), req.method()) {