diff --git a/crates/common/src/config/network.rs b/crates/common/src/config/network.rs index 6f0c31d1..daf51d58 100644 --- a/crates/common/src/config/network.rs +++ b/crates/common/src/config/network.rs @@ -239,6 +239,7 @@ impl AsnGeoLookupConfig { } .into() } + "disabled" | "none" | "false" => AsnGeoLookupConfig::Disabled.into(), _ => { config.new_build_error("server.asn.type", "Invalid value"); None diff --git a/crates/common/src/config/smtp/session.rs b/crates/common/src/config/smtp/session.rs index 69c11635..525bc6bd 100644 --- a/crates/common/src/config/smtp/session.rs +++ b/crates/common/src/config/smtp/session.rs @@ -130,7 +130,6 @@ pub enum AddressMapping { #[derive(Clone)] pub struct Data { pub script: IfBlock, - pub pipe_commands: Vec, // Limits pub max_messages: IfBlock, @@ -146,14 +145,6 @@ pub struct Data { pub add_date: IfBlock, } -// Ceci n'est pas une pipe -#[derive(Clone)] -pub struct Pipe { - pub command: IfBlock, - pub arguments: IfBlock, - pub timeout: IfBlock, -} - #[derive(Clone)] pub struct Milter { pub enable: IfBlock, @@ -229,13 +220,6 @@ impl SessionConfig { .into_iter() .filter_map(|id| parse_hooks(config, &id, &has_rcpt_vars)) .collect(); - session.data.pipe_commands = config - .sub_keys("session.data.pipe", "") - .map(|s| s.to_string()) - .collect::>() - .into_iter() - .filter_map(|id| parse_pipe(config, &id, &has_rcpt_vars)) - .collect(); session.throttle = SessionThrottle::parse(config); session.mta_sts_policy = Policy::try_parse(config); @@ -519,17 +503,6 @@ impl SessionThrottle { } } -fn parse_pipe(config: &mut Config, id: &str, token_map: &TokenMap) -> Option { - Some(Pipe { - command: IfBlock::try_parse(config, ("session.data.pipe", id, "command"), token_map)?, - arguments: IfBlock::try_parse(config, ("session.data.pipe", id, "arguments"), token_map)?, - timeout: IfBlock::try_parse(config, ("session.data.pipe", id, "timeout"), token_map) - .unwrap_or_else(|| { - IfBlock::new::<()>(format!("session.data.pipe.{id}.timeout"), [], "30s") - }), - }) -} - fn parse_milter(config: &mut Config, id: &str, token_map: &TokenMap) -> Option { let hostname = config .value_require(("session.milter", id, "hostname"))? @@ -800,7 +773,6 @@ impl Default for SessionConfig { }, data: Data { script: IfBlock::empty("session.data.script"), - pipe_commands: Default::default(), max_messages: IfBlock::new::<()>("session.data.limits.messages", [], "10"), max_message_size: IfBlock::new::<()>("session.data.limits.size", [], "104857600"), max_received_headers: IfBlock::new::<()>( diff --git a/crates/common/src/config/spamfilter.rs b/crates/common/src/config/spamfilter.rs index 3e074dc8..0799ef33 100644 --- a/crates/common/src/config/spamfilter.rs +++ b/crates/common/src/config/spamfilter.rs @@ -20,7 +20,7 @@ use super::{if_block::IfBlock, tokenizer::TokenMap}; pub struct SpamFilterConfig { pub enabled: bool, pub dnsbl: DnsBlConfig, - pub rules: Vec, + pub rules: SpamFilterRules, pub lists: SpamFilterLists, pub pyzor: Option, pub reputation: Option, @@ -112,10 +112,15 @@ pub struct PyzorConfig { pub ratio: f64, } -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct SpamFilterRule { - pub rule: IfBlock, - pub scope: Element, +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct SpamFilterRules { + pub url: Vec, + pub domain: Vec, + pub email: Vec, + pub ip: Vec, + pub header: Vec, + pub body: Vec, + pub any: Vec, } #[derive(Debug, Clone, Default, PartialEq, Eq)] @@ -197,7 +202,7 @@ impl SpamFilterConfig { .property_or_default("spam-filter.enable", "true") .unwrap_or(true), dnsbl: DnsBlConfig::parse(config), - rules: parse_rules(config), + rules: SpamFilterRules::parse(config), lists: SpamFilterLists::parse(config), pyzor: PyzorConfig::parse(config).await, reputation: ReputationConfig::parse(config), @@ -209,23 +214,46 @@ impl SpamFilterConfig { } } -fn parse_rules(config: &mut Config) -> Vec { - let mut rules = vec![]; - for id in config - .sub_keys("spam-filter.rule", ".scope") - .map(|k| k.to_string()) - .collect::>() - { - if let Some(rule) = SpamFilterRule::parse(config, id) { - rules.push(rule); +impl SpamFilterRules { + pub fn parse(config: &mut Config) -> SpamFilterRules { + let mut rules = vec![]; + for id in config + .sub_keys("spam-filter.rule", ".scope") + .map(|k| k.to_string()) + .collect::>() + { + if let Some(rule) = SpamFilterRule::parse(config, id) { + rules.push(rule); + } } + rules.sort_by(|a, b| a.priority.cmp(&b.priority)); + + let mut result = SpamFilterRules::default(); + + for rule in rules { + match rule.scope { + Element::Url => result.url.push(rule.rule), + Element::Domain => result.domain.push(rule.rule), + Element::Email => result.email.push(rule.rule), + Element::Ip => result.ip.push(rule.rule), + Element::Header => result.header.push(rule.rule), + Element::Body => result.body.push(rule.rule), + Element::Any => result.any.push(rule.rule), + } + } + + result } - rules.sort_by(|a, b| a.1.cmp(&b.1)); - rules.into_iter().map(|(rule, _)| rule).collect() +} + +struct SpamFilterRule { + rule: IfBlock, + priority: i32, + scope: Element, } impl SpamFilterRule { - pub fn parse(config: &mut Config, id: String) -> Option<(Self, i32)> { + pub fn parse(config: &mut Config, id: String) -> Option { let id = id.as_str(); if !config .property_or_default(("spam-filter.rule", id, "enable"), "true") @@ -240,18 +268,16 @@ impl SpamFilterRule { .property_or_default::(("spam-filter.rule", id, "scope"), "any") .unwrap_or_default(); - ( - SpamFilterRule { - rule: IfBlock::try_parse( - config, - ("spam-filter.rule", id, "condition"), - &scope.token_map(), - )?, - scope, - }, + SpamFilterRule { + rule: IfBlock::try_parse( + config, + ("spam-filter.rule", id, "condition"), + &scope.token_map(), + )?, + scope, priority, - ) - .into() + } + .into() } } diff --git a/crates/common/src/enterprise/config.rs b/crates/common/src/enterprise/config.rs index 3cb70889..244572eb 100644 --- a/crates/common/src/enterprise/config.rs +++ b/crates/common/src/enterprise/config.rs @@ -92,10 +92,13 @@ impl Enterprise { .keys .insert("enterprise.license-key".to_string(), license.clone()); if let Err(err) = config_manager - .set([ConfigKey { - key: "enterprise.license-key".to_string(), - value: license.to_string(), - }]) + .set( + [ConfigKey { + key: "enterprise.license-key".to_string(), + value: license.to_string(), + }], + true, + ) .await { trc::error!(err @@ -205,8 +208,8 @@ impl Enterprise { impl SpamFilterLlmConfig { pub fn parse(config: &mut Config, models: &AHashMap>) -> Option { - if config - .property_or_default("spam-filter.llm.enable", "false") + if !config + .property_or_default::("spam-filter.llm.enable", "false") .unwrap_or_default() { return None; diff --git a/crates/common/src/listener/acme/cache.rs b/crates/common/src/listener/acme/cache.rs index c7269fc7..e79df89a 100644 --- a/crates/common/src/listener/acme/cache.rs +++ b/crates/common/src/listener/acme/cache.rs @@ -98,10 +98,13 @@ impl Server { self.core .storage .config - .set([ConfigKey { - key: self.build_key(provider, class, items), - value: URL_SAFE_NO_PAD.encode(contents.as_ref()), - }]) + .set( + [ConfigKey { + key: self.build_key(provider, class, items), + value: URL_SAFE_NO_PAD.encode(contents.as_ref()), + }], + true, + ) .await } diff --git a/crates/common/src/listener/blocked.rs b/crates/common/src/listener/blocked.rs index 26710280..33df6664 100644 --- a/crates/common/src/listener/blocked.rs +++ b/crates/common/src/listener/blocked.rs @@ -234,10 +234,13 @@ impl Server { self.core .storage .config - .set([ConfigKey { - key: format!("{}.{}", BLOCKED_IP_KEY, ip), - value: String::new(), - }]) + .set( + [ConfigKey { + key: format!("{}.{}", BLOCKED_IP_KEY, ip), + value: String::new(), + }], + true, + ) .await?; // Increment version diff --git a/crates/common/src/manager/boot.rs b/crates/common/src/manager/boot.rs index 15fc349d..2d3bd126 100644 --- a/crates/common/src/manager/boot.rs +++ b/crates/common/src/manager/boot.rs @@ -319,7 +319,7 @@ impl BootManager { config.keys.insert(item.key.clone(), item.value.clone()); } - if let Err(err) = manager.set(insert_keys).await { + if let Err(err) = manager.set(insert_keys, true).await { config .new_build_error("*", format!("Failed to update configuration: {err}")); } diff --git a/crates/common/src/manager/config.rs b/crates/common/src/manager/config.rs index 3a8c0b0e..c93240b8 100644 --- a/crates/common/src/manager/config.rs +++ b/crates/common/src/manager/config.rs @@ -173,7 +173,7 @@ impl ConfigManager { Ok(results) } - pub async fn set(&self, keys: I) -> trc::Result<()> + pub async fn set(&self, keys: I, overwrite: bool) -> trc::Result<()> where I: IntoIterator, T: Into, @@ -183,10 +183,13 @@ impl ConfigManager { for key in keys { let key = key.into(); - if self.cfg_local_patterns.is_local_key(&key.key) { - local_batch.push(key); - } else { - batch.set(ValueClass::Config(key.key.into_bytes()), key.value); + + if overwrite || self.get(&key.key).await?.is_none() || key.key.starts_with("version.") { + if self.cfg_local_patterns.is_local_key(&key.key) { + local_batch.push(key); + } else { + batch.set(ValueClass::Config(key.key.into_bytes()), key.value); + } } } @@ -342,7 +345,7 @@ impl ConfigManager { .await? .map_or(true, |v| v != external.version) { - self.set(external.keys).await?; + self.set(external.keys, false).await?; trc::event!( Config(trc::ConfigEvent::ImportExternal), @@ -382,11 +385,11 @@ impl ConfigManager { external.id.clone_from(&key); external.version.clone_from(&value); external.keys.push(ConfigKey::from((key, value))); - } else if key.starts_with("queue.quota.") + } else if key.starts_with("spam-filter.") + || key.starts_with("server.asn.") + || key.starts_with("queue.quota.") || key.starts_with("queue.throttle.") || key.starts_with("session.throttle.") - || (key.starts_with("lookup.") && !key.starts_with("lookup.default.")) - || key.starts_with("sieve.trusted.scripts.") { external.keys.push(ConfigKey::from((key, value))); } else { diff --git a/crates/common/src/manager/mod.rs b/crates/common/src/manager/mod.rs index 4db19fa7..76cf2d4b 100644 --- a/crates/common/src/manager/mod.rs +++ b/crates/common/src/manager/mod.rs @@ -20,7 +20,8 @@ pub mod reload; pub mod restore; pub mod webadmin; -const DEFAULT_SPAMFILTER_URL: &str = "https://get.stalw.art/resources/config/spamfilter.toml"; +const DEFAULT_SPAMFILTER_URL: &str = + "https://raw.githubusercontent.com/stalwartlabs/spam-filter/refs/heads/main/spam-filter.toml"; const DEFAULT_WEBADMIN_URL: &str = "https://github.com/stalwartlabs/webadmin/releases/latest/download/webadmin.zip"; pub const WEBADMIN_KEY: &[u8] = "STALWART_WEBADMIN".as_bytes(); diff --git a/crates/jmap/src/api/management/dkim.rs b/crates/jmap/src/api/management/dkim.rs index 5fcbe4eb..2b4a140b 100644 --- a/crates/jmap/src/api/management/dkim.rs +++ b/crates/jmap/src/api/management/dkim.rs @@ -244,7 +244,7 @@ impl DkimManagement for Server { "Message-ID".to_string(), ), (format!("signature.{id}.report"), "false".to_string()), - ]) + ], true) .await } } diff --git a/crates/jmap/src/api/management/principal.rs b/crates/jmap/src/api/management/principal.rs index 3513bd52..f1bc2877 100644 --- a/crates/jmap/src/api/management/principal.rs +++ b/crates/jmap/src/api/management/principal.rs @@ -626,7 +626,7 @@ impl PrincipalManager for Server { self.core .storage .config - .set([("authentication.fallback-admin.secret", password)]) + .set([("authentication.fallback-admin.secret", password)], true) .await?; // Remove entries from cache diff --git a/crates/jmap/src/api/management/settings.rs b/crates/jmap/src/api/management/settings.rs index ccabef51..fcec4d13 100644 --- a/crates/jmap/src/api/management/settings.rs +++ b/crates/jmap/src/api/management/settings.rs @@ -312,14 +312,17 @@ impl ManageSettings for Server { self.core .storage .config - .set(values.into_iter().map(|(key, value)| ConfigKey { - key: if let Some(prefix) = &prefix { - format!("{prefix}.{key}") - } else { - key - }, - value, - })) + .set( + values.into_iter().map(|(key, value)| ConfigKey { + key: if let Some(prefix) = &prefix { + format!("{prefix}.{key}") + } else { + key + }, + value, + }), + true, + ) .await?; } } diff --git a/crates/jmap/src/api/management/sieve.rs b/crates/jmap/src/api/management/sieve.rs index ccc1b337..bccffc19 100644 --- a/crates/jmap/src/api/management/sieve.rs +++ b/crates/jmap/src/api/management/sieve.rs @@ -49,7 +49,7 @@ impl SieveHandler for Server { &self, req: &HttpRequest, path: Vec<&str>, - body: Option>, + _body: Option>, access_token: &AccessToken, ) -> trc::Result { // Validate the access token @@ -78,8 +78,7 @@ impl SieveHandler for Server { .duration_since(SystemTime::UNIX_EPOCH) .map_or(0, |d| d.as_secs()), ) - .set_variable("test", true) - .with_message(body.as_deref().unwrap_or_default()); + .set_variable("test", true); let mut envelope_to = Vec::new(); for (key, value) in UrlParams::new(req.uri().query()).into_inner() { diff --git a/crates/jmap/src/email/ingest.rs b/crates/jmap/src/email/ingest.rs index 057ab01c..9303b7cb 100644 --- a/crates/jmap/src/email/ingest.rs +++ b/crates/jmap/src/email/ingest.rs @@ -121,23 +121,22 @@ impl EmailIngest for Server { // Check for Spam headers let mut is_spam = false; - let todo = "true"; - /*if let (IngestSource::Smtp, Some((header_name, header_value))) = - (params.source, &self.core.jmap.spam_header) + if let (IngestSource::Smtp, Some(header_name)) = + (params.source, &self.core.spam.headers.status) { if params.mailbox_ids == [INBOX_ID] && message.root_part().headers().iter().any(|header| { - &header.name == header_name + header.name() == header_name && header .value() .as_text() - .map_or(false, |value| value.contains(header_value)) + .map_or(false, |value| value.contains("Yes")) }) { params.mailbox_ids[0] = JUNK_ID; is_spam = true; } - }*/ + } // Obtain message references and thread name let mut message_id = String::new(); diff --git a/crates/smtp/Cargo.toml b/crates/smtp/Cargo.toml index 8830a331..42b04b41 100644 --- a/crates/smtp/Cargo.toml +++ b/crates/smtp/Cargo.toml @@ -17,6 +17,7 @@ utils = { path = "../utils" } nlp = { path = "../nlp" } directory = { path = "../directory" } common = { path = "../common" } +spam-filter = { path = "../spam-filter" } trc = { path = "../trc" } mail-auth = { version = "0.5" } mail-send = { version = "0.4", default-features = false, features = ["cram-md5", "ring", "tls12"] } @@ -56,6 +57,7 @@ chrono = "0.4" [features] test_mode = [] +enterprise = [] #[[bench]] #name = "hash" diff --git a/crates/smtp/src/inbound/data.rs b/crates/smtp/src/inbound/data.rs index 70e9f629..e4826057 100644 --- a/crates/smtp/src/inbound/data.rs +++ b/crates/smtp/src/inbound/data.rs @@ -6,13 +6,14 @@ use std::{ borrow::Cow, - process::Stdio, - sync::Arc, time::{Duration, Instant, SystemTime}, }; use common::{ - config::smtp::{auth::VerifyStrategy, session::Stage}, + config::{ + smtp::{auth::VerifyStrategy, session::Stage}, + spamfilter::SpamFilterAction, + }, listener::SessionStream, psl, scripts::ScriptModification, @@ -22,12 +23,12 @@ use mail_auth::{ dmarc, AuthenticatedMessage, AuthenticationResults, DkimResult, DmarcResult, ReceivedSpf, }; use mail_builder::headers::{date::Date, message_id::generate_message_id_header}; +use mail_parser::MessageParser; use sieve::runtime::Variable; use smtp_proto::{ MAIL_BY_RETURN, RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_NEVER, RCPT_NOTIFY_SUCCESS, }; use store::write::now; -use tokio::{io::AsyncWriteExt, process::Command}; use trc::SmtpEvent; use utils::config::Rate; @@ -43,22 +44,31 @@ use super::{ArcSeal, AuthResult, DkimSign}; impl Session { pub async fn queue_message(&mut self) -> Cow<'static, [u8]> { - // Authenticate message - let raw_message = Arc::new(std::mem::take(&mut self.data.message)); - let auth_message = if let Some(auth_message) = AuthenticatedMessage::parse_with_opts( - &raw_message, - self.server.core.smtp.mail_auth.dkim.strict, - ) { - auth_message - } else { - trc::event!( - Smtp(SmtpEvent::MessageParseFailed), - SpanId = self.data.session_id, - ); + // Parse message + let raw_message = std::mem::take(&mut self.data.message); + let parsed_message = match MessageParser::new() + .parse(&raw_message) + .filter(|p| p.headers().iter().any(|h| !h.name.is_other())) + { + Some(parsed_message) => parsed_message, + None => { + trc::event!( + Smtp(SmtpEvent::MessageParseFailed), + SpanId = self.data.session_id, + ); - return (&b"550 5.7.7 Failed to parse message.\r\n"[..]).into(); + return (&b"550 5.7.7 Failed to parse message.\r\n"[..]).into(); + } }; + // Authenticate message + let auth_message = AuthenticatedMessage::from_parsed( + &parsed_message, + self.server.core.smtp.mail_auth.dkim.strict, + ); + let has_date_header = auth_message.has_date_header(); + let has_message_id_header = auth_message.has_message_id_header(); + // Loop detection let dc = &self.server.core.smtp.session.data; let ac = &self.server.core.smtp.mail_auth; @@ -312,11 +322,38 @@ impl Session { // Analyze reports if is_report { - self.server - .analyze_report(raw_message.clone(), self.data.session_id); if !rc.analysis.forward { + self.server.analyze_report( + mail_parser::Message { + html_body: parsed_message.html_body, + text_body: parsed_message.text_body, + attachments: parsed_message.attachments, + parts: parsed_message + .parts + .into_iter() + .map(|p| p.into_owned()) + .collect(), + raw_message: b"".into(), + }, + self.data.session_id, + ); self.data.messages_sent += 1; return (b"250 2.0.0 Message queued for delivery.\r\n"[..]).into(); + } else { + self.server.analyze_report( + mail_parser::Message { + html_body: parsed_message.html_body.clone(), + text_body: parsed_message.text_body.clone(), + attachments: parsed_message.attachments.clone(), + parts: parsed_message + .parts + .iter() + .map(|p| p.clone().into_owned()) + .collect(), + raw_message: b"".into(), + }, + self.data.session_id, + ); } } @@ -383,6 +420,35 @@ impl Session { } } + // Run SPAM filter + if self.server.core.spam.enabled { + match self + .spam_classify( + &parsed_message, + &dkim_output, + (&arc_output).into(), + dmarc_result.as_ref(), + dmarc_policy.as_ref(), + ) + .await + { + SpamFilterAction::Allow(spam_headers) => { + if !spam_headers.is_empty() { + headers.extend_from_slice(spam_headers.as_bytes()); + } + } + SpamFilterAction::Discard => { + self.data.messages_sent += 1; + return (b"250 2.0.0 Message queued for delivery.\r\n"[..]).into(); + } + SpamFilterAction::Reject => { + self.data.messages_sent += 1; + return (b"550 5.7.1 Message rejected due to excessive spam score.\r\n"[..]) + .into(); + } + } + } + // Run Milter filters let mut modifications = Vec::new(); match self.run_milters(Stage::Data, (&auth_message).into()).await { @@ -420,117 +486,6 @@ impl Session { None }; - // Pipe message - for pipe in &dc.pipe_commands { - if let Some(command_) = self - .server - .eval_if::(&pipe.command, self, self.data.session_id) - .await - { - let piped_message = edited_message.as_ref().unwrap_or(&raw_message).clone(); - let timeout = self - .server - .eval_if(&pipe.timeout, self, self.data.session_id) - .await - .unwrap_or_else(|| Duration::from_secs(30)); - - let mut command = Command::new(&command_); - for argument in self - .server - .eval_if::, _>(&pipe.arguments, self, self.data.session_id) - .await - .unwrap_or_default() - { - command.arg(argument); - } - let time = Instant::now(); - match command - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .kill_on_drop(true) - .spawn() - { - Ok(mut child) => { - if let Some(mut stdin) = child.stdin.take() { - match tokio::time::timeout(timeout, stdin.write_all(&piped_message)) - .await - { - Ok(Ok(_)) => { - drop(stdin); - match tokio::time::timeout(timeout, child.wait_with_output()) - .await - { - Ok(Ok(output)) => { - if output.status.success() - && !output.stdout.is_empty() - && output.stdout[..] != piped_message[..] - { - edited_message = output.stdout.into(); - } - - trc::event!( - Smtp(SmtpEvent::PipeSuccess), - SpanId = self.data.session_id, - Path = command_, - Result = output.status.to_string(), - Elapsed = time.elapsed(), - ); - } - Ok(Err(err)) => { - trc::event!( - Smtp(SmtpEvent::PipeError), - SpanId = self.data.session_id, - Reason = err.to_string(), - Elapsed = time.elapsed(), - ); - } - Err(_) => { - trc::event!( - Smtp(SmtpEvent::PipeError), - SpanId = self.data.session_id, - Reason = "Timeout", - Elapsed = time.elapsed(), - ); - } - } - } - Ok(Err(err)) => { - trc::event!( - Smtp(SmtpEvent::PipeError), - SpanId = self.data.session_id, - Reason = err.to_string(), - Elapsed = time.elapsed(), - ); - } - Err(_) => { - trc::event!( - Smtp(SmtpEvent::PipeError), - SpanId = self.data.session_id, - Reason = "Stdin timeout", - Elapsed = time.elapsed(), - ); - } - } - } else { - trc::event!( - Smtp(SmtpEvent::PipeError), - SpanId = self.data.session_id, - Reason = "Stdin not available", - Elapsed = time.elapsed(), - ); - } - } - Err(err) => { - trc::event!( - Smtp(SmtpEvent::PipeError), - SpanId = self.data.session_id, - Reason = err.to_string(), - ); - } - } - } - } - // Sieve filtering if let Some((script, script_id)) = self .server @@ -544,7 +499,6 @@ impl Session { { let params = self .build_script_parameters("data") - .with_message(edited_message.as_ref().unwrap_or(&raw_message)) .with_auth_headers(&headers) .set_variable( "arc.result", @@ -589,7 +543,8 @@ impl Session { .as_ref() .map(|a| a.as_str()) .unwrap_or_default(), - ); + ) + .with_message(parsed_message); let modifications = match self.run_script(script_id, script.clone(), params).await { ScriptResult::Accept { modifications } => modifications, @@ -646,7 +601,7 @@ impl Session { } // Add any missing headers - if !auth_message.has_date_header() + if !has_date_header && self .server .eval_if(&dc.add_date, self, self.data.session_id) @@ -657,7 +612,7 @@ impl Session { headers.extend_from_slice(Date::now().to_rfc822().as_bytes()); headers.extend_from_slice(b"\r\n"); } - if !auth_message.has_message_id_header() + if !has_message_id_header && self .server .eval_if(&dc.add_message_id, self, self.data.session_id) @@ -670,9 +625,7 @@ impl Session { } // DKIM sign - let raw_message = edited_message - .as_deref() - .unwrap_or_else(|| raw_message.as_slice()); + let raw_message = edited_message.as_deref().unwrap_or(raw_message.as_slice()); for signer in self .server .eval_if::, _>(&ac.dkim.sign, self, self.data.session_id) diff --git a/crates/smtp/src/inbound/spam.rs b/crates/smtp/src/inbound/spam.rs index c2587844..f56890a9 100644 --- a/crates/smtp/src/inbound/spam.rs +++ b/crates/smtp/src/inbound/spam.rs @@ -4,14 +4,128 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::listener::SessionStream; +use common::{config::spamfilter::SpamFilterAction, listener::SessionStream}; use mail_auth::{dmarc::Policy, ArcOutput, DkimOutput, DmarcResult}; use mail_parser::Message; -use spam_filter::SpamFilterInput; +use spam_filter::{ + analysis::{ + bayes::SpamFilterAnalyzeBayes, date::SpamFilterAnalyzeDate, dmarc::SpamFilterAnalyzeDmarc, + domain::SpamFilterAnalyzeDomain, ehlo::SpamFilterAnalyzeEhlo, from::SpamFilterAnalyzeFrom, + headers::SpamFilterAnalyzeHeaders, html::SpamFilterAnalyzeHtml, init::SpamFilterInit, + ip::SpamFilterAnalyzeIp, llm::SpamFilterAnalyzeLlm, messageid::SpamFilterAnalyzeMid, + mime::SpamFilterAnalyzeMime, pyzor::SpamFilterAnalyzePyzor, + received::SpamFilterAnalyzeReceived, recipient::SpamFilterAnalyzeRecipient, + replyto::SpamFilterAnalyzeReplyTo, reputation::SpamFilterAnalyzeReputation, + rules::SpamFilterAnalyzeRules, score::SpamFilterAnalyzeScore, + subject::SpamFilterAnalyzeSubject, trusted_reply::SpamFilterAnalyzeTrustedReply, + url::SpamFilterAnalyzeUrl, + }, + SpamFilterInput, +}; use crate::core::Session; impl Session { + pub async fn spam_classify<'x>( + &'x self, + message: &'x Message<'x>, + dkim_result: &'x [DkimOutput<'x>], + arc_result: Option<&'x ArcOutput<'x>>, + dmarc_result: Option<&'x DmarcResult>, + dmarc_policy: Option<&'x Policy>, + ) -> SpamFilterAction { + let server = &self.server; + let mut ctx = server.spam_filter_init(self.build_spam_input( + message, + dkim_result, + arc_result, + dmarc_result, + dmarc_policy, + )); + + if !self.is_authenticated() { + // IP address analysis + server.spam_filter_analyze_ip(&mut ctx).await; + + // DMARC/SPF/DKIM/ARC analysis + server.spam_filter_analyze_dmarc(&mut ctx).await; + + // EHLO hostname analysis + server.spam_filter_analyze_ehlo(&mut ctx).await; + + // Generic header analysis + server.spam_filter_analyze_headers(&mut ctx).await; + + // Received headers analysis + server.spam_filter_analyze_received(&mut ctx).await; + + // Message-ID analysis + server.spam_filter_analyze_message_id(&mut ctx).await; + + // Date header analysis + server.spam_filter_analyze_date(&mut ctx).await; + + // Subject analysis + server.spam_filter_analyze_subject(&mut ctx).await; + + // From and Envelope From analysis + server.spam_filter_analyze_from(&mut ctx).await; + + // Reply-To analysis + server.spam_filter_analyze_reply_to(&mut ctx).await; + + // Recipient analysis + server.spam_filter_analyze_recipient(&mut ctx).await; + + // E-mail and domain analysis + server.spam_filter_analyze_domain(&mut ctx).await; + + // URL analysis + server.spam_filter_analyze_url(&mut ctx).await; + + // MIME part analysis + server.spam_filter_analyze_mime(&mut ctx).await; + + // HTML content analysis + server.spam_filter_analyze_html(&mut ctx).await; + + // LLM classification + server.spam_filter_analyze_llm(&mut ctx).await; + + // Trusted reply analysis + server.spam_filter_analyze_reply_in(&mut ctx).await; + + // Spam trap + server.spam_filter_analyze_spam_trap(&mut ctx).await; + + // Pyzor checks + server.spam_filter_analyze_pyzor(&mut ctx).await; + + // Bayes classification + server.spam_filter_analyze_bayes_classify(&mut ctx).await; + + // User-defined rules + server.spam_filter_analyze_rules(&mut ctx).await; + + // Calculate score + match server.spam_filter_score(&mut ctx).await { + SpamFilterAction::Allow(_) => (), + SpamFilterAction::Discard => return SpamFilterAction::Discard, + SpamFilterAction::Reject => return SpamFilterAction::Reject, + } + + // Reputation tracking and adjust score + server.spam_filter_analyze_reputation(&mut ctx).await; + + // Final score calculation + server.spam_filter_finalize(&mut ctx).await + } else { + // Trusted reply tracking + server.spam_filter_analyze_reply_out(&mut ctx).await; + SpamFilterAction::Allow(String::new()) + } + } + pub fn build_spam_input<'x>( &'x self, message: &'x Message<'x>, diff --git a/crates/smtp/src/reporting/analysis.rs b/crates/smtp/src/reporting/analysis.rs index ee6b4230..6439a96d 100644 --- a/crates/smtp/src/reporting/analysis.rs +++ b/crates/smtp/src/reporting/analysis.rs @@ -8,7 +8,6 @@ use std::{ borrow::Cow, collections::hash_map::Entry, io::{Cursor, Read}, - sync::Arc, }; use ahash::AHashMap; @@ -18,7 +17,7 @@ use mail_auth::{ report::{tlsrpt::TlsReport, ActionDisposition, DmarcResult, Feedback, Report}, zip, }; -use mail_parser::{MessageParser, MimeHeaders, PartType}; +use mail_parser::{Message, MimeHeaders, PartType}; use store::{ write::{now, BatchBuilder, Bincode, ReportClass, ValueClass}, @@ -53,23 +52,13 @@ pub struct IncomingReport { } pub trait AnalyzeReport: Sync + Send { - fn analyze_report(&self, message: Arc>, session_id: u64); + fn analyze_report(&self, message: Message<'static>, session_id: u64); } impl AnalyzeReport for Server { - fn analyze_report(&self, message: Arc>, session_id: u64) { + fn analyze_report(&self, message: Message<'static>, session_id: u64) { let core = self.clone(); tokio::spawn(async move { - let message = if let Some(message) = MessageParser::default().parse(message.as_ref()) { - message - } else { - trc::event!( - IncomingReport(IncomingReportEvent::MessageParseFailed), - SpanId = session_id - ); - - return; - }; let from = message .from() .and_then(|a| a.last()) diff --git a/crates/smtp/src/scripts/event_loop.rs b/crates/smtp/src/scripts/event_loop.rs index 9d9ac226..57e5f291 100644 --- a/crates/smtp/src/scripts/event_loop.rs +++ b/crates/smtp/src/scripts/event_loop.rs @@ -8,6 +8,7 @@ use std::{borrow::Cow, future::Future, sync::Arc, time::Instant}; use common::{scripts::plugins::PluginContext, Server}; use mail_auth::common::headers::HeaderWriter; +use mail_parser::{Encoding, Message, MessagePart, PartType}; use sieve::{ compiler::grammar::actions::action_redirect::{ByMode, ByTime, Notify, NotifyItem, Ret}, Event, Input, MatchAs, Recipient, Sieve, @@ -47,7 +48,19 @@ impl RunScript for Server { .core .sieve .trusted_runtime - .filter(params.message.unwrap_or_default()) + .filter_parsed(params.message.unwrap_or_else(|| Message { + parts: vec![MessagePart { + headers: vec![], + is_encoding_problem: false, + body: PartType::Text("".into()), + encoding: Encoding::None, + offset_header: 0, + offset_body: 0, + offset_end: 0, + }], + raw_message: b""[..].into(), + ..Default::default() + })) .with_vars_env(params.variables) .with_envelope_list(params.envelope) .with_user_address(¶ms.from_addr) diff --git a/crates/smtp/src/scripts/mod.rs b/crates/smtp/src/scripts/mod.rs index 4be80a80..543d6116 100644 --- a/crates/smtp/src/scripts/mod.rs +++ b/crates/smtp/src/scripts/mod.rs @@ -10,6 +10,7 @@ use ahash::AHashMap; use common::{ auth::AccessToken, expr::functions::ResolveVariable, scripts::ScriptModification, Server, }; +use mail_parser::Message; use sieve::{runtime::Variable, Envelope}; pub mod envelope; @@ -30,7 +31,7 @@ pub enum ScriptResult { } pub struct ScriptParameters<'x> { - message: Option<&'x [u8]>, + message: Option>, headers: Option<&'x [u8]>, variables: AHashMap, Variable>, envelope: Vec<(Envelope, Variable)>, @@ -82,7 +83,7 @@ impl<'x> ScriptParameters<'x> { self } - pub fn with_message(self, message: &'x [u8]) -> Self { + pub fn with_message(self, message: Message<'x>) -> Self { Self { message: message.into(), ..self diff --git a/crates/spam-filter/src/analysis/bayes.rs b/crates/spam-filter/src/analysis/bayes.rs index fd854890..6d77a2b6 100644 --- a/crates/spam-filter/src/analysis/bayes.rs +++ b/crates/spam-filter/src/analysis/bayes.rs @@ -19,7 +19,7 @@ pub trait SpamFilterAnalyzeBayes: Sync + Send { fn spam_filter_analyze_spam_trap( &self, ctx: &mut SpamFilterContext<'_>, - ) -> impl Future + Send; + ) -> impl Future + Send; } impl SpamFilterAnalyzeBayes for Server { @@ -43,7 +43,7 @@ impl SpamFilterAnalyzeBayes for Server { } } - async fn spam_filter_analyze_spam_trap(&self, ctx: &mut SpamFilterContext<'_>) { + async fn spam_filter_analyze_spam_trap(&self, ctx: &mut SpamFilterContext<'_>) -> bool { if ctx .output .env_to_addr @@ -51,6 +51,9 @@ impl SpamFilterAnalyzeBayes for Server { .any(|addr| self.core.spam.lists.spamtraps.contains(&addr.address)) { ctx.result.add_tag("SPAM_TRAP"); + true + } else { + false } } } diff --git a/crates/spam-filter/src/analysis/ip.rs b/crates/spam-filter/src/analysis/ip.rs index d880a596..185d7ab3 100644 --- a/crates/spam-filter/src/analysis/ip.rs +++ b/crates/spam-filter/src/analysis/ip.rs @@ -123,12 +123,13 @@ impl SpamFilterAnalyzeIp for Server { continue; } + let ip_resolver = IpResolver::new(ip.element); for dnsbl in &self.core.spam.dnsbl.servers { if dnsbl.scope == Element::Ip { if let Some(tag) = is_dnsbl( self, dnsbl, - SpamFilterResolver::new(ctx, &IpResolver::new(ip.element), ip.location), + SpamFilterResolver::new(ctx, &ip_resolver, ip.location), ) .await { diff --git a/crates/spam-filter/src/analysis/llm.rs b/crates/spam-filter/src/analysis/llm.rs index 271b8f80..bd7053a7 100644 --- a/crates/spam-filter/src/analysis/llm.rs +++ b/crates/spam-filter/src/analysis/llm.rs @@ -36,6 +36,7 @@ impl SpamFilterAnalyzeLlm for Server { "{}\n\nSubject: {}\n\n{}", config.prompt, ctx.output.subject, body ); + match config .model .send_request(prompt, config.temperature.into()) @@ -68,7 +69,7 @@ impl SpamFilterAnalyzeLlm for Server { confidence = Some(value); } } else if config.index_explanation.map_or(false, |i| i == idx) { - explanation = Some(value); + explanation = Some(value.replace('\n', " ")); } } } @@ -85,10 +86,12 @@ impl SpamFilterAnalyzeLlm for Server { _ => return, }; - if let Some(explanation) = explanation { - ctx.result.llm_header = - format!("X-Spam-Llm-Explanation: {category} ({explanation})\r\n",) - .into(); + if let (Some(header), Some(mut explanation)) = + (&self.core.spam.headers.llm, explanation) + { + explanation.truncate(512); + ctx.result.header = + format!("{header}: {category} ({explanation})\r\n",).into(); } } Err(err) => { diff --git a/crates/spam-filter/src/analysis/pyzor.rs b/crates/spam-filter/src/analysis/pyzor.rs index 6e38a241..f9f756ad 100644 --- a/crates/spam-filter/src/analysis/pyzor.rs +++ b/crates/spam-filter/src/analysis/pyzor.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ - use std::future::Future; +use std::{future::Future, time::Instant}; use common::Server; @@ -20,20 +20,33 @@ pub trait SpamFilterAnalyzePyzor: Sync + Send { impl SpamFilterAnalyzePyzor for Server { async fn spam_filter_analyze_pyzor(&self, ctx: &mut SpamFilterContext<'_>) { if let Some(config) = &self.core.spam.pyzor { + let time = Instant::now(); match pyzor_check(ctx.input.message, config).await { Ok(Some(result)) => { - if result.code == 200 + let is_spam = result.code == 200 && result.count > config.min_count && (result.wl_count < config.min_wl_count - || (result.wl_count as f64 / result.count as f64) < config.ratio) - { + || (result.wl_count as f64 / result.count as f64) < config.ratio); + if is_spam { ctx.result.add_tag("PYZOR"); } - let todo = "log time"; + trc::event!( + Spam(trc::SpamEvent::Pyzor), + Result = is_spam, + Details = vec![ + trc::Value::from(result.code), + trc::Value::from(result.count), + trc::Value::from(result.wl_count) + ], + SpanId = ctx.input.span_id, + Elapsed = time.elapsed() + ); } Ok(None) => {} Err(err) => { - trc::error!(err.span_id(ctx.input.span_id)); + trc::error!(err + .span_id(ctx.input.span_id) + .ctx(trc::Key::Elapsed, time.elapsed())); } } } diff --git a/crates/spam-filter/src/analysis/reputation.rs b/crates/spam-filter/src/analysis/reputation.rs index 3a37625b..bac8573a 100644 --- a/crates/spam-filter/src/analysis/reputation.rs +++ b/crates/spam-filter/src/analysis/reputation.rs @@ -85,7 +85,7 @@ impl SpamFilterAnalyzeReputation for Server { let mut reputation = 0.0; for (rep_type, key) in types { - let mut token = match key_get::( + let token = match key_get::( self, ctx.input.span_id, KeyValue::<()>::build_key(rep_type.prefix(), key.as_ref()), @@ -115,16 +115,25 @@ impl SpamFilterAnalyzeReputation for Server { }; // Update reputation - token.score = (token.count + 1) as f64 + let updated_score = (token.count + 1) as f64 * (ctx.result.score + config.token_score * token.score) / (config.token_score * token.count as f64 + 1.0); - token.count += 1; + let updated_count = token.count + 1; + if !ctx.input.is_test { key_set( self, ctx.input.span_id, - KeyValue::with_prefix(rep_type.prefix(), key.as_ref(), token.serialize()) - .expires(config.expiry), + KeyValue::with_prefix( + rep_type.prefix(), + key.as_ref(), + Reputation { + count: updated_count, + score: updated_score, + } + .serialize(), + ) + .expires(config.expiry), ) .await; } @@ -136,10 +145,8 @@ impl SpamFilterAnalyzeReputation for Server { Type::Domain => config.domain_weight, Type::Asn => config.asn_weight, }; - let c = println!("{rep_type:?} {weight}"); reputation += token.score / token.count as f64 * weight; - let c = println!("{rep_type:?} {weight}: {reputation}"); } // Adjust score diff --git a/crates/spam-filter/src/analysis/rules.rs b/crates/spam-filter/src/analysis/rules.rs index 47cc1a15..500c9d8e 100644 --- a/crates/spam-filter/src/analysis/rules.rs +++ b/crates/spam-filter/src/analysis/rules.rs @@ -6,10 +6,7 @@ use std::future::Future; -use common::{ - config::spamfilter::{Element, Location}, - Server, -}; +use common::{config::spamfilter::Location, Server}; use crate::{ modules::expression::{EmailHeader, IpResolver, SpamFilterResolver, StringResolver}, @@ -25,152 +22,13 @@ pub trait SpamFilterAnalyzeRules: Sync + Send { impl SpamFilterAnalyzeRules for Server { async fn spam_filter_analyze_rules(&self, ctx: &mut SpamFilterContext<'_>) { - for rule in &self.core.spam.rules { - match rule.scope { - Element::Url => { - for url in &ctx.output.urls { - if let Some(tag) = self - .eval_if::( - &rule.rule, - &SpamFilterResolver::new(ctx, &url.element, url.location), - ctx.input.span_id, - ) - .await - { - ctx.result.tags.insert(tag); - } - } - } - Element::Domain => { - for domain in &ctx.output.domains { - if let Some(tag) = self - .eval_if::( - &rule.rule, - &SpamFilterResolver::new( - ctx, - &StringResolver(domain.element.as_str()), - domain.location, - ), - ctx.input.span_id, - ) - .await - { - ctx.result.tags.insert(tag); - } - } - } - Element::Email => { - for email in &ctx.output.emails { - if let Some(tag) = self - .eval_if::( - &rule.rule, - &SpamFilterResolver::new(ctx, &email.element, email.location), - ctx.input.span_id, - ) - .await - { - ctx.result.tags.insert(tag); - } - } - - for (rcpt, location) in [ - (&ctx.output.recipients_to, Location::HeaderTo), - (&ctx.output.recipients_cc, Location::HeaderCc), - (&ctx.output.recipients_bcc, Location::HeaderBcc), - ] { - for email in rcpt { - if let Some(tag) = self - .eval_if::( - &rule.rule, - &SpamFilterResolver::new(ctx, email, location), - ctx.input.span_id, - ) - .await - { - ctx.result.tags.insert(tag); - } - } - } - } - Element::Ip => { - for ip in &ctx.output.ips { - if let Some(tag) = self - .eval_if::( - &rule.rule, - &SpamFilterResolver::new( - ctx, - &IpResolver::new(ip.element), - ip.location, - ), - ctx.input.span_id, - ) - .await - { - ctx.result.tags.insert(tag); - } - } - } - Element::Header => { - for header in ctx.input.message.headers() { - let raw = String::from_utf8_lossy( - ctx.input - .message - .raw_message() - .get(header.offset_start..header.offset_end) - .unwrap_or_default(), - ); - - if let Some(tag) = self - .eval_if::( - &rule.rule, - &SpamFilterResolver::new( - ctx, - &EmailHeader { - header, - raw: raw.as_ref(), - }, - Location::BodyText, - ), - ctx.input.span_id, - ) - .await - { - ctx.result.tags.insert(tag); - } - } - } - Element::Body => { - for (idx, part) in ctx.output.text_parts.iter().enumerate() { - let text = match part { - TextPart::Plain { text_body, .. } => *text_body, - TextPart::Html { text_body, .. } => text_body.as_str(), - TextPart::None => continue, - }; - let location = if ctx.input.message.text_body.contains(&idx) { - Location::BodyText - } else if ctx.input.message.html_body.contains(&idx) { - Location::BodyHtml - } else { - Location::Attachment - }; - - if let Some(tag) = self - .eval_if::( - &rule.rule, - &SpamFilterResolver::new(ctx, &StringResolver(text), location), - ctx.input.span_id, - ) - .await - { - ctx.result.tags.insert(tag); - } - } - } - Element::Any => { + if !self.core.spam.rules.url.is_empty() { + for url in &ctx.output.urls { + for rule in &self.core.spam.rules.url { if let Some(tag) = self .eval_if::( - &rule.rule, - &SpamFilterResolver::new(ctx, &StringResolver(""), Location::BodyText), + rule, + &SpamFilterResolver::new(ctx, &url.element, url.location), ctx.input.span_id, ) .await @@ -180,5 +38,157 @@ impl SpamFilterAnalyzeRules for Server { } } } + + if !self.core.spam.rules.domain.is_empty() { + for domain in &ctx.output.domains { + let resolver = StringResolver(domain.element.as_str()); + + for rule in &self.core.spam.rules.domain { + if let Some(tag) = self + .eval_if::( + rule, + &SpamFilterResolver::new(ctx, &resolver, domain.location), + ctx.input.span_id, + ) + .await + { + ctx.result.tags.insert(tag); + } + } + } + } + + if !self.core.spam.rules.email.is_empty() { + for email in &ctx.output.emails { + for rule in &self.core.spam.rules.email { + if let Some(tag) = self + .eval_if::( + rule, + &SpamFilterResolver::new(ctx, &email.element, email.location), + ctx.input.span_id, + ) + .await + { + ctx.result.tags.insert(tag); + } + } + } + + for (rcpt, location) in [ + (&ctx.output.recipients_to, Location::HeaderTo), + (&ctx.output.recipients_cc, Location::HeaderCc), + (&ctx.output.recipients_bcc, Location::HeaderBcc), + ] { + for email in rcpt { + for rule in &self.core.spam.rules.email { + if let Some(tag) = self + .eval_if::( + rule, + &SpamFilterResolver::new(ctx, email, location), + ctx.input.span_id, + ) + .await + { + ctx.result.tags.insert(tag); + } + } + } + } + } + + if !self.core.spam.rules.ip.is_empty() { + for ip in &ctx.output.ips { + let ip_resolver = IpResolver::new(ip.element); + + for rule in &self.core.spam.rules.ip { + if let Some(tag) = self + .eval_if::( + rule, + &SpamFilterResolver::new(ctx, &ip_resolver, ip.location), + ctx.input.span_id, + ) + .await + { + ctx.result.tags.insert(tag); + } + } + } + } + + if !self.core.spam.rules.header.is_empty() { + for header in ctx.input.message.headers() { + let raw = String::from_utf8_lossy( + ctx.input + .message + .raw_message() + .get(header.offset_start..header.offset_end) + .unwrap_or_default(), + ); + let header_resolver = EmailHeader { + header, + raw: raw.as_ref(), + }; + + for rule in &self.core.spam.rules.header { + if let Some(tag) = self + .eval_if::( + rule, + &SpamFilterResolver::new(ctx, &header_resolver, Location::BodyText), + ctx.input.span_id, + ) + .await + { + ctx.result.tags.insert(tag); + } + } + } + } + + if !self.core.spam.rules.body.is_empty() { + for (idx, part) in ctx.output.text_parts.iter().enumerate() { + let text = match part { + TextPart::Plain { text_body, .. } => *text_body, + TextPart::Html { text_body, .. } => text_body.as_str(), + TextPart::None => continue, + }; + let location = if ctx.input.message.text_body.contains(&idx) { + Location::BodyText + } else if ctx.input.message.html_body.contains(&idx) { + Location::BodyHtml + } else { + Location::Attachment + }; + let string_resolver = StringResolver(text); + + for rule in &self.core.spam.rules.body { + if let Some(tag) = self + .eval_if::( + rule, + &SpamFilterResolver::new(ctx, &string_resolver, location), + ctx.input.span_id, + ) + .await + { + ctx.result.tags.insert(tag); + } + } + } + } + + if !self.core.spam.rules.any.is_empty() { + let dummy_resolver = StringResolver(""); + for rule in &self.core.spam.rules.any { + if let Some(tag) = self + .eval_if::( + rule, + &SpamFilterResolver::new(ctx, &dummy_resolver, Location::BodyText), + ctx.input.span_id, + ) + .await + { + ctx.result.tags.insert(tag); + } + } + } } } diff --git a/crates/spam-filter/src/analysis/score.rs b/crates/spam-filter/src/analysis/score.rs index 5a061846..f490b968 100644 --- a/crates/spam-filter/src/analysis/score.rs +++ b/crates/spam-filter/src/analysis/score.rs @@ -13,17 +13,16 @@ pub trait SpamFilterAnalyzeScore: Sync + Send { fn spam_filter_score( &self, ctx: &mut SpamFilterContext<'_>, - ) -> impl Future> + Send; + ) -> impl Future> + Send; fn spam_filter_finalize( &self, ctx: &mut SpamFilterContext<'_>, - header: String, ) -> impl Future> + Send; } impl SpamFilterAnalyzeScore for Server { - async fn spam_filter_score(&self, ctx: &mut SpamFilterContext<'_>) -> SpamFilterAction { + async fn spam_filter_score(&self, ctx: &mut SpamFilterContext<'_>) -> SpamFilterAction<()> { let mut results = vec![]; let mut header_len = 60; @@ -47,7 +46,10 @@ impl SpamFilterAnalyzeScore for Server { // Write results header sorted by score if let Some(header_name) = &self.core.spam.headers.result { - let mut header = String::with_capacity(header_name.len() + header_len + 2); + let mut header = ctx + .result + .header + .get_or_insert_with(|| String::with_capacity(header_name.len() + header_len + 2)); results.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap().then_with(|| a.0.cmp(b.0))); header.push_str(header_name); header.push_str(": "); @@ -59,16 +61,15 @@ impl SpamFilterAnalyzeScore for Server { } header.push_str("\r\n"); - SpamFilterAction::Allow(header) + SpamFilterAction::Allow(()) } else { - SpamFilterAction::Allow(String::new()) + SpamFilterAction::Allow(()) } } async fn spam_filter_finalize( &self, ctx: &mut SpamFilterContext<'_>, - mut header: String, ) -> SpamFilterAction { // Train Bayes classifier if let Some(config) = self.core.spam.bayes.as_ref().filter(|c| c.auto_learn) { @@ -94,6 +95,7 @@ impl SpamFilterAnalyzeScore for Server { { SpamFilterAction::Discard } else { + let mut header = std::mem::take(&mut ctx.result.header).unwrap_or_default(); if let Some(header_name) = &self.core.spam.headers.status { let _ = write!( &mut header, diff --git a/crates/spam-filter/src/lib.rs b/crates/spam-filter/src/lib.rs index 3818821f..8119f540 100644 --- a/crates/spam-filter/src/lib.rs +++ b/crates/spam-filter/src/lib.rs @@ -99,7 +99,7 @@ pub struct SpamFilterResult { pub rbl_domain_checks: usize, pub rbl_url_checks: usize, pub rbl_email_checks: usize, - pub llm_header: Option, + pub header: Option, } pub struct SpamFilterContext<'x> { diff --git a/crates/spam-filter/src/modules/bayes.rs b/crates/spam-filter/src/modules/bayes.rs index 0ad3d108..df83b46e 100644 --- a/crates/spam-filter/src/modules/bayes.rs +++ b/crates/spam-filter/src/modules/bayes.rs @@ -161,7 +161,7 @@ pub(crate) async fn bayes_classify( // Make sure we have enough training data if spam_learns < classifier.min_learns || ham_learns < classifier.min_learns { trc::event!( - Spam(trc::SpamEvent::NotEnoughTrainingData), + Spam(trc::SpamEvent::ClassifyError), SpanId = ctx.input.span_id, Details = vec![ trc::Value::from(spam_learns), diff --git a/crates/spam-filter/src/modules/dnsbl.rs b/crates/spam-filter/src/modules/dnsbl.rs index 95c1ebf7..5fedd95c 100644 --- a/crates/spam-filter/src/modules/dnsbl.rs +++ b/crates/spam-filter/src/modules/dnsbl.rs @@ -50,12 +50,10 @@ pub(crate) async fn is_dnsbl( } } - let todo = "use proper event error"; - match server.core.smtp.resolvers.dns.ipv4_lookup(&zone).await { Ok(result) => { trc::event!( - Spam(SpamEvent::Classify), + Spam(SpamEvent::Dnsbl), Result = result .iter() .map(|ip| trc::Value::from(ip.to_string())) @@ -84,7 +82,7 @@ pub(crate) async fn is_dnsbl( } Err(Error::DnsRecordNotFound(_)) => { trc::event!( - Spam(SpamEvent::Classify), + Spam(SpamEvent::Dnsbl), Result = trc::Value::None, Elapsed = time.elapsed() ); @@ -93,7 +91,7 @@ pub(crate) async fn is_dnsbl( } Err(err) => { trc::event!( - Spam(SpamEvent::Classify), + Spam(SpamEvent::DnsblError), Elapsed = time.elapsed(), CausedBy = err.to_string() ); diff --git a/crates/spam-filter/src/modules/remote_list.rs b/crates/spam-filter/src/modules/remote_list.rs index a4dc5c47..05c5b9d6 100644 --- a/crates/spam-filter/src/modules/remote_list.rs +++ b/crates/spam-filter/src/modules/remote_list.rs @@ -70,8 +70,6 @@ async fn is_in_remote_list_( } } - let todo = "update RuntimeError with SpamEvent error"; - match server.inner.data.remote_lists.read().get(&config.id) { Some(remote_list) if remote_list.expires < Instant::now() => { return Ok(remote_list.entries.contains(item)) @@ -79,6 +77,7 @@ async fn is_in_remote_list_( _ => {} } + let time = Instant::now(); let response = reqwest::Client::builder() .timeout(config.timeout) .user_agent(USER_AGENT) @@ -88,7 +87,7 @@ async fn is_in_remote_list_( .send() .await .map_err(|err| { - trc::SieveEvent::RuntimeError + trc::SpamEvent::RemoteListError .into_err() .reason(err) .ctx(trc::Key::Url, config.url.to_string()) @@ -100,16 +99,18 @@ async fn is_in_remote_list_( .bytes_with_limit(config.max_size) .await .map_err(|err| { - trc::SieveEvent::RuntimeError + trc::SpamEvent::RemoteListError .into_err() .reason(err) .ctx(trc::Key::Url, config.url.to_string()) + .ctx(trc::Key::Elapsed, time.elapsed()) .details("Failed to fetch resource") })? .ok_or_else(|| { - trc::SieveEvent::RuntimeError + trc::SpamEvent::RemoteListError .into_err() .ctx(trc::Key::Url, config.url.to_string()) + .ctx(trc::Key::Elapsed, time.elapsed()) .details("Resource is too large") })?; @@ -135,10 +136,11 @@ async fn is_in_remote_list_( for (pos, line) in BufReader::new(reader).lines().enumerate() { let line_ = line.map_err(|err| { - trc::SieveEvent::RuntimeError + trc::SpamEvent::RemoteListError .into_err() .reason(err) .ctx(trc::Key::Url, config.url.to_string()) + .ctx(trc::Key::Elapsed, time.elapsed()) .details("Failed to read line") })?; // Clear list once the first entry has been successfully fetched, decompressed and UTF8-decoded @@ -194,9 +196,10 @@ async fn is_in_remote_list_( } trc::event!( - Spam(trc::SpamEvent::ListUpdated), + Spam(trc::SpamEvent::RemoteList), Url = config.url.to_string(), Total = list.entries.len(), + Elapsed = time.elapsed(), SpanId = span_id ); @@ -204,10 +207,11 @@ async fn is_in_remote_list_( list.expires = Instant::now() + config.refresh; Ok(list.entries.contains(item)) } else { - trc::bail!(trc::SieveEvent::RuntimeError + trc::bail!(trc::SpamEvent::RemoteListError .into_err() .ctx(trc::Key::Code, response.status().as_u16()) .ctx(trc::Key::Url, config.url.to_string()) + .ctx(trc::Key::Elapsed, time.elapsed()) .details("Failed to fetch remote list")); } } diff --git a/crates/trc/src/event/description.rs b/crates/trc/src/event/description.rs index fb2eb07f..dbbedf55 100644 --- a/crates/trc/src/event/description.rs +++ b/crates/trc/src/event/description.rs @@ -409,8 +409,6 @@ impl SmtpEvent { SmtpEvent::MessageParseFailed => "Message parsing failed", SmtpEvent::MessageTooLarge => "Message too large", SmtpEvent::LoopDetected => "Mail loop detected", - SmtpEvent::PipeSuccess => "Pipe command succeeded", - SmtpEvent::PipeError => "Pipe command failed", SmtpEvent::DkimPass => "DKIM verification passed", SmtpEvent::DkimFail => "DKIM verification failed", SmtpEvent::ArcPass => "ARC verification passed", @@ -504,8 +502,6 @@ impl SmtpEvent { SmtpEvent::LoopDetected => { "A mail loop was detected, the message contains too many Received headers" } - SmtpEvent::PipeSuccess => "The pipe command succeeded", - SmtpEvent::PipeError => "The pipe command failed", SmtpEvent::DkimPass => "Successful DKIM verification", SmtpEvent::DkimFail => "Failed to verify DKIM signature", SmtpEvent::ArcPass => "Successful ARC verification", @@ -1019,29 +1015,33 @@ impl PushSubscriptionEvent { impl SpamEvent { pub fn description(&self) -> &'static str { match self { + SpamEvent::Pyzor => "Pyzor success", SpamEvent::PyzorError => "Pyzor error", - SpamEvent::ListUpdated => "Spam list updated", + SpamEvent::RemoteList => "Remote list updated", + SpamEvent::RemoteListError => "Error updating remote list", SpamEvent::Train => "Training spam filter", SpamEvent::TrainBalance => "Balancing spam filter training data", SpamEvent::TrainError => "Error training spam filter", SpamEvent::Classify => "Classifying message for spam", - SpamEvent::ClassifyError => "Error classifying message for spam", - SpamEvent::NotEnoughTrainingData => "Not enough training data for spam filter", + SpamEvent::ClassifyError => "Not enough training data for spam filter", + SpamEvent::Dnsbl => "DNSBL query", + SpamEvent::DnsblError => "Error querying DNSBL", } } pub fn explain(&self) -> &'static str { match self { SpamEvent::PyzorError => "An error occurred with Pyzor", - SpamEvent::ListUpdated => "The spam list has been updated", SpamEvent::Train => "The spam filter is being trained with the message", SpamEvent::TrainBalance => "The spam filter training data is being balanced", SpamEvent::TrainError => "An error occurred while training the spam filter", SpamEvent::Classify => "The message is being classified for spam", - SpamEvent::ClassifyError => "An error occurred while classifying the message for spam", - SpamEvent::NotEnoughTrainingData => { - "There is not enough training data for the spam filter" - } + SpamEvent::ClassifyError => "There is not enough training data for the spam filter", + SpamEvent::Pyzor => "Pyzor query successful", + SpamEvent::RemoteList => "The remote list was updated", + SpamEvent::RemoteListError => "An error occurred while updating the remote list", + SpamEvent::Dnsbl => "The DNSBL query was successful", + SpamEvent::DnsblError => "An error occurred while querying the DNSBL", } } } diff --git a/crates/trc/src/event/level.rs b/crates/trc/src/event/level.rs index 8a900818..692d2cf1 100644 --- a/crates/trc/src/event/level.rs +++ b/crates/trc/src/event/level.rs @@ -155,8 +155,6 @@ impl EventType { | SmtpEvent::InvalidParameter | SmtpEvent::UnsupportedParameter | SmtpEvent::SyntaxError - | SmtpEvent::PipeSuccess - | SmtpEvent::PipeError | SmtpEvent::Error => Level::Debug, SmtpEvent::MissingLocalHostname | SmtpEvent::RemoteIdNotFound => Level::Warn, SmtpEvent::ConcurrencyLimitExceeded @@ -337,14 +335,17 @@ impl EventType { | SieveEvent::ActionReject => Level::Debug, }, EventType::Spam(event) => match event { - SpamEvent::PyzorError | SpamEvent::TrainError | SpamEvent::ClassifyError => { - Level::Warn - } - SpamEvent::Train + SpamEvent::PyzorError + | SpamEvent::TrainError + | SpamEvent::DnsblError + | SpamEvent::RemoteListError => Level::Warn, + SpamEvent::Pyzor + | SpamEvent::Train | SpamEvent::Classify - | SpamEvent::NotEnoughTrainingData - | SpamEvent::TrainBalance => Level::Debug, - SpamEvent::ListUpdated => Level::Info, + | SpamEvent::ClassifyError + | SpamEvent::TrainBalance + | SpamEvent::Dnsbl + | SpamEvent::RemoteList => Level::Debug, }, EventType::Http(event) => match event { HttpEvent::ConnectionStart | HttpEvent::ConnectionEnd => Level::Debug, diff --git a/crates/trc/src/ipc/metrics.rs b/crates/trc/src/ipc/metrics.rs index d8e92b19..ca0ad4b3 100644 --- a/crates/trc/src/ipc/metrics.rs +++ b/crates/trc/src/ipc/metrics.rs @@ -577,12 +577,12 @@ impl EventType { ) => true, EventType::Spam( SpamEvent::PyzorError - | SpamEvent::ListUpdated + | SpamEvent::RemoteListError | SpamEvent::Train | SpamEvent::TrainError | SpamEvent::Classify | SpamEvent::ClassifyError - | SpamEvent::NotEnoughTrainingData, + | SpamEvent::DnsblError, ) => true, EventType::PushSubscription(_) => true, EventType::Cluster( diff --git a/crates/trc/src/lib.rs b/crates/trc/src/lib.rs index c7d9eb22..cd26ace0 100644 --- a/crates/trc/src/lib.rs +++ b/crates/trc/src/lib.rs @@ -360,8 +360,6 @@ pub enum SmtpEvent { MessageParseFailed, MessageTooLarge, LoopDetected, - PipeSuccess, - PipeError, DkimPass, DkimFail, ArcPass, @@ -602,14 +600,17 @@ pub enum PushSubscriptionEvent { #[event_type] pub enum SpamEvent { + Pyzor, PyzorError, - ListUpdated, + RemoteList, + RemoteListError, + Dnsbl, + DnsblError, Train, TrainBalance, TrainError, Classify, ClassifyError, - NotEnoughTrainingData, } #[event_type] diff --git a/crates/trc/src/serializers/binary.rs b/crates/trc/src/serializers/binary.rs index 2ddd8e4a..170c1ed0 100644 --- a/crates/trc/src/serializers/binary.rs +++ b/crates/trc/src/serializers/binary.rs @@ -763,8 +763,6 @@ impl EventType { EventType::Smtp(SmtpEvent::MtPriorityInvalid) => 455, EventType::Smtp(SmtpEvent::MultipleMailFrom) => 456, EventType::Smtp(SmtpEvent::Noop) => 457, - EventType::Smtp(SmtpEvent::PipeError) => 458, - EventType::Smtp(SmtpEvent::PipeSuccess) => 459, EventType::Smtp(SmtpEvent::Quit) => 460, EventType::Smtp(SmtpEvent::RateLimitExceeded) => 461, EventType::Smtp(SmtpEvent::RawInput) => 462, @@ -797,8 +795,8 @@ impl EventType { EventType::Smtp(SmtpEvent::VrfyNotFound) => 489, EventType::Spam(SpamEvent::Classify) => 490, EventType::Spam(SpamEvent::ClassifyError) => 491, - EventType::Spam(SpamEvent::ListUpdated) => 492, - EventType::Spam(SpamEvent::NotEnoughTrainingData) => 493, + EventType::Spam(SpamEvent::RemoteList) => 492, + EventType::Spam(SpamEvent::RemoteListError) => 493, EventType::Spam(SpamEvent::PyzorError) => 494, EventType::Spam(SpamEvent::Train) => 495, EventType::Spam(SpamEvent::TrainBalance) => 496, @@ -867,6 +865,9 @@ impl EventType { EventType::Store(StoreEvent::AzureError) => 559, EventType::TlsRpt(TlsRptEvent::RecordNotFound) => 560, EventType::Smtp(SmtpEvent::RcptToGreylisted) => 561, + EventType::Spam(SpamEvent::Dnsbl) => 562, + EventType::Spam(SpamEvent::DnsblError) => 563, + EventType::Spam(SpamEvent::Pyzor) => 564, } } @@ -1366,8 +1367,6 @@ impl EventType { 455 => Some(EventType::Smtp(SmtpEvent::MtPriorityInvalid)), 456 => Some(EventType::Smtp(SmtpEvent::MultipleMailFrom)), 457 => Some(EventType::Smtp(SmtpEvent::Noop)), - 458 => Some(EventType::Smtp(SmtpEvent::PipeError)), - 459 => Some(EventType::Smtp(SmtpEvent::PipeSuccess)), 460 => Some(EventType::Smtp(SmtpEvent::Quit)), 461 => Some(EventType::Smtp(SmtpEvent::RateLimitExceeded)), 462 => Some(EventType::Smtp(SmtpEvent::RawInput)), @@ -1400,8 +1399,8 @@ impl EventType { 489 => Some(EventType::Smtp(SmtpEvent::VrfyNotFound)), 490 => Some(EventType::Spam(SpamEvent::Classify)), 491 => Some(EventType::Spam(SpamEvent::ClassifyError)), - 492 => Some(EventType::Spam(SpamEvent::ListUpdated)), - 493 => Some(EventType::Spam(SpamEvent::NotEnoughTrainingData)), + 492 => Some(EventType::Spam(SpamEvent::RemoteList)), + 493 => Some(EventType::Spam(SpamEvent::RemoteListError)), 494 => Some(EventType::Spam(SpamEvent::PyzorError)), 495 => Some(EventType::Spam(SpamEvent::Train)), 496 => Some(EventType::Spam(SpamEvent::TrainBalance)), @@ -1474,6 +1473,9 @@ impl EventType { 559 => Some(EventType::Store(StoreEvent::AzureError)), 560 => Some(EventType::TlsRpt(TlsRptEvent::RecordNotFound)), 561 => Some(EventType::Smtp(SmtpEvent::RcptToGreylisted)), + 562 => Some(EventType::Spam(SpamEvent::Dnsbl)), + 563 => Some(EventType::Spam(SpamEvent::DnsblError)), + 564 => Some(EventType::Spam(SpamEvent::Pyzor)), _ => None, } } diff --git a/tests/resources/smtp/antispam/combined.test b/tests/resources/smtp/antispam/combined.test index a32bbb59..899e5da2 100644 --- a/tests/resources/smtp/antispam/combined.test +++ b/tests/resources/smtp/antispam/combined.test @@ -6,8 +6,8 @@ spf.result none spf_ehlo.result none dmarc.result none remote_ip 195.210.29.48 -expect_score 8 -expect rdns_none auth_na dmarc_na helo_nores_a_or_mx once_received mid_rhs_match_from spf_na has_data_uri arc_na subject_has_exclaim subject_ends_exclaim mime_html_only html_short_link_img_1 to_dn_none rcpt_count_one to_match_envrcpt_all fromhost_nores_a_or_mx rcvd_count_zero from_eq_envfrom dkim_na rcvd_no_tls_last from_has_dn date_in_past +expect_header X-Spam-Result: ARC_NA (0.00), DKIM_NA (0.00), DMARC_NA (0.00), FROM_EQ_ENVFROM (0.00), FROM_HAS_DN (0.00), HAS_DATA_URI (0.00), RCPT_COUNT_ONE (0.00), RCVD_COUNT_ZERO (0.00), SPF_NA (0.00), SUBJECT_ENDS_EXCLAIM (0.00), TO_DN_NONE (0.00), TO_MATCH_ENVRCPT_ALL (0.00), ONCE_RECEIVED (0.10), RCVD_NO_TLS_LAST (0.10), MIME_HTML_ONLY (0.20), HELO_NORES_A_OR_MX (0.30), AUTH_NA (1.00), DATE_IN_PAST (1.00), MID_RHS_MATCH_FROM (1.00), RDNS_NONE (1.00), FROMHOST_NORES_A_OR_MX (1.50), HTML_SHORT_LINK_IMG_1 (2.00), PYZOR (3.50) +expect_header X-Spam-Status: Yes, score=11.70 From: Client Services To: licensing@stalw.art @@ -49,8 +49,8 @@ dkim.domains tenthrevolution.com dmarc.result pass remote_ip 185.58.86.181 tls.version TLSv1.3 -expect_score 3 -expect from_eq_envfrom from_has_dn helo_nores_a_or_mx forged_rcvd_trail date_in_past arc_na uri_count_odd dkim_signed has_attachment spf_allow rcvd_tls_last rcpt_count_one mime_good subject_ends_spaces fromhost_nores_a_or_mx to_dn_eq_addr_all dkim_allow dmarc_policy_allow rcvd_count_three to_match_envrcpt_all +expect_header X-Spam-Result: DMARC_POLICY_ALLOW (-0.50), DKIM_ALLOW (-0.20), SPF_ALLOW (-0.20), MIME_GOOD (-0.10), ARC_NA (0.00), DKIM_SIGNED (0.00), FROM_EQ_ENVFROM (0.00), FROM_HAS_DN (0.00), HAS_ATTACHMENT (0.00), RCPT_COUNT_ONE (0.00), RCVD_COUNT_THREE (0.00), TO_DN_EQ_ADDR_ALL (0.00), TO_MATCH_ENVRCPT_ALL (0.00), RCVD_NO_TLS_LAST (0.10), HELO_NORES_A_OR_MX (0.30), SUBJECT_ENDS_SPACES (0.50), URI_COUNT_ODD (0.50), DATE_IN_PAST (1.00), FORGED_RCVD_TRAIL (1.00), FROMHOST_NORES_A_OR_MX (1.50) +expect_header X-Spam-Status: No, score=3.90 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=tenthrevolution.com; s=mimecast20200102; t=1669138703; @@ -87,57 +87,8 @@ Message-ID: remote_ip 10.0.0.1 score 2.0 -final_score 1.5 +final_score 1.45 expect From: user@domain.org @@ -20,7 +20,7 @@ Test remote_ip 10.0.0.1 score 3.0 -final_score 2.2525252525252526 +final_score 2.1772727272727272 expect From: user@domain.org @@ -29,7 +29,7 @@ Test remote_ip 10.0.0.1 score -5.0 -final_score -1.4949494949494948 +final_score -1.5954545454545457 expect From: user@domain.org diff --git a/tests/resources/smtp/pipe/pipe_me.sh b/tests/resources/smtp/pipe/pipe_me.sh deleted file mode 100644 index d3d8c6e8..00000000 --- a/tests/resources/smtp/pipe/pipe_me.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -if [[ $1 == "hello" ]] && [[ $2 == "world" ]]; then - echo "X-My-Header: true" - while read line - do - echo "$line" - done < /dev/stdin - exit 0; -else - echo "Invalid parameters!" - exit 1; -fi - diff --git a/tests/src/smtp/inbound/antispam.rs b/tests/src/smtp/inbound/antispam.rs index a5d9406a..d35de182 100644 --- a/tests/src/smtp/inbound/antispam.rs +++ b/tests/src/smtp/inbound/antispam.rs @@ -1,16 +1,20 @@ use std::{ - borrow::Cow, fs, path::PathBuf, sync::Arc, time::{Duration, Instant}, }; -use ahash::AHashSet; +use ahash::{AHashMap, AHashSet}; use common::{ auth::AccessToken, - enterprise::llm::{ - AiApiConfig, ChatCompletionChoice, ChatCompletionRequest, ChatCompletionResponse, Message, + config::spamfilter::SpamFilterAction, + enterprise::{ + llm::{ + AiApiConfig, ChatCompletionChoice, ChatCompletionRequest, ChatCompletionResponse, + Message, + }, + SpamFilterLlmConfig, }, Core, }; @@ -21,7 +25,6 @@ use mail_auth::{ IprevResult, SpfOutput, SpfResult, MX, }; use mail_parser::MessageParser; -use sieve::runtime::Variable; use smtp::core::{Session, SessionAddress}; use smtp_proto::{MAIL_BODY_8BITMIME, MAIL_SMTPUTF8}; use spam_filter::{ @@ -182,10 +185,12 @@ async fn antispam() { let mut core = Core::parse(&mut config, stores, Default::default()) .await .enable_enterprise(); - core.enterprise.as_mut().unwrap().ai_apis.insert( + let ai_apis = AHashMap::from_iter([( "dummy".to_string(), AiApiConfig::parse(&mut config, "dummy").unwrap().into(), - ); + )]); + core.enterprise.as_mut().unwrap().spam_filter_llm = + SpamFilterLlmConfig::parse(&mut config, &ai_apis); crate::AssertConfig::assert_no_errors(config); // Add mock DNS entries @@ -279,7 +284,8 @@ async fn antispam() { .join("smtp") .join("antispam"); for test_name in [ - /*"ip", + "combined", + "ip", "helo", "received", "messageid", @@ -298,11 +304,10 @@ async fn antispam() { "replies_out", "replies_in", "spamtrap", - "bayes_classify",*/ + "bayes_classify", "reputation", "pyzor", "llm", - "combined", ] { /*if test_name != "combined" { continue; @@ -324,7 +329,7 @@ async fn antispam() { let mut dmarc_result = None; let mut dmarc_policy = None; let mut expected_tags = AHashSet::new(); - let mut score_expect = 0.0; + let mut expect_headers = String::new(); let mut score_set = 0.0; let mut score_final = 0.0; let mut body_params = 0; @@ -420,8 +425,14 @@ async fn antispam() { expected_tags .extend(value.split_ascii_whitespace().map(|v| v.to_uppercase())); } - "expect_score" => { - score_expect = value.parse::().unwrap(); + "expect_header" => { + let value = value.trim(); + if !value.is_empty() { + if !expect_headers.is_empty() { + expect_headers.push(' '); + } + expect_headers.push_str(value); + } } "score" => { score_set = value.parse::().unwrap(); @@ -478,6 +489,39 @@ async fn antispam() { } } let parsed_message = MessageParser::new().parse(&message).unwrap(); + + // Combined tests + if test_name == "combined" { + match session + .spam_classify( + &parsed_message, + &dkim_domains, + arc_result.as_ref(), + dmarc_result.as_ref(), + dmarc_policy.as_ref(), + ) + .await + { + SpamFilterAction::Allow(header) => { + let mut last_ch = 'x'; + let mut result = String::with_capacity(header.len()); + for ch in header.chars() { + if !ch.is_whitespace() { + if last_ch.is_whitespace() { + result.push(' '); + } + result.push(ch); + } + last_ch = ch; + } + assert_eq!(result, expect_headers); + } + other => panic!("Unexpected action {other:?}"), + } + continue; + } + + // Initialize filter let mut spam_input = session.build_spam_input( &parsed_message, &dkim_domains, @@ -486,8 +530,6 @@ async fn antispam() { dmarc_policy.as_ref(), ); spam_input.is_tls = is_tls; - - // Initialize filter let mut spam_ctx = server.spam_filter_init(spam_input); match test_name { "html" => { @@ -564,9 +606,7 @@ async fn antispam() { } "spamtrap" => { server.spam_filter_analyze_spam_trap(&mut spam_ctx).await; - server - .spam_filter_finalize(&mut spam_ctx, String::new()) - .await; + server.spam_filter_finalize(&mut spam_ctx).await; } "bayes_classify" => { server @@ -584,9 +624,6 @@ async fn antispam() { "llm" => { server.spam_filter_analyze_llm(&mut spam_ctx).await; } - "combined" => { - todo!("combined"); - } _ => panic!("Invalid test {test_name:?}"), } diff --git a/tests/src/smtp/inbound/auth.rs b/tests/src/smtp/inbound/auth.rs index 5f042ebc..fee57498 100644 --- a/tests/src/smtp/inbound/auth.rs +++ b/tests/src/smtp/inbound/auth.rs @@ -20,14 +20,14 @@ use smtp::core::{Session, State}; const CONFIG: &str = r#" [storage] -data = "sqlite" -lookup = "sqlite" -blob = "sqlite" -fts = "sqlite" +data = "rocksdb" +lookup = "rocksdb" +blob = "rocksdb" +fts = "rocksdb" directory = "local" -[store."sqlite"] -type = "sqlite" +[store."rocksdb"] +type = "rocksdb" path = "{TMP}/queue.db" [directory."local"] diff --git a/tests/src/smtp/inbound/data.rs b/tests/src/smtp/inbound/data.rs index d4415bb6..7b7552a7 100644 --- a/tests/src/smtp/inbound/data.rs +++ b/tests/src/smtp/inbound/data.rs @@ -20,16 +20,19 @@ use smtp::core::Session; const CONFIG: &str = r#" [storage] -data = "sqlite" -lookup = "sqlite" -blob = "sqlite" -fts = "sqlite" +data = "rocksdb" +lookup = "rocksdb" +blob = "rocksdb" +fts = "rocksdb" directory = "local" -[store."sqlite"] -type = "sqlite" +[store."rocksdb"] +type = "rocksdb" path = "{TMP}/queue.db" +[spam-filter] +enable = false + [directory."local"] type = "memory" @@ -125,12 +128,7 @@ async fn data() { // Send broken message session - .send_message( - "john@doe.org", - &["bill@foobar.org"], - "From: john", - "550 5.7.7", - ) + .send_message("john@doe.org", &["bill@foobar.org"], "invalid", "550 5.7.7") .await; // Naive Loop detection diff --git a/tests/src/smtp/inbound/dmarc.rs b/tests/src/smtp/inbound/dmarc.rs index 951c9c66..82eca839 100644 --- a/tests/src/smtp/inbound/dmarc.rs +++ b/tests/src/smtp/inbound/dmarc.rs @@ -27,13 +27,13 @@ use smtp::core::Session; const CONFIG: &str = r#" [storage] -data = "sqlite" -lookup = "sqlite" -blob = "sqlite" -fts = "sqlite" +data = "rocksdb" +lookup = "rocksdb" +blob = "rocksdb" +fts = "rocksdb" -[store."sqlite"] -type = "sqlite" +[store."rocksdb"] +type = "rocksdb" path = "{TMP}/queue.db" [directory."local"] diff --git a/tests/src/smtp/inbound/mail.rs b/tests/src/smtp/inbound/mail.rs index 2b692cdb..7cde61b3 100644 --- a/tests/src/smtp/inbound/mail.rs +++ b/tests/src/smtp/inbound/mail.rs @@ -21,13 +21,13 @@ use crate::smtp::{ const CONFIG: &str = r#" [storage] -data = "sqlite" -lookup = "sqlite" -blob = "sqlite" -fts = "sqlite" +data = "rocksdb" +lookup = "rocksdb" +blob = "rocksdb" +fts = "rocksdb" -[store."sqlite"] -type = "sqlite" +[store."rocksdb"] +type = "rocksdb" path = "{TMP}/data.db" [session.ehlo] diff --git a/tests/src/smtp/inbound/milter.rs b/tests/src/smtp/inbound/milter.rs index d898197d..5dd711bb 100644 --- a/tests/src/smtp/inbound/milter.rs +++ b/tests/src/smtp/inbound/milter.rs @@ -51,13 +51,13 @@ struct HeaderTest { const CONFIG_MILTER: &str = r#" [storage] -data = "sqlite" -lookup = "sqlite" -blob = "sqlite" -fts = "sqlite" +data = "rocksdb" +lookup = "rocksdb" +blob = "rocksdb" +fts = "rocksdb" -[store."sqlite"] -type = "sqlite" +[store."rocksdb"] +type = "rocksdb" path = "{TMP}/queue.db" [session.rcpt] @@ -77,13 +77,13 @@ stages = ["data"] const CONFIG_JMILTER: &str = r#" [storage] -data = "sqlite" -lookup = "sqlite" -blob = "sqlite" -fts = "sqlite" +data = "rocksdb" +lookup = "rocksdb" +blob = "rocksdb" +fts = "rocksdb" -[store."sqlite"] -type = "sqlite" +[store."rocksdb"] +type = "rocksdb" path = "{TMP}/queue.db" [session.rcpt] diff --git a/tests/src/smtp/inbound/rcpt.rs b/tests/src/smtp/inbound/rcpt.rs index 2f972174..764478e7 100644 --- a/tests/src/smtp/inbound/rcpt.rs +++ b/tests/src/smtp/inbound/rcpt.rs @@ -21,13 +21,13 @@ use crate::smtp::{ const CONFIG: &str = r#" [storage] -data = "sqlite" -lookup = "sqlite" -blob = "sqlite" -fts = "sqlite" +data = "rocksdb" +lookup = "rocksdb" +blob = "rocksdb" +fts = "rocksdb" -[store."sqlite"] -type = "sqlite" +[store."rocksdb"] +type = "rocksdb" path = "{TMP}/queue.db" [directory."local"] diff --git a/tests/src/smtp/inbound/scripts.rs b/tests/src/smtp/inbound/scripts.rs index 951b631a..4240939b 100644 --- a/tests/src/smtp/inbound/scripts.rs +++ b/tests/src/smtp/inbound/scripts.rs @@ -42,11 +42,8 @@ max-connections = 10 min-connections = 0 idle-timeout = "5m" -[session.data.pipe."test"] -command = [ { if = "remote_ip = '10.0.0.123'", then = "'/bin/bash'" }, - { else = false } ] -arguments = "['{CFG_PATH}/pipe_me.sh', 'hello', 'world']" -timeout = "10s" +[spam-filter] +enable = false [sieve.trusted] from-name = "'Sieve Daemon'" @@ -99,7 +96,6 @@ email = ["john@localdomain.org", "jdoe@localdomain.org", "john.doe@localdomain.o email-list = ["info@localdomain.org"] member-of = ["sales"] - "#; #[tokio::test] @@ -134,23 +130,8 @@ async fn sieve_scripts() { } // Prepare config - let tmp_dir = TempDir::new("smtp_sieve_test", true); - let mut config = Config::new( - tmp_dir.update_config( - config.replace( - "{CFG_PATH}", - PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("resources") - .join("smtp") - .join("pipe") - .as_path() - .to_str() - .unwrap(), - ), - ), - ) - .unwrap(); + let mut config = Config::new(tmp_dir.update_config(config)).unwrap(); config.resolve_all_macros().await; let stores = Stores::parse_all(&mut config).await; let core = Core::parse(&mut config, stores, Default::default()).await; @@ -392,24 +373,4 @@ async fn sieve_scripts() { .assert_contains("Received: ") .assert_contains("Authentication-Results: "); qr.assert_no_events(); - - // Test pipes - session.data.remote_ip_str = "10.0.0.123".parse().unwrap(); - session.data.remote_ip = session.data.remote_ip_str.parse().unwrap(); - session - .send_message( - "test@example.net", - &["pipe@foobar.com"], - "test:no_dkim", - "250", - ) - .await; - - qr.expect_message() - .await - .read_lines(&qr) - .await - .assert_contains("X-My-Header: true") - .assert_contains("Authentication-Results"); - qr.assert_no_events(); } diff --git a/tests/src/smtp/inbound/sign.rs b/tests/src/smtp/inbound/sign.rs index 5ccc2dae..56f31b97 100644 --- a/tests/src/smtp/inbound/sign.rs +++ b/tests/src/smtp/inbound/sign.rs @@ -75,13 +75,13 @@ set-body-length = false const CONFIG: &str = r#" [storage] -data = "sqlite" -lookup = "sqlite" -blob = "sqlite" -fts = "sqlite" +data = "rocksdb" +lookup = "rocksdb" +blob = "rocksdb" +fts = "rocksdb" -[store."sqlite"] -type = "sqlite" +[store."rocksdb"] +type = "rocksdb" path = "{TMP}/queue.db" [directory."local"] diff --git a/tests/src/smtp/inbound/throttle.rs b/tests/src/smtp/inbound/throttle.rs index 414ec302..54dbf706 100644 --- a/tests/src/smtp/inbound/throttle.rs +++ b/tests/src/smtp/inbound/throttle.rs @@ -14,13 +14,13 @@ use utils::config::Config; const CONFIG: &str = r#" [storage] -data = "sqlite" -lookup = "sqlite" -blob = "sqlite" -fts = "sqlite" +data = "rocksdb" +lookup = "rocksdb" +blob = "rocksdb" +fts = "rocksdb" -[store."sqlite"] -type = "sqlite" +[store."rocksdb"] +type = "rocksdb" path = "{TMP}/data.db" [[session.throttle]] diff --git a/tests/src/smtp/inbound/vrfy.rs b/tests/src/smtp/inbound/vrfy.rs index 9345dad1..516b899f 100644 --- a/tests/src/smtp/inbound/vrfy.rs +++ b/tests/src/smtp/inbound/vrfy.rs @@ -21,14 +21,14 @@ use crate::{ const CONFIG: &str = r#" [storage] -data = "sqlite" -lookup = "sqlite" -blob = "sqlite" -fts = "sqlite" +data = "rocksdb" +lookup = "rocksdb" +blob = "rocksdb" +fts = "rocksdb" directory = "local" -[store."sqlite"] -type = "sqlite" +[store."rocksdb"] +type = "rocksdb" path = "{TMP}/data.db" [directory."local"] diff --git a/tests/src/smtp/lookup/sql.rs b/tests/src/smtp/lookup/sql.rs index bca69908..1a5c5d45 100644 --- a/tests/src/smtp/lookup/sql.rs +++ b/tests/src/smtp/lookup/sql.rs @@ -48,7 +48,6 @@ emails = "SELECT address FROM emails WHERE name = ? AND type != 'list' ORDER BY verify = "SELECT address FROM emails WHERE address LIKE '%' || ? || '%' AND type = 'primary' ORDER BY address LIMIT 5" expand = "SELECT p.address FROM emails AS p JOIN emails AS l ON p.name = l.name WHERE p.type = 'primary' AND l.address = ? AND l.type = 'list' ORDER BY p.address LIMIT 50" domains = "SELECT 1 FROM emails WHERE address LIKE '%@' || ? LIMIT 1" -is_ip_allowed = "SELECT addr FROM allowed_ips WHERE addr = ? LIMIT 1" [directory."sql"] type = "sql" @@ -73,7 +72,7 @@ relay = false errors.wait = "5ms" [session.extensions] -requiretls = [{if = "key_exists('sql/is_ip_allowed', remote_ip)", then = true}, +requiretls = [{if = "sql_query('sql', 'SELECT addr FROM allowed_ips WHERE addr = ? LIMIT 1', remote_ip)", then = true}, {else = false}] expn = true vrfy = true @@ -140,18 +139,6 @@ async fn lookup_sql() { handle .create_test_user_with_email("mike@foobar.net", "098765", "Mike") .await; - /*handle - .link_test_address("jane@foobar.org", "sales@foobar.org", "list") - .await; - handle - .link_test_address("john@foobar.org", "sales@foobar.org", "list") - .await; - handle - .link_test_address("bill@foobar.org", "sales@foobar.org", "list") - .await; - handle - .link_test_address("mike@foobar.net", "support@foobar.org", "list") - .await;*/ for query in [ "CREATE TABLE domains (name TEXT PRIMARY KEY, description TEXT);", diff --git a/tests/src/smtp/mod.rs b/tests/src/smtp/mod.rs index f79005ae..bf70df35 100644 --- a/tests/src/smtp/mod.rs +++ b/tests/src/smtp/mod.rs @@ -25,7 +25,7 @@ use crate::AssertConfig; pub mod config; pub mod inbound; pub mod lookup; -pub mod management; +//pub mod management; pub mod outbound; pub mod queue; pub mod reporting; @@ -128,13 +128,13 @@ cert = '%{file:{CERT}}%' private-key = '%{file:{PK}}%' [storage] -data = "sqlite" -lookup = "sqlite" -blob = "sqlite" -fts = "sqlite" +data = "rocksdb" +lookup = "rocksdb" +blob = "rocksdb" +fts = "rocksdb" -[store."sqlite"] -type = "sqlite" +[store."rocksdb"] +type = "rocksdb" path = "{TMP}/queue.db" "#;