From e7298df6104153c7a7bf4b71345d97435468d07b Mon Sep 17 00:00:00 2001 From: mdecimus Date: Thu, 19 Dec 2024 18:12:47 +0100 Subject: [PATCH] Port Spam filter to Rust - part 9 --- crates/common/src/config/jmap/settings.rs | 13 - crates/common/src/config/mod.rs | 2 +- crates/common/src/config/spamfilter.rs | 81 +- crates/common/src/enterprise/config.rs | 2 +- crates/common/src/expr/mod.rs | 2 +- crates/common/src/scripts/functions/html.rs | 422 ------ crates/common/src/scripts/functions/mod.rs | 9 +- crates/common/src/scripts/functions/text.rs | 5 + crates/common/src/scripts/plugins/text.rs | 3 +- crates/jmap/src/email/ingest.rs | 5 +- crates/nlp/src/tokenizers/types.rs | 2 +- crates/smtp/src/inbound/mod.rs | 1 + crates/smtp/src/inbound/spam.rs | 61 + crates/smtp/src/scripts/event_loop.rs | 27 - crates/smtp/src/scripts/mod.rs | 13 - crates/spam-filter/src/analysis/bayes.rs | 3 +- crates/spam-filter/src/analysis/date.rs | 41 +- crates/spam-filter/src/analysis/dmarc.rs | 118 +- crates/spam-filter/src/analysis/domain.rs | 44 +- crates/spam-filter/src/analysis/from.rs | 2 +- crates/spam-filter/src/analysis/headers.rs | 3 +- crates/spam-filter/src/analysis/html.rs | 11 +- crates/spam-filter/src/analysis/init.rs | 47 +- crates/spam-filter/src/analysis/ip.rs | 28 +- crates/spam-filter/src/analysis/messageid.rs | 31 +- crates/spam-filter/src/analysis/mime.rs | 27 +- crates/spam-filter/src/analysis/received.rs | 14 +- crates/spam-filter/src/analysis/recipient.rs | 76 +- crates/spam-filter/src/analysis/reputation.rs | 56 +- crates/spam-filter/src/analysis/rules.rs | 6 +- crates/spam-filter/src/analysis/score.rs | 50 +- crates/spam-filter/src/analysis/subject.rs | 18 +- crates/spam-filter/src/analysis/url.rs | 90 +- crates/spam-filter/src/lib.rs | 21 +- crates/spam-filter/src/modules/bayes.rs | 12 +- crates/spam-filter/src/modules/dnsbl.rs | 29 +- crates/spam-filter/src/modules/expression.rs | 43 +- crates/spam-filter/src/modules/mod.rs | 2 +- crates/spam-filter/src/modules/remote_list.rs | 10 +- tests/resources/smtp/antispam/bounce.test | 15 +- tests/resources/smtp/antispam/combined.test | 9 +- tests/resources/smtp/antispam/from.test | 9 +- tests/resources/smtp/antispam/headers.test | 6 +- tests/resources/smtp/antispam/html.test | 2 +- tests/resources/smtp/antispam/messageid.test | 6 + tests/resources/smtp/antispam/mime.test | 2 +- tests/resources/smtp/antispam/rbl.test | 19 +- tests/resources/smtp/antispam/received.test | 9 + tests/resources/smtp/antispam/recipient.test | 4 +- tests/resources/smtp/antispam/replyto.test | 6 +- tests/resources/smtp/antispam/subject.test | 27 +- tests/resources/smtp/antispam/url.test | 18 +- tests/src/smtp/inbound/antispam.rs | 1296 ++++++++++------- 53 files changed, 1506 insertions(+), 1352 deletions(-) delete mode 100644 crates/common/src/scripts/functions/html.rs create mode 100644 crates/smtp/src/inbound/spam.rs diff --git a/crates/common/src/config/jmap/settings.rs b/crates/common/src/config/jmap/settings.rs index 78f4c4a0..cf6b105e 100644 --- a/crates/common/src/config/jmap/settings.rs +++ b/crates/common/src/config/jmap/settings.rs @@ -7,7 +7,6 @@ use std::{str::FromStr, time::Duration}; use jmap_proto::request::capability::BaseCapabilities; -use mail_parser::HeaderName; use nlp::language::Language; use utils::config::{cron::SimpleCron, utils::ParseValue, Config, Rate}; @@ -65,7 +64,6 @@ pub struct JmapConfig { pub fallback_admin: Option<(String, String)>, pub master_user: Option<(String, String)>, - pub spam_header: Option<(HeaderName<'static>, String)>, pub default_folders: Vec, pub shared_folder: String, @@ -334,17 +332,6 @@ impl JmapConfig { encrypt_append: config .property_or_default("storage.encryption.append", "false") .unwrap_or(false), - spam_header: config - .property_or_default::>("spam.header.is-spam", "X-Spam-Status: Yes") - .unwrap_or_default() - .and_then(|v| { - v.split_once(':').map(|(k, v)| { - ( - mail_parser::HeaderName::parse(k.trim().to_string()).unwrap(), - v.trim().to_string(), - ) - }) - }), http_use_forwarded: config .property("server.http.use-x-forwarded") .unwrap_or(false), diff --git a/crates/common/src/config/mod.rs b/crates/common/src/config/mod.rs index f22fed07..d5ed0255 100644 --- a/crates/common/src/config/mod.rs +++ b/crates/common/src/config/mod.rs @@ -185,7 +185,7 @@ impl Core { oauth: OAuthConfig::parse(config), acme: AcmeProviders::parse(config), metrics: Metrics::parse(config), - spam: SpamFilterConfig::parse(config), + spam: SpamFilterConfig::parse(config).await, storage: Storage { data, blob, diff --git a/crates/common/src/config/spamfilter.rs b/crates/common/src/config/spamfilter.rs index 5c7577f3..3e074dc8 100644 --- a/crates/common/src/config/spamfilter.rs +++ b/crates/common/src/config/spamfilter.rs @@ -8,6 +8,7 @@ use std::{net::SocketAddr, time::Duration}; use ahash::AHashSet; use nlp::bayes::BayesClassifier; +use tokio::net::lookup_host; use utils::{ config::{utils::ParseValue, Config}, glob::{GlobMap, GlobSet}, @@ -26,6 +27,14 @@ pub struct SpamFilterConfig { pub bayes: Option, pub scores: SpamFilterScoreConfig, pub expiry: SpamFilterExpiryConfig, + pub headers: SpamFilterHeaderConfig, +} + +#[derive(Debug, Clone)] +pub struct SpamFilterHeaderConfig { + pub status: Option, + pub result: Option, + pub llm: Option, } #[derive(Debug, Clone, Default)] @@ -182,7 +191,7 @@ pub enum RemoteListFormat { } impl SpamFilterConfig { - pub fn parse(config: &mut Config) -> Self { + pub async fn parse(config: &mut Config) -> Self { SpamFilterConfig { enabled: config .property_or_default("spam-filter.enable", "true") @@ -190,11 +199,12 @@ impl SpamFilterConfig { dnsbl: DnsBlConfig::parse(config), rules: parse_rules(config), lists: SpamFilterLists::parse(config), - pyzor: PyzorConfig::parse(config), + pyzor: PyzorConfig::parse(config).await, reputation: ReputationConfig::parse(config), bayes: BayesConfig::parse(config), scores: SpamFilterScoreConfig::parse(config), expiry: SpamFilterExpiryConfig::parse(config), + headers: SpamFilterHeaderConfig::parse(config), } } } @@ -217,7 +227,7 @@ fn parse_rules(config: &mut Config) -> Vec { impl SpamFilterRule { pub fn parse(config: &mut Config, id: String) -> Option<(Self, i32)> { let id = id.as_str(); - if config + if !config .property_or_default(("spam-filter.rule", id, "enable"), "true") .unwrap_or(true) { @@ -249,7 +259,7 @@ impl DnsBlConfig { pub fn parse(config: &mut Config) -> Self { let mut servers = vec![]; for id in config - .sub_keys("spam-filter.dnsbl.server", ".url") + .sub_keys("spam-filter.dnsbl.server", ".scope") .map(|k| k.to_string()) .collect::>() { @@ -280,7 +290,7 @@ impl DnsBlServer { pub fn parse(config: &mut Config, id: String) -> Option { let id_ = id.as_str(); - if config + if !config .property_or_default(("spam-filter.dnsbl.server", id_, "enable"), "true") .unwrap_or(true) { @@ -300,7 +310,7 @@ impl DnsBlServer { tags: IfBlock::try_parse( config, ("spam-filter.dnsbl.server", id_, "tag"), - &Element::Domain.token_map(), + &Element::Ip.token_map(), )?, id, } @@ -308,6 +318,32 @@ impl DnsBlServer { } } +impl SpamFilterHeaderConfig { + pub fn parse(config: &mut Config) -> Self { + let mut header = SpamFilterHeaderConfig::default(); + + for (typ, var) in [ + ("status", &mut header.status), + ("result", &mut header.result), + ("llm", &mut header.llm), + ] { + if config + .property_or_default(("spam-filter.header", typ, "enable"), "true") + .unwrap_or(true) + { + if let Some(value) = config.value(("spam-filter.header", typ, "name")) { + let value = value.trim(); + if !value.is_empty() { + *var = value.to_string().into(); + } + } + } + } + + header + } +} + impl SpamFilterLists { pub fn parse(config: &mut Config) -> Self { let mut lists = SpamFilterLists { @@ -511,7 +547,7 @@ impl SpamFilterLists { } impl PyzorConfig { - pub fn parse(config: &mut Config) -> Option { + pub async fn parse(config: &mut Config) -> Option { if !config .property_or_default("spam-filter.pyzor.enable", "true") .unwrap_or(true) @@ -525,8 +561,18 @@ impl PyzorConfig { let host = config .value("spam-filter.pyzor.host") .unwrap_or("public.pyzor.org"); - let address = match format!("{host}:{port}").parse() { - Ok(address) => address, + let address = match lookup_host(format!("{host}:{port}")) + .await + .map(|mut a| a.next()) + { + Ok(Some(address)) => address, + Ok(None) => { + config.new_build_error( + "spam-filter.pyzor.host", + "Invalid address: No addresses found.", + ); + return None; + } Err(err) => { config.new_build_error( "spam-filter.pyzor.host", @@ -716,14 +762,23 @@ impl Location { } } +impl Default for SpamFilterHeaderConfig { + fn default() -> Self { + SpamFilterHeaderConfig { + status: "X-Spam-Status".to_string().into(), + result: "X-Spam-Result".to_string().into(), + llm: "X-Spam-LLM".to_string().into(), + } + } +} + pub const V_SPAM_REMOTE_IP: u32 = 100; pub const V_SPAM_REMOTE_IP_PTR: u32 = 101; pub const V_SPAM_EHLO_DOMAIN: u32 = 102; pub const V_SPAM_AUTH_AS: u32 = 103; pub const V_SPAM_ASN: u32 = 104; pub const V_SPAM_COUNTRY: u32 = 105; -pub const V_SPAM_TLS_VERSION: u32 = 106; -pub const V_SPAM_TLS_CIPHER: u32 = 107; +pub const V_SPAM_IS_TLS: u32 = 106; pub const V_SPAM_ENV_FROM: u32 = 108; pub const V_SPAM_ENV_FROM_LOCAL: u32 = 109; pub const V_SPAM_ENV_FROM_DOMAIN: u32 = 110; @@ -794,8 +849,7 @@ impl Element { ("auth_as", V_SPAM_AUTH_AS), ("asn", V_SPAM_ASN), ("country", V_SPAM_COUNTRY), - ("tls_version", V_SPAM_TLS_VERSION), - ("tls_cipher", V_SPAM_TLS_CIPHER), + ("is_tls", V_SPAM_IS_TLS), ("env_from", V_SPAM_ENV_FROM), ("env_from.local", V_SPAM_ENV_FROM_LOCAL), ("env_from.domain", V_SPAM_ENV_FROM_DOMAIN), @@ -844,6 +898,7 @@ impl Element { ]), Element::Email => map.with_variables_map([ ("email", V_RCPT_EMAIL), + ("value", V_RCPT_EMAIL), ("name", V_RCPT_NAME), ("local", V_RCPT_LOCAL), ("domain", V_RCPT_DOMAIN), diff --git a/crates/common/src/enterprise/config.rs b/crates/common/src/enterprise/config.rs index 6cc294e8..3cb70889 100644 --- a/crates/common/src/enterprise/config.rs +++ b/crates/common/src/enterprise/config.rs @@ -233,7 +233,7 @@ impl SpamFilterLlmConfig { .unwrap_or_default() .chars() .next() - .unwrap_or('|'), + .unwrap_or(','), index_category: config .property("spam-filter.llm.index.category") .unwrap_or_default(), diff --git a/crates/common/src/expr/mod.rs b/crates/common/src/expr/mod.rs index c537f3b4..de0b3757 100644 --- a/crates/common/src/expr/mod.rs +++ b/crates/common/src/expr/mod.rs @@ -99,7 +99,7 @@ pub enum ExpressionItem { ArrayBuild(u32), } -#[derive(Debug)] +#[derive(Debug, Clone)] pub enum Variable<'x> { String(Cow<'x, str>), Integer(i64), diff --git a/crates/common/src/scripts/functions/html.rs b/crates/common/src/scripts/functions/html.rs deleted file mode 100644 index 5f565f5c..00000000 --- a/crates/common/src/scripts/functions/html.rs +++ /dev/null @@ -1,422 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use std::borrow::Cow; - -use mail_parser::decoders::html::{add_html_token, html_to_text}; -use sieve::{runtime::Variable, Context}; - -pub fn fn_html_to_text<'x>(_: &'x Context<'x>, v: Vec) -> Variable { - html_to_text(v[0].to_string().as_ref()).into() -} - -pub fn fn_html_has_tag<'x>(_: &'x Context<'x>, v: Vec) -> Variable { - v[0].as_array() - .map(|arr| { - let token = v[1].to_string(); - arr.iter().any(|v| { - v.to_string() - .as_ref() - .strip_prefix('<') - .map_or(false, |tag| tag.starts_with(token.as_ref())) - }) - }) - .unwrap_or_default() - .into() -} - -pub fn fn_html_attr_size<'x>(_: &'x Context<'x>, v: Vec) -> Variable { - let t = v[0].to_string(); - let mut dimension = None; - - if let Some(value) = get_attribute(t.as_ref(), v[1].to_string().as_ref()) { - let value = value.trim(); - if let Some(pct) = value.strip_suffix('%') { - if let Ok(pct) = pct.trim().parse::() { - dimension = ((v[2].to_integer() * pct as i64) / 100).into(); - } - } else if let Ok(value) = value.parse::() { - dimension = (value as i64).into(); - } - } - - dimension.map(Variable::Integer).unwrap_or_default() -} - -pub fn fn_html_attrs<'x>(_: &'x Context<'x>, v: Vec) -> Variable { - html_attr_tokens( - v[0].to_string().as_ref(), - v[1].to_string().as_ref(), - v[2].to_string_array(), - ) - .into() -} - -pub fn fn_html_attr<'x>(_: &'x Context<'x>, v: Vec) -> Variable { - get_attribute(v[0].to_string().as_ref(), v[1].to_string().as_ref()) - .map(Variable::from) - .unwrap_or_default() -} - -pub fn html_to_tokens(input: &str) -> Vec { - let input = input.as_bytes(); - let mut iter = input.iter().enumerate(); - let mut tags = vec![]; - - let mut is_token_start = true; - let mut is_after_space = false; - let mut is_new_line = true; - - let mut token_start = 0; - let mut token_end = 0; - - let mut text = String::from("_"); - - while let Some((pos, &ch)) = iter.next() { - match ch { - b'<' => { - if !is_token_start { - add_html_token( - &mut text, - &input[token_start..token_end + 1], - is_after_space, - ); - is_after_space = false; - is_token_start = true; - } - if text.len() > 1 { - tags.push(Variable::String(text.into())); - text = String::from("_"); - } - - let mut tag = vec![b'<']; - if matches!(input.get(pos + 1..pos + 4), Some(b"!--")) { - let mut last_ch: u8 = 0; - for (_, &ch) in iter.by_ref() { - match ch { - b'>' if tag.len() > 3 - && matches!(tag.last(), Some(b'-')) - && matches!(tag.get(tag.len() - 2), Some(b'-')) => - { - break; - } - b' ' | b'\t' | b'\r' | b'\n' => { - if last_ch != b' ' { - tag.push(b' '); - } else { - last_ch = b' '; - } - continue; - } - _ => { - tag.push(ch); - } - } - last_ch = ch; - } - } else { - let mut in_quote = false; - let mut last_ch = b' '; - for (_, &ch) in iter.by_ref() { - match ch { - b'>' if !in_quote => { - break; - } - b'"' => { - in_quote = !in_quote; - tag.push(b'"'); - } - b' ' | b'\t' | b'\r' | b'\n' if !in_quote => { - if last_ch != b' ' { - tag.push(b' '); - last_ch = b' '; - } - continue; - } - b'/' if !in_quote => { - tag.push(b'/'); - last_ch = b' '; - continue; - } - _ => { - tag.push(if in_quote { - ch - } else { - ch.to_ascii_lowercase() - }); - } - } - last_ch = ch; - } - } - tags.push(Variable::String( - String::from_utf8(tag).unwrap_or_default().into(), - )); - continue; - } - b' ' | b'\t' | b'\r' | b'\n' => { - if !is_token_start { - add_html_token( - &mut text, - &input[token_start..token_end + 1], - is_after_space && !is_new_line, - ); - is_new_line = false; - } - is_after_space = true; - is_token_start = true; - continue; - } - b'&' if !is_token_start => { - add_html_token( - &mut text, - &input[token_start..token_end + 1], - is_after_space && !is_new_line, - ); - is_new_line = false; - is_token_start = true; - is_after_space = false; - } - b';' if !is_token_start => { - add_html_token( - &mut text, - &input[token_start..pos + 1], - is_after_space && !is_new_line, - ); - is_token_start = true; - is_after_space = false; - is_new_line = false; - continue; - } - _ => (), - } - - if is_token_start { - token_start = pos; - is_token_start = false; - } - token_end = pos; - } - - if !is_token_start { - add_html_token( - &mut text, - &input[token_start..token_end + 1], - is_after_space && !is_new_line, - ); - } - if text.len() > 1 { - tags.push(Variable::String(text.into())); - } - - tags -} - -pub fn html_attr_tokens(input: &str, tag: &str, attrs: Vec>) -> Vec { - let input = input.as_bytes(); - let mut iter = input.iter().enumerate().peekable(); - let mut tags = vec![]; - - while let Some((mut pos, &ch)) = iter.next() { - if ch == b'<' { - if !matches!(input.get(pos + 1..pos + 4), Some(b"!--")) { - let mut in_quote = false; - let mut last_ch_pos: usize = 0; - - while matches!(iter.peek(), Some((_, &ch)) if ch.is_ascii_whitespace()) { - pos += 1; - iter.next(); - } - - let found_tag = tag.is_empty() - || (matches!(input.get(pos + 1..pos + tag.len() + 1), Some(t) if t.eq_ignore_ascii_case(tag.as_bytes())) - && matches!(input.get(pos + tag.len() + 1), Some(ch) if ch.is_ascii_whitespace())); - - 'outer: while let Some((pos, &ch)) = iter.next() { - match ch { - b'>' if !in_quote => { - break; - } - b'"' => { - in_quote = !in_quote; - } - b'=' if found_tag - && !in_quote - && attrs.iter().any(|attr| matches!(input.get(last_ch_pos.saturating_sub(attr.len()) + 1..last_ch_pos + 1), Some(a) if a.eq_ignore_ascii_case(attr.as_bytes()))) - && matches!(input.get(last_ch_pos + 1), Some(ch) if ch.is_ascii_whitespace() || *ch == b'=') => - { - while matches!(iter.peek(), Some((_, &ch)) if ch.is_ascii_whitespace()) - { - iter.next(); - } - let mut tag = vec![]; - - for (_, &ch) in iter.by_ref() { - match ch { - b'>' if !in_quote => { - if !tag.is_empty() { - tags.push(Variable::String( - String::from_utf8(tag).unwrap_or_default().into(), - )); - } - break 'outer; - } - b'"' => { - if in_quote { - in_quote = false; - break; - } else { - in_quote = true; - } - } - b' ' | b'\t' | b'\r' | b'\n' if !in_quote => { - break; - } - _ => { - tag.push(ch); - } - } - } - - if !tag.is_empty() { - tags.push(Variable::String( - String::from_utf8(tag).unwrap_or_default().into(), - )); - } - } - b' ' | b'\t' | b'\r' | b'\n' => {} - _ => { - last_ch_pos = pos; - } - } - } - } else { - let mut last_ch: u8 = 0; - let mut before_last_ch: u8 = 0; - - for (_, &ch) in iter.by_ref() { - if ch == b'>' && last_ch == b'-' && before_last_ch == b'-' { - break; - } - before_last_ch = last_ch; - last_ch = ch; - } - } - } - } - - tags -} - -pub fn html_img_area(arr: &[Variable]) -> u32 { - arr.iter() - .filter_map(|v| { - let t = v.to_string(); - if t.starts_with("() { - let size = if idx == 0 { 800 } else { 600 }; - dimensions[idx] = (size * pct) / 100; - } - } else if let Ok(value) = value.parse::() { - dimensions[idx] = value; - } - } - } - - Some(dimensions[0].saturating_mul(dimensions[1])) - } else { - None - } - }) - .sum::() -} - -pub fn get_attribute<'x>(tag: &'x str, attr_name: &str) -> Option<&'x str> { - let tag = tag.as_bytes(); - let attr_name = attr_name.as_bytes(); - let mut iter = tag.iter().enumerate().peekable(); - let mut in_quote = false; - let mut start_pos = usize::MAX; - let mut end_pos = usize::MAX; - - while let Some((pos, ch)) = iter.next() { - match ch { - b'=' if !in_quote => { - if start_pos != usize::MAX - && end_pos != usize::MAX - && tag - .get(start_pos..end_pos + 1) - .map_or(false, |name| name == attr_name) - { - let mut token_start = 0; - let mut token_end = 0; - - for (pos, ch) in iter.by_ref() { - match ch { - b'"' => { - if !in_quote { - token_start = pos + 1; - in_quote = true; - } else { - token_end = pos; - break; - } - } - b' ' if !in_quote => { - if token_start != 0 { - token_end = pos; - break; - } - } - _ => { - if token_start == 0 { - token_start = pos; - } - } - } - } - - return if token_start > 0 { - if token_end == 0 { - token_end = tag.len(); - } - Some(std::str::from_utf8(&tag[token_start..token_end]).unwrap_or_default()) - } else { - None - }; - } else { - start_pos = usize::MAX; - end_pos = usize::MAX; - } - } - b'"' => { - in_quote = !in_quote; - } - b' ' => { - if !in_quote && !matches!(iter.peek(), Some((_, b'='))) { - start_pos = usize::MAX; - end_pos = usize::MAX; - } - } - _ => { - if !in_quote { - if start_pos == usize::MAX { - start_pos = pos; - } - end_pos = pos; - } - } - } - } - - None -} diff --git a/crates/common/src/scripts/functions/mod.rs b/crates/common/src/scripts/functions/mod.rs index f348dbc6..e0f27e67 100644 --- a/crates/common/src/scripts/functions/mod.rs +++ b/crates/common/src/scripts/functions/mod.rs @@ -7,7 +7,6 @@ pub mod array; mod email; mod header; -pub mod html; pub mod image; pub mod misc; pub mod text; @@ -16,9 +15,7 @@ pub mod url; use sieve::{runtime::Variable, FunctionMap}; -use self::{ - array::*, email::*, header::*, html::*, image::*, misc::*, text::*, unicode::*, url::*, -}; +use self::{array::*, email::*, header::*, image::*, misc::*, text::*, unicode::*, url::*}; pub fn register_functions_trusted() -> FunctionMap { FunctionMap::new() @@ -70,10 +67,6 @@ pub fn register_functions_trusted() -> FunctionMap { .with_function_args("cosine_similarity", fn_cosine_similarity, 2) .with_function_args("jaccard_similarity", fn_jaccard_similarity, 2) .with_function_args("levenshtein_distance", fn_levenshtein_distance, 2) - .with_function_args("html_has_tag", fn_html_has_tag, 2) - .with_function_args("html_attr", fn_html_attr, 2) - .with_function_args("html_attrs", fn_html_attrs, 3) - .with_function_args("html_attr_size", fn_html_attr_size, 3) .with_function_args("uri_part", fn_uri_part, 2) .with_function_args("substring", fn_substring, 3) .with_function_args("split", fn_split, 2) diff --git a/crates/common/src/scripts/functions/text.rs b/crates/common/src/scripts/functions/text.rs index f46e50fb..610db889 100644 --- a/crates/common/src/scripts/functions/text.rs +++ b/crates/common/src/scripts/functions/text.rs @@ -4,6 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use mail_parser::decoders::html::html_to_text; use sieve::{runtime::Variable, Context}; use super::ApplyString; @@ -310,3 +311,7 @@ pub fn fn_detect_language<'x>(_: &'x Context<'x>, v: Vec) -> Variable .unwrap_or("unknown") .into() } + +pub fn fn_html_to_text<'x>(_: &'x Context<'x>, v: Vec) -> Variable { + html_to_text(v[0].to_string().as_ref()).into() +} diff --git a/crates/common/src/scripts/plugins/text.rs b/crates/common/src/scripts/plugins/text.rs index 922c89b6..8ea351be 100644 --- a/crates/common/src/scripts/plugins/text.rs +++ b/crates/common/src/scripts/plugins/text.rs @@ -7,7 +7,7 @@ use nlp::tokenizers::types::{TokenType, TypesTokenizer}; use sieve::{runtime::Variable, FunctionMap}; -use crate::scripts::functions::{html::html_to_tokens, text::tokenize_words, ApplyString}; +use crate::scripts::functions::{text::tokenize_words, ApplyString}; use super::PluginContext; @@ -22,7 +22,6 @@ pub fn register_domain_part(plugin_id: u32, fnc_map: &mut FunctionMap) { pub fn exec_tokenize(ctx: PluginContext<'_>) -> trc::Result { let mut v = ctx.arguments; let (urls, urls_without_scheme, emails) = match v[1].to_string().as_ref() { - "html" => return Ok(html_to_tokens(v[0].to_string().as_ref()).into()), "words" => return Ok(tokenize_words(&v[0])), "uri" | "url" => (true, true, true), "uri_strict" | "url_strict" => (true, false, false), diff --git a/crates/jmap/src/email/ingest.rs b/crates/jmap/src/email/ingest.rs index e287332c..057ab01c 100644 --- a/crates/jmap/src/email/ingest.rs +++ b/crates/jmap/src/email/ingest.rs @@ -121,7 +121,8 @@ impl EmailIngest for Server { // Check for Spam headers let mut is_spam = false; - if let (IngestSource::Smtp, Some((header_name, header_value))) = + let todo = "true"; + /*if let (IngestSource::Smtp, Some((header_name, header_value))) = (params.source, &self.core.jmap.spam_header) { if params.mailbox_ids == [INBOX_ID] @@ -136,7 +137,7 @@ impl EmailIngest for Server { params.mailbox_ids[0] = JUNK_ID; is_spam = true; } - } + }*/ // Obtain message references and thread name let mut message_id = String::new(); diff --git a/crates/nlp/src/tokenizers/types.rs b/crates/nlp/src/tokenizers/types.rs index ecfe4719..a5c088d7 100644 --- a/crates/nlp/src/tokenizers/types.rs +++ b/crates/nlp/src/tokenizers/types.rs @@ -752,7 +752,7 @@ impl> TokenType { TokenType::Url(url) => url.as_ref().trim().to_lowercase().into(), TokenType::UrlNoScheme(url) if !with_scheme_only => { let url = url.as_ref(); - format!("http:s//{}", url.trim().to_lowercase()) + format!("https://{}", url.trim().to_lowercase()) .to_lowercase() .into() } diff --git a/crates/smtp/src/inbound/mod.rs b/crates/smtp/src/inbound/mod.rs index 83ee1ceb..cdf9762b 100644 --- a/crates/smtp/src/inbound/mod.rs +++ b/crates/smtp/src/inbound/mod.rs @@ -20,6 +20,7 @@ pub mod mail; pub mod milter; pub mod rcpt; pub mod session; +pub mod spam; pub mod spawn; pub mod vrfy; diff --git a/crates/smtp/src/inbound/spam.rs b/crates/smtp/src/inbound/spam.rs new file mode 100644 index 00000000..c2587844 --- /dev/null +++ b/crates/smtp/src/inbound/spam.rs @@ -0,0 +1,61 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use common::listener::SessionStream; +use mail_auth::{dmarc::Policy, ArcOutput, DkimOutput, DmarcResult}; +use mail_parser::Message; +use spam_filter::SpamFilterInput; + +use crate::core::Session; + +impl Session { + pub fn build_spam_input<'x>( + &'x self, + message: &'x Message<'x>, + dkim_result: &'x [DkimOutput<'x>], + arc_result: Option<&'x ArcOutput>, + dmarc_result: Option<&'x DmarcResult>, + dmarc_policy: Option<&'x Policy>, + ) -> SpamFilterInput<'x> { + SpamFilterInput { + message, + span_id: self.data.session_id, + arc_result, + spf_ehlo_result: self.data.spf_ehlo.as_ref(), + spf_mail_from_result: self.data.spf_mail_from.as_ref(), + dkim_result, + dmarc_result, + dmarc_policy, + iprev_result: self.data.iprev.as_ref(), + remote_ip: self.data.remote_ip, + ehlo_domain: self.data.helo_domain.as_str().into(), + authenticated_as: self.data.authenticated_as.as_ref().map(|a| a.name.as_str()), + asn: self.data.asn_geo_data.asn.as_ref().map(|a| a.id), + country: self.data.asn_geo_data.country.as_ref().map(|c| c.as_str()), + is_tls: self.stream.is_tls(), + env_from: self + .data + .mail_from + .as_ref() + .map(|m| m.address_lcase.as_str()) + .unwrap_or_default(), + env_from_flags: self + .data + .mail_from + .as_ref() + .map(|m| m.flags) + .unwrap_or_default(), + env_rcpt_to: self + .data + .rcpt_to + .iter() + .map(|r| r.address_lcase.as_str()) + .collect(), + account_id: None, + is_test: false, + } + } +} diff --git a/crates/smtp/src/scripts/event_loop.rs b/crates/smtp/src/scripts/event_loop.rs index ebc7fab3..9d9ac226 100644 --- a/crates/smtp/src/scripts/event_loop.rs +++ b/crates/smtp/src/scripts/event_loop.rs @@ -359,33 +359,6 @@ impl RunScript for Server { } } - // Assert global variables - #[cfg(feature = "test_mode")] - if let Some(expected_variables) = params.expected_variables { - for var_name in instance.global_variable_names() { - if instance.global_variable(var_name).unwrap().to_bool() - && !expected_variables.contains_key(var_name) - { - panic!( - "Unexpected variable {var_name:?} with value {:?}\nExpected {:?}\nFound: {:?}", - instance.global_variable(var_name).unwrap(), - expected_variables.keys().collect::>(), - instance.global_variable_names().collect::>() - ); - } - } - - for (name, expected) in &expected_variables { - if let Some(value) = instance.global_variable(name.as_str()) { - assert_eq!(value, expected, "Variable {name:?} has unexpected value"); - } else { - panic!("Missing variable {name:?} with value {expected:?}\nExpected {:?}\nFound: {:?}", - expected_variables.keys().collect::>(), - instance.global_variable_names().collect::>()); - } - } - } - // Keep id // 0 = use original message // MAX = implicit keep diff --git a/crates/smtp/src/scripts/mod.rs b/crates/smtp/src/scripts/mod.rs index 901bf6b3..4be80a80 100644 --- a/crates/smtp/src/scripts/mod.rs +++ b/crates/smtp/src/scripts/mod.rs @@ -38,8 +38,6 @@ pub struct ScriptParameters<'x> { from_name: String, return_path: String, sign: Vec, - #[cfg(feature = "test_mode")] - expected_variables: Option>, access_token: Option<&'x AccessToken>, session_id: u64, } @@ -51,8 +49,6 @@ impl<'x> ScriptParameters<'x> { envelope: Vec::with_capacity(6), message: None, headers: None, - #[cfg(feature = "test_mode")] - expected_variables: None, from_addr: Default::default(), from_name: Default::default(), return_path: Default::default(), @@ -123,15 +119,6 @@ impl<'x> ScriptParameters<'x> { self.session_id = session_id; self } - - #[cfg(feature = "test_mode")] - pub fn with_expected_variables( - mut self, - expected_variables: AHashMap, - ) -> Self { - self.expected_variables = expected_variables.into(); - self - } } impl Default for ScriptParameters<'_> { diff --git a/crates/spam-filter/src/analysis/bayes.rs b/crates/spam-filter/src/analysis/bayes.rs index 9ec14ced..fd854890 100644 --- a/crates/spam-filter/src/analysis/bayes.rs +++ b/crates/spam-filter/src/analysis/bayes.rs @@ -27,13 +27,14 @@ impl SpamFilterAnalyzeBayes for Server { if let Some(config) = &self.core.spam.bayes { if !ctx.result.has_tag("SPAM_TRAP") && !ctx.result.has_tag("TRUSTED_REPLY") { match bayes_classify(self, ctx).await { - Ok(score) => { + Ok(Some(score)) => { if score > config.score_spam { ctx.result.add_tag("BAYES_SPAM"); } else if score < config.score_ham { ctx.result.add_tag("BAYES_HAM"); } } + Ok(None) => (), Err(err) => { trc::error!(err.span_id(ctx.input.span_id).caused_by(trc::location!())); } diff --git a/crates/spam-filter/src/analysis/date.rs b/crates/spam-filter/src/analysis/date.rs index 0f25d349..8a618f51 100644 --- a/crates/spam-filter/src/analysis/date.rs +++ b/crates/spam-filter/src/analysis/date.rs @@ -4,9 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ - use std::future::Future; +use std::future::Future; use common::Server; +use mail_parser::HeaderName; use store::write::now; use crate::SpamFilterContext; @@ -20,23 +21,35 @@ pub trait SpamFilterAnalyzeDate: Sync + Send { impl SpamFilterAnalyzeDate for Server { async fn spam_filter_analyze_date(&self, ctx: &mut SpamFilterContext<'_>) { - if let Some(date) = ctx.input.message.date() { - let date = date.to_timestamp(); - if date != 0 { - let date_diff = now() as i64 - date; + match ctx + .input + .message + .header(HeaderName::Date) + .map(|h| h.as_datetime()) + { + Some(Some(date)) => { + let date = date.to_timestamp(); + if date != 0 { + let date_diff = now() as i64 - date; - if date_diff > 86400 { - // Older than a day - ctx.result.add_tag("DATE_IN_PAST"); - } else if -date_diff > 7200 { - //# More than 2 hours in the future - ctx.result.add_tag("DATE_IN_FUTURE"); + if date_diff > 86400 { + // Older than a day + ctx.result.add_tag("DATE_IN_PAST"); + } else if -date_diff > 7200 { + //# More than 2 hours in the future + ctx.result.add_tag("DATE_IN_FUTURE"); + } + } else { + ctx.result.add_tag("INVALID_DATE"); } - } else { + } + Some(None) => { ctx.result.add_tag("INVALID_DATE"); } - } else { - ctx.result.add_tag("MISSING_DATE"); + + None => { + ctx.result.add_tag("MISSING_DATE"); + } } } } diff --git a/crates/spam-filter/src/analysis/dmarc.rs b/crates/spam-filter/src/analysis/dmarc.rs index 676e9878..eeddba79 100644 --- a/crates/spam-filter/src/analysis/dmarc.rs +++ b/crates/spam-filter/src/analysis/dmarc.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ - use std::future::Future; +use std::future::Future; use common::Server; use mail_auth::{ @@ -22,16 +22,19 @@ pub trait SpamFilterAnalyzeDmarc: Sync + Send { impl SpamFilterAnalyzeDmarc for Server { async fn spam_filter_analyze_dmarc(&self, ctx: &mut SpamFilterContext<'_>) { - ctx.result - .add_tag(match ctx.input.spf_mail_from_result.result() { - SpfResult::Pass => "SPF_ALLOW", - SpfResult::Fail => "SPF_FAIL", - SpfResult::SoftFail => "SPF_SOFTFAIL", - SpfResult::Neutral => "SPF_NEUTRAL", - SpfResult::TempError => "SPF_DNSFAIL", - SpfResult::PermError => "SPF_PERMFAIL", - SpfResult::None => "SPF_NA", - }); + ctx.result.add_tag( + ctx.input + .spf_mail_from_result + .map_or("SPF_NA", |r| match r.result() { + SpfResult::Pass => "SPF_ALLOW", + SpfResult::Fail => "SPF_FAIL", + SpfResult::SoftFail => "SPF_SOFTFAIL", + SpfResult::Neutral => "SPF_NEUTRAL", + SpfResult::TempError => "SPF_DNSFAIL", + SpfResult::PermError => "SPF_PERMFAIL", + SpfResult::None => "SPF_NA", + }), + ); ctx.result.add_tag( match ctx @@ -51,25 +54,30 @@ impl SpamFilterAnalyzeDmarc for Server { }, ); - ctx.result.add_tag(match ctx.input.arc_result.result() { - DkimResult::Pass => "ARC_ALLOW", - DkimResult::Fail(_) => "ARC_REJECT", - DkimResult::PermError(_) => "ARC_INVALID", - DkimResult::TempError(_) => "ARC_DNSFAIL", - DkimResult::Neutral(_) | DkimResult::None => "ARC_NA", - }); + ctx.result + .add_tag(ctx.input.arc_result.map_or("ARC_NA", |r| match r.result() { + DkimResult::Pass => "ARC_ALLOW", + DkimResult::Fail(_) => "ARC_REJECT", + DkimResult::PermError(_) => "ARC_INVALID", + DkimResult::TempError(_) => "ARC_DNSFAIL", + DkimResult::Neutral(_) | DkimResult::None => "ARC_NA", + })); - ctx.result.add_tag(match ctx.input.dmarc_result { - DmarcResult::Pass => "DMARC_POLICY_ALLOW", - DmarcResult::TempError(_) => "DMARC_DNSFAIL", - DmarcResult::PermError(_) => "DMARC_BAD_POLICY", - DmarcResult::None => "DMARC_NA", - DmarcResult::Fail(_) => match ctx.input.dmarc_policy { - Policy::Quarantine => "DMARC_POLICY_QUARANTINE", - Policy::Reject => "DMARC_POLICY_REJECT", - Policy::Unspecified | Policy::None => "DMARC_POLICY_SOFTFAIL", - }, - }); + ctx.result + .add_tag(ctx.input.dmarc_result.map_or("DMARC_NA", |r| match r { + DmarcResult::Pass => "DMARC_POLICY_ALLOW", + DmarcResult::TempError(_) => "DMARC_DNSFAIL", + DmarcResult::PermError(_) => "DMARC_BAD_POLICY", + DmarcResult::None => "DMARC_NA", + DmarcResult::Fail(_) => ctx.input.dmarc_policy.map_or( + "DMARC_POLICY_SOFTFAIL", + |p| match p { + Policy::Quarantine => "DMARC_POLICY_QUARANTINE", + Policy::Reject => "DMARC_POLICY_REJECT", + Policy::Unspecified | Policy::None => "DMARC_POLICY_SOFTFAIL", + }, + ), + })); for header in ctx.input.message.headers() { let header_name = header.name(); @@ -83,37 +91,43 @@ impl SpamFilterAnalyzeDmarc for Server { if self .core .spam - .lists.dmarc_allow + .lists + .dmarc_allow .contains(&ctx.output.from.email.domain_part.fqdn) { - if matches!(ctx.input.dmarc_result, DmarcResult::Pass) { + if matches!(ctx.input.dmarc_result, Some(DmarcResult::Pass)) { ctx.result.add_tag("ALLOWLIST_DMARC"); - } else { + } else if ctx.input.dmarc_result.is_some() { ctx.result.add_tag("BLOCKLIST_DMARC"); } } else if self .core .spam - .lists.spf_dkim_allow + .lists + .spf_dkim_allow .contains(&ctx.output.from.email.domain_part.fqdn) { - let is_dkim_pass = matches!(ctx.input.arc_result.result(), DkimResult::Pass) - || ctx.input.dkim_result.iter().any(|r| { - matches!(r.result(), DkimResult::Pass) - && r.signature().map_or(false, |s| { - s.domain().to_lowercase() == ctx.output.from.email.domain_part.fqdn - }) - }); - let is_spf_pass = matches!(ctx.input.spf_mail_from_result.result(), SpfResult::Pass); + let spf = ctx + .input + .spf_mail_from_result + .map(|r| r.result()) + .unwrap_or(SpfResult::None); + let is_dkim_pass = matches!( + ctx.input.arc_result.map(|r| r.result()), + Some(DkimResult::Pass) + ) || ctx.input.dkim_result.iter().any(|r| { + matches!(r.result(), DkimResult::Pass) + && r.signature().map_or(false, |s| { + s.domain().to_lowercase() == ctx.output.from.email.domain_part.fqdn + }) + }); + let is_spf_pass = matches!(spf, SpfResult::Pass); if is_dkim_pass && is_spf_pass { ctx.result.add_tag("ALLOWLIST_SPF_DKIM"); } else if is_dkim_pass { ctx.result.add_tag("ALLOWLIST_DKIM"); - if !matches!( - ctx.input.spf_mail_from_result.result(), - SpfResult::TempError - ) { + if !matches!(spf, SpfResult::TempError) { ctx.result.add_tag("BLOCKLIST_SPF"); } } else if is_spf_pass { @@ -126,14 +140,12 @@ impl SpamFilterAnalyzeDmarc for Server { { ctx.result.add_tag("BLOCKLIST_DKIM"); } - } else if !matches!( - ctx.input.spf_mail_from_result.result(), - SpfResult::TempError - ) && !ctx - .input - .dkim_result - .iter() - .any(|r| matches!(r.result(), DkimResult::TempError(_))) + } else if !matches!(spf, SpfResult::TempError) + && !ctx + .input + .dkim_result + .iter() + .any(|r| matches!(r.result(), DkimResult::TempError(_))) { ctx.result.add_tag("BLOCKLIST_SPF_DKIM"); } diff --git a/crates/spam-filter/src/analysis/domain.rs b/crates/spam-filter/src/analysis/domain.rs index edb258fd..45381fa3 100644 --- a/crates/spam-filter/src/analysis/domain.rs +++ b/crates/spam-filter/src/analysis/domain.rs @@ -11,6 +11,7 @@ use common::{ Server, }; use mail_auth::DkimResult; +use mail_parser::{HeaderName, HeaderValue, Host}; use nlp::tokenizers::types::TokenType; use crate::{ @@ -19,7 +20,7 @@ use crate::{ expression::{SpamFilterResolver, StringResolver}, html::{HtmlToken, A, HREF}, }, - Email, Recipient, SpamFilterContext, TextPart, + Email, Hostname, Recipient, SpamFilterContext, TextPart, }; use super::{is_trusted_domain, ElementLocation}; @@ -49,11 +50,31 @@ impl SpamFilterAnalyzeDomain for Server { } } + // Add Received headers + for header in ctx.input.message.headers() { + if let (HeaderName::Received, HeaderValue::Received(received)) = + (&header.name, &header.value) + { + for host in [&received.from, &received.helo, &received.by] + .into_iter() + .flatten() + { + if let Host::Name(name) = host { + if let Some(name) = Hostname::new(name.as_ref()).sld { + domains.insert(ElementLocation::new(name, Location::HeaderReceived)); + } + } + } + } + } + // Add EHLO domain - domains.insert(ElementLocation::new( - ctx.output.ehlo_host.fqdn.clone(), - Location::Ehlo, - )); + if !ctx.output.ehlo_host.fqdn.is_empty() { + domains.insert(ElementLocation::new( + ctx.output.ehlo_host.fqdn.clone(), + Location::Ehlo, + )); + } // Add PTR if let Some(ptr) = &ctx.output.iprev_ptr { @@ -161,12 +182,13 @@ impl SpamFilterAnalyzeDomain for Server { // Validate email for email in &emails { // Skip trusted domains - if is_trusted_domain( - self, - &email.element.email.domain_part.fqdn, - ctx.input.span_id, - ) - .await + if !email.element.email.is_valid() + || is_trusted_domain( + self, + &email.element.email.domain_part.fqdn, + ctx.input.span_id, + ) + .await { continue; } diff --git a/crates/spam-filter/src/analysis/from.rs b/crates/spam-filter/src/analysis/from.rs index 8e9ca853..7321cfb5 100644 --- a/crates/spam-filter/src/analysis/from.rs +++ b/crates/spam-filter/src/analysis/from.rs @@ -125,7 +125,7 @@ impl SpamFilterAnalyzeFrom for Server { } if (!env_from_empty && ctx.output.env_from_addr.address == from_addr.address) - || (!ctx.output.env_from_postmaster + || (ctx.output.env_from_postmaster && from_addr_is_valid && from_addr.domain_part.sld == ctx.output.ehlo_host.sld) { diff --git a/crates/spam-filter/src/analysis/headers.rs b/crates/spam-filter/src/analysis/headers.rs index 68f0173c..7085da78 100644 --- a/crates/spam-filter/src/analysis/headers.rs +++ b/crates/spam-filter/src/analysis/headers.rs @@ -28,7 +28,7 @@ impl SpamFilterAnalyzeHeaders for Server { for header in ctx.input.message.headers() { // Add header exists tag let hdr_name = header.name(); - let mut tag = String::with_capacity(hdr_name.len() + 5); + let mut tag: String = String::with_capacity(hdr_name.len() + 5); tag.push_str("X_HDR_"); for ch in hdr_name.chars() { if ch.is_ascii_alphanumeric() { @@ -37,6 +37,7 @@ impl SpamFilterAnalyzeHeaders for Server { tag.push('_'); } } + ctx.result.add_tag(tag); match &header.name { HeaderName::ContentType diff --git a/crates/spam-filter/src/analysis/html.rs b/crates/spam-filter/src/analysis/html.rs index f5e1ccdf..4329f2c2 100644 --- a/crates/spam-filter/src/analysis/html.rs +++ b/crates/spam-filter/src/analysis/html.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ - use std::future::Future; +use std::future::Future; use common::Server; use hyper::Uri; @@ -20,6 +20,7 @@ pub trait SpamFilterAnalyzeHtml: Sync + Send { ) -> impl Future + Send; } +#[derive(Debug)] struct Href { url_parsed: Option, host: Option, @@ -119,7 +120,7 @@ impl SpamFilterAnalyzeHtml for Server { if src.starts_with("data:") && src.contains(";base64,") { // Has Data URI encoding - ctx.result.add_tag("Has Data URI encoding"); + ctx.result.add_tag("HAS_DATA_URI"); } continue; } @@ -185,7 +186,7 @@ impl SpamFilterAnalyzeHtml for Server { has_rel_style = true; } } else if *attr == HREF - && value.to_ascii_lowercase().ends_with(".css") + && value.to_ascii_lowercase().contains(".css") { has_href_css = true; } @@ -217,7 +218,7 @@ impl SpamFilterAnalyzeHtml for Server { } _ => (), }, - HtmlToken::Text { text } if in_head > 0 => { + HtmlToken::Text { text } if in_head == 0 => { if let Some((href_url, href_host)) = last_href .as_ref() .and_then(|href| Some((href.url_parsed.as_ref()?, href.host.as_ref()?))) @@ -313,7 +314,7 @@ impl SpamFilterAnalyzeHtml for Server { } if (!has_link_to_img || html_text_chars >= 2048) - && html_img_words as f64 / (html_words as f64 + html_img_words as f64) > 0.5 + && (html_img_words as f64 / (html_words as f64 + html_img_words as f64) > 0.5) { // Message contains more images than text ctx.result.add_tag("HTML_TEXT_IMG_RATIO"); diff --git a/crates/spam-filter/src/analysis/init.rs b/crates/spam-filter/src/analysis/init.rs index fc5627a6..a398cf74 100644 --- a/crates/spam-filter/src/analysis/init.rs +++ b/crates/spam-filter/src/analysis/init.rs @@ -9,7 +9,7 @@ use mail_parser::{parsers::fields::thread::thread_name, HeaderName, PartType}; use nlp::tokenizers::types::{TokenType, TypesTokenizer}; use crate::{ - modules::html::{html_to_tokens, HtmlToken}, + modules::html::{html_to_tokens, HtmlToken, HEAD}, Email, Hostname, Recipient, SpamFilterContext, SpamFilterInput, SpamFilterOutput, SpamFilterResult, TextPart, }; @@ -92,7 +92,6 @@ impl SpamFilterInit for Server { .tokenize_emails(true) .map(|t| t.word) .collect::>(); - let subject = subject.to_lowercase(); // Tokenize and convert text parts let mut text_parts = Vec::new(); @@ -124,15 +123,25 @@ impl SpamFilterInit for Server { }) .sum(); let mut text_body = String::with_capacity(text_body_len); + let mut in_head = false; for token in &html_tokens { - if let HtmlToken::Text { text } = token { - if !text_body.is_empty() - && !text_body.ends_with(' ') - && text.starts_with(' ') - { - text_body.push(' '); + match token { + HtmlToken::StartTag { name: HEAD, .. } => { + in_head = true; } - text_body.push_str(text) + HtmlToken::EndTag { name: HEAD } => { + in_head = false; + } + HtmlToken::Text { text } if !in_head => { + if !text_body.is_empty() + && !text_body.ends_with(' ') + && text.starts_with(' ') + { + text_body.push(' '); + } + text_body.push_str(text) + } + _ => {} } } @@ -190,17 +199,17 @@ impl SpamFilterInit for Server { } text_parts.extend(text_parts_nested); - let subject_thread = thread_name(&subject).to_string(); + let subject_thread = thread_name(subject).to_string(); let env_from_addr = Email::new(input.env_from); SpamFilterContext { output: SpamFilterOutput { - ehlo_host: Hostname::new(input.ehlo_domain), - iprev_ptr: input - .iprev_result - .ptr - .as_ref() - .and_then(|ptr| ptr.first()) - .map(|ptr| ptr.strip_suffix('.').unwrap_or(ptr).to_lowercase()), + ehlo_host: Hostname::new(input.ehlo_domain.unwrap_or("unknown")), + iprev_ptr: input.iprev_result.and_then(|r| { + r.ptr + .as_ref() + .and_then(|ptr| ptr.first()) + .map(|ptr| ptr.strip_suffix('.').unwrap_or(ptr).to_lowercase()) + }), env_from_postmaster: env_from_addr.address.is_empty() || POSTMASTER_ADDRESSES.contains(&env_from_addr.local_part.as_str()), env_from_addr, @@ -215,9 +224,9 @@ impl SpamFilterInit for Server { }, reply_to, subject_thread_lc: subject_thread.trim().to_lowercase(), - subject_lc: subject.trim().to_lowercase(), subject_thread, - subject, + subject_lc: subject.trim().to_lowercase(), + subject: subject.to_string(), subject_tokens, recipients_to, recipients_cc, diff --git a/crates/spam-filter/src/analysis/ip.rs b/crates/spam-filter/src/analysis/ip.rs index 32a0a9a9..d880a596 100644 --- a/crates/spam-filter/src/analysis/ip.rs +++ b/crates/spam-filter/src/analysis/ip.rs @@ -44,18 +44,22 @@ impl SpamFilterAnalyzeIp for Server { (&header.name, &header.value) { if let Some(ip) = received.from_ip() { - ctx.output - .ips - .insert(ElementLocation::new(ip, Location::HeaderReceived)); + if !ip.is_loopback() && !self.is_ip_allowed(&ip) { + ctx.output + .ips + .insert(ElementLocation::new(ip, Location::HeaderReceived)); + } } for host in [&received.from, &received.helo, &received.by] .into_iter() .flatten() { if let Host::IpAddr(ip) = host { - ctx.output - .ips - .insert(ElementLocation::new(*ip, Location::HeaderReceived)); + if !ip.is_loopback() && !self.is_ip_allowed(ip) { + ctx.output + .ips + .insert(ElementLocation::new(*ip, Location::HeaderReceived)); + } } } } @@ -124,7 +128,7 @@ impl SpamFilterAnalyzeIp for Server { if let Some(tag) = is_dnsbl( self, dnsbl, - SpamFilterResolver::new(ctx, &IpResolver(ip.element), ip.location), + SpamFilterResolver::new(ctx, &IpResolver::new(ip.element), ip.location), ) .await { @@ -139,10 +143,12 @@ impl SpamFilterAnalyzeIp for Server { } // Reverse DNS validation - match &ctx.input.iprev_result.result { - IprevResult::TempError(_) => ctx.result.add_tag("RDNS_DNSFAIL"), - IprevResult::Fail(_) | IprevResult::PermError(_) => ctx.result.add_tag("RDNS_DNSFAIL"), - IprevResult::Pass | IprevResult::None => (), + if let Some(iprev) = ctx.input.iprev_result { + match &iprev.result { + IprevResult::TempError(_) => ctx.result.add_tag("RDNS_DNSFAIL"), + IprevResult::Fail(_) | IprevResult::PermError(_) => ctx.result.add_tag("RDNS_NONE"), + IprevResult::Pass | IprevResult::None => (), + } } } } diff --git a/crates/spam-filter/src/analysis/messageid.rs b/crates/spam-filter/src/analysis/messageid.rs index f9820be2..7a8ef78d 100644 --- a/crates/spam-filter/src/analysis/messageid.rs +++ b/crates/spam-filter/src/analysis/messageid.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ - use std::future::Future; +use std::future::Future; use common::Server; use mail_parser::HeaderName; @@ -20,20 +20,23 @@ pub trait SpamFilterAnalyzeMid: Sync + Send { impl SpamFilterAnalyzeMid for Server { async fn spam_filter_analyze_message_id(&self, ctx: &mut SpamFilterContext<'_>) { - let mid_raw = ctx - .input - .message - .header_raw(HeaderName::MessageId) - .unwrap_or_default() - .trim(); + let mut mid = ""; + let mut mid_raw = ""; - if !mid_raw.is_empty() { - let mid = ctx - .input - .message - .message_id() + for header in ctx.input.message.headers() { + if let (HeaderName::MessageId, value) = (&header.name, &header.value) { + mid = value.as_text().unwrap_or_default(); + mid_raw = std::str::from_utf8( + &ctx.input.message.raw_message()[header.offset_start..header.offset_end], + ) .unwrap_or_default() - .to_lowercase(); + .trim(); + break; + } + } + + if !mid.is_empty() { + let mid = mid.to_lowercase(); if let Some(mid_host) = mid.rsplit_once('@').map(|(_, host)| Hostname::new(host)) { if mid_host.ip.is_some() { if mid_host.fqdn.starts_with('[') { @@ -81,7 +84,7 @@ impl SpamFilterAnalyzeMid for Server { ctx.result.add_tag("INVALID_MSGID"); } - if !mid_raw.starts_with('<') || !mid_raw.ends_with('>') { + if !mid_raw.starts_with('<') || !mid_raw.contains('>') { ctx.result.add_tag("MID_MISSING_BRACKETS"); } } else { diff --git a/crates/spam-filter/src/analysis/mime.rs b/crates/spam-filter/src/analysis/mime.rs index 3f212b70..576d4ee2 100644 --- a/crates/spam-filter/src/analysis/mime.rs +++ b/crates/spam-filter/src/analysis/mime.rs @@ -83,7 +83,7 @@ impl SpamFilterAnalyzeMime for Server { if !has_mime_version && (has_ct || has_cte) { ctx.result.add_tag("MISSING_MIME_VERSION"); } - if has_ct && !is_plain_text && !has_cte && !had_cd { + if has_ct && !is_plain_text && !has_cte && !had_cd && !has_mime_version { // Only Content-Type header without other MIME headers ctx.result.add_tag("MIME_HEADER_CTYPE_ONLY"); } @@ -323,10 +323,7 @@ impl SpamFilterAnalyzeMime for Server { if !is_encrypted && !has_content_id && cd.map_or(true, |cd| { - cd.attribute("type") - .unwrap_or_default() - .to_ascii_lowercase() - != "attachment" + !cd.c_type.eq_ignore_ascii_case("attachment") && !cd.has_attribute("filename") }) { @@ -346,20 +343,16 @@ impl SpamFilterAnalyzeMime for Server { if is_attachment { // Has a MIME attachment ctx.result.add_tag("HAS_ATTACHMENT"); - match &part.body { - PartType::Binary(bytes) | PartType::InlineBinary(bytes) => { - if let Some(t) = infer::get(bytes.as_ref()) { - if t.mime_type() != ct_full { - // Known content-type - ctx.result.add_tag("MIME_GOOD"); - } else if ct_full != "application/octet-stream" { - // Known bad content-type - ctx.result.add_tag("MIME_BAD"); - } + if ct_full != "application/octet-stream" { + if let Some(t) = infer::get(part.contents()) { + if t.mime_type() == ct_full { + // Known content-type + ctx.result.add_tag("MIME_GOOD"); + } else { + // Known bad content-type + ctx.result.add_tag("MIME_BAD"); } } - - _ => (), } } diff --git a/crates/spam-filter/src/analysis/received.rs b/crates/spam-filter/src/analysis/received.rs index 71c8296f..8c7d8b91 100644 --- a/crates/spam-filter/src/analysis/received.rs +++ b/crates/spam-filter/src/analysis/received.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ - use std::future::Future; +use std::future::Future; use common::Server; use mail_parser::{HeaderName, Host}; @@ -41,10 +41,10 @@ impl SpamFilterAnalyzeReceived for Server { } if let Some(received) = header.value().as_received() { - let helo_domain = received.helo(); + let helo_domain = received.from().or_else(|| received.helo()); let ip_rev = received.from_iprev(); - if matches!(&helo_domain, Some(Host::Name(hostname)) if hostname.eq_ignore_ascii_case("localhost")) + if matches!(&helo_domain, Some(Host::Name(hostname)) if hostname.eq_ignore_ascii_case("user")) { // HELO domain is "user" ctx.result.add_tag("RCVD_HELO_USER"); @@ -68,7 +68,7 @@ impl SpamFilterAnalyzeReceived for Server { } } - if received.from_ip().is_some() { + if matches!(received.from, Some(Host::IpAddr(_))) { // Received from an IP address rather than a FQDN rcvd_from_ip += 1; } @@ -102,14 +102,14 @@ impl SpamFilterAnalyzeReceived for Server { } // Received from an authenticated user - if !ctx.input.authenticated_as.is_empty() { + if ctx.input.authenticated_as.is_some() { ctx.result.add_tag("RCVD_VIA_SMTP_AUTH"); } // Received with TLS checks - if rcvd_count > 0 && rcvd_count == tls_count && !ctx.input.tls_version.is_empty() { + if rcvd_count > 0 && rcvd_count == tls_count && ctx.input.is_tls { ctx.result.add_tag("RCVD_TLS_ALL"); - } else if !ctx.input.tls_version.is_empty() { + } else if ctx.input.is_tls { ctx.result.add_tag("RCVD_TLS_LAST"); } else { ctx.result.add_tag("RCVD_NO_TLS_LAST"); diff --git a/crates/spam-filter/src/analysis/recipient.rs b/crates/spam-filter/src/analysis/recipient.rs index 84fba142..5a285073 100644 --- a/crates/spam-filter/src/analysis/recipient.rs +++ b/crates/spam-filter/src/analysis/recipient.rs @@ -242,46 +242,50 @@ impl SpamFilterAnalyzeRecipient for Server { ctx.result.add_tag("RCPT_BOUNCEMOREONE"); } - for rcpts in [&ctx.output.recipients_to, &ctx.output.recipients_cc] { - let mut is_sorted = false; - if rcpts.len() >= 6 { - // Check if the recipients list is sorted - let mut sorted = true; - for i in 1..rcpts.len() { - if rcpts[i - 1].email.address > rcpts[i].email.address { - sorted = false; - break; - } + let rcpts = ctx + .output + .recipients_to + .iter() + .chain(ctx.output.recipients_cc.iter()) + .collect::>(); + + let mut is_sorted = false; + if rcpts.len() >= 6 { + // Check if the recipients list is sorted + let mut sorted = true; + for i in 1..rcpts.len() { + if rcpts[i - 1].email.address > rcpts[i].email.address { + sorted = false; + break; } - if sorted { - ctx.result.add_tag("SORTED_RECIPS"); - is_sorted = true; + } + if sorted { + ctx.result.add_tag("SORTED_RECIPS"); + is_sorted = true; + } + } + + if !is_sorted && rcpt_count >= 5 { + // Look for similar recipients + let mut hits = 0; + let mut combinations = 0; + for i in 0..rcpts.len() { + for j in i + 1..rcpts.len() { + let a = &rcpts[i].email; + let b = &rcpts[j].email; + + if levenshtein_distance(&a.local_part, &b.local_part) < 3 + || (a.domain_part.fqdn != b.domain_part.fqdn + && levenshtein_distance(&a.domain_part.fqdn, &b.domain_part.fqdn) < 4) + { + hits += 1; + } + combinations += 1; } } - if !is_sorted && rcpt_count >= 5 { - // Look for similar recipients - let mut hits = 0; - let mut combinations = 0; - for i in 0..rcpts.len() { - for j in i + 1..rcpts.len() { - let a = &rcpts[i].email; - let b = &rcpts[j].email; - - if levenshtein_distance(&a.local_part, &b.local_part) < 3 - || (a.domain_part.fqdn != b.domain_part.fqdn - && levenshtein_distance(&a.domain_part.fqdn, &b.domain_part.fqdn) - < 4) - { - hits += 1; - } - combinations += 1; - } - } - - if hits as f64 / combinations as f64 > 0.65 { - ctx.result.add_tag("SUSPICIOUS_RECIPS"); - } + if hits as f64 / combinations as f64 > 0.65 { + ctx.result.add_tag("SUSPICIOUS_RECIPS"); } } } diff --git a/crates/spam-filter/src/analysis/reputation.rs b/crates/spam-filter/src/analysis/reputation.rs index f35a5f84..3a37625b 100644 --- a/crates/spam-filter/src/analysis/reputation.rs +++ b/crates/spam-filter/src/analysis/reputation.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ - use std::{borrow::Cow, future::Future}; +use std::{borrow::Cow, future::Future}; use common::{ ip_to_bytes, Server, KV_REPUTATION_ASN, KV_REPUTATION_DOMAIN, KV_REPUTATION_FROM, @@ -25,6 +25,7 @@ pub trait SpamFilterAnalyzeReputation: Sync + Send { ) -> impl Future + Send; } +#[derive(Debug)] enum Type { Ip, From, @@ -48,7 +49,7 @@ impl SpamFilterAnalyzeReputation for Server { }; // Do not penalize forged domains - let is_dmarc_pass = matches!(ctx.input.dmarc_result, DmarcResult::Pass); + let is_dmarc_pass = matches!(ctx.input.dmarc_result, Some(DmarcResult::Pass)); let mut types = vec![ (Type::Ip, Cow::Owned(ip_to_bytes(&ctx.input.remote_ip))), @@ -84,29 +85,34 @@ impl SpamFilterAnalyzeReputation for Server { let mut reputation = 0.0; for (rep_type, key) in types { - let mut token = - match key_get::(self, ctx.input.span_id, key.clone()).await { - Ok(Some(token)) => token, - Ok(None) if !ctx.input.is_test => { - key_set( - self, - ctx.input.span_id, - KeyValue::with_prefix( - rep_type.prefix(), - key.as_ref(), - Reputation { - count: 1, - score: ctx.result.score, - } - .serialize(), - ) - .expires(config.expiry), + let mut token = match key_get::( + self, + ctx.input.span_id, + KeyValue::<()>::build_key(rep_type.prefix(), key.as_ref()), + ) + .await + { + Ok(Some(token)) => token, + Ok(None) if !ctx.input.is_test => { + key_set( + self, + ctx.input.span_id, + KeyValue::with_prefix( + rep_type.prefix(), + key.as_ref(), + Reputation { + count: 1, + score: ctx.result.score, + } + .serialize(), ) - .await; - continue; - } - _ => continue, - }; + .expires(config.expiry), + ) + .await; + continue; + } + Ok(None) | Err(_) => continue, + }; // Update reputation token.score = (token.count + 1) as f64 @@ -130,8 +136,10 @@ impl SpamFilterAnalyzeReputation for Server { Type::Domain => config.domain_weight, Type::Asn => config.asn_weight, }; + let c = println!("{rep_type:?} {weight}"); reputation += token.score / token.count as f64 * weight; + let c = println!("{rep_type:?} {weight}: {reputation}"); } // Adjust score diff --git a/crates/spam-filter/src/analysis/rules.rs b/crates/spam-filter/src/analysis/rules.rs index 5a5e6aab..47cc1a15 100644 --- a/crates/spam-filter/src/analysis/rules.rs +++ b/crates/spam-filter/src/analysis/rules.rs @@ -97,7 +97,11 @@ impl SpamFilterAnalyzeRules for Server { if let Some(tag) = self .eval_if::( &rule.rule, - &SpamFilterResolver::new(ctx, &IpResolver(ip.element), ip.location), + &SpamFilterResolver::new( + ctx, + &IpResolver::new(ip.element), + ip.location, + ), ctx.input.span_id, ) .await diff --git a/crates/spam-filter/src/analysis/score.rs b/crates/spam-filter/src/analysis/score.rs index 5fa7e154..5a061846 100644 --- a/crates/spam-filter/src/analysis/score.rs +++ b/crates/spam-filter/src/analysis/score.rs @@ -45,19 +45,24 @@ impl SpamFilterAnalyzeScore for Server { } } - // Sort by score - let mut header = String::with_capacity(header_len); - results.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap().then_with(|| a.0.cmp(b.0))); - header.push_str("X-Spam-Result: "); - for (idx, (tag, score)) in results.into_iter().enumerate() { - if idx > 0 { - header.push_str(",\r\n\t"); + // Write results header sorted by score + if let Some(header_name) = &self.core.spam.headers.result { + let mut header = String::with_capacity(header_name.len() + header_len + 2); + results.sort_by(|a, b| a.1.partial_cmp(&b.1).unwrap().then_with(|| a.0.cmp(b.0))); + header.push_str(header_name); + header.push_str(": "); + for (idx, (tag, score)) in results.into_iter().enumerate() { + if idx > 0 { + header.push_str(",\r\n\t"); + } + let _ = write!(&mut header, "{} ({:.2})", tag, score); } - let _ = write!(&mut header, "{} ({:.2})", tag, score); - } - header.push_str("\r\n"); + header.push_str("\r\n"); - SpamFilterAction::Allow(header) + SpamFilterAction::Allow(header) + } else { + SpamFilterAction::Allow(String::new()) + } } async fn spam_filter_finalize( @@ -89,16 +94,19 @@ impl SpamFilterAnalyzeScore for Server { { SpamFilterAction::Discard } else { - let _ = write!( - &mut header, - "X-Spam-Status: {}, score={:.2}\r\n", - if ctx.result.score >= self.core.spam.scores.spam_threshold { - "Yes" - } else { - "No" - }, - ctx.result.score - ); + if let Some(header_name) = &self.core.spam.headers.status { + let _ = write!( + &mut header, + "{}: {}, score={:.2}\r\n", + header_name, + if ctx.result.score >= self.core.spam.scores.spam_threshold { + "Yes" + } else { + "No" + }, + ctx.result.score + ); + } SpamFilterAction::Allow(header) } } diff --git a/crates/spam-filter/src/analysis/subject.rs b/crates/spam-filter/src/analysis/subject.rs index f4438159..a4eae2d6 100644 --- a/crates/spam-filter/src/analysis/subject.rs +++ b/crates/spam-filter/src/analysis/subject.rs @@ -32,6 +32,7 @@ impl SpamFilterAnalyzeSubject for Server { .raw_message() .get(header.offset_start..header.offset_end) .unwrap_or_default(); + break; } } @@ -46,7 +47,6 @@ impl SpamFilterAnalyzeSubject for Server { let mut lower_count = 0; let mut last_ch = ' '; - let mut last_ch_trimmed = ' '; let mut is_ascii = true; for ch in ctx.output.subject_thread.chars() { @@ -69,8 +69,6 @@ impl SpamFilterAnalyzeSubject for Server { } } } - - last_ch_trimmed = ch; } if !ch.is_ascii() { @@ -80,14 +78,12 @@ impl SpamFilterAnalyzeSubject for Server { last_ch = ch; } - if last_ch.is_whitespace() { - if last_ch_trimmed.is_whitespace() { - // Subject is empty - ctx.result.add_tag("EMPTY_SUBJECT"); - } else { - // Subject ends with whitespace - ctx.result.add_tag("SUBJECT_ENDS_SPACES"); - } + if ctx.output.subject_lc.is_empty() { + // Subject is empty + ctx.result.add_tag("EMPTY_SUBJECT"); + } else if ctx.output.subject.ends_with(' ') { + // Subject ends with whitespace + ctx.result.add_tag("SUBJECT_ENDS_SPACES"); } if ctx.output.subject_thread.len() >= 10 diff --git a/crates/spam-filter/src/analysis/url.rs b/crates/spam-filter/src/analysis/url.rs index 887f63e9..ede510ce 100644 --- a/crates/spam-filter/src/analysis/url.rs +++ b/crates/spam-filter/src/analysis/url.rs @@ -187,7 +187,7 @@ impl SpamFilterAnalyzeUrl for Server { if let Some(tag) = is_dnsbl( self, dnsbl, - SpamFilterResolver::new(ctx, &IpResolver(ip), url.location), + SpamFilterResolver::new(ctx, &IpResolver::new(ip), url.location), ) .await { @@ -201,46 +201,52 @@ impl SpamFilterAnalyzeUrl for Server { // Check for redirectors ctx.result.add_tag("REDIRECTOR_URL"); - let mut redirect_count = 0; - let mut url_redirect = Cow::Borrowed(url.element.as_str()); + if !ctx.result.has_tag("URL_REDIRECTOR_NESTED") { + let mut redirect_count = 1; + let mut url_redirect = Cow::Borrowed(url.element.as_str()); - while redirect_count <= 0 { - match http_get_header(url_redirect.as_ref(), LOCATION, Duration::from_secs(5)) + while redirect_count <= 3 { + match http_get_header( + url_redirect.as_ref(), + LOCATION, + Duration::from_secs(5), + ) .await - { - Ok(Some(location)) => { - if let Ok(location_parsed) = location.parse::() { - let host = - Hostname::new(location_parsed.host().unwrap_or_default()); - if self - .core - .spam - .lists - .url_redirectors - .contains(host.sld_or_default()) - { - url_redirect = Cow::Owned(location); - redirect_count += 1; - continue; - } else { - ctx.output.urls.insert(ElementLocation::new( - UrlParts::new(location.to_lowercase()) - .with_parts(location_parsed, host), - url.location, - )); + { + Ok(Some(location)) => { + if let Ok(location_parsed) = location.parse::() { + let host = + Hostname::new(location_parsed.host().unwrap_or_default()); + if self + .core + .spam + .lists + .url_redirectors + .contains(host.sld_or_default()) + { + url_redirect = Cow::Owned(location); + redirect_count += 1; + continue; + } else { + ctx.output.urls.insert(ElementLocation::new( + UrlParts::new(location.to_lowercase()) + .with_parts(location_parsed, host), + url.location, + )); + } } } + Ok(None) => {} + Err(err) => { + trc::error!(err.span_id(ctx.input.span_id)); + } } - Ok(None) => {} - Err(err) => { - trc::error!(err.span_id(ctx.input.span_id)); - } + break; } - break; - } - if redirect_count > 5 { - ctx.result.add_tag("URL_REDIRECTOR_NESTED"); + if redirect_count > 3 { + ctx.result.add_tag("URL_REDIRECTOR_NESTED"); + } } } @@ -270,10 +276,10 @@ impl SpamFilterAnalyzeUrl for Server { { ctx.result.add_tag("HOMOGRAPH_URL"); } + } - if !cured_host.is_single_script() { - ctx.result.add_tag("MIXED_CHARSET_URL"); - } + if !host.fqdn.is_single_script() { + ctx.result.add_tag("MIXED_CHARSET_URL"); } } @@ -333,11 +339,21 @@ impl SpamFilterAnalyzeUrl for Server { } } +#[allow(unreachable_code)] +#[allow(unused_variables)] async fn http_get_header( url: &str, header: hyper::header::HeaderName, timeout: Duration, ) -> trc::Result> { + #[cfg(feature = "test_mode")] + { + return if url.contains("redirect.") { + Ok(url.split_once("/?").unwrap().1.to_string().into()) + } else { + Ok(None) + }; + } reqwest::Client::builder() .user_agent("Mozilla/5.0 (X11; Linux i686; rv:109.0) Gecko/20100101 Firefox/118.0") .timeout(timeout) diff --git a/crates/spam-filter/src/lib.rs b/crates/spam-filter/src/lib.rs index 66546004..3818821f 100644 --- a/crates/spam-filter/src/lib.rs +++ b/crates/spam-filter/src/lib.rs @@ -24,29 +24,28 @@ pub struct SpamFilterInput<'x> { pub span_id: u64, // Sender authentication - pub arc_result: &'x ArcOutput<'x>, - pub spf_ehlo_result: &'x SpfOutput, - pub spf_mail_from_result: &'x SpfOutput, + pub arc_result: Option<&'x ArcOutput<'x>>, + pub spf_ehlo_result: Option<&'x SpfOutput>, + pub spf_mail_from_result: Option<&'x SpfOutput>, pub dkim_result: &'x [DkimOutput<'x>], - pub dmarc_result: &'x DmarcResult, - pub dmarc_policy: &'x Policy, - pub iprev_result: &'x IprevOutput, + pub dmarc_result: Option<&'x DmarcResult>, + pub dmarc_policy: Option<&'x Policy>, + pub iprev_result: Option<&'x IprevOutput>, // Session details pub remote_ip: IpAddr, - pub ehlo_domain: &'x str, - pub authenticated_as: &'x str, + pub ehlo_domain: Option<&'x str>, + pub authenticated_as: Option<&'x str>, pub asn: Option, pub country: Option<&'x str>, // TLS - pub tls_version: &'x str, - pub tls_cipher: &'x str, + pub is_tls: bool, // Envelope pub env_from: &'x str, pub env_from_flags: u64, - pub env_rcpt_to: &'x [&'x str], + pub env_rcpt_to: Vec<&'x str>, pub account_id: Option, pub is_test: bool, diff --git a/crates/spam-filter/src/modules/bayes.rs b/crates/spam-filter/src/modules/bayes.rs index 823d2351..0ad3d108 100644 --- a/crates/spam-filter/src/modules/bayes.rs +++ b/crates/spam-filter/src/modules/bayes.rs @@ -136,11 +136,11 @@ pub(crate) async fn bayes_train( pub(crate) async fn bayes_classify( server: &Server, ctx: &SpamFilterContext<'_>, -) -> trc::Result { +) -> trc::Result> { let classifier = if let Some(config) = &server.core.spam.bayes { &config.classifier } else { - return Ok(0.0); + return Ok(None); }; // Obtain training counts @@ -169,7 +169,7 @@ pub(crate) async fn bayes_classify( trc::Value::from(classifier.min_learns) ], ); - return Ok(0.0); + return Ok(None); } // Classify the text @@ -265,10 +265,10 @@ pub(crate) async fn bayes_classify( trc::Value::from(ham_learns), trc::Value::from(classifier.min_learns) ], - Result = result.unwrap_or_default() + Result = result.map(trc::Value::from).unwrap_or_default() ); - Ok(result.unwrap_or_default()) + Ok(result) } pub(crate) async fn bayes_is_balanced( @@ -357,7 +357,7 @@ const P_REMOTE_IP: u8 = 4; impl SpamFilterContext<'_> { pub fn spam_tokens(&self) -> HashSet> { let mut tokens = HashSet::new(); - if matches!(self.input.dmarc_result, DmarcResult::Pass) { + if matches!(self.input.dmarc_result, Some(DmarcResult::Pass)) { for addr in [&self.output.env_from_addr, &self.output.from.email] { if !addr.address.is_empty() { tokens.insert(add_prefix(P_FROM_EMAIL, addr.address.as_bytes())); diff --git a/crates/spam-filter/src/modules/dnsbl.rs b/crates/spam-filter/src/modules/dnsbl.rs index 1c83fd92..95c1ebf7 100644 --- a/crates/spam-filter/src/modules/dnsbl.rs +++ b/crates/spam-filter/src/modules/dnsbl.rs @@ -23,6 +23,33 @@ pub(crate) async fn is_dnsbl( let zone = server .eval_if::(&config.zone, &resolver, resolver.ctx.input.span_id) .await?; + + #[cfg(feature = "test_mode")] + { + if zone.contains(".11.20.") { + let parts = zone.split('.').collect::>(); + + return if config.tags.if_then.iter().any(|i| i.expr.items.len() == 3) && parts[0] != "2" + { + None + } else { + server + .eval_if( + &config.tags, + &SpamFilterResolver::new( + resolver.ctx, + &IpResolver::new( + format!("127.0.{}.{}", parts[1], parts[0]).parse().unwrap(), + ), + resolver.location, + ), + resolver.ctx.input.span_id, + ) + .await + }; + } + } + let todo = "use proper event error"; match server.core.smtp.resolvers.dns.ipv4_lookup(&zone).await { @@ -41,7 +68,7 @@ pub(crate) async fn is_dnsbl( &config.tags, &SpamFilterResolver::new( resolver.ctx, - &IpResolver( + &IpResolver::new( result .iter() .copied() diff --git a/crates/spam-filter/src/modules/expression.rs b/crates/spam-filter/src/modules/expression.rs index 3ed5a1ed..b7b23391 100644 --- a/crates/spam-filter/src/modules/expression.rs +++ b/crates/spam-filter/src/modules/expression.rs @@ -44,11 +44,10 @@ impl ResolveVariable for SpamFilterResolver<'_, T> { .unwrap_or_default() .into(), V_SPAM_EHLO_DOMAIN => self.ctx.output.ehlo_host.fqdn.as_str().into(), - V_SPAM_AUTH_AS => self.ctx.input.authenticated_as.into(), + V_SPAM_AUTH_AS => self.ctx.input.authenticated_as.unwrap_or_default().into(), V_SPAM_ASN => self.ctx.input.asn.unwrap_or_default().into(), V_SPAM_COUNTRY => self.ctx.input.country.unwrap_or_default().into(), - V_SPAM_TLS_VERSION => self.ctx.input.tls_version.into(), - V_SPAM_TLS_CIPHER => self.ctx.input.tls_cipher.into(), + V_SPAM_IS_TLS => self.ctx.input.is_tls.into(), V_SPAM_ENV_FROM => self.ctx.output.env_from_addr.address.as_str().into(), V_SPAM_ENV_FROM_LOCAL => self.ctx.output.env_from_addr.local_part.as_str().into(), V_SPAM_ENV_FROM_DOMAIN => self @@ -442,14 +441,36 @@ impl ResolveVariable for StringListResolver<'_> { } } -pub struct IpResolver(pub IpAddr); +pub struct IpResolver { + ip: IpAddr, + ip_string: String, + reverse: String, + octets: Variable<'static>, +} impl ResolveVariable for IpResolver { fn resolve_variable(&self, variable: u32) -> Variable<'_> { match variable { - V_IP => Variable::String(self.0.to_string().into()), - V_IP_REVERSE => Variable::String(self.0.to_reverse_name().into()), - V_IP_OCTETS => Variable::Array(match self.0 { + V_IP => Variable::String(self.ip_string.as_str().into()), + V_IP_REVERSE => Variable::String(self.reverse.as_str().into()), + V_IP_OCTETS => self.octets.clone(), + V_IP_IS_V4 => Variable::Integer(self.ip.is_ipv4() as _), + V_IP_IS_V6 => Variable::Integer(self.ip.is_ipv6() as _), + _ => Variable::Integer(0), + } + } + + fn resolve_global(&self, _: &str) -> Variable<'_> { + Variable::Integer(0) + } +} + +impl IpResolver { + pub fn new(ip: IpAddr) -> Self { + Self { + ip_string: ip.to_string(), + reverse: ip.to_reverse_name(), + octets: Variable::Array(match ip { IpAddr::V4(ipv4_addr) => ipv4_addr .octets() .iter() @@ -461,13 +482,7 @@ impl ResolveVariable for IpResolver { .map(|o| Variable::Integer(*o as _)) .collect(), }), - V_IP_IS_V4 => Variable::Integer(self.0.is_ipv4() as _), - V_IP_IS_V6 => Variable::Integer(self.0.is_ipv6() as _), - _ => Variable::Integer(0), + ip, } } - - fn resolve_global(&self, _: &str) -> Variable<'_> { - Variable::Integer(0) - } } diff --git a/crates/spam-filter/src/modules/mod.rs b/crates/spam-filter/src/modules/mod.rs index 994fab9f..e7e7d6a7 100644 --- a/crates/spam-filter/src/modules/mod.rs +++ b/crates/spam-filter/src/modules/mod.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ - use common::Server; +use common::Server; use store::{dispatch::lookup::KeyValue, Deserialize, Value}; pub mod bayes; diff --git a/crates/spam-filter/src/modules/remote_list.rs b/crates/spam-filter/src/modules/remote_list.rs index ae0c0edc..a4dc5c47 100644 --- a/crates/spam-filter/src/modules/remote_list.rs +++ b/crates/spam-filter/src/modules/remote_list.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ - use std::{ +use std::{ collections::HashSet, io::{BufRead, BufReader}, time::Instant, @@ -19,12 +19,20 @@ use common::{ }; use mail_auth::flate2; +#[allow(unused_variables)] +#[allow(unreachable_code)] pub async fn is_in_remote_list( server: &Server, config: &RemoteListConfig, item: &str, span_id: u64, ) -> bool { + #[cfg(feature = "test_mode")] + { + return (config.url.contains("open") && item.contains("open")) + || (config.url.contains("tank") && item.contains("tank")); + } + match is_in_remote_list_(server, config, item, span_id).await { Ok(result) => result, Err(err) => { diff --git a/tests/resources/smtp/antispam/bounce.test b/tests/resources/smtp/antispam/bounce.test index 784a731a..76886e67 100644 --- a/tests/resources/smtp/antispam/bounce.test +++ b/tests/resources/smtp/antispam/bounce.test @@ -1,11 +1,12 @@ -expect SUBJ_BOUNCE_WORDS +expect SUBJ_BOUNCE_WORDS SINGLE_SHORT_PART Subject: Delivery Status Notification (Failure) Test -expect BOUNCE +expect BOUNCE SINGLE_SHORT_PART IS_DSN +MIME-Version: 1.0 Content-Type: multipart/report; report-type="delivery-status"; boundary="176e677bbd667276_87a2ed9cf1f4ecb_a49e592dab77f72e" @@ -18,8 +19,9 @@ Your message could not be delivered. --176e677bbd667276_87a2ed9cf1f4ecb_a49e592dab77f72e-- envelope_from spammer@domain.com -expect +expect SINGLE_SHORT_PART IS_DSN +MIME-Version: 1.0 Content-Type: multipart/report; report-type="delivery-status"; boundary="176e677bbd667276_87a2ed9cf1f4ecb_a49e592dab77f72e" @@ -31,7 +33,7 @@ Your message could not be delivered. --176e677bbd667276_87a2ed9cf1f4ecb_a49e592dab77f72e-- -expect BOUNCE +expect BOUNCE SINGLE_SHORT_PART From: MDaemon X-MDDSN-Message: True @@ -40,7 +42,7 @@ Subject: Something went wrong Your message could not be delivered. -expect BOUNCE SUBJ_BOUNCE_WORDS +expect BOUNCE SUBJ_BOUNCE_WORDS SINGLE_SHORT_PART From: Automated Subject: Delivery failure @@ -48,8 +50,9 @@ Subject: Delivery failure Your message could not be delivered. -expect BOUNCE +expect BOUNCE HAS_ATTACHMENT HAS_MESSAGE_PARTS +MIME-Version: 1.0 From: Automated Subject: Something unexpected happened Content-Type: multipart/mixed; diff --git a/tests/resources/smtp/antispam/combined.test b/tests/resources/smtp/antispam/combined.test index 4db30df6..a32bbb59 100644 --- a/tests/resources/smtp/antispam/combined.test +++ b/tests/resources/smtp/antispam/combined.test @@ -6,8 +6,7 @@ spf.result none spf_ehlo.result none dmarc.result none remote_ip 195.210.29.48 -expect_header X-Spam-Status Yes, score=8. -expect_header X-Spam-Result +expect_score 8 expect rdns_none auth_na dmarc_na helo_nores_a_or_mx once_received mid_rhs_match_from spf_na has_data_uri arc_na subject_has_exclaim subject_ends_exclaim mime_html_only html_short_link_img_1 to_dn_none rcpt_count_one to_match_envrcpt_all fromhost_nores_a_or_mx rcvd_count_zero from_eq_envfrom dkim_na rcvd_no_tls_last from_has_dn date_in_past From: Client Services @@ -50,8 +49,7 @@ dkim.domains tenthrevolution.com dmarc.result pass remote_ip 185.58.86.181 tls.version TLSv1.3 -expect_header X-Spam-Status No, score=3. -expect_header X-Spam-Result +expect_score 3 expect from_eq_envfrom from_has_dn helo_nores_a_or_mx forged_rcvd_trail date_in_past arc_na uri_count_odd dkim_signed has_attachment spf_allow rcvd_tls_last rcpt_count_one mime_good subject_ends_spaces fromhost_nores_a_or_mx to_dn_eq_addr_all dkim_allow dmarc_policy_allow rcvd_count_three to_match_envrcpt_all DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=tenthrevolution.com; @@ -625,8 +623,7 @@ dmarc.result fail dmarc.policy reject remote_ip 51.89.165.39 tls.version TLS1_2 -expect_header X-Spam-Status Yes, score=13. -expect_header X-Spam-Result +expect_score 13 expect has_replyto violated_direct_spf replyto_addr_eq_from uri_count_odd once_received r_parts_differ mid_rhs_match_from fromhost_nores_a_or_mx from_has_dn dkim_allow date_in_past to_match_envrcpt_all html_short_link_img_1 rcpt_count_one arc_na helo_nores_a_or_mx spf_softfail rcvd_tls_last rcvd_count_zero replyto_dom_eq_from_dom to_dn_none has_list_unsub dkim_signed rdns_none from_eq_envfrom dmarc_policy_reject DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; s=sectionalism; d=grupokonecta.net; diff --git a/tests/resources/smtp/antispam/from.test b/tests/resources/smtp/antispam/from.test index d8b54be5..e63bb522 100644 --- a/tests/resources/smtp/antispam/from.test +++ b/tests/resources/smtp/antispam/from.test @@ -20,7 +20,7 @@ From: test Test envelope_from www-data@domain.org -expect FROM_SERVICE_ACCT ENVFROM_SERVICE_ACCT FROM_HAS_DN FROM_EQ_ENVFROM +expect FROM_SERVICE_ACCT FROM_HAS_DN FROM_EQ_ENVFROM From: "WWW DATA" @@ -150,12 +150,11 @@ Disposition-Notification-To: Test -envelope_from hello@domain.org +envelope_from anonymous@domain.org expect FROM_SERVICE_ACCT WWW_DOT_DOMAIN FROM_EQ_ENVFROM FROM_HAS_DN -From: "Hello" -Sender: -Reply-to: +From: "Hello" +Reply-to: Test diff --git a/tests/resources/smtp/antispam/headers.test b/tests/resources/smtp/antispam/headers.test index 54eaa816..3aa7bc6a 100644 --- a/tests/resources/smtp/antispam/headers.test +++ b/tests/resources/smtp/antispam/headers.test @@ -16,13 +16,11 @@ To: test@test.com Test -expect XM_CASE HAS_LIST_UNSUB HAS_XOIP HAS_ORG_HEADER PRECEDENCE_BULK MULTIPLE_UNIQUE_HEADERS +expect XM_CASE HAS_LIST_UNSUB PRECEDENCE_BULK MULTIPLE_UNIQUE_HEADERS X-mailer: my mailer 1 -X-Originating-IP: 127.0.0.1 List-Unsubscribe: Precedence: bulk -Organization: my org Subject: first subject Subject: second subject @@ -38,7 +36,7 @@ Subject: test Test -expect X_PHP_EVAL HAS_X_POS HAS_X_SOURCE HAS_X_PHP_SCRIPT PHP_XPS_PATTERN HIDDEN_SOURCE_OBJ HAS_X_GMSV HAS_X_ANTIABUSE HAS_X_AS HAS_XAW +expect X_PHP_EVAL HIDDEN_SOURCE_OBJ HAS_X_GMSV HAS_X_AS X-PHP-Script: sendmail.php X-PHP-Originating-Script: eval() diff --git a/tests/resources/smtp/antispam/html.test b/tests/resources/smtp/antispam/html.test index 616f4ab6..d4504b1c 100644 --- a/tests/resources/smtp/antispam/html.test +++ b/tests/resources/smtp/antispam/html.test @@ -284,7 +284,7 @@ Content-Transfer-Encoding: 8bit -

some text

+

some text and a lovely explanation to avoid the text to image ratio tag

Red dot Click me for a hello message diff --git a/tests/resources/smtp/antispam/messageid.test b/tests/resources/smtp/antispam/messageid.test index 253837c4..037347b7 100644 --- a/tests/resources/smtp/antispam/messageid.test +++ b/tests/resources/smtp/antispam/messageid.test @@ -45,6 +45,12 @@ expect INVALID_MSGID Message-ID: (hello world) +Test + +expect MID_RHS_TOO_LONG + +Message-ID: + Test expect MID_MISSING_BRACKETS diff --git a/tests/resources/smtp/antispam/mime.test b/tests/resources/smtp/antispam/mime.test index 71a63952..0b6bfe78 100644 --- a/tests/resources/smtp/antispam/mime.test +++ b/tests/resources/smtp/antispam/mime.test @@ -149,7 +149,7 @@ Content-Type: multipart/alternative; Content-Type: text/plain; charset="utf-8" Content-Transfer-Encoding: 7bit -Lorem ipsum dolor sit amet, Rcnsectetur Radipiscing elit, Rsed do Reiusmod tempor +Lorem ipsum dolor sit Ramet, Rcnsectetur Radipiscing elit, Rsed do Reiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore diff --git a/tests/resources/smtp/antispam/rbl.test b/tests/resources/smtp/antispam/rbl.test index 0d638537..52799ee1 100644 --- a/tests/resources/smtp/antispam/rbl.test +++ b/tests/resources/smtp/antispam/rbl.test @@ -1,12 +1,12 @@ -remote_ip 192.168.0.1 -expect RCVD_IN_DNSWL_LOW RBL_SPAMHAUS +remote_ip 20.11.0.1 +expect RCVD_IN_DNSWL_LOW Subject: test test -remote_ip 192.168.0.2 +remote_ip 20.11.0.2 expect RBL_SENDERSCORE RBL_NIXSPAM RBL_SEM RBL_SPAMHAUS_SBL RBL_BARRACUDA RBL_BLOCKLISTDE RBL_VIRUSFREE_BOTNET RBL_SPAMCOP RCVD_IN_DNSWL_MED Subject: test @@ -14,14 +14,14 @@ Subject: test test -remote_ip 192.168.0.14 -expect RWL_MAILSPIKE_NEUTRAL RECEIVED_SPAMHAUS_SBL RBL_SPAMHAUS RECEIVED_SPAMHAUS_XBL RECEIVED_BLOCKLISTDE RCVD_IN_DNSWL_MED +remote_ip 20.11.0.14 +expect RWL_MAILSPIKE_NEUTRAL RECEIVED_SPAMHAUS_SBL RECEIVED_SPAMHAUS_XBL RECEIVED_BLOCKLISTDE RCVD_IN_DNSWL_MED -Received: from Agni (localhost [192.168.0.5]) (TLS: TLSv1/SSLv3, 168bits,DES-CBC3-SHA) by agni.forevermore.net +Received: from Agni (localhost [20.11.0.5]) (TLS: TLSv1/SSLv3, 168bits,DES-CBC3-SHA) by agni.forevermore.net with esmtp; Mon, 28 Oct 2002 14:48:52 -0800 -Received: from [192.168.0.14] (79.sub-174-252-72.myvzw.com [192.168.0.8]) by mx.google.com +Received: from [20.11.0.14] (79.sub-174-252-72.myvzw.com [20.11.0.8]) by mx.google.com with ESMTPS id m16sm345129qck.28.2011.06.15.07.42.02 (version=TLSv1/SSLv3 cipher=OTHER); Wed, 15 Jun 2011 07:42:08 -0700 (PDT) -Received: from user (192.168.0.2) by DB6PR07MB3384.eurprd07.prod.outlook.com ([192.168.0.2]) +Received: from user (20.11.0.2) by DB6PR07MB3384.eurprd07.prod.outlook.com ([20.11.0.2]) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.1143.11; Thu, 13 Sep 2018 14:47:44 +0000 Subject: test @@ -39,6 +39,7 @@ And my website is https://sem-fresh15.com/offers.html Try cheating with a trusted domain user@dkimtrusted.org +dkim.result pass dkim.domains dkimtrusted.org expect DWL_DNSWL_HI @@ -57,7 +58,7 @@ Subject: test test -expect SURBL_HASHBL_ABUSE SURBL_HASHBL_MALWARE SURBL_HASHBL_PHISH +expect SURBL_HASHBL_ABUSE SURBL_HASHBL_MALWARE SURBL_HASHBL_PHISH URL_ONLY From: spammer@spamcorp.net Reply-To: User diff --git a/tests/resources/smtp/antispam/received.test b/tests/resources/smtp/antispam/received.test index 8f430440..183d14c9 100644 --- a/tests/resources/smtp/antispam/received.test +++ b/tests/resources/smtp/antispam/received.test @@ -38,6 +38,8 @@ Received: from Agni (localhost [::ffff:127.0.0.1]) (TLS: TLSv1/SSLv3, 168bits,DE with esmtp; Mon, 28 Oct 2002 14:48:52 -0800 Received: from [10.231.252.223] (79.sub-174-252-72.myvzw.com [174.252.72.79]) by mx.google.com with ESMTPS id m16sm345129qck.28.2011.06.15.07.42.02 (version=TLSv1/SSLv3 cipher=OTHER); Wed, 15 Jun 2011 07:42:08 -0700 (PDT) +Received: from other.myvzw.com (79.sub-174-252-72.myvzw.com [174.252.72.79]) by mx.google.com + with ESMTPS id m16sm345129qck.28.2011.06.15.07.42.02 (version=TLSv1/SSLv3 cipher=OTHER); Wed, 15 Jun 2011 07:42:08 -0700 (PDT) Received: from user (10.175.233.33) by DB6PR07MB3384.eurprd07.prod.outlook.com (10.175.234.11) with Microsoft SMTP Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.1143.11; Thu, 13 Sep 2018 14:47:44 +0000 Received: from [94.198.96.74] (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange ECDHE (P-256) server-signature RSA-PSS (2048 bits) server-digest SHA256) (No client certificate requested) @@ -55,3 +57,10 @@ X-Mailer: MUA Subject: test test + +expect RCVD_UNPARSABLE RCVD_NO_TLS_LAST RCVD_COUNT_ONE + +To: user@domain.com +Received: invalid + +test diff --git a/tests/resources/smtp/antispam/recipient.test b/tests/resources/smtp/antispam/recipient.test index 434e4b4e..e7d43b40 100644 --- a/tests/resources/smtp/antispam/recipient.test +++ b/tests/resources/smtp/antispam/recipient.test @@ -24,7 +24,7 @@ Cc: other@user.org Test -expect RCPT_ADDR_IN_SUBJECT TO_DN_NONE RCPT_COUNT_ONE +expect RCPT_IN_SUBJECT TO_DN_NONE RCPT_COUNT_ONE To: hello@world.com Subject: Special offer for HELLO@world.com @@ -117,7 +117,7 @@ Cc: otheruser@guerrillamail.com Test envelope_from test@test.com -expect FREEMAIL_CC DISPOSABLE_TO RCPT_COUNT_THREE TO_DN_NONE +expect FREEMAIL_CC DISPOSABLE_TO DISPOSABLE_CC RCPT_COUNT_THREE TO_DN_NONE To: otheruser@guerrillamail.com Cc: user@gmail.com diff --git a/tests/resources/smtp/antispam/replyto.test b/tests/resources/smtp/antispam/replyto.test index 5980fe97..79100316 100644 --- a/tests/resources/smtp/antispam/replyto.test +++ b/tests/resources/smtp/antispam/replyto.test @@ -72,21 +72,21 @@ Test expect REPLYTO_EXCESS_QP REPLYTO_DOM_EQ_FROM_DOM HAS_REPLYTO From: hello@domain.org -Reply-to: =?iso-8859-1?Q?Die_Hasen_und_die_Froesche?= +Reply-to: =?iso-8859-1?Q?Die_Hasen_und_die_Froesche?= Test expect REPLYTO_EXCESS_BASE64 REPLYTO_DOM_EQ_FROM_DOM HAS_REPLYTO From: hello@domain.org -Reply-to: "=?iso-8859-1?B?RGllIEhhc2VuIHVuIGRpZSBGcm9lc2NoZQ==?=" +Reply-to: "=?iso-8859-1?B?RGllIEhhc2VuIHVuIGRpZSBGcm9lc2NoZQ==?=" Test expect REPLYTO_EMAIL_HAS_TITLE REPLYTO_DOM_EQ_FROM_DOM HAS_REPLYTO From: hello@domain.org -Reply-to: "Mr. Hello" +Reply-to: "Mr. Hello" Test diff --git a/tests/resources/smtp/antispam/subject.test b/tests/resources/smtp/antispam/subject.test index 945015e0..a98d973a 100644 --- a/tests/resources/smtp/antispam/subject.test +++ b/tests/resources/smtp/antispam/subject.test @@ -27,7 +27,7 @@ Subject: thís líné shóúld bé éncódéd Test -param.body 8bitmime +param.8bitmime 1 expect Subject: thís líné shóúld bé éncódéd @@ -103,15 +103,36 @@ Subject: =?iso-8859-1?Q?Die_Hasen_und_die_Fr=F6sche_?= Test param.smtputf8 1 -expect SUBJECT_HAS_CURRENCY SUBJECT_ENDS_EXCLAIM SUBJECT_HAS_EXCLAIM +expect SUBJECT_HAS_CURRENCY SUBJECT_ENDS_EXCLAIM Subject: You have won £200! Test param.smtputf8 1 -expect SUBJECT_HAS_CURRENCY SUBJECT_HAS_QUESTION SUBJECT_ENDS_QUESTION +expect SUBJECT_HAS_CURRENCY SUBJECT_ENDS_QUESTION Subject: Have you won $200? Test + +expect RCPT_IN_SUBJECT + +To: hello@world.org +Subject: Great offers for hello@world.org + +Test + +expect RCPT_DOMAIN_IN_SUBJECT + +To: hello@world.org +Subject: Great offers for world.org + +Test + +expect + +To: hello@world.org +Subject: Question about other@domain.net + +Test diff --git a/tests/resources/smtp/antispam/url.test b/tests/resources/smtp/antispam/url.test index 63cc4c97..7256e9fd 100644 --- a/tests/resources/smtp/antispam/url.test +++ b/tests/resources/smtp/antispam/url.test @@ -46,7 +46,7 @@ Subject: redirect to omograph login to https://www.redirect.com/?https://xn--twiter-507b.com -expect HAS_ONION_URI +expect HAS_ONION_URI HAS_ANON_DOMAIN Subject: url in title darkweb.onion/login @@ -54,26 +54,36 @@ test expect HAS_IPFS_GATEWAY_URL HAS_WP_URI URI_HIDDEN_PATH +Content-Type: text/html; charset="utf-8" Subject: html test - + -expect HAS_GUC_PROXY_URI HAS_GOOGLE_FIREBASE_URL HAS_GOOGLE_REDIR +expect HAS_GUC_PROXY_URI HAS_GOOGLE_FIREBASE_URL HAS_GOOGLE_REDIR HAS_ANON_DOMAIN URL_ONLY +Content-Type: text/html; charset="utf-8" Subject: mixed urls googleusercontent.com/proxy/url google.com/url?otherurl.org -expect WP_COMPROMISED HAS_WP_URI +expect WP_COMPROMISED Subject: plain test http://url.com/Well-known/../assetlinks.json http://wp.com/WP-content/content.pdf + +expect HAS_WP_URI + +Subject: plain test + +http://url.com/Well-known/../assetlinks.json +http://wp.com/WP-other/content.pdf + expect PHISHED_OPENPHISH PHISHED_PHISHTANK diff --git a/tests/src/smtp/inbound/antispam.rs b/tests/src/smtp/inbound/antispam.rs index 9e4beb2a..a5d9406a 100644 --- a/tests/src/smtp/inbound/antispam.rs +++ b/tests/src/smtp/inbound/antispam.rs @@ -1,83 +1,82 @@ use std::{ borrow::Cow, - collections::HashMap, fs, path::PathBuf, sync::Arc, time::{Duration, Instant}, }; -use ahash::AHashMap; +use ahash::AHashSet; use common::{ auth::AccessToken, enterprise::llm::{ AiApiConfig, ChatCompletionChoice, ChatCompletionRequest, ChatCompletionResponse, Message, }, - scripts::{ - functions::html::{get_attribute, html_attr_tokens, html_img_area, html_to_tokens}, - ScriptModification, - }, Core, }; use hyper::Method; use jmap::api::{http::ToHttpResponse, JsonResponse}; -use mail_auth::{dmarc::Policy, DkimResult, DmarcResult, IprevResult, SpfResult, MX}; +use mail_auth::{ + dkim::Signature, dmarc::Policy, ArcOutput, DkimOutput, DkimResult, DmarcResult, IprevOutput, + IprevResult, SpfOutput, SpfResult, MX, +}; +use mail_parser::MessageParser; use sieve::runtime::Variable; -use smtp::{ - core::{Session, SessionAddress}, - inbound::AuthResult, - scripts::{event_loop::RunScript, ScriptResult}, +use smtp::core::{Session, SessionAddress}; +use smtp_proto::{MAIL_BODY_8BITMIME, MAIL_SMTPUTF8}; +use spam_filter::{ + analysis::{ + bayes::SpamFilterAnalyzeBayes, date::SpamFilterAnalyzeDate, dmarc::SpamFilterAnalyzeDmarc, + domain::SpamFilterAnalyzeDomain, ehlo::SpamFilterAnalyzeEhlo, from::SpamFilterAnalyzeFrom, + headers::SpamFilterAnalyzeHeaders, html::SpamFilterAnalyzeHtml, init::SpamFilterInit, + ip::SpamFilterAnalyzeIp, llm::SpamFilterAnalyzeLlm, messageid::SpamFilterAnalyzeMid, + mime::SpamFilterAnalyzeMime, pyzor::SpamFilterAnalyzePyzor, + received::SpamFilterAnalyzeReceived, recipient::SpamFilterAnalyzeRecipient, + replyto::SpamFilterAnalyzeReplyTo, reputation::SpamFilterAnalyzeReputation, + rules::SpamFilterAnalyzeRules, score::SpamFilterAnalyzeScore, + subject::SpamFilterAnalyzeSubject, trusted_reply::SpamFilterAnalyzeTrustedReply, + url::SpamFilterAnalyzeUrl, + }, + modules::html::{html_to_tokens, HtmlToken}, }; use store::Stores; use utils::config::Config; use crate::{ http_server::{spawn_mock_http_server, HttpMessage}, - jmap::enterprise::EnterpriseCore, + //jmap::enterprise::EnterpriseCore, smtp::{session::TestSession, TempDir, TestSMTP}, }; const CONFIG: &str = r#" -[spam.header] -is-spam = "X-Spam-Status: Yes" +[spam-filter.bayes.classify] +balance = "0.0" +learns = 10 -[lookup.spam-config] -add-spam = true -add-spam-result = true -learn-enable = true -#learn-balance = "0.9" -learn-balance = "0.0" -learn-ham-replies = true -learn-ham-threshold = "-0.5" -learn-spam-threshold = "6.0" -threshold-spam = "5.0" -threshold-discard = 0 -threshold-reject = 0 -directory = "" -lookup = "" -llm-model = "dummy" -llm-prompt = "You are an AI assistant specialized in analyzing email content to detect unsolicited, commercial, or harmful messages. Format your response as follows, separated by commas: Category,Confidence,Explanation -Here's the email to analyze, please provide your analysis based on the above instructions, ensuring your response is in the specified comma-separated format:" -add-llm-result = false +[spam-filter.bayes.auto-learn.threshold] +ham = "-0.5" +spam = "6.0" + +[spam-filter.score] +spam = "5.0" + +[spam-filter.llm] +enable = true +model = "dummy" +prompt = "You are an AI assistant specialized in analyzing email content to detect unsolicited, commercial, or harmful messages. Format your response as follows, separated by commas: Category,Confidence,Explanation +Here's the email to analyze, please provide your analysis based on the above instructions, ensuring your response is in the specified comma-separated format." +separator = "," +categories = ["Unsolicited", "Commercial", "Harmful", "Legitimate"] +confidence = ["High", "Medium", "Low"] + +[spam-filter.llm.index] +category = 0 +confidence = 1 +explanation = 2 [session.rcpt] relay = true -[sieve.trusted] -from-name = "'Sieve Daemon'" -from-addr = "'sieve@foobar.org'" -return-path = "" -hostname = "mx.foobar.org" -no-capability-check = true - -[sieve.trusted.limits] -redirects = 3 -out-messages = 5 -received-headers = 50 -cpu = 500000 -nested-includes = 5 -duplicate-expiry = "7d" - [storage] data = "spamdb" lookup = "spamdb" @@ -90,7 +89,7 @@ type = "internal" store = "spamdb" [store."spamdb"] -type = "sqlite" +type = "rocksdb" path = "{PATH}/test_antispam.db" #[store."redis"] @@ -103,112 +102,78 @@ type = "chat" model = "gpt-dummy" allow-invalid-certs = true -[lookup] -"spam-free" = {"gmail.com", "googlemail.com", "yahoomail.com", "*freemail.org"} -"spam-disposable" = {"guerrillamail.com", "*disposable.org"} -"spam-redirect" = {"bit.ly", "redirect.io", "redirect.me", "redirect.org", "redirect.com", "redirect.net", "t.ly", "tinyurl.com"} -"spam-dmarc" = {"dmarc-allow.org"} -"spam-spdk" = {"spf-dkim-allow.org"} -"spam-mime" = { "html" = "text/html|BAD", +[spam-filter.list] +"freemail-providers" = {"gmail.com", "googlemail.com", "yahoomail.com", "*freemail.org"} +"disposable-providers" = {"guerrillamail.com", "*disposable.org"} +"url-redirectors" = {"bit.ly", "redirect.io", "redirect.me", "redirect.org", "redirect.com", "redirect.net", "t.ly", "tinyurl.com"} +"dmarc-allow" = {"dmarc-allow.org"} +"spf-dkim-allow" = {"spf-dkim-allow.org"} +"spam-traps" = {"spamtrap@*"} +"trusted-domains" = {"stalw.art"} +"file-extensions" = { "html" = "text/html|BAD", "pdf" = "application/pdf|NZ", "txt" = "text/plain|message/disposition-notification|text/rfc822-headers", "zip" = "AR", "js" = "BAD|NZ", "hta" = "BAD|NZ" } -"spam-trap" = {"spamtrap@*"} -"spam-allow" = {"stalw.art"} - -[sieve.trusted.scripts] "#; +pub trait EnterpriseCore { + fn enable_enterprise(self) -> Self; +} + +impl EnterpriseCore for Core { + fn enable_enterprise(mut self) -> Self { + let todo = "remove"; + self.enterprise = common::enterprise::Enterprise { + license: common::enterprise::license::LicenseKey { + valid_to: store::write::now() + 3600, + valid_from: store::write::now() - 3600, + domain: String::new(), + accounts: 100, + }, + undelete: None, + trace_store: None, + metrics_store: None, + metrics_alerts: vec![], + logo_url: None, + ai_apis: Default::default(), + spam_filter_llm: None, + } + .into(); + self + } +} + #[tokio::test(flavor = "multi_thread")] async fn antispam() { // Enable logging crate::enable_logging(); // Prepare config - let tests = [ - "html", - "subject", - "bounce", - "received", - "messageid", - "date", - "from", - "replyto", - "recipient", - "mime", - "headers", - "url", - "dmarc", - "ip", - "helo", - "rbl", - "replies_out", - "replies_in", - "spamtrap", - "bayes_classify", - "reputation", - "pyzor", - "llm", - ]; let tmp_dir = TempDir::new("smtp_antispam_test", true); - let base_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .parent() - .unwrap() - .to_path_buf() - .join("resources") - .join("config") - .join("spamfilter"); - let mut config = CONFIG - .replace("{PATH}", tmp_dir.temp_dir.as_path().to_str().unwrap()) - .replace( - "{LIST_PATH}", - PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("resources") - .join("smtp") - .join("lists") - .to_str() - .unwrap(), - ); - let scores = fs::read_to_string(base_path.join("maps").join("scores.map")).unwrap(); - let base_path = base_path.join("scripts"); - let script_config = fs::read_to_string(base_path.join("config.sieve")).unwrap(); - let script_prelude = fs::read_to_string(base_path.join("prelude.sieve")).unwrap(); - let mut all_scripts = script_config.clone() + "\n" + script_prelude.as_str(); - for test_name in tests { - let mut script = fs::read_to_string(base_path.join(format!("{test_name}.sieve"))).unwrap(); - if !["reputation", "replies_out", "pyzor"].contains(&test_name) { - all_scripts = all_scripts + "\n" + script.as_str(); + let mut config = CONFIG.replace("{PATH}", tmp_dir.temp_dir.as_path().to_str().unwrap()); + let base_path = PathBuf::from( + std::env::var("SPAM_RULES_DIR") + .unwrap_or_else(|_| "/Users/me/code/spam-filter".to_string()), + ); + for section in ["rules", "lists"] { + for entry in fs::read_dir(base_path.join(section)).unwrap() { + let entry = entry.unwrap(); + let path = entry.path(); + if path.is_file() { + let file_name = path.file_name().unwrap().to_str().unwrap(); + if file_name.ends_with(".toml") + && ((section == "rules" && file_name != "llm.toml") + || (section == "lists" && file_name == "scores.toml")) + { + let contents = fs::read_to_string(&path).unwrap(); + config.push_str("\n\n"); + config.push_str(&contents); + } + } } - - if test_name == "reputation" { - script = "let \"score\" \"env.score\";\n\n".to_string() - + script.as_str() - + concat!( - "\n\nif eval \"score != env.final_score\" ", - "{let \"t.INVALID_SCORE\" \"score\";}\n" - ); - } else if test_name == "bayes_classify" { - script = script.replace("200", "10"); - } - - config.push_str(&format!( - "{test_name}.contents = '''{script_config}\n{script_prelude}\n{script}\n'''\n" - )); } - for test_name in ["composites", "scores", "epilogue"] { - all_scripts = all_scripts - + "\n" - + fs::read_to_string(base_path.join(format!("{test_name}.sieve"))) - .unwrap() - .as_str(); - } - - config.push_str(&format!( - "combined.contents = '''{all_scripts}\n'''\n[lookup]\n" - )); - config.push_str(&scores); // Parse config let mut config = Config::new(&config).unwrap(); @@ -313,19 +278,36 @@ async fn antispam() { .join("resources") .join("smtp") .join("antispam"); - for &test_name in tests.iter().chain(&["combined"]) { + for test_name in [ + /*"ip", + "helo", + "received", + "messageid", + "date", + "from", + "subject", + "replyto", + "recipient", + "headers", + "url", + "html", + "mime", + "bounce", + "dmarc", + "rbl", + "replies_out", + "replies_in", + "spamtrap", + "bayes_classify",*/ + "reputation", + "pyzor", + "llm", + "combined", + ] { /*if test_name != "combined" { continue; }*/ println!("===== {test_name} ====="); - let script = server - .core - .sieve - .trusted_scripts - .get(test_name) - .cloned() - .unwrap(); - let contents = fs::read_to_string(base_path.join(format!("{test_name}.test"))).unwrap(); let mut lines = contents.lines(); let mut has_more = true; @@ -333,12 +315,21 @@ async fn antispam() { while has_more { let mut message = String::new(); let mut in_params = true; - let mut variables: HashMap = HashMap::new(); - let mut expected_variables = AHashMap::new(); - let mut expected_headers = AHashMap::new(); // Build session let mut session = Session::test(server.clone()); + let mut arc_result = None; + let mut dkim_result = None; + let mut dkim_signatures = vec![]; + let mut dmarc_result = None; + let mut dmarc_policy = None; + let mut expected_tags = AHashSet::new(); + let mut score_expect = 0.0; + let mut score_set = 0.0; + let mut score_final = 0.0; + let mut body_params = 0; + let mut is_tls = false; + for line in lines.by_ref() { if in_params { if line.is_empty() { @@ -362,32 +353,43 @@ async fn antispam() { })); } "spf.result" | "spf_ehlo.result" => { - variables.insert( - param.to_string(), - SpfResult::from_str(value).as_str().to_string().into(), - ); + session.data.spf_mail_from = + Some(SpfOutput::default().with_result(SpfResult::from_str(value))); } "iprev.result" => { - variables.insert( - param.to_string(), - IprevResult::from_str(value).as_str().to_string().into(), - ); + session + .data + .iprev + .get_or_insert(IprevOutput { + result: IprevResult::None, + ptr: None, + }) + .result = IprevResult::from_str(value); } - "dkim.result" | "arc.result" => { - variables.insert( - param.to_string(), - DkimResult::from_str(value).as_str().to_string().into(), - ); + "dkim.result" => { + dkim_result = match DkimResult::from_str(value) { + DkimResult::Pass => DkimOutput::pass(), + DkimResult::Neutral(error) => DkimOutput::neutral(error), + DkimResult::Fail(error) => DkimOutput::fail(error), + DkimResult::PermError(error) => DkimOutput::perm_err(error), + DkimResult::TempError(error) => DkimOutput::temp_err(error), + DkimResult::None => unreachable!(), + } + .into(); + } + "arc.result" => { + arc_result = ArcOutput::default() + .with_result(DkimResult::from_str(value)) + .into(); } "dkim.domains" => { - variables.insert( - param.to_string(), - value - .split_ascii_whitespace() - .map(|s| Variable::from(s.to_string())) - .collect::>() - .into(), - ); + dkim_signatures = value + .split_ascii_whitespace() + .map(|s| Signature { + d: s.to_lowercase(), + ..Default::default() + }) + .collect(); } "envelope_from" => { session.data.mail_from = Some(SessionAddress::new(value.to_string())); @@ -399,50 +401,42 @@ async fn antispam() { .push(SessionAddress::new(value.to_string())); } "iprev.ptr" => { - variables.insert(param.to_string(), value.to_string().into()); + session + .data + .iprev + .get_or_insert(IprevOutput { + result: IprevResult::None, + ptr: None, + }) + .ptr = Some(Arc::new(vec![value.to_string()])); } "dmarc.result" => { - variables.insert( - param.to_string(), - DmarcResult::from_str(value).as_str().to_string().into(), - ); + dmarc_result = DmarcResult::from_str(value).into(); } "dmarc.policy" => { - variables.insert( - param.to_string(), - Policy::from_str(value).as_str().to_string().into(), - ); + dmarc_policy = Policy::from_str(value).into(); } "expect" => { - expected_variables.extend(value.split_ascii_whitespace().map(|v| { - v.split_once('=') - .map(|(k, v)| { - ( - k.to_lowercase(), - if v.contains('.') { - Variable::Float(v.parse().unwrap()) - } else { - Variable::Integer(v.parse().unwrap()) - }, - ) - }) - .unwrap_or((v.to_lowercase(), Variable::Integer(1))) - })); + expected_tags + .extend(value.split_ascii_whitespace().map(|v| v.to_uppercase())); } - "expect_header" => { - if let Some((header, value)) = value.split_once(' ') { - expected_headers - .insert(header.to_string(), value.trim().to_string()); - } else { - expected_headers.insert(value.to_string(), String::new()); - } + "expect_score" => { + score_expect = value.parse::().unwrap(); } - "score" | "final_score" => { - variables - .insert(param.to_string(), value.parse::().unwrap().into()); + "score" => { + score_set = value.parse::().unwrap(); } - _ if param.starts_with("param.") | param.starts_with("tls.") => { - variables.insert(param.to_string(), value.to_string().into()); + "final_score" => { + score_final = value.parse::().unwrap(); + } + "param.smtputf8" => { + body_params |= MAIL_SMTPUTF8; + } + "param.8bitmime" => { + body_params |= MAIL_BODY_8BITMIME; + } + "tls.version" => { + is_tls = true; } _ => panic!("Invalid parameter {param:?}"), } @@ -461,295 +455,163 @@ async fn antispam() { panic!("No message found"); } - // Build script params - let mut expected = expected_variables.keys().collect::>(); - expected.sort_unstable_by(|a, b| b.cmp(a)); - println!("Testing tags {:?}", expected); - let mut params = session - .build_script_parameters("data") - .with_expected_variables(expected_variables) - .with_message(message.as_bytes()); - for (name, value) in variables { - params = params.set_variable(name, value); + if body_params != 0 { + session + .data + .mail_from + .get_or_insert_with(|| SessionAddress::new("".to_string())) + .flags = body_params; } - // Run script - let server_ = server.clone(); - let script = script.clone(); - match server_.run_script("test".to_string(), script, params).await { - ScriptResult::Accept { modifications } => { - if modifications.len() != expected_headers.len() { - panic!( - "Expected {:?} headers, got {:?}", - expected_headers, modifications - ); - } - for modification in modifications { - if let ScriptModification::AddHeader { name, value } = modification { - if let Some(expected_value) = expected_headers.remove(name.as_str()) { - if !expected_value.is_empty() - && !value.starts_with(expected_value.as_str()) - { - panic!( - "Expected header {:?} to be {:?}, got {:?}", - name, expected_value, value - ); - } - } else { - panic!("Unexpected header {:?}", name); - } - } else { - panic!("Unexpected modification {:?}", modification); - } + // Build input + let mut dkim_domains = vec![]; + if let Some(dkim_result) = dkim_result { + if dkim_signatures.is_empty() { + dkim_signatures.push(Signature { + d: "unknown.org".to_string(), + ..Default::default() + }); + } + + for signature in &dkim_signatures { + dkim_domains.push(dkim_result.clone().with_signature(signature)); + } + } + let parsed_message = MessageParser::new().parse(&message).unwrap(); + let mut spam_input = session.build_spam_input( + &parsed_message, + &dkim_domains, + arc_result.as_ref(), + dmarc_result.as_ref(), + dmarc_policy.as_ref(), + ); + spam_input.is_tls = is_tls; + + // Initialize filter + let mut spam_ctx = server.spam_filter_init(spam_input); + match test_name { + "html" => { + server.spam_filter_analyze_html(&mut spam_ctx).await; + } + "subject" => { + server.spam_filter_analyze_headers(&mut spam_ctx).await; + spam_ctx.result.tags.retain(|t| t.starts_with("X_HDR_")); + server.spam_filter_analyze_subject(&mut spam_ctx).await; + server.spam_filter_analyze_rules(&mut spam_ctx).await; + spam_ctx.result.tags.retain(|t| !t.starts_with("X_HDR_")); + } + "received" => { + server.spam_filter_analyze_received(&mut spam_ctx).await; + } + "messageid" => { + server.spam_filter_analyze_message_id(&mut spam_ctx).await; + } + "date" => { + server.spam_filter_analyze_date(&mut spam_ctx).await; + } + "from" => { + server.spam_filter_analyze_from(&mut spam_ctx).await; + server.spam_filter_analyze_rules(&mut spam_ctx).await; + } + "replyto" => { + server.spam_filter_analyze_reply_to(&mut spam_ctx).await; + server.spam_filter_analyze_rules(&mut spam_ctx).await; + } + "recipient" => { + server.spam_filter_analyze_headers(&mut spam_ctx).await; + spam_ctx.result.tags.retain(|t| t.starts_with("X_HDR_")); + server.spam_filter_analyze_recipient(&mut spam_ctx).await; + server.spam_filter_analyze_rules(&mut spam_ctx).await; + spam_ctx.result.tags.retain(|t| !t.starts_with("X_HDR_")); + } + "mime" => { + server.spam_filter_analyze_mime(&mut spam_ctx).await; + } + "headers" => { + server.spam_filter_analyze_headers(&mut spam_ctx).await; + server.spam_filter_analyze_rules(&mut spam_ctx).await; + spam_ctx.result.tags.retain(|t| !t.starts_with("X_HDR_")); + } + "url" => { + server.spam_filter_analyze_url(&mut spam_ctx).await; + server.spam_filter_analyze_rules(&mut spam_ctx).await; + } + "dmarc" => { + server.spam_filter_analyze_dmarc(&mut spam_ctx).await; + } + "ip" => { + server.spam_filter_analyze_ip(&mut spam_ctx).await; + } + "helo" => { + server.spam_filter_analyze_ehlo(&mut spam_ctx).await; + } + "bounce" => { + server.spam_filter_analyze_mime(&mut spam_ctx).await; + server.spam_filter_analyze_headers(&mut spam_ctx).await; + server.spam_filter_analyze_rules(&mut spam_ctx).await; + spam_ctx.result.tags.retain(|t| !t.starts_with("X_HDR_")); + } + "rbl" => { + server.spam_filter_analyze_url(&mut spam_ctx).await; + server.spam_filter_analyze_ip(&mut spam_ctx).await; + server.spam_filter_analyze_domain(&mut spam_ctx).await; + } + "replies_out" => { + server.spam_filter_analyze_reply_out(&mut spam_ctx).await; + } + "replies_in" => { + server.spam_filter_analyze_reply_in(&mut spam_ctx).await; + } + "spamtrap" => { + server.spam_filter_analyze_spam_trap(&mut spam_ctx).await; + server + .spam_filter_finalize(&mut spam_ctx, String::new()) + .await; + } + "bayes_classify" => { + server + .spam_filter_analyze_bayes_classify(&mut spam_ctx) + .await; + } + "reputation" => { + spam_ctx.result.score = score_set; + server.spam_filter_analyze_reputation(&mut spam_ctx).await; + assert_eq!(spam_ctx.result.score, score_final); + } + "pyzor" => { + server.spam_filter_analyze_pyzor(&mut spam_ctx).await; + } + "llm" => { + server.spam_filter_analyze_llm(&mut spam_ctx).await; + } + "combined" => { + todo!("combined"); + } + _ => panic!("Invalid test {test_name:?}"), + } + + // Compare tags + if spam_ctx.result.tags != expected_tags { + for tag in &spam_ctx.result.tags { + if !expected_tags.contains(tag) { + println!("Unexpected tag: {tag:?}"); } } - ScriptResult::Reject(message) => panic!("{}", message), - ScriptResult::Replace { - message, - modifications, - } => println!( - "Replace: {} with modifications {:?}", - String::from_utf8_lossy(&message), - modifications - ), - ScriptResult::Discard => println!("Discard"), + + for tag in &expected_tags { + if !spam_ctx.result.tags.contains(tag) { + println!("Missing tag: {tag:?}"); + } + } + + panic!("Tags mismatch, expected {expected_tags:?}"); + } else { + println!("Tags matched: {expected_tags:?}"); } } } } -#[test] -fn html_tokens() { - for (input, expected) in [ - ( - "hello
world
", - vec![ - Variable::from("using <>
", - vec![ - Variable::from("".to_string()), - Variable::from("
tag
", - vec![ - Variable::from("_test".to_string()), - Variable::from("< >>hello world< br \n />", - vec![ - Variable::from("<".to_string()), - Variable::from("<".to_string()), - Variable::from("hello world".to_string()), - Variable::from("
ignore headxyz", - "

<body>

" - ), - vec![ - Variable::from("".to_string()), - Variable::from("what is ♥?

ßĂΒγ ", - "don't hurt me.

" - ), - vec![ - Variable::from("", - "this is the actual text" - ), - vec![ - Variable::from( - concat!( - "", - "text", - "< a href = test ignore>text", - "< a href = fudge href ignore>text", - " a href = \"unknown\" ", - ), - vec![ - Variable::from("hello world".to_string()), - Variable::from("test".to_string()), - Variable::from("fudge".to_string()), - Variable::from("foobar".to_string()), - ], - ), - ] { - assert_eq!( - html_attr_tokens(input, "a", vec![Cow::from("href")]), - expected, - "Failed for '{:?}'", - input - ); - } - - for (tag, attr_name, expected) in [ - ("", - "20", - "30", - "", - "" - ))), - 92600 - ); -} - trait ParseConfigValue: Sized { fn from_str(value: &str) -> Self; } @@ -819,3 +681,465 @@ impl ParseConfigValue for Policy { } } } + +#[test] +fn html_tokens() { + for (input, expected) in [ + ( + concat!("hello
world
"), + vec![ + HtmlToken::StartTag { + name: 1819112552, + attributes: vec![], + is_self_closing: false, + }, + HtmlToken::Text { + text: "hello".to_string(), + }, + HtmlToken::StartTag { + name: 29282, + attributes: vec![], + is_self_closing: true, + }, + HtmlToken::Text { + text: "world".to_string(), + }, + HtmlToken::StartTag { + name: 29282, + attributes: vec![], + is_self_closing: true, + }, + HtmlToken::EndTag { name: 1819112552 }, + ], + ), + ( + concat!("using <>
"), + vec![ + HtmlToken::StartTag { + name: 1819112552, + attributes: vec![], + is_self_closing: false, + }, + HtmlToken::Text { + text: "using <>".to_string(), + }, + HtmlToken::StartTag { + name: 29282, + attributes: vec![], + is_self_closing: true, + }, + HtmlToken::EndTag { name: 1819112552 }, + ], + ), + ( + concat!("test tag
"), + vec![ + HtmlToken::Text { + text: "test".to_string(), + }, + HtmlToken::StartTag { + name: 7630702, + attributes: vec![(29282, None)], + is_self_closing: true, + }, + HtmlToken::Text { + text: " tag".to_string(), + }, + HtmlToken::StartTag { + name: 29282, + attributes: vec![], + is_self_closing: true, + }, + ], + ), + ( + concat!("<>< >>hello world< br \n />"), + vec![ + HtmlToken::StartTag { + name: 6775156, + attributes: vec![], + is_self_closing: true, + }, + HtmlToken::Text { + text: ">hello world".to_string(), + }, + HtmlToken::StartTag { + name: 29282, + attributes: vec![], + is_self_closing: true, + }, + ], + ), + ( + concat!( + "ignore headxyz

<body><", + "/h1>" + ), + vec![ + HtmlToken::StartTag { + name: 1684104552, + attributes: vec![], + is_self_closing: false, + }, + HtmlToken::StartTag { + name: 435611265396, + attributes: vec![], + is_self_closing: false, + }, + HtmlToken::Text { + text: "ignore head".to_string(), + }, + HtmlToken::EndTag { name: 435611265396 }, + HtmlToken::StartTag { + name: 7630702, + attributes: vec![(1684104552, None)], + is_self_closing: false, + }, + HtmlToken::Text { + text: "xyz".to_string(), + }, + HtmlToken::EndTag { name: 7630702 }, + HtmlToken::EndTag { name: 1684104552 }, + HtmlToken::StartTag { + name: 12648, + attributes: vec![], + is_self_closing: false, + }, + HtmlToken::Text { + text: "".to_string(), + }, + HtmlToken::EndTag { name: 12648 }, + ], + ), + ( + concat!( + "

what is ♥?

ß&", + "Abreve;Βγ don't hurt me.", + "

" + ), + vec![ + HtmlToken::StartTag { + name: 112, + attributes: vec![], + is_self_closing: false, + }, + HtmlToken::Text { + text: "what is ♥?".to_string(), + }, + HtmlToken::EndTag { name: 112 }, + HtmlToken::StartTag { + name: 112, + attributes: vec![], + is_self_closing: false, + }, + HtmlToken::Text { + text: "ßĂΒγ don't hurt me.".to_string(), + }, + HtmlToken::EndTag { name: 112 }, + ], + ), + ( + concat!( + "this is the actual text" + ), + vec![ + HtmlToken::Comment { + text: concat!( + "!--[if mso]> < < < < ignore > -> here --".to_string(), + }, + HtmlToken::Text { + text: " the actual".to_string(), + }, + HtmlToken::Comment { + text: "!--".to_string(), + }, + HtmlToken::Text { + text: " text".to_string(), + }, + ], + ), + ( + concat!( + " < p > hello < / p > < p > world < / ", + "p > !!! < br > " + ), + vec![ + HtmlToken::StartTag { + name: 112, + attributes: vec![], + is_self_closing: false, + }, + HtmlToken::Text { + text: "hello".to_string(), + }, + HtmlToken::EndTag { name: 112 }, + HtmlToken::StartTag { + name: 112, + attributes: vec![], + is_self_closing: false, + }, + HtmlToken::Text { + text: " world".to_string(), + }, + HtmlToken::EndTag { name: 112 }, + HtmlToken::Text { + text: " !!!".to_string(), + }, + HtmlToken::StartTag { + name: 29282, + attributes: vec![], + is_self_closing: false, + }, + ], + ), + ( + concat!("

please unsubscribe here.

"), + vec![ + HtmlToken::StartTag { + name: 112, + attributes: vec![], + is_self_closing: false, + }, + HtmlToken::Text { + text: "please unsubscribe".to_string(), + }, + HtmlToken::StartTag { + name: 97, + attributes: vec![(1717924456, Some("#".to_string()))], + is_self_closing: false, + }, + HtmlToken::Text { + text: " here".to_string(), + }, + HtmlToken::EndTag { name: 97 }, + HtmlToken::Text { + text: ".".to_string(), + }, + HtmlToken::EndTag { name: 112 }, + ], + ), + ( + concat!( + "texttexttexttext", + "< a href = \"e\" >texttext< anchor href = \"x\">t", + "ext" + ), + vec![ + HtmlToken::StartTag { + name: 97, + attributes: vec![(1717924456, Some("a".to_string()))], + is_self_closing: false, + }, + HtmlToken::Text { + text: "text".to_string(), + }, + HtmlToken::EndTag { name: 97 }, + HtmlToken::StartTag { + name: 97, + attributes: vec![(1717924456, Some("b".to_string()))], + is_self_closing: false, + }, + HtmlToken::Text { + text: "text".to_string(), + }, + HtmlToken::EndTag { name: 97 }, + HtmlToken::StartTag { + name: 97, + attributes: vec![(1717924456, Some("c".to_string()))], + is_self_closing: false, + }, + HtmlToken::Text { + text: "text".to_string(), + }, + HtmlToken::EndTag { name: 97 }, + HtmlToken::StartTag { + name: 97, + attributes: vec![(1717924456, Some("d".to_string()))], + is_self_closing: false, + }, + HtmlToken::Text { + text: "text".to_string(), + }, + HtmlToken::EndTag { name: 97 }, + HtmlToken::StartTag { + name: 97, + attributes: vec![(1717924456, Some("e".to_string()))], + is_self_closing: false, + }, + HtmlToken::Text { + text: "text".to_string(), + }, + HtmlToken::EndTag { name: 97 }, + HtmlToken::StartTag { + name: 97, + attributes: vec![(125779835187816, Some("ignore".to_string()))], + is_self_closing: false, + }, + HtmlToken::Text { + text: "text".to_string(), + }, + HtmlToken::EndTag { name: 97 }, + HtmlToken::StartTag { + name: 125822818283105, + attributes: vec![(1717924456, Some("x".to_string()))], + is_self_closing: false, + }, + HtmlToken::Text { + text: "text".to_string(), + }, + HtmlToken::EndTag { name: 97 }, + ], + ), + ( + concat!( + "texttexttexttext< a ", + "href = e >textt", + "exttext" + ), + vec![ + HtmlToken::StartTag { + name: 97, + attributes: vec![(1717924456, Some("a".to_string()))], + is_self_closing: false, + }, + HtmlToken::Text { + text: "text".to_string(), + }, + HtmlToken::EndTag { name: 97 }, + HtmlToken::StartTag { + name: 97, + attributes: vec![(1717924456, Some("b".to_string()))], + is_self_closing: false, + }, + HtmlToken::Text { + text: "text".to_string(), + }, + HtmlToken::EndTag { name: 97 }, + HtmlToken::StartTag { + name: 97, + attributes: vec![(1717924456, Some("c".to_string()))], + is_self_closing: false, + }, + HtmlToken::Text { + text: "text".to_string(), + }, + HtmlToken::EndTag { name: 97 }, + HtmlToken::StartTag { + name: 97, + attributes: vec![(1717924456, Some("d".to_string()))], + is_self_closing: false, + }, + HtmlToken::Text { + text: "text".to_string(), + }, + HtmlToken::EndTag { name: 97 }, + HtmlToken::StartTag { + name: 97, + attributes: vec![(1717924456, Some("e".to_string()))], + is_self_closing: false, + }, + HtmlToken::Text { + text: "text".to_string(), + }, + HtmlToken::EndTag { name: 97 }, + HtmlToken::StartTag { + name: 97, + attributes: vec![(125779835187816, Some("ignore".to_string()))], + is_self_closing: false, + }, + HtmlToken::Text { + text: "text".to_string(), + }, + HtmlToken::EndTag { name: 97 }, + HtmlToken::StartTag { + name: 125822818283105, + attributes: vec![(1717924456, Some("x".to_string()))], + is_self_closing: false, + }, + HtmlToken::Text { + text: "text".to_string(), + }, + HtmlToken::EndTag { name: 97 }, + ], + ), + ( + concat!( + "text< a href = test igno", + "re>text< a href = fudge href ignor", + "e>text a href = \"unkn", + "own\" " + ), + vec![ + HtmlToken::Comment { + text: "!-- texttext--text--" + .to_string(), + }, + HtmlToken::StartTag { + name: 97, + attributes: vec![(1717924456, Some("hello world".to_string()))], + is_self_closing: false, + }, + HtmlToken::Text { + text: "text".to_string(), + }, + HtmlToken::EndTag { name: 97 }, + HtmlToken::StartTag { + name: 97, + attributes: vec![ + (1717924456, Some("test".to_string())), + (111542170183529, None), + ], + is_self_closing: false, + }, + HtmlToken::Text { + text: "text".to_string(), + }, + HtmlToken::EndTag { name: 97 }, + HtmlToken::StartTag { + name: 97, + attributes: vec![ + (1717924456, Some("fudge".to_string())), + (1717924456, None), + (111542170183529, None), + ], + is_self_closing: false, + }, + HtmlToken::Text { + text: "text".to_string(), + }, + HtmlToken::EndTag { name: 97 }, + HtmlToken::StartTag { + name: 97, + attributes: vec![(1717924456, Some("foobar".to_string()))], + is_self_closing: false, + }, + HtmlToken::Text { + text: "a href = \"unknown\"".to_string(), + }, + HtmlToken::EndTag { name: 97 }, + ], + ), + ] { + assert_eq!(expected, html_to_tokens(input), "failed for {input:?}"); + } +}