From 3a2df77eb4ff89e5aaae823d0229815bde8451c3 Mon Sep 17 00:00:00 2001 From: mdecimus Date: Thu, 2 Jan 2025 15:43:00 +0100 Subject: [PATCH] Spam filter improvements (closes #947) --- crates/common/src/config/spamfilter.rs | 16 +- crates/common/src/manager/mod.rs | 2 +- crates/nlp/src/bayes/tokenize.rs | 7 +- crates/nlp/src/tokenizers/types.rs | 64 ++--- crates/spam-filter/src/analysis/domain.rs | 186 ++++++------- crates/spam-filter/src/analysis/from.rs | 8 +- crates/spam-filter/src/analysis/init.rs | 66 ++++- crates/spam-filter/src/analysis/ip.rs | 86 +++--- crates/spam-filter/src/analysis/messageid.rs | 2 +- crates/spam-filter/src/analysis/mime.rs | 67 ++--- crates/spam-filter/src/analysis/subject.rs | 53 ++-- crates/spam-filter/src/analysis/url.rs | 267 ++++++++----------- crates/spam-filter/src/lib.rs | 15 +- crates/spam-filter/src/modules/bayes.rs | 33 ++- crates/spam-filter/src/modules/dnsbl.rs | 60 ++++- crates/spam-filter/src/modules/expression.rs | 39 ++- tests/Cargo.toml | 3 +- tests/resources/smtp/antispam/combined.test | 239 ++++++++++++++++- tests/resources/smtp/antispam/from.test | 56 ++-- tests/resources/smtp/antispam/messageid.test | 2 +- tests/resources/smtp/antispam/rbl.test | 12 +- tests/resources/smtp/antispam/recipient.test | 7 + tests/resources/smtp/antispam/url.test | 8 + tests/src/smtp/inbound/antispam.rs | 10 +- 24 files changed, 813 insertions(+), 495 deletions(-) diff --git a/crates/common/src/config/spamfilter.rs b/crates/common/src/config/spamfilter.rs index 98d41f69..0d72f36f 100644 --- a/crates/common/src/config/spamfilter.rs +++ b/crates/common/src/config/spamfilter.rs @@ -155,6 +155,8 @@ pub enum Location { HeaderTo, HeaderCc, HeaderBcc, + HeaderMid, + HeaderDnt, Ehlo, BodyText, BodyHtml, @@ -271,16 +273,16 @@ impl DnsBlConfig { DnsBlConfig { max_ip_checks: config - .property_or_default("spam-filter.dnsbl.max-check.ip", "20") + .property_or_default("spam-filter.dnsbl.max-check.ip", "50") .unwrap_or(20), max_domain_checks: config - .property_or_default("spam-filter.dnsbl.max-check.domain", "20") + .property_or_default("spam-filter.dnsbl.max-check.domain", "50") .unwrap_or(20), max_email_checks: config - .property_or_default("spam-filter.dnsbl.max-check.email", "20") + .property_or_default("spam-filter.dnsbl.max-check.email", "50") .unwrap_or(20), max_url_checks: config - .property_or_default("spam-filter.dnsbl.max-check.url", "20") + .property_or_default("spam-filter.dnsbl.max-check.url", "50") .unwrap_or(20), servers, } @@ -634,6 +636,8 @@ impl Location { Location::HeaderTo => "to", Location::HeaderCc => "cc", Location::HeaderBcc => "bcc", + Location::HeaderMid => "message_id", + Location::HeaderDnt => "dnt", Location::Ehlo => "ehlo", Location::BodyText => "body_text", Location::BodyHtml => "body_html", @@ -691,6 +695,8 @@ pub const V_SPAM_BODY_RAW: u32 = 134; pub const V_SPAM_SUBJECT: u32 = 135; pub const V_SPAM_SUBJECT_THREAD: u32 = 136; pub const V_SPAM_LOCATION: u32 = 137; +pub const V_WORDS_SUBJECT: u32 = 138; +pub const V_WORDS_BODY: u32 = 139; pub const V_RCPT_EMAIL: u32 = 0; pub const V_RCPT_NAME: u32 = 1; @@ -759,9 +765,11 @@ impl Element { ("body", V_SPAM_BODY_TEXT), ("body.text", V_SPAM_BODY_TEXT), ("body.html", V_SPAM_BODY_HTML), + ("body.words", V_WORDS_BODY), ("body.raw", V_SPAM_BODY_RAW), ("subject", V_SPAM_SUBJECT), ("subject.thread", V_SPAM_SUBJECT_THREAD), + ("subject.words", V_WORDS_SUBJECT), ("location", V_SPAM_LOCATION), ]); diff --git a/crates/common/src/manager/mod.rs b/crates/common/src/manager/mod.rs index 2fdfb689..342adf86 100644 --- a/crates/common/src/manager/mod.rs +++ b/crates/common/src/manager/mod.rs @@ -22,7 +22,7 @@ pub mod restore; pub mod webadmin; const DEFAULT_SPAMFILTER_URL: &str = - "https://raw.githubusercontent.com/stalwartlabs/spam-filter/refs/heads/main/spam-filter.toml"; + "https://github.com/stalwartlabs/spam-filter/releases/latest/download/spam-filter.toml"; const DEFAULT_WEBADMIN_URL: &str = "https://github.com/stalwartlabs/webadmin/releases/latest/download/webadmin.zip"; pub const WEBADMIN_KEY: &[u8] = "STALWART_WEBADMIN".as_bytes(); diff --git a/crates/nlp/src/bayes/tokenize.rs b/crates/nlp/src/bayes/tokenize.rs index 0c4402fd..d336f090 100644 --- a/crates/nlp/src/bayes/tokenize.rs +++ b/crates/nlp/src/bayes/tokenize.rs @@ -118,7 +118,7 @@ impl> Iterator for BayesTokenizer { } } -impl> TokenType { +impl, E: AsRef, U: AsRef, I: AsRef> TokenType { pub fn to_bayes_token(&self) -> Option { match self { TokenType::Alphabetic(word) => { @@ -138,7 +138,10 @@ impl> TokenType { TokenType::UrlNoScheme(word) => { BayesInputToken::Raw(url_host_as_bytes(word.as_ref())).into() } - TokenType::Alphanumeric(word) | TokenType::Email(word) | TokenType::UrlNoHost(word) => { + TokenType::Alphanumeric(word) | TokenType::UrlNoHost(word) => { + BayesInputToken::Raw(word.as_ref().to_lowercase().into_bytes()).into() + } + TokenType::Email(word) => { BayesInputToken::Raw(word.as_ref().to_lowercase().into_bytes()).into() } TokenType::Other(ch) => { diff --git a/crates/nlp/src/tokenizers/types.rs b/crates/nlp/src/tokenizers/types.rs index cd269800..659d5126 100644 --- a/crates/nlp/src/tokenizers/types.rs +++ b/crates/nlp/src/tokenizers/types.rs @@ -12,7 +12,7 @@ use super::Token; pub struct TypesTokenizer<'x> { text: &'x str, iter: CharIndices<'x>, - tokens: Vec>>, + tokens: Vec>>, peek_pos: usize, last_ch_is_space: bool, last_token_is_dot: bool, @@ -24,7 +24,7 @@ pub struct TypesTokenizer<'x> { } #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum TokenType { +pub enum TokenType { Alphabetic(T), Alphanumeric(T), Integer(T), @@ -33,18 +33,18 @@ pub enum TokenType { Space, // Detected types - Url(T), - UrlNoScheme(T), + Url(U), + UrlNoScheme(U), UrlNoHost(T), - IpAddr(T), - Email(T), + IpAddr(I), + Email(E), Float(T), } -impl Copy for Token> {} +impl Copy for Token> {} impl<'x> Iterator for TypesTokenizer<'x> { - type Item = Token>; + type Item = Token>; fn next(&mut self) -> Option { let token = self.peek()?; @@ -207,7 +207,7 @@ impl<'x> TypesTokenizer<'x> { } } - fn next_(&mut self) -> Option>> { + fn next_(&mut self) -> Option>> { if self.tokens.is_empty() && !self.eof { self.consume(); } @@ -218,7 +218,7 @@ impl<'x> TypesTokenizer<'x> { } } - fn peek(&mut self) -> Option>> { + fn peek(&mut self) -> Option>> { while self.tokens.len() <= self.peek_pos && !self.eof { self.consume(); } @@ -242,8 +242,8 @@ impl<'x> TypesTokenizer<'x> { fn try_parse_url( &mut self, - scheme_token: Option>>, - ) -> Option>> { + scheme_token: Option>>, + ) -> Option>> { let (has_scheme, allow_blank_host) = scheme_token.as_ref().map_or((false, false), |t| { ( true, @@ -459,7 +459,7 @@ impl<'x> TypesTokenizer<'x> { .into() } - fn try_parse_email(&mut self) -> Option>> { + fn try_parse_email(&mut self) -> Option>> { // Start token is a valid local part atom let start_token = self.peek()?; let mut last_is_dot = false; @@ -602,7 +602,7 @@ impl<'x> TypesTokenizer<'x> { None } - fn try_parse_number(&mut self) -> Option>> { + fn try_parse_number(&mut self) -> Option>> { self.peek_rewind(); let mut start_pos = usize::MAX; let mut end_pos = usize::MAX; @@ -685,7 +685,7 @@ impl<'x> TypesTokenizer<'x> { } } -impl TokenType { +impl TokenType { fn is_email_atom(&self) -> bool { matches!( self, @@ -727,40 +727,6 @@ impl TokenType { } } -impl> TokenType { - pub fn hostname(&self) -> Option<&str> { - match self { - TokenType::Url(url) => url.as_ref().split_once("://").map(|(_, host)| { - host.split_once('/') - .map_or(host, |(h, _)| h.split_once(':').map_or(h, |(h, _)| h)) - }), - TokenType::UrlNoScheme(url) => { - let url = url.as_ref(); - url.split_once('/').map_or(url, |(host, _)| host).into() - } - TokenType::Email(email) => email.as_ref().rsplit_once('@').map(|(_, domain)| domain), - _ => None, - } - } - - pub fn hostname_sld(&self) -> Option<&str> { - self.hostname().and_then(|host| psl::domain_str(host)) - } - - pub fn url_lowercase(&self, with_scheme_only: bool) -> Option { - match self { - TokenType::Url(url) => url.as_ref().trim().to_lowercase().into(), - TokenType::UrlNoScheme(url) if !with_scheme_only => { - let url = url.as_ref(); - format!("https://{}", url.trim().to_lowercase()) - .to_lowercase() - .into() - } - _ => None, - } - } -} - #[cfg(test)] mod test { diff --git a/crates/spam-filter/src/analysis/domain.rs b/crates/spam-filter/src/analysis/domain.rs index 1cb1b920..e47df425 100644 --- a/crates/spam-filter/src/analysis/domain.rs +++ b/crates/spam-filter/src/analysis/domain.rs @@ -11,13 +11,13 @@ use common::{ Server, }; use mail_auth::DkimResult; -use mail_parser::{HeaderName, HeaderValue, Host}; +use mail_parser::{parsers::MessageStream, HeaderName, HeaderValue, Host}; use nlp::tokenizers::types::TokenType; use crate::{ modules::{ - dnsbl::is_dnsbl, - expression::{SpamFilterResolver, StringResolver}, + dnsbl::check_dnsbl, + expression::StringResolver, html::{HtmlToken, A, HREF}, }, Email, Hostname, Recipient, SpamFilterContext, TextPart, @@ -52,19 +52,63 @@ impl SpamFilterAnalyzeDomain for Server { // Add Received headers for header in ctx.input.message.headers() { - if let (HeaderName::Received, HeaderValue::Received(received)) = - (&header.name, &header.value) - { - for host in [&received.from, &received.helo, &received.by] - .into_iter() - .flatten() - { - if let Host::Name(name) = host { - if let Some(name) = Hostname::new(name.as_ref()).sld { - domains.insert(ElementLocation::new(name, Location::HeaderReceived)); + match (&header.name, &header.value) { + (HeaderName::Received, HeaderValue::Received(received)) => { + for host in [&received.from, &received.helo, &received.by] + .into_iter() + .flatten() + { + if let Host::Name(name) = host { + if let Some(name) = Hostname::new(name.as_ref()).sld { + domains + .insert(ElementLocation::new(name, Location::HeaderReceived)); + } } } } + (HeaderName::MessageId, value) => { + if let Some(mid_domain) = value + .as_text() + .and_then(|s| s.rsplit_once('@')) + .and_then(|(_, d)| { + let host = Hostname::new(d); + if host.sld.is_some() { + Some(host) + } else { + None + } + }) + { + domains.insert(ElementLocation::new(mid_domain.fqdn, Location::HeaderMid)); + } + } + (HeaderName::Other(name), _) + if name.eq_ignore_ascii_case("Disposition-Notification-To") => + { + if let Some(address) = MessageStream::new( + ctx.input + .message + .raw_message + .get(header.offset_start..header.offset_end) + .unwrap_or_default(), + ) + .parse_address() + .as_address() + { + for addr in address.iter() { + if let Some(email) = addr.address() { + emails.insert(ElementLocation::new( + Recipient { + email: Email::new(email), + name: None, + }, + Location::HeaderDnt, + )); + } + } + } + } + _ => (), } } @@ -104,46 +148,13 @@ impl SpamFilterAnalyzeDomain for Server { for (part_id, part) in ctx.output.text_parts.iter().enumerate() { let is_body = ctx.input.message.text_body.contains(&part_id) || ctx.input.message.html_body.contains(&part_id); - match part { - TextPart::Plain { tokens, .. } => emails.extend(tokens.iter().filter_map(|t| { - if let TokenType::Email(email) = t { - Some(ElementLocation::new( - Recipient { - email: Email::new(email), - name: None, - }, - if is_body { - Location::BodyText - } else { - Location::Attachment - }, - )) - } else { - None - } - })), + let tokens = match part { + TextPart::Plain { tokens, .. } => tokens, TextPart::Html { tokens, html_tokens, .. } => { - emails.extend(tokens.iter().filter_map(|t| { - if let TokenType::Email(email) = t { - Some(ElementLocation::new( - Recipient { - email: Email::new(email), - name: None, - }, - if is_body { - Location::BodyHtml - } else { - Location::Attachment - }, - )) - } else { - None - } - })); emails.extend(html_tokens.iter().filter_map(|token| { if let HtmlToken::StartTag { name: A, @@ -174,8 +185,34 @@ impl SpamFilterAnalyzeDomain for Server { None } })); + tokens + } + TextPart::None => continue, + }; + + for token in tokens { + if let TokenType::Email(email) = token { + if is_body && !ctx.result.has_tag("RCPT_IN_BODY") { + for rcpt in ctx.output.all_recipients() { + if rcpt.email.address == email.address { + ctx.result.add_tag("RCPT_IN_BODY"); + break; + } + } + } + + emails.insert(ElementLocation::new( + Recipient { + email: email.clone(), + name: None, + }, + if is_body { + Location::BodyText + } else { + Location::Attachment + }, + )); } - TextPart::None => (), } } @@ -194,22 +231,7 @@ impl SpamFilterAnalyzeDomain for Server { } // Check Email DNSBL - if ctx.result.rbl_email_checks < self.core.spam.dnsbl.max_email_checks { - for dnsbl in &self.core.spam.dnsbl.servers { - if dnsbl.scope == Element::Email { - if let Some(tag) = is_dnsbl( - self, - dnsbl, - SpamFilterResolver::new(ctx, &email.element, email.location), - ) - .await - { - ctx.result.add_tag(tag); - } - } - } - ctx.result.rbl_email_checks += 1; - } + check_dnsbl(self, ctx, &email.element, Element::Email, email.location).await; domains.insert(ElementLocation::new( email.element.email.domain_part.fqdn.clone(), @@ -220,30 +242,16 @@ impl SpamFilterAnalyzeDomain for Server { // Validate domains for domain in &domains { // Skip trusted domains - if is_trusted_domain(self, &domain.element, ctx.input.span_id).await { - continue; - } - - // Check Domain DNSBL - if ctx.result.rbl_domain_checks < self.core.spam.dnsbl.max_domain_checks { - for dnsbl in &self.core.spam.dnsbl.servers { - if dnsbl.scope == Element::Domain { - if let Some(tag) = is_dnsbl( - self, - dnsbl, - SpamFilterResolver::new( - ctx, - &StringResolver(domain.element.as_str()), - domain.location, - ), - ) - .await - { - ctx.result.add_tag(tag); - } - } - } - ctx.result.rbl_domain_checks += 1; + if !is_trusted_domain(self, &domain.element, ctx.input.span_id).await { + // Check Domain DNSBL + check_dnsbl( + self, + ctx, + &StringResolver(domain.element.as_str()), + Element::Domain, + domain.location, + ) + .await; } } ctx.output.emails = emails; diff --git a/crates/spam-filter/src/analysis/from.rs b/crates/spam-filter/src/analysis/from.rs index b6385702..a3f292c5 100644 --- a/crates/spam-filter/src/analysis/from.rs +++ b/crates/spam-filter/src/analysis/from.rs @@ -123,13 +123,13 @@ impl SpamFilterAnalyzeFrom for Server { } if !env_from_empty && ctx.output.env_from_addr.address == from_addr.address { - ctx.result.add_tag("FROM_EQ_ENVFROM"); + ctx.result.add_tag("FROM_EQ_ENV_FROM"); } else if from_addr_is_valid { if from_addr.domain_part.sld == ctx.output.ehlo_host.sld { - ctx.result.add_tag("FROMTLD_EQ_ENVFROMTLD"); + ctx.result.add_tag("FROMTLD_EQ_ENV_FROMTLD"); } else if !ctx.output.env_from_postmaster { ctx.result.add_tag("FORGED_SENDER"); - ctx.result.add_tag("FROM_NEQ_ENVFROM"); + ctx.result.add_tag("FROM_NEQ_ENV_FROM"); } } @@ -212,7 +212,7 @@ impl SpamFilterAnalyzeFrom for Server { ctx.result.add_tag("FROMHOST_NORES_A_OR_MX"); } } else { - ctx.result.add_tag("ENVFROM_INVALID"); + ctx.result.add_tag("ENV_FROM_INVALID"); } // Check whether disposition notification address is different to return path diff --git a/crates/spam-filter/src/analysis/init.rs b/crates/spam-filter/src/analysis/init.rs index d23e6317..f51229ba 100644 --- a/crates/spam-filter/src/analysis/init.rs +++ b/crates/spam-filter/src/analysis/init.rs @@ -10,10 +10,12 @@ use nlp::tokenizers::types::{TokenType, TypesTokenizer}; use crate::{ modules::html::{html_to_tokens, HtmlToken, HEAD}, - Email, Hostname, Recipient, SpamFilterContext, SpamFilterInput, SpamFilterOutput, + Email, Hostname, IpParts, Recipient, SpamFilterContext, SpamFilterInput, SpamFilterOutput, SpamFilterResult, TextPart, }; +use super::url::UrlParts; + pub trait SpamFilterInit { fn spam_filter_init<'x>(&self, input: SpamFilterInput<'x>) -> SpamFilterContext<'x>; } @@ -90,7 +92,22 @@ impl SpamFilterInit for Server { .tokenize_urls(true) .tokenize_urls_without_scheme(true) .tokenize_emails(true) - .map(|t| t.word) + .map(|t| match t.word { + TokenType::Alphabetic(s) => TokenType::Alphabetic(s.into()), + TokenType::Alphanumeric(s) => TokenType::Alphanumeric(s.into()), + TokenType::Integer(s) => TokenType::Integer(s.into()), + TokenType::Other(s) => TokenType::Other(s), + TokenType::Punctuation(s) => TokenType::Punctuation(s), + TokenType::Space => TokenType::Space, + TokenType::Url(url) => TokenType::Url(UrlParts::new(url)), + TokenType::UrlNoHost(s) => TokenType::UrlNoHost(s.into()), + TokenType::UrlNoScheme(s) => { + TokenType::UrlNoScheme(UrlParts::new(format!("https://{}", s.trim()))) + } + TokenType::IpAddr(i) => TokenType::IpAddr(IpParts::new(i)), + TokenType::Email(e) => TokenType::Email(Email::new(e)), + TokenType::Float(s) => TokenType::Float(s.into()), + }) .collect::>(); // Tokenize and convert text parts @@ -110,7 +127,22 @@ impl SpamFilterInit for Server { .tokenize_urls(true) .tokenize_urls_without_scheme(true) .tokenize_emails(true) - .map(|t| t.word) + .map(|t| match t.word { + TokenType::Alphabetic(s) => TokenType::Alphabetic(s.into()), + TokenType::Alphanumeric(s) => TokenType::Alphanumeric(s.into()), + TokenType::Integer(s) => TokenType::Integer(s.into()), + TokenType::Other(s) => TokenType::Other(s), + TokenType::Punctuation(s) => TokenType::Punctuation(s), + TokenType::Space => TokenType::Space, + TokenType::Url(url) => TokenType::Url(UrlParts::new(url)), + TokenType::UrlNoHost(s) => TokenType::UrlNoHost(s.into()), + TokenType::UrlNoScheme(s) => TokenType::UrlNoScheme(UrlParts::new( + format!("https://{}", s.trim()), + )), + TokenType::IpAddr(i) => TokenType::IpAddr(IpParts::new(i)), + TokenType::Email(e) => TokenType::Email(Email::new(e)), + TokenType::Float(s) => TokenType::Float(s.into()), + }) .collect::>(), }, PartType::Html(html) => { @@ -153,23 +185,31 @@ impl SpamFilterInit for Server { .tokenize_emails(true) .map(|t| match t.word { TokenType::Alphabetic(s) => { - TokenType::Alphabetic(s.to_string()) + TokenType::Alphabetic(s.to_string().into()) } TokenType::Alphanumeric(s) => { - TokenType::Alphanumeric(s.to_string()) + TokenType::Alphanumeric(s.to_string().into()) + } + TokenType::Integer(s) => { + TokenType::Integer(s.to_string().into()) } - TokenType::Integer(s) => TokenType::Integer(s.to_string()), TokenType::Other(s) => TokenType::Other(s), TokenType::Punctuation(s) => TokenType::Punctuation(s), TokenType::Space => TokenType::Space, - TokenType::Url(s) => TokenType::Url(s.to_string()), - TokenType::UrlNoScheme(s) => { - TokenType::UrlNoScheme(s.to_string()) + TokenType::Url(url) => { + TokenType::Url(UrlParts::new(url.to_string())) } - TokenType::UrlNoHost(s) => TokenType::UrlNoHost(s.to_string()), - TokenType::IpAddr(s) => TokenType::IpAddr(s.to_string()), - TokenType::Email(s) => TokenType::Email(s.to_string()), - TokenType::Float(s) => TokenType::Float(s.to_string()), + TokenType::UrlNoHost(s) => { + TokenType::UrlNoHost(s.to_string().into()) + } + TokenType::UrlNoScheme(s) => TokenType::UrlNoScheme( + UrlParts::new(format!("https://{}", s.trim())), + ), + TokenType::IpAddr(i) => { + TokenType::IpAddr(IpParts::new(i.to_string())) + } + TokenType::Email(e) => TokenType::Email(Email::new(e)), + TokenType::Float(s) => TokenType::Float(s.to_string().into()), }) .collect::>(), html_tokens, diff --git a/crates/spam-filter/src/analysis/ip.rs b/crates/spam-filter/src/analysis/ip.rs index 5e0e3ca0..291d6a30 100644 --- a/crates/spam-filter/src/analysis/ip.rs +++ b/crates/spam-filter/src/analysis/ip.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{future::Future, net::IpAddr}; +use std::{borrow::Cow, future::Future}; use common::{ config::spamfilter::{Element, IpResolver, Location}, @@ -13,11 +13,9 @@ use common::{ use mail_auth::IprevResult; use mail_parser::{HeaderName, HeaderValue, Host}; use nlp::tokenizers::types::TokenType; +use store::ahash::AHashSet; -use crate::{ - modules::{dnsbl::is_dnsbl, expression::SpamFilterResolver}, - SpamFilterContext, TextPart, -}; +use crate::{modules::dnsbl::check_dnsbl, IpParts, SpamFilterContext, TextPart}; use super::ElementLocation; @@ -31,9 +29,9 @@ pub trait SpamFilterAnalyzeIp: Sync + Send { impl SpamFilterAnalyzeIp for Server { async fn spam_filter_analyze_ip(&self, ctx: &mut SpamFilterContext<'_>) { // IP Address RBL - ctx.output - .ips - .insert(ElementLocation::new(ctx.input.remote_ip, Location::Tcp)); + let mut ips = AHashSet::new(); + + ips.insert(ElementLocation::new(ctx.input.remote_ip, Location::Tcp)); // Obtain IP addresses from Received headers for header in ctx.input.message.headers() { @@ -42,9 +40,7 @@ impl SpamFilterAnalyzeIp for Server { { if let Some(ip) = received.from_ip() { if !ip.is_loopback() && !self.is_ip_allowed(&ip) { - ctx.output - .ips - .insert(ElementLocation::new(ip, Location::HeaderReceived)); + ips.insert(ElementLocation::new(ip, Location::HeaderReceived)); } } for host in [&received.from, &received.helo, &received.by] @@ -53,9 +49,7 @@ impl SpamFilterAnalyzeIp for Server { { if let Host::IpAddr(ip) = host { if !ip.is_loopback() && !self.is_ip_allowed(ip) { - ctx.output - .ips - .insert(ElementLocation::new(*ip, Location::HeaderReceived)); + ips.insert(ElementLocation::new(*ip, Location::HeaderReceived)); } } } @@ -67,10 +61,10 @@ impl SpamFilterAnalyzeIp for Server { let is_body = ctx.input.message.text_body.contains(&part_id) || ctx.input.message.html_body.contains(&part_id); match part { - TextPart::Plain { tokens, .. } => { - ctx.output.ips.extend(tokens.iter().filter_map(|t| { + TextPart::Plain { tokens, .. } | TextPart::Html { tokens, .. } => { + ips.extend(tokens.iter().filter_map(|t| { if let TokenType::IpAddr(ip) = t { - ip.parse::().ok().map(|ip| { + ip.ip.map(|ip| { ElementLocation::new( ip, if is_body { @@ -85,30 +79,13 @@ impl SpamFilterAnalyzeIp for Server { } })) } - TextPart::Html { tokens, .. } => { - ctx.output.ips.extend(tokens.iter().filter_map(|t| { - if let TokenType::IpAddr(ip) = t { - ip.parse::().ok().map(|ip| { - ElementLocation::new( - ip, - if is_body { - Location::BodyHtml - } else { - Location::Attachment - }, - ) - }) - } else { - None - } - })) - } + TextPart::None => (), } } // Validate IP addresses - for ip in &ctx.output.ips { + for ip in &ips { if ip.element.is_loopback() || ip.element.is_multicast() || ip.element.is_unspecified() @@ -120,25 +97,16 @@ impl SpamFilterAnalyzeIp for Server { continue; } - let ip_resolver = IpResolver::new(ip.element); - for dnsbl in &self.core.spam.dnsbl.servers { - if dnsbl.scope == Element::Ip { - if let Some(tag) = is_dnsbl( - self, - dnsbl, - SpamFilterResolver::new(ctx, &ip_resolver, ip.location), - ) - .await - { - ctx.result.add_tag(tag); - } - } - } - ctx.result.rbl_ip_checks += 1; - if ctx.result.rbl_ip_checks >= self.core.spam.dnsbl.max_ip_checks { - break; - } + check_dnsbl( + self, + ctx, + &IpResolver::new(ip.element), + Element::Ip, + ip.location, + ) + .await; } + ctx.output.ips = ips; // Reverse DNS validation if let Some(iprev) = ctx.input.iprev_result { @@ -150,3 +118,13 @@ impl SpamFilterAnalyzeIp for Server { } } } + +impl<'x> IpParts<'x> { + pub fn new(text: impl Into>) -> IpParts<'x> { + let text = text.into(); + IpParts { + ip: text.parse().ok(), + text, + } + } +} diff --git a/crates/spam-filter/src/analysis/messageid.rs b/crates/spam-filter/src/analysis/messageid.rs index 4c9208db..4540b5d5 100644 --- a/crates/spam-filter/src/analysis/messageid.rs +++ b/crates/spam-filter/src/analysis/messageid.rs @@ -61,7 +61,7 @@ impl SpamFilterAnalyzeMid for Server { // From address present in Message-ID checks for (part, sender) in [ ("FROM", &ctx.output.from.email), - ("ENVFROM", &ctx.output.env_from_addr), + ("ENV_FROM", &ctx.output.env_from_addr), ] { if !sender.address.is_empty() { if mid.contains(&sender.address) { diff --git a/crates/spam-filter/src/analysis/mime.rs b/crates/spam-filter/src/analysis/mime.rs index 9e67538b..1ea34ad5 100644 --- a/crates/spam-filter/src/analysis/mime.rs +++ b/crates/spam-filter/src/analysis/mime.rs @@ -10,12 +10,11 @@ use common::{ scripts::functions::{array::cosine_similarity, unicode::CharUtils}, Server, }; -use hyper::Uri; use mail_parser::{HeaderName, MimeHeaders, PartType}; use nlp::tokenizers::types::TokenType; use unicode_security::MixedScript; -use crate::{Hostname, SpamFilterContext, TextPart}; +use crate::{SpamFilterContext, TextPart}; pub trait SpamFilterAnalyzeMime: Sync + Send { fn spam_filter_analyze_mime( @@ -171,25 +170,36 @@ impl SpamFilterAnalyzeMime for Server { let mut html_part_uris = 0; for text_part in part_ids.iter().map(|id| &ctx.output.text_parts[*id]) { - match text_part { + let (tokens, words, uri_count) = match text_part { TextPart::Plain { tokens, .. } if !has_plain_part => { - words_and_uris( - tokens, - &mut text_part_words, - &mut text_part_uris, - ); has_plain_part = true; + (tokens, &mut text_part_words, &mut text_part_uris) } TextPart::Html { tokens, .. } if !has_html_part => { - words_and_uris( - tokens, - &mut html_part_words, - &mut html_part_uris, - ); has_html_part = true; + (tokens, &mut html_part_words, &mut html_part_uris) + } + _ => continue, + }; + + let mut uris = HashSet::new(); + for token in tokens { + match token { + TokenType::Alphabetic(v) | TokenType::Alphanumeric(v) => { + words.push(v.as_ref()); + } + TokenType::Url(v) => { + if let Some(host) = + v.url_parsed.as_ref().map(|uri| &uri.host) + { + uris.insert(host.sld_or_default()); + } + } + _ => (), } - _ => (), } + + *uri_count = uris.len(); } // Multipart message mostly text/html MIME @@ -418,32 +428,3 @@ impl SpamFilterAnalyzeMime for Server { } } } - -fn words_and_uris<'x, T: AsRef>( - tokens: &'x [TokenType], - words: &mut Vec<&'x str>, - uri_count: &mut usize, -) { - let mut uris = HashSet::new(); - - for token in tokens { - match token { - TokenType::Alphabetic(v) | TokenType::Alphanumeric(v) => { - words.push(v.as_ref()); - } - TokenType::Url(v) => { - if let Some(host) = v - .as_ref() - .parse::() - .ok() - .and_then(|uri| uri.host().map(Hostname::new)) - { - uris.insert(host.sld.unwrap_or(host.fqdn)); - } - } - _ => (), - } - } - - *uri_count = uris.len(); -} diff --git a/crates/spam-filter/src/analysis/subject.rs b/crates/spam-filter/src/analysis/subject.rs index a4eae2d6..e98a670b 100644 --- a/crates/spam-filter/src/analysis/subject.rs +++ b/crates/spam-filter/src/analysis/subject.rs @@ -11,7 +11,7 @@ use mail_parser::HeaderName; use nlp::tokenizers::types::TokenType; use smtp_proto::{MAIL_BODY_8BITMIME, MAIL_BODY_BINARYMIME, MAIL_SMTPUTF8}; -use crate::{Email, SpamFilterContext}; +use crate::SpamFilterContext; pub trait SpamFilterAnalyzeSubject: Sync + Send { fn spam_filter_analyze_subject( @@ -97,36 +97,55 @@ impl SpamFilterAnalyzeSubject for Server { for token in &ctx.output.subject_tokens { match token { - TokenType::Url(_) => { + TokenType::Url(url) => { // Subject contains URL ctx.result.add_tag("URL_IN_SUBJECT"); + + if let Some(url_parsed) = &url.url_parsed { + let host = url_parsed.host.sld_or_default(); + for rcpt in ctx.output.all_recipients() { + if rcpt.email.domain_part.sld_or_default() == host { + ctx.result.add_tag("RCPT_DOMAIN_IN_SUBJECT"); + break; + } + } + } } - TokenType::Email(address) => { + TokenType::UrlNoScheme(url) => { + if let Some(url_parsed) = &url.url_parsed { + let host = url_parsed.host.sld_or_default(); + for rcpt in ctx.output.all_recipients() { + if rcpt.email.domain_part.sld_or_default() == host { + ctx.result.add_tag("RCPT_DOMAIN_IN_SUBJECT"); + break; + } + } + } + } + TokenType::Email(email) => { // Subject contains recipient - let email = Email::new(address); - if ctx.output.env_to_addr.contains(&email) + if ctx.output.env_to_addr.contains(email) || ctx .output .all_recipients() .any(|r| r.email.address == email.address) { ctx.result.add_tag("RCPT_IN_SUBJECT"); + } else { + let host = email.domain_part.sld_or_default(); + for rcpt in ctx.output.all_recipients() { + if rcpt.email.address == email.address { + ctx.result.add_tag("RCPT_IN_SUBJECT"); + break; + } else if rcpt.email.domain_part.sld_or_default() == host { + ctx.result.add_tag("RCPT_DOMAIN_IN_SUBJECT"); + break; + } + } } - continue; } _ => {} } - - if let Some(hostname) = token.hostname_sld() { - let hostname = Some(hostname.to_lowercase()); - if ctx - .output - .all_recipients() - .any(|r| r.email.domain_part.sld == hostname) - { - ctx.result.add_tag("RCPT_DOMAIN_IN_SUBJECT"); - } - } } // Validate encoding diff --git a/crates/spam-filter/src/analysis/url.rs b/crates/spam-filter/src/analysis/url.rs index b0e21b50..c0cb62e5 100644 --- a/crates/spam-filter/src/analysis/url.rs +++ b/crates/spam-filter/src/analysis/url.rs @@ -16,8 +16,8 @@ use nlp::tokenizers::types::TokenType; use reqwest::redirect::Policy; use unicode_security::MixedScript; -use crate::modules::dnsbl::is_dnsbl; -use crate::modules::expression::{SpamFilterResolver, StringResolver}; +use crate::modules::dnsbl::check_dnsbl; +use crate::modules::expression::StringResolver; use crate::modules::html::SRC; use crate::{ modules::html::{HtmlToken, A, HREF}, @@ -33,11 +33,14 @@ pub trait SpamFilterAnalyzeUrl: Sync + Send { ) -> impl Future + Send; } -pub struct UrlParts { +#[derive(Clone, Debug)] +pub struct UrlParts<'x> { pub url: String, + pub url_original: Cow<'x, str>, pub url_parsed: Option, } +#[derive(Clone, Debug)] pub struct UrlParsed { pub parts: Uri, pub host: Hostname, @@ -46,35 +49,20 @@ pub struct UrlParsed { impl SpamFilterAnalyzeUrl for Server { async fn spam_filter_analyze_url(&self, ctx: &mut SpamFilterContext<'_>) { // Extract URLs - let mut urls: HashSet> = HashSet::from_iter( - ctx.output - .subject_tokens - .iter() - .filter_map(|t| t.url_lowercase(false)) - .map(|url| ElementLocation::new(url, Location::HeaderSubject)), - ); + let mut urls: HashSet>> = + HashSet::from_iter(ctx.output.subject_tokens.iter().filter_map(|t| match t { + TokenType::Url(url) | TokenType::UrlNoScheme(url) => Some(ElementLocation::new( + url.to_owned(), + Location::HeaderSubject, + )), + _ => None, + })); for (part_id, part) in ctx.output.text_parts.iter().enumerate() { let is_body = ctx.input.message.text_body.contains(&part_id) || ctx.input.message.html_body.contains(&part_id); - match part { - TextPart::Plain { tokens, .. } => { - urls.extend( - tokens - .iter() - .filter_map(|t| t.url_lowercase(false)) - .map(|url| { - ElementLocation::new( - url, - if is_body { - Location::BodyText - } else { - Location::Attachment - }, - ) - }), - ); - } + let tokens = match part { + TextPart::Plain { tokens, .. } => tokens, TextPart::Html { html_tokens, tokens, @@ -86,7 +74,7 @@ impl SpamFilterAnalyzeUrl for Server { match value { Some(value) if [HREF, SRC].contains(attr) => { urls.insert(ElementLocation::new( - value.trim().to_lowercase(), + UrlParts::new(value.trim().to_string()), if is_body { Location::BodyHtml } else { @@ -99,23 +87,37 @@ impl SpamFilterAnalyzeUrl for Server { } } } - urls.extend( - tokens - .iter() - .filter_map(|t| t.url_lowercase(false)) - .map(|url| { - ElementLocation::new( - url, - if is_body { - Location::BodyHtml - } else { - Location::Attachment - }, - ) - }), - ); + tokens + } + TextPart::None => &[][..], + }; + + for token in tokens { + match token { + TokenType::Url(url) | TokenType::UrlNoScheme(url) => { + if is_body && !ctx.result.has_tag("RCPT_DOMAIN_IN_BODY") { + if let Some(url_parsed) = &url.url_parsed { + let host = url_parsed.host.sld_or_default(); + for rcpt in ctx.output.all_recipients() { + if rcpt.email.domain_part.sld_or_default() == host { + ctx.result.add_tag("RCPT_DOMAIN_IN_BODY"); + break; + } + } + } + } + + urls.insert(ElementLocation::new( + url.to_owned(), + if is_body { + Location::BodyHtml + } else { + Location::Attachment + }, + )); + } + _ => {} } - TextPart::None => {} } if is_body { @@ -135,8 +137,9 @@ impl SpamFilterAnalyzeUrl for Server { } } - for url in urls { - for ch in url.element.chars() { + let mut redirected_urls = HashSet::new(); + for url in &urls { + for ch in url.element.url.chars() { if ch.is_zwsp() { ctx.result.add_tag("ZERO_WIDTH_SPACE_URL"); } @@ -147,29 +150,17 @@ impl SpamFilterAnalyzeUrl for Server { } // Skip non-URLs such as 'data:' and 'mailto:' - if !url.element.contains("://") { - ctx.output.urls.insert(ElementLocation::new( - UrlParts::new(url.element), - url.location, - )); + if !url.element.url.contains("://") { continue; } - // Parse url - let url_parsed = match url.element.parse::() { - Ok(url_parsed) if url_parsed.host().is_some() => UrlParsed { - host: Hostname::new(url_parsed.host().unwrap()), - parts: url_parsed, - }, - _ => { - // URL could not be parsed - ctx.output.urls.insert(ElementLocation::new( - UrlParts::new(url.element), - url.location, - )); - ctx.result.add_tag("R_UNPARSABLE_URL"); - continue; - } + // Obtain parse url + let url_parsed = if let Some(url_parsed) = &url.element.url_parsed { + url_parsed + } else { + // URL could not be parsed + ctx.result.add_tag("R_UNPARSABLE_URL"); + continue; }; let host_sld = url_parsed.host.sld_or_default(); @@ -180,29 +171,14 @@ impl SpamFilterAnalyzeUrl for Server { if let Some(ip) = url_parsed.host.ip { // Check IP DNSBL - if ctx.result.rbl_ip_checks < self.core.spam.dnsbl.max_ip_checks { - for dnsbl in &self.core.spam.dnsbl.servers { - if dnsbl.scope == Element::Ip { - if let Some(tag) = is_dnsbl( - self, - dnsbl, - SpamFilterResolver::new(ctx, &IpResolver::new(ip), url.location), - ) - .await - { - ctx.result.add_tag(tag); - } - } - } - ctx.result.rbl_ip_checks += 1; - } + check_dnsbl(self, ctx, &IpResolver::new(ip), Element::Ip, url.location).await; } else if is_url_redirector(self, host_sld, ctx.input.span_id).await { // Check for redirectors ctx.result.add_tag("REDIRECTOR_URL"); if !ctx.result.has_tag("URL_REDIRECTOR_NESTED") { let mut redirect_count = 1; - let mut url_redirect = Cow::Borrowed(url.element.as_str()); + let mut url_redirect = Cow::Borrowed(url.element.url.as_str()); while redirect_count <= 3 { match http_get_header( @@ -213,25 +189,21 @@ impl SpamFilterAnalyzeUrl for Server { .await { Ok(Some(location)) => { - if let Ok(location_parsed) = location.parse::() { - let host = - Hostname::new(location_parsed.host().unwrap_or_default()); + let location = UrlParts::new(location); + if let Some(location_parsed) = &location.url_parsed { if is_url_redirector( self, - host.sld_or_default(), + location_parsed.host.sld_or_default(), ctx.input.span_id, ) .await { - url_redirect = Cow::Owned(location); + url_redirect = Cow::Owned(location.url); redirect_count += 1; continue; } else { - ctx.output.urls.insert(ElementLocation::new( - UrlParts::new(location.to_lowercase()) - .with_parts(location_parsed, host), - url.location, - )); + redirected_urls + .insert(ElementLocation::new(location, url.location)); } } } @@ -248,15 +220,11 @@ impl SpamFilterAnalyzeUrl for Server { } } } - - // Add URL - ctx.output.urls.insert(ElementLocation::new( - UrlParts::new(url.element).with_parsed(url_parsed), - url.location, - )); } - for (el, url_parsed) in ctx.output.urls.iter().filter_map(|el| { + urls.extend(redirected_urls); + + for (el, url_parsed) in urls.iter().filter_map(|el| { el.element .url_parsed .as_ref() @@ -282,49 +250,25 @@ impl SpamFilterAnalyzeUrl for Server { } // Check Domain DNSBL - if ctx.result.rbl_domain_checks < self.core.spam.dnsbl.max_domain_checks { - for dnsbl in &self.core.spam.dnsbl.servers { - if matches!(dnsbl.scope, Element::Domain) { - if let Some(tag) = is_dnsbl( - self, - dnsbl, - SpamFilterResolver::new( - ctx, - &StringResolver(host.sld_or_default()), - el.location, - ), - ) - .await - { - ctx.result.add_tag(tag); - } - } - } - ctx.result.rbl_domain_checks += 1; - } + check_dnsbl( + self, + ctx, + &StringResolver(host.sld_or_default()), + Element::Domain, + el.location, + ) + .await; } else { // URL is an ip address ctx.result.add_tag("R_SUSPICIOUS_URL"); } // Check URL DNSBL - if ctx.result.rbl_url_checks < self.core.spam.dnsbl.max_url_checks { - for dnsbl in &self.core.spam.dnsbl.servers { - if matches!(dnsbl.scope, Element::Url) { - if let Some(tag) = is_dnsbl( - self, - dnsbl, - SpamFilterResolver::new(ctx, &el.element, el.location), - ) - .await - { - ctx.result.add_tag(tag); - } - } - } - ctx.result.rbl_url_checks += 1; - } + check_dnsbl(self, ctx, &el.element, Element::Url, el.location).await; } + + // Update context + ctx.output.urls = urls; } } @@ -373,7 +317,7 @@ async fn http_get_header( }) } -fn is_single_url>(tokens: &[TokenType]) -> bool { +fn is_single_url(tokens: &[TokenType]) -> bool { let mut url_count = 0; let mut word_count = 0; @@ -396,7 +340,10 @@ fn is_single_url>(tokens: &[TokenType]) -> bool { url_count == 1 && word_count <= 1 } -fn is_single_html_url>(html_tokens: &[HtmlToken], tokens: &[TokenType]) -> bool { +fn is_single_html_url( + html_tokens: &[HtmlToken], + tokens: &[TokenType], +) -> bool { let mut url_count = 0; let mut word_count = 0; @@ -432,38 +379,46 @@ fn is_single_html_url>(html_tokens: &[HtmlToken], tokens: &[TokenT url_count == 1 } -impl PartialEq for UrlParts { +impl PartialEq for UrlParts<'_> { fn eq(&self, other: &Self) -> bool { self.url == other.url } } -impl Eq for UrlParts {} +impl Eq for UrlParts<'_> {} -impl Hash for UrlParts { +impl Hash for UrlParts<'_> { fn hash(&self, state: &mut H) { self.url.hash(state); } } -impl UrlParts { - pub fn new(url: String) -> Self { +impl<'x> UrlParts<'x> { + pub fn new(url: impl Into>) -> Self { + let url_original = url.into(); + let url = url_original.trim().to_lowercase(); + Self { + url_parsed: url.parse::().ok().and_then(|url_parsed| { + if url_parsed.host().is_some() { + Some(UrlParsed { + host: Hostname::new(url_parsed.host().unwrap()), + parts: url_parsed, + }) + } else { + None + } + }), url, - url_parsed: None, + url_original, } } - pub fn with_parsed(mut self, url_parsed: UrlParsed) -> Self { - self.url_parsed = Some(url_parsed); - self - } - - pub fn with_parts(mut self, url_parsed: Uri, host: Hostname) -> Self { - self.url_parsed = Some(UrlParsed { - parts: url_parsed, - host, - }); - self + pub fn to_owned(&self) -> UrlParts<'static> { + UrlParts { + url: self.url.clone(), + url_original: Cow::Owned(self.url_original.clone().into_owned()), + url_parsed: self.url_parsed.clone(), + } } } diff --git a/crates/spam-filter/src/lib.rs b/crates/spam-filter/src/lib.rs index c1f39014..79552c48 100644 --- a/crates/spam-filter/src/lib.rs +++ b/crates/spam-filter/src/lib.rs @@ -7,6 +7,7 @@ pub mod analysis; pub mod modules; +use std::borrow::Cow; use std::collections::HashSet; use std::hash::{Hash, Hasher}; use std::net::{IpAddr, Ipv4Addr}; @@ -68,25 +69,31 @@ pub struct SpamFilterOutput<'x> { pub subject_lc: String, pub subject_thread: String, pub subject_thread_lc: String, - pub subject_tokens: Vec>, + pub subject_tokens: Vec, Email, UrlParts<'x>, IpParts<'x>>>, pub ips: AHashSet>, - pub urls: HashSet>, + pub urls: HashSet>>, pub emails: HashSet>, pub domains: HashSet>, pub text_parts: Vec>, } +#[derive(Debug)] +pub struct IpParts<'x> { + ip: Option, + text: Cow<'x, str>, +} + pub enum TextPart<'x> { Plain { text_body: &'x str, - tokens: Vec>, + tokens: Vec, Email, UrlParts<'x>, IpParts<'x>>>, }, Html { html_tokens: Vec, text_body: String, - tokens: Vec>, + tokens: Vec, Email, UrlParts<'x>, IpParts<'x>>>, }, None, } diff --git a/crates/spam-filter/src/modules/bayes.rs b/crates/spam-filter/src/modules/bayes.rs index f167b74a..b2e0184f 100644 --- a/crates/spam-filter/src/modules/bayes.rs +++ b/crates/spam-filter/src/modules/bayes.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{collections::HashSet, future::Future, time::Duration}; +use std::{borrow::Cow, collections::HashSet, future::Future, time::Duration}; use common::{ip_to_bytes, Server, KV_BAYES_MODEL_GLOBAL, KV_BAYES_MODEL_USER}; use mail_auth::DmarcResult; @@ -22,7 +22,7 @@ use store::dispatch::lookup::KeyValue; use trc::AddContext; use utils::cache::TtlEntry; -use crate::{SpamFilterContext, TextPart}; +use crate::{analysis::url::UrlParts, Email, IpParts, SpamFilterContext, TextPart}; pub trait BayesClassifier { fn bayes_train( @@ -97,10 +97,7 @@ impl BayesClassifier for Server { }) => { model.train( OsbTokenizer::new( - BayesTokenizer::new( - text_body, - tokens.iter().filter_map(to_bayes_token_owned), - ), + BayesTokenizer::new(text_body, tokens.iter().filter_map(to_bayes_token)), 5, ), is_spam, @@ -252,7 +249,7 @@ impl BayesClassifier for Server { text_body, tokens, .. }) => { for token in OsbTokenizer::<_, TokenHash>::new( - BayesTokenizer::new(text_body, tokens.iter().filter_map(to_bayes_token_owned)), + BayesTokenizer::new(text_body, tokens.iter().filter_map(to_bayes_token)), 5, ) { let weights = self @@ -455,10 +452,26 @@ fn add_prefix(prefix: u8, key: &[u8]) -> Vec { buf } -fn to_bayes_token(token: &TokenType<&str>) -> Option { +fn to_bayes_token( + token: &TokenType, Email, UrlParts<'_>, IpParts<'_>>, +) -> Option { token.to_bayes_token() } -fn to_bayes_token_owned(token: &TokenType) -> Option { - token.to_bayes_token() +impl AsRef for Email { + fn as_ref(&self) -> &str { + &self.address + } +} + +impl AsRef for UrlParts<'_> { + fn as_ref(&self) -> &str { + &self.url + } +} + +impl AsRef for IpParts<'_> { + fn as_ref(&self) -> &str { + &self.text + } } diff --git a/crates/spam-filter/src/modules/dnsbl.rs b/crates/spam-filter/src/modules/dnsbl.rs index 2f52d104..653319e3 100644 --- a/crates/spam-filter/src/modules/dnsbl.rs +++ b/crates/spam-filter/src/modules/dnsbl.rs @@ -11,19 +11,73 @@ use std::{ }; use common::{ - config::spamfilter::{DnsBlServer, IpResolver}, + config::spamfilter::{DnsBlServer, Element, IpResolver, Location}, expr::functions::ResolveVariable, Server, }; use mail_auth::{common::resolver::IntoFqdn, Error}; use trc::SpamEvent; +use crate::SpamFilterContext; + use super::expression::SpamFilterResolver; -pub(crate) async fn is_dnsbl( +pub(crate) async fn check_dnsbl( + server: &Server, + ctx: &mut SpamFilterContext<'_>, + resolver: &impl ResolveVariable, + scope: Element, + location: Location, +) { + let (mut checks, max_checks) = match scope { + Element::Email => ( + ctx.result.rbl_email_checks, + server.core.spam.dnsbl.max_email_checks, + ), + Element::Ip => ( + ctx.result.rbl_ip_checks, + server.core.spam.dnsbl.max_ip_checks, + ), + Element::Url => ( + ctx.result.rbl_url_checks, + server.core.spam.dnsbl.max_url_checks, + ), + Element::Domain => ( + ctx.result.rbl_domain_checks, + server.core.spam.dnsbl.max_domain_checks, + ), + Element::Header | Element::Body | Element::Any => unreachable!(), + }; + + for dnsbl in &server.core.spam.dnsbl.servers { + if dnsbl.scope == scope && checks < max_checks { + if let Some(tag) = is_dnsbl( + server, + dnsbl, + SpamFilterResolver::new(ctx, resolver, location), + &mut checks, + ) + .await + { + ctx.result.add_tag(tag); + } + } + } + + match scope { + Element::Email => ctx.result.rbl_email_checks = checks, + Element::Ip => ctx.result.rbl_ip_checks = checks, + Element::Url => ctx.result.rbl_url_checks = checks, + Element::Domain => ctx.result.rbl_domain_checks = checks, + Element::Header | Element::Body | Element::Any => unreachable!(), + } +} + +async fn is_dnsbl( server: &Server, config: &DnsBlServer, resolver: SpamFilterResolver<'_, impl ResolveVariable>, + checks: &mut usize, ) -> Option { let time = Instant::now(); let zone = server @@ -60,6 +114,8 @@ pub(crate) async fn is_dnsbl( Some(Some(result)) => result, Some(None) => return None, None => { + *checks += 1; + match server .core .smtp diff --git a/crates/spam-filter/src/modules/expression.rs b/crates/spam-filter/src/modules/expression.rs index 2e7f8a5e..df6aacfa 100644 --- a/crates/spam-filter/src/modules/expression.rs +++ b/crates/spam-filter/src/modules/expression.rs @@ -9,6 +9,7 @@ use common::{ expr::{functions::ResolveVariable, Variable}, }; use mail_parser::{Header, HeaderValue}; +use nlp::tokenizers::types::TokenType; use crate::{analysis::url::UrlParts, Recipient, SpamFilterContext, TextPart}; @@ -224,6 +225,42 @@ impl ResolveVariable for SpamFilterResolver<'_, T> { V_SPAM_SUBJECT => self.ctx.output.subject_lc.as_str().into(), V_SPAM_SUBJECT_THREAD => self.ctx.output.subject_thread_lc.as_str().into(), V_SPAM_LOCATION => self.location.as_str().into(), + V_WORDS_SUBJECT => self + .ctx + .output + .subject_tokens + .iter() + .filter_map(|w| match w { + TokenType::Alphabetic(w) + | TokenType::Alphanumeric(w) + | TokenType::Integer(w) + | TokenType::Float(w) => Some(Variable::String(w.as_ref().into())), + _ => None, + }) + .collect::>() + .into(), + V_WORDS_BODY => self + .ctx + .input + .message + .html_body + .first() + .and_then(|idx| self.ctx.output.text_parts.get(*idx)) + .map(|part| match part { + TextPart::Plain { tokens, .. } | TextPart::Html { tokens, .. } => tokens + .iter() + .filter_map(|w| match w { + TokenType::Alphabetic(w) + | TokenType::Alphanumeric(w) + | TokenType::Integer(w) + | TokenType::Float(w) => Some(Variable::String(w.as_ref().into())), + _ => None, + }) + .collect::>(), + TextPart::None => vec![], + }) + .unwrap_or_default() + .into(), _ => Variable::Integer(0), } } @@ -346,7 +383,7 @@ impl ResolveVariable for Recipient { } } -impl ResolveVariable for UrlParts { +impl ResolveVariable for UrlParts<'_> { fn resolve_variable(&self, variable: u32) -> Variable<'_> { match variable { V_URL_FULL => Variable::String(self.url.as_str().into()), diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 938381c2..ccddd211 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -6,7 +6,8 @@ resolver = "2" [features] #default = ["sqlite", "postgres", "mysql", "rocks", "elastic", "s3", "redis", "azure", "foundationdb"] -default = ["rocks"] +default = ["sqlite", "postgres", "mysql", "rocks", "s3", "redis", "foundationdb"] +#default = ["rocks"] sqlite = ["store/sqlite"] foundationdb = ["store/foundation", "common/foundation"] postgres = ["store/postgres"] diff --git a/tests/resources/smtp/antispam/combined.test b/tests/resources/smtp/antispam/combined.test index 87cdae76..d5f65693 100644 --- a/tests/resources/smtp/antispam/combined.test +++ b/tests/resources/smtp/antispam/combined.test @@ -6,8 +6,8 @@ spf.result none spf_ehlo.result none dmarc.result none remote_ip 195.210.29.48 -expect_header X-Spam-Result: ARC_NA (0.00), DKIM_NA (0.00), DMARC_NA (0.00), FROM_EQ_ENVFROM (0.00), FROM_HAS_DN (0.00), HAS_DATA_URI (0.00), MID_RHS_MATCH_ENVFROM (0.00), RCPT_COUNT_ONE (0.00), RCVD_COUNT_ZERO (0.00), SPF_NA (0.00), SUBJECT_ENDS_EXCLAIM (0.00), TO_DN_NONE (0.00), TO_MATCH_ENVRCPT_ALL (0.00), ONCE_RECEIVED (0.10), RCVD_NO_TLS_LAST (0.10), MIME_HTML_ONLY (0.20), HELO_NORES_A_OR_MX (0.30), AUTH_NA (1.00), DATE_IN_PAST (1.00), MID_RHS_MATCH_FROM (1.00), RDNS_NONE (1.00), FROMHOST_NORES_A_OR_MX (1.50), HTML_SHORT_LINK_IMG_1 (2.00), PYZOR (3.50) -expect_header X-Spam-Status: Yes, score=11.70 +expect_header X-Spam-Result: ARC_NA (0.00), DKIM_NA (0.00), FROM_EQ_ENV_FROM (0.00), FROM_HAS_DN (0.00), HAS_DATA_URI (0.00), MID_RHS_MATCH_ENV_FROM (0.00), RCPT_COUNT_ONE (0.00), RCVD_COUNT_ZERO (0.00), SPF_NA (0.00), SUBJECT_ENDS_EXCLAIM (0.00), TO_DN_NONE (0.00), TO_MATCH_ENVRCPT_ALL (0.00), ONCE_RECEIVED (0.10), RCVD_NO_TLS_LAST (0.10), MIME_HTML_ONLY (0.20), HELO_NORES_A_OR_MX (0.30), AUTH_NA (1.00), DATE_IN_PAST (1.00), DMARC_NA (1.00), MID_RHS_MATCH_FROM (1.00), FROMHOST_NORES_A_OR_MX (1.50), HTML_SHORT_LINK_IMG_1 (2.00), RDNS_NONE (2.00), PYZOR (3.50) +expect_header X-Spam-Status: Yes, score=13.70 From: Client Services To: licensing@stalw.art @@ -49,7 +49,7 @@ dkim.domains tenthrevolution.com dmarc.result pass remote_ip 185.58.86.181 tls.version TLSv1.3 -expect_header X-Spam-Result: DMARC_POLICY_ALLOW (-0.50), DKIM_ALLOW (-0.20), SPF_ALLOW (-0.20), MIME_GOOD (-0.10), ARC_NA (0.00), DKIM_SIGNED (0.00), FROM_EQ_ENVFROM (0.00), FROM_HAS_DN (0.00), HAS_ATTACHMENT (0.00), RCPT_COUNT_ONE (0.00), RCVD_COUNT_THREE (0.00), TO_DN_EQ_ADDR_ALL (0.00), TO_MATCH_ENVRCPT_ALL (0.00), RCVD_NO_TLS_LAST (0.10), HELO_NORES_A_OR_MX (0.30), SUBJECT_ENDS_SPACES (0.50), URI_COUNT_ODD (0.50), DATE_IN_PAST (1.00), FORGED_RCVD_TRAIL (1.00), FROMHOST_NORES_A_OR_MX (1.50) +expect_header X-Spam-Result: DMARC_POLICY_ALLOW (-0.50), DKIM_ALLOW (-0.20), SPF_ALLOW (-0.20), MIME_GOOD (-0.10), ARC_NA (0.00), DKIM_SIGNED (0.00), FROM_EQ_ENV_FROM (0.00), FROM_HAS_DN (0.00), HAS_ATTACHMENT (0.00), RCPT_COUNT_ONE (0.00), RCVD_COUNT_THREE (0.00), TO_DN_EQ_ADDR_ALL (0.00), TO_MATCH_ENVRCPT_ALL (0.00), RCVD_NO_TLS_LAST (0.10), HELO_NORES_A_OR_MX (0.30), SUBJECT_ENDS_SPACES (0.50), URI_COUNT_ODD (0.50), DATE_IN_PAST (1.00), FORGED_RCVD_TRAIL (1.00), FROMHOST_NORES_A_OR_MX (1.50) expect_header X-Spam-Status: No, score=3.90 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=tenthrevolution.com; @@ -574,8 +574,8 @@ dmarc.result fail dmarc.policy reject remote_ip 51.89.165.39 tls.version TLS1_2 -expect_header X-Spam-Result: DKIM_ALLOW (-0.20), HAS_LIST_UNSUB (-0.01), ARC_NA (0.00), DKIM_SIGNED (0.00), FROM_EQ_ENVFROM (0.00), FROM_HAS_DN (0.00), HAS_REPLYTO (0.00), MID_RHS_MATCH_ENVFROM (0.00), RCPT_COUNT_ONE (0.00), RCVD_COUNT_ZERO (0.00), REPLYTO_ADDR_EQ_FROM (0.00), REPLYTO_EQ_FROM (0.00), SPF_SOFTFAIL (0.00), TO_DN_NONE (0.00), TO_MATCH_ENVRCPT_ALL (0.00), ONCE_RECEIVED (0.10), RCVD_NO_TLS_LAST (0.10), HELO_NORES_A_OR_MX (0.30), DATE_IN_PAST (1.00), MID_RHS_MATCH_FROM (1.00), RDNS_NONE (1.00), R_PARTS_DIFFER (1.00), FROMHOST_NORES_A_OR_MX (1.50), DMARC_POLICY_REJECT (2.00), HTML_SHORT_LINK_IMG_1 (2.00), VIOLATED_DIRECT_SPF (3.50) -expect_header X-Spam-Status: Yes, score=13.29 +expect_header X-Spam-Result: DKIM_ALLOW (-0.20), HAS_LIST_UNSUB (-0.01), ARC_NA (0.00), DKIM_SIGNED (0.00), FROM_EQ_ENV_FROM (0.00), FROM_HAS_DN (0.00), HAS_REPLYTO (0.00), MID_RHS_MATCH_ENV_FROM (0.00), RCPT_COUNT_ONE (0.00), RCVD_COUNT_ZERO (0.00), REPLYTO_ADDR_EQ_FROM (0.00), REPLYTO_EQ_FROM (0.00), SPF_SOFTFAIL (0.00), TO_DN_NONE (0.00), TO_MATCH_ENVRCPT_ALL (0.00), ONCE_RECEIVED (0.10), RCVD_NO_TLS_LAST (0.10), HELO_NORES_A_OR_MX (0.30), DATE_IN_PAST (1.00), MID_RHS_MATCH_FROM (1.00), R_PARTS_DIFFER (1.00), FROMHOST_NORES_A_OR_MX (1.50), HTML_SHORT_LINK_IMG_1 (2.00), RDNS_NONE (2.00), VIOLATED_DIRECT_SPF (3.50), DMARC_POLICY_REJECT (4.00) +expect_header X-Spam-Status: Yes, score=16.29 DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; s=sectionalism; d=grupokonecta.net; h=To:Subject:Message-ID:Date:From:Reply-To:MIME-Version:List-Unsubscribe: @@ -778,4 +778,233 @@ e; CLICK AQUÍ CLICK AQUÍ CLICK AQUÍ CLICK AQUÍ"= --b1_3d217f30a568faa9ce3dd7dc73399561-- + +envelope_from miah.join@outlook.com +envelope_to hello@stalw.art +helo_domain HK2PR02CU002.outbound.protection.outlook.com +iprev.result pass +dkim.result pass +dkim.domains outlook.com +spf.result pass +spf_ehlo.result pass +dmarc.result pass +dmarc.policy reject +remote_ip 52.103.64.5 +tls.version TLS1_2 +expect_header X-Spam-Result: DMARC_POLICY_ALLOW (-0.50), DKIM_ALLOW (-0.20), SPF_ALLOW (-0.20), ARC_NA (0.00), ARC_SIGNED (0.00), DKIM_SIGNED (0.00), FREEMAIL_FROM (0.00), FROM_EQ_ENV_FROM (0.00), FROM_HAS_DN (0.00), HAS_SEO_WORD (0.00), HAS_X_PRIO_ONE (0.00), MID_RHS_MATCH_ENV_FROMTLD (0.00), MID_RHS_MATCH_FROMTLD (0.00), RCPT_COUNT_ONE (0.00), RCPT_IN_BODY (0.00), RCVD_COUNT_TWO (0.00), TO_DN_EQ_ADDR_ALL (0.00), TO_MATCH_ENVRCPT_ALL (0.00), RCVD_NO_TLS_LAST (0.10), HELO_NORES_A_OR_MX (0.30), DATE_IN_PAST (1.00), HEADER_EMPTY_DELIMITER (1.00), FROMHOST_NORES_A_OR_MX (1.50), SEO_SPAM (5.00) +expect_header X-Spam-Status: Yes, score=8.00 + +Return-Path: +ARC-Seal: i=1; a=rsa-sha256; s=arcselector10001; d=microsoft.com; cv=none; + b=sTW55J00fLHM5CSFAdYk6Kpyecib7sSXQWU51a+Eo6514pesoEtpNxM3eYurQfYQY7j+MMcwJ50u9fzJPOUm0JInaQMoDrUWJ5dObEglZtxbN1fpwHLOOP5rjWm+zd9p02jLCCpvoHnu4rIZmog1MO/pCiVRMWemUMzJ2O7mk2zbmode8ryb9tT1ho8XNeCYK9zKmoHwCl2p6TjO4HFQ4SU2hYIWd3//6gfnPDN2qIOgw6Z51zgsEtUYYENIKuHswZFWjt7925Wq380r5Fi+fsaKT8xAWFTq9igFNWKDVU2k7ZL6QlCsXpTRS57rrl1dBYAod1byHHbCOqa+g+VOAA== +ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=microsoft.com; + s=arcselector10001; + h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-AntiSpam-MessageData-ChunkCount:X-MS-Exchange-AntiSpam-MessageData-0:X-MS-Exchange-AntiSpam-MessageData-1; + bh=SVZW4LFWvuTy4mbM+Q+E+yH3a0DZrHFN3U3wHGbAHMQ=; + b=NavrW7s0URdfDuEFuzkxV7EUwJtiynvH1o9mzF39USQEfd9l1KyQpzOxo9Po8Dar1qqa/4ECNeUSmetx+NBDArmlTpBak+BKYXAXVRlHheyxILyU/f0RX01+7aifIzLj7LWv7Sx66b9D9/DjaVDbtMvFGPFUzk0JtiATahe7ZU0iKBvsRbGJjS9r0Sq2vHY/SQEUxOxKXUhUQBepSf9k7ibBZK27OhSz9v/jjSDCL/mh5MoOgbq7S8lbxUGS356c/Rm3ZWEInIRcbVqI+P75abEvzRNhRyBDId74h9IZnv+wz9QfGnk8TaFAExRBJ5BzIKlDibTZ+Kzuc+7mvOAKLw== +ARC-Authentication-Results: i=1; mx.microsoft.com 1; spf=none; dmarc=none; + dkim=none; arc=none +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=outlook.com; + s=selector1; + h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-SenderADCheck; + bh=SVZW4LFWvuTy4mbM+Q+E+yH3a0DZrHFN3U3wHGbAHMQ=; + b=Q6QpEIbyyvkSgmfTsPeVdPuTyh+6lA/+qAoEm5k5gEDuyqmLjwVELDsOJQAZzwfQfmxN02O5dpbD0mDWKLFR7Ft//121jF9EV06fbGMNXuuBpfZJ24npu+bPbHs66D7USSMEE6zvuf4bnlhVV0iTTxWwhNEawPfaFpuukvlVO1GtPOjH0SeymOnfHM3LrGSkwYpw5aeEGjrJLFQRSN+k8mD7PyoOkJFFBUyqySWdkRsQ5aw9+7f3wbHbDOb4rqkmkC6fUZSMcqpTSpFFS3fDlQQrcnwhh8ir/tq74AuVYyMoUMns81tExoILI78twEHxyGN2zgLw7K9QojJCf+IIEQ== +Received: from SEYPR04MB7496.apcprd04.prod.outlook.com (2603:1096:101:1db::7) + by PUZPR04MB6246.apcprd04.prod.outlook.com (2603:1096:301:ec::5) with + Microsoft SMTP Server (version=TLS1_2, + cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.8293.20; Tue, 31 Dec + 2024 08:24:54 +0000 +Received: from SEYPR04MB7496.apcprd04.prod.outlook.com + ([fe80::1aab:cf90:44d8:af4e]) by SEYPR04MB7496.apcprd04.prod.outlook.com + ([fe80::1aab:cf90:44d8:af4e%6]) with mapi id 15.20.8293.000; Tue, 31 Dec 2024 + 08:24:54 +0000 +From: Miah Join +To: "hello@stalw.art" +Subject: SEO-Inquiry +Thread-Topic: SEO-Inquiry +Thread-Index: AdtbXSTUEOZ2HAOVQCqvPiaCPhJ+xw== +Importance: high +X-Priority: 1 +Sensitivity: private +Date: Tue, 31 Dec 2024 08:24:22 +0000 +Message-ID: + +Accept-Language: en-US +Content-Language: en-US +X-MS-Has-Attach: +X-MS-TNEF-Correlator: +x-ms-publictraffictype: Email +x-ms-traffictypediagnostic: SEYPR04MB7496:EE_|PUZPR04MB6246:EE_ +x-ms-office365-filtering-correlation-id: d8480a0c-4e45-4278-ad1b-08dd29749a89 +x-ms-exchange-slblob-mailprops: + SoURN12IA8vYLoDX/njsxmIBKrA5xww7SXj4BirxWi/11KcMwu1z/THqGhGNTFhVj0W89DLwoSD+IqZucSTc96MoKe9ia4xsvj73oWgcR/6suK5u7GoMSUHmwhrgVtaAkaceSYHenT+iTqqsr6L31R3bTGzhR1r7TEKkTLaJO6pXyfeXy8wUEdMEUOpfggfX/BBwA6KLTGvIE9euGA810m3+uPbUX+cE3WNT2yFsiC+H/YFGbcISQcbwlqT/7fAjiNT6fore4X1HnccC1CzArvmHrHF37pjc/JTEQFUxg7woH0GKJtvsUFbDUPvXjKp9rTRqLHTb8fYVSRdeivWB4lJocoW5VqJ9YichWUGp1ELEDuto7T/ourT4i6Q/Z+D+3/Q67J0SAdD8+XAXHfYKG2rq14F+tz4KxbO6EPp7O6HpaHMSbpzps5/sOJ2gmeHucnICmy81mKE9aq2sfJPPdymPBczv4hB7QO+xQDHFNpypNedZ8cTbtW5MlVn18lgmP0BuDBEgU4WcbKu2bMzgl6HJRQctSSldfkkyfGI/URoo69SEmOZiDuuCzhKQQ09ho6xRG3kpsD9DyRAUQyyo3Wq4J1UgLuRyRVFYwPI4zL+e47La1ilj199UD/SJrbHMMTt6itlRBpnH4N7Ev1ELztRfH5VjQ5aG +x-microsoft-antispam: + BCL:0;ARA:14566002|8060799006|7092599003|8062599003|5062599005|15080799006|461199028|19110799003|440099028|3412199025|102099032; +x-microsoft-antispam-message-info: + =?us-ascii?Q?sSaF+W5EA6WGixAhyYLVcDu/rxF60TPo55Pe5lDbuGq4SQKPFczK5OPYJQHf?= + =?us-ascii?Q?6w0tFaoHE+b0IsosJe0H/dJXTmVvChLHdVnaTZiPcDKrZVOcLyvm2WnEoemj?= + =?us-ascii?Q?m21GgioEUF/NQdwaXt6AAYv18zt1GxKufXRYIpLkmKM8gV1XQMXPJteH53ax?= + =?us-ascii?Q?vifrj3RXruNdM31oe0LrZdIbsbKuvKDSdByf1mB2PVfCgEXIjvfFLgJidnah?= + =?us-ascii?Q?d30fWN6LcYhQvtVNfpiJwtroUR5oox/HxRtDbU+fkvn0LLoFX7nxdECueURA?= + =?us-ascii?Q?/67CsMYT5/NKrE+oaYmPjDYkLmS+YDBmWdUyuQDJIQ0Lb7vKisWAOFrweDgc?= + =?us-ascii?Q?pJUyUtAA5UliC4AO+gI+AH3cv2JaMn+Z7gCf9WpcTCk8PkOEtg4Y1661t4T3?= + =?us-ascii?Q?rOHV1Q7ibkz6JYCiIVGl1macRWWcIl1GeUWLUiHduVR/ax3PLUYLHYblCHv0?= + =?us-ascii?Q?XVrSCAAa8+vce6+c7ymVd8uBAvfTT/hfYKwSGR2R6V9clhEDjE5TXsIjx1qc?= + =?us-ascii?Q?AsCWjvexOp/pWalj3jwJzOE/xKHehhzyIeOJ0kN8snZBJiZpD9VnhYE6Lt8I?= + =?us-ascii?Q?4ttzIth0+B9mw24tVTPg2cZ3N9yFsznBvKxtuFYl3Oj3YzacJVbwk+XyRsjQ?= + =?us-ascii?Q?LXmGP3d8NPm2Nhy0yfrt4/wxf2th/e+/I3siWDuHcwJiDFdhvoT/AOuUoFl9?= + =?us-ascii?Q?5Vs6Amo3CmVo4TGEf+EQNPoLsXjVAMzpk/5TgnKQ2sanXVv1zPGrTT7aErrv?= + =?us-ascii?Q?m2fE/VhGQOqKiizPYaMaC6W4YPdrAYOOrpZFOgzFfMoNmyDmIfIucke8DJTg?= + =?us-ascii?Q?K630mHxneVrofGWREkohZiDUe/tQsjp/0Hy2x2792Pg2aumt5nc1Aw/QJrwr?= + =?us-ascii?Q?VHaKD7O2hL1QLJFbojSSKlWyGQ/CE0DsyUVihC0dWqfdWa8gPlHZ3uzzMN5J?= + =?us-ascii?Q?BazCEPgYBVnmiQw/ejBmBpm5iKIA0zgNKGva/8tDikzhviHpCe9lijBuNsu3?= + =?us-ascii?Q?OIfW8bDcJpyKB+g5S7EVo9K+lwYq0i86di3dpDvmSBfB99PFK9kqzaAEfAIU?= + =?us-ascii?Q?Duh11rVb5jqL5hm7Zhsb9T2kGOiAorAjmrAHQFSyCNO910vF+8s=3D?= +x-ms-exchange-antispam-messagedata-chunkcount: 1 +x-ms-exchange-antispam-messagedata-0: + =?us-ascii?Q?9c/kgRHXQSmmVIMMaHuipN3QMSRVkf870g3+luQav+fyRaGeQw2TxAqSMXIQ?= + =?us-ascii?Q?MqoXTcYcUTW8oy80k0fPln9Hz61fG51WDZ2vzG1wnYvfHer1zkAdp/njN/D2?= + =?us-ascii?Q?jXEtZZP4w7Fi1Tf3oyRzBLQhMtORyV3RQU/uWIRpjvP/jlu3zfVeIyFtON/q?= + =?us-ascii?Q?oJqPzYVyxeLv832rNiqEVsAPLOar26932L6xXlandRXwk6WZTN7J76uUF+i0?= + =?us-ascii?Q?FpKfqvx2IHOT/Qc8dJLf3H/iBywKPBINfmUdTdX83EzX9DYFmCJ0mJHhgmFV?= + =?us-ascii?Q?2r/JQEN6c+ziSN/sqFBrNvpWaEUGzyB7k4HT+HV9bv/8nx0jVYVdb8Jg/Lxl?= + =?us-ascii?Q?nt9FxkqJPCdTjYYQwCzM+74raMHz5o3URpFYGfEYsgBBrqZPa6oNgxZBrLU6?= + =?us-ascii?Q?w/m7VsQI9AbLh/jteycxh6INOvPy6SS3m7+FGFpFjwNAfSQLEyghWO0hyyk3?= + =?us-ascii?Q?75mJwXHJ53u+J6jMQQPEz6h2SFHN64nrGRbPq6ruSdyEhSj9BdkuEnKb/yAR?= + =?us-ascii?Q?07pJ7iBfQVTfQ04xXejXErXvrDZGCcVy3/1AOajGLmcW1iTgoRwF3KKTmKct?= + =?us-ascii?Q?g+/IaxelPpT95uCPhSWREdVLoxuQW6pBltFeXod8ou3YpiOoss2h0EplB4Xa?= + =?us-ascii?Q?TKPKZSDdPBeCbOwx9GQUhySu57WwarR4Q74+MQwx8zDHdlytk9bLFZrCSCnt?= + =?us-ascii?Q?OixF+n8R+5U1D+Vub74mpSguv+2efQNt2lwJei3iDd1mC6WRXdGaC4+WdCEt?= + =?us-ascii?Q?uPRtELZCDsLSUE+097ixYl7uMLCe8nHUFuECfu41T1yX9PPMEmpadpxazQCd?= + =?us-ascii?Q?USBa0u5BAMvXiZJHfPgf3y255JPI079k+DdhGkhF2cDwLciCHELy5rM52TGT?= + =?us-ascii?Q?IrCNmfxb/RmOWBVEsvbhV4glTLJlORemnfACjPfh8SzldzzIj2W8zUUrxJQt?= + =?us-ascii?Q?6bdoxj9FbxsJ3wvYQTDqF9olDDPh/2z1g28uXkknEgvcdU/XvlzE+bz3ACnC?= + =?us-ascii?Q?yoxmdm3P7/9aIojcU9CeOsnQBDHgBYOxXIK5vNogdrZV9Ew4G2w/gOevLmvA?= + =?us-ascii?Q?HAbDE1IaI2hhX0zhg2D/QCZZNpvlDgJzESkffQNSOYsIWlHeWBzOJeq7iM0Y?= + =?us-ascii?Q?pXEltktrRe3doxEsnAYKPirbULWz4u+XJXzAOddWBWiYGNi6ua5wuyv7pyRX?= + =?us-ascii?Q?CMc1g/sPb+5aGiPpbK4Si7KWv6HpNO3v0ACS0qrmdaaPWSx+wL8tsxO4KBI?= + =?us-ascii?Q?=3D?= +Content-Type: multipart/alternative; + boundary="_000_SEYPR04MB74966F6A34B2B0AC5DA8E6DDFD0A2SEYPR04MB7496apcp_" +MIME-Version: 1.0 +X-OriginatorOrg: outlook.com +X-MS-Exchange-CrossTenant-AuthAs: Internal +X-MS-Exchange-CrossTenant-AuthSource: SEYPR04MB7496.apcprd04.prod.outlook.com +X-MS-Exchange-CrossTenant-RMS-PersistedConsumerOrg: 00000000-0000-0000-0000-000000000000 +X-MS-Exchange-CrossTenant-Network-Message-Id: d8480a0c-4e45-4278-ad1b-08dd29749a89 +X-MS-Exchange-CrossTenant-originalarrivaltime: 31 Dec 2024 08:24:22.5882 + (UTC) +X-MS-Exchange-CrossTenant-fromentityheader: Hosted +X-MS-Exchange-CrossTenant-id: 84df9e7f-e9f6-40af-b435-aaaaaaaaaaaa +X-MS-Exchange-CrossTenant-rms-persistedconsumerorg: 00000000-0000-0000-0000-000000000000 +X-MS-Exchange-Transport-CrossTenantHeadersStamped: PUZPR04MB6246 + +--_000_SEYPR04MB74966F6A34B2B0AC5DA8E6DDFD0A2SEYPR04MB7496apcp_ +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: quoted-printable + +Hello, hello@stalw.art + + + +I'm just checking with you to see if you're interested in (SEO) search engi= +ne optimization, or if you're interested in Google 1st page for better busi= +ness. + + + +If so, I'd love to tell you a little bit more about my abilities and show y= +ou some of my work. I am a very + +Skilled SEO expert with various abilities and can (1st Page on Google). any= +thing. + + + +I look forward to hearing from you. + + + +Thanks, + +Miah + +--_000_SEYPR04MB74966F6A34B2B0AC5DA8E6DDFD0A2SEYPR04MB7496apcp_ +Content-Type: text/html; charset="us-ascii" +Content-Transfer-Encoding: quoted-printable + + + + + + + + +
+

Hello, hello@stalw.art  = +;    

+

 

+

I’m just checking with= + you to see if you’re interested in +(SEO) search engi= +ne optimization, or if you’re interes= +ted in Google 1st page for better business.  +

+

 

+

If so, I’d love to tel= +l you a little bit more about my abilities and show you some of my work. I = +am a very  +

+

Skilled SEO expert with vari= +ous abilities and can (1st Page on Google). anything.  +

+

 

+

I look forward to hearing fr= +om you.  +

+

 

+

Thanks,

+

Miah

+
+ + + +--_000_SEYPR04MB74966F6A34B2B0AC5DA8E6DDFD0A2SEYPR04MB7496apcp_-- diff --git a/tests/resources/smtp/antispam/from.test b/tests/resources/smtp/antispam/from.test index ad49442f..16e4c53f 100644 --- a/tests/resources/smtp/antispam/from.test +++ b/tests/resources/smtp/antispam/from.test @@ -5,7 +5,7 @@ X-From: hello@domain.org Test envelope_from hello@domain.org -expect MULTIPLE_FROM FROM_EQ_ENVFROM FROM_NO_DN +expect MULTIPLE_FROM FROM_EQ_ENV_FROM FROM_NO_DN From: hello@domain.org From: hello@domain.org @@ -13,70 +13,70 @@ From: hello@domain.org Test envelope_from test -expect FROM_INVALID ENVFROM_INVALID +expect FROM_INVALID ENV_FROM_INVALID From: test Test envelope_from www-data@domain.org -expect FROM_SERVICE_ACCT FROM_HAS_DN FROM_EQ_ENVFROM +expect FROM_SERVICE_ACCT FROM_HAS_DN FROM_EQ_ENV_FROM From: "WWW DATA" Test envelope_from hello@domain.org -expect FROM_DN_EQ_ADDR FROM_EQ_ENVFROM +expect FROM_DN_EQ_ADDR FROM_EQ_ENV_FROM From: "hello@domain.org" Test envelope_from hello@domain.org -expect SPOOF_DISPLAY_NAME FROM_EQ_ENVFROM FROM_HAS_DN +expect SPOOF_DISPLAY_NAME FROM_EQ_ENV_FROM FROM_HAS_DN From: "hello@otherdomain.org" Test envelope_from hello@domain.co.uk -expect FROM_NEQ_DISPLAY_NAME FROM_EQ_ENVFROM FROM_HAS_DN +expect FROM_NEQ_DISPLAY_NAME FROM_EQ_ENV_FROM FROM_HAS_DN From: "hello@other.domain.co.uk" Test helo_domain mx.domain.co.uk -expect FROMTLD_EQ_ENVFROMTLD FROM_NEQ_DISPLAY_NAME FROM_HAS_DN FROM_BOUNCE +expect FROMTLD_EQ_ENV_FROMTLD FROM_NEQ_DISPLAY_NAME FROM_HAS_DN FROM_BOUNCE From: "postmaster@mx.domain.co.uk" Test helo_domain mx.domain.co.uk -expect FROMTLD_EQ_ENVFROMTLD FROM_HAS_DN FROM_BOUNCE +expect FROMTLD_EQ_ENV_FROMTLD FROM_HAS_DN FROM_BOUNCE From: "Mailer Daemon" Test envelope_from mrspammer@domain.org -expect FROM_NAME_HAS_TITLE FROM_NAME_EXCESS_SPACE FROM_EQ_ENVFROM FROM_HAS_DN +expect FROM_NAME_HAS_TITLE FROM_NAME_EXCESS_SPACE FROM_EQ_ENV_FROM FROM_HAS_DN From: "Mr. Money Maker" Test envelope_from hello+world@domain.org -expect TAGGED_FROM FROM_EQ_ENVFROM FROM_NO_DN +expect TAGGED_FROM FROM_EQ_ENV_FROM FROM_NO_DN From: hello+world@domain.org Test envelope_from hello@domain.org -expect TO_EQ_FROM FROM_EQ_ENVFROM FROM_NO_DN +expect TO_EQ_FROM FROM_EQ_ENV_FROM FROM_NO_DN From: hello@domain.org To: hello@domain.org @@ -84,7 +84,7 @@ To: hello@domain.org Test envelope_from hello@domain.org -expect FROM_EQ_ENVFROM FROM_NO_DN +expect FROM_EQ_ENV_FROM FROM_NO_DN From: hello@domain.org To: hello@domain.org, bye@domain.org @@ -92,7 +92,7 @@ To: hello@domain.org, bye@domain.org Test envelope_from hello@domain.org -expect FROM_NEEDS_ENCODING FROM_EQ_ENVFROM FROM_HAS_DN +expect FROM_NEEDS_ENCODING FROM_EQ_ENV_FROM FROM_HAS_DN From: "Hélló" @@ -100,42 +100,42 @@ Test param.smtputf8 1 envelope_from hello@domain.org -expect FROM_EQ_ENVFROM FROM_HAS_DN +expect FROM_EQ_ENV_FROM FROM_HAS_DN From: "Hélló" Test envelope_from hello@domain.org -expect FROM_EXCESS_QP FROM_EQ_ENVFROM FROM_HAS_DN +expect FROM_EXCESS_QP FROM_EQ_ENV_FROM FROM_HAS_DN From: =?iso-8859-1?Q?Die_Hasen_und_die_Froesche?= Test envelope_from hello@domain.org -expect FROM_EXCESS_BASE64 FROM_EQ_ENVFROM FROM_HAS_DN +expect FROM_EXCESS_BASE64 FROM_EQ_ENV_FROM FROM_HAS_DN From: "=?iso-8859-1?B?RGllIEhhc2VuIHVuIGRpZSBGcm9lc2NoZQ==?=" Test envelope_from hello@domain.org -expect FROM_EQ_ENVFROM FROM_HAS_DN +expect FROM_EQ_ENV_FROM FROM_HAS_DN From: "=?iso-8859-1?Q?Die_Hasen_und_die_Fr=F6sche?=" Test envelope_from hello@domain.org -expect R_NO_SPACE_IN_FROM FROM_EQ_ENVFROM FROM_HAS_DN +expect R_NO_SPACE_IN_FROM FROM_EQ_ENV_FROM FROM_HAS_DN From: "Hello" Test envelope_from hello@domain.org -expect HEADER_RCONFIRM_MISMATCH FROM_EQ_ENVFROM FROM_HAS_DN +expect HEADER_RCONFIRM_MISMATCH FROM_EQ_ENV_FROM FROM_HAS_DN From: "Hello" X-Confirm-Reading-To: @@ -143,7 +143,7 @@ X-Confirm-Reading-To: Test envelope_from hello@domain.org -expect HEADER_FORGED_MDN FROM_EQ_ENVFROM FROM_HAS_DN +expect HEADER_FORGED_MDN FROM_EQ_ENV_FROM FROM_HAS_DN From: "Hello" Disposition-Notification-To: @@ -151,7 +151,7 @@ Disposition-Notification-To: Test envelope_from anonymous@domain.org -expect FROM_SERVICE_ACCT WWW_DOT_DOMAIN FROM_EQ_ENVFROM FROM_HAS_DN +expect FROM_SERVICE_ACCT WWW_DOT_DOMAIN FROM_EQ_ENV_FROM FROM_HAS_DN From: "Hello" Reply-to: @@ -159,49 +159,49 @@ Reply-to: Test envelope_from hello@custom.disposable.org -expect FREEMAIL_FROM DISPOSABLE_ENV_FROM FROM_NEQ_ENVFROM FROM_NO_DN FORGED_SENDER +expect FREEMAIL_FROM DISPOSABLE_ENV_FROM FROM_NEQ_ENV_FROM FROM_NO_DN FORGED_SENDER From: hello@gmail.com Test envelope_from hello@gmail.com -expect DISPOSABLE_FROM FREEMAIL_ENV_FROM FROM_NEQ_ENVFROM FROM_NO_DN FORGED_SENDER +expect DISPOSABLE_FROM FREEMAIL_ENV_FROM FROM_NEQ_ENV_FROM FROM_NO_DN FORGED_SENDER From: hello@custom.disposable.org Test envelope_from hello@nomx.org -expect FROMHOST_NORES_A_OR_MX FROM_EQ_ENVFROM FROM_NO_DN +expect FROMHOST_NORES_A_OR_MX FROM_EQ_ENV_FROM FROM_NO_DN From: hello@nomx.org Test envelope_from baz@domain.org -expect SPOOF_DISPLAY_NAME FROM_HAS_DN FROM_EQ_ENVFROM +expect SPOOF_DISPLAY_NAME FROM_HAS_DN FROM_EQ_ENV_FROM From: "Foo (foo@bar.com)" Test envelope_from baz@domain.org -expect SPOOF_DISPLAY_NAME FROM_HAS_DN FROM_EQ_ENVFROM +expect SPOOF_DISPLAY_NAME FROM_HAS_DN FROM_EQ_ENV_FROM From: Foo (foo@bar.com) Test envelope_from baz@domain.org -expect SPOOF_DISPLAY_NAME FROM_HAS_DN FROM_EQ_ENVFROM +expect SPOOF_DISPLAY_NAME FROM_HAS_DN FROM_EQ_ENV_FROM From: "Foo foo@bar.com" Test envelope_from baz@domain.org -expect SPOOF_DISPLAY_NAME FROM_HAS_DN FROM_EQ_ENVFROM +expect SPOOF_DISPLAY_NAME FROM_HAS_DN FROM_EQ_ENV_FROM From: "Foo 'foo@bar.com'" diff --git a/tests/resources/smtp/antispam/messageid.test b/tests/resources/smtp/antispam/messageid.test index deff8460..80e2cd46 100644 --- a/tests/resources/smtp/antispam/messageid.test +++ b/tests/resources/smtp/antispam/messageid.test @@ -81,7 +81,7 @@ Message-ID: <1234@host.domain.co.uk> Test envelope_from hello@domain.co.uk -expect MID_RHS_MATCH_ENVFROMTLD +expect MID_RHS_MATCH_ENV_FROMTLD Message-ID: <1234@host.domain.co.uk> diff --git a/tests/resources/smtp/antispam/rbl.test b/tests/resources/smtp/antispam/rbl.test index 52799ee1..89afc4b4 100644 --- a/tests/resources/smtp/antispam/rbl.test +++ b/tests/resources/smtp/antispam/rbl.test @@ -1,5 +1,5 @@ remote_ip 20.11.0.1 -expect RCVD_IN_DNSWL_LOW +expect RCVD_IN_DNSWL_LOW RBL_SENDERSCORE_REPUT_0 Subject: test @@ -7,7 +7,7 @@ test remote_ip 20.11.0.2 -expect RBL_SENDERSCORE RBL_NIXSPAM RBL_SEM RBL_SPAMHAUS_SBL RBL_BARRACUDA RBL_BLOCKLISTDE RBL_VIRUSFREE_BOTNET RBL_SPAMCOP RCVD_IN_DNSWL_MED +expect RBL_SENDERSCORE_REPUT_0 RBL_NIXSPAM RBL_SEM RBL_SPAMHAUS_SBL RBL_BARRACUDA RBL_BLOCKLISTDE RBL_VIRUSFREE_BOTNET RBL_SPAMCOP RCVD_IN_DNSWL_MED Subject: test @@ -15,7 +15,7 @@ test remote_ip 20.11.0.14 -expect RWL_MAILSPIKE_NEUTRAL RECEIVED_SPAMHAUS_SBL RECEIVED_SPAMHAUS_XBL RECEIVED_BLOCKLISTDE RCVD_IN_DNSWL_MED +expect RBL_SENDERSCORE_REPUT_1 RWL_MAILSPIKE_NEUTRAL RECEIVED_SPAMHAUS_SBL RECEIVED_SPAMHAUS_XBL RECEIVED_BLOCKLISTDE RCVD_IN_DNSWL_MED Received: from Agni (localhost [20.11.0.5]) (TLS: TLSv1/SSLv3, 168bits,DES-CBC3-SHA) by agni.forevermore.net with esmtp; Mon, 28 Oct 2002 14:48:52 -0800 @@ -58,7 +58,7 @@ Subject: test test -expect SURBL_HASHBL_ABUSE SURBL_HASHBL_MALWARE SURBL_HASHBL_PHISH URL_ONLY +expect SURBL_HASHBL_ABUSE SURBL_HASHBL_MALWARE SURBL_HASHBL_PHISH URL_ONLY REDIRECTOR_URL From: spammer@spamcorp.net Reply-To: User @@ -66,6 +66,6 @@ Subject: test Content-Type: text/html; charset="utf-8" -test -https://phishing.net/logintest +https://lnkiy.in/other/path?query=true diff --git a/tests/resources/smtp/antispam/recipient.test b/tests/resources/smtp/antispam/recipient.test index 7d84faae..d56f90b6 100644 --- a/tests/resources/smtp/antispam/recipient.test +++ b/tests/resources/smtp/antispam/recipient.test @@ -189,3 +189,10 @@ To: user@test.org Cc: "Test" Test + +expect RCPT_IN_BODY TO_DN_NONE RCPT_COUNT_ONE + +To: hello@world.com +Subject: Special offer + +An offer for hello@world.com diff --git a/tests/resources/smtp/antispam/url.test b/tests/resources/smtp/antispam/url.test index e6da8d67..60d64481 100644 --- a/tests/resources/smtp/antispam/url.test +++ b/tests/resources/smtp/antispam/url.test @@ -114,3 +114,11 @@ Partner: 12345678
Portal: IP-Sperre einsehen + +expect RCPT_DOMAIN_IN_BODY + +To: hello@world.com +Subject: Special offer + +An offer for world.com + diff --git a/tests/src/smtp/inbound/antispam.rs b/tests/src/smtp/inbound/antispam.rs index 468e9cfe..b6572112 100644 --- a/tests/src/smtp/inbound/antispam.rs +++ b/tests/src/smtp/inbound/antispam.rs @@ -117,8 +117,9 @@ allow-invalid-certs = true "known-dmarc-domains" = {"dmarc-allow.org"} "spam-traps" = {"spamtrap@*"} "trusted-domains" = {"stalw.art"} -"freemail-providers" = {"gmail.com", "googlemail.com", "yahoomail.com", "*freemail.org"} +"freemail-providers" = {"gmail.com", "googlemail.com", "yahoomail.com", "outlook.com", "*freemail.org"} "disposable-providers" = {"guerrillamail.com", "*disposable.org"} +"surbl-hashbl" = {"bit.ly", "drive.google.com", "lnkiy.in"} "#; #[tokio::test(flavor = "multi_thread")] @@ -188,15 +189,15 @@ async fn antispam() { "127.0.1.3", ), ( - "94c57fe69a113e875f772bdea55bf2c3.hashbl.surbl.org", + "ba76e47680ba70a0cbff8d6c92139683.hashbl.surbl.org", "127.0.0.16", ), ( - "64aca53deb83db2ba30a59604ada2d80.hashbl.surbl.org", + "0ac5b387a1c6d8461a78bbf7b172a2a1.hashbl.surbl.org", "127.0.0.64", ), ( - "02159eed92622b2fb8c83c659f269007.hashbl.surbl.org", + "637d6717761b5de0c84108c894bb68f2.hashbl.surbl.org", "127.0.0.8", ), ] { @@ -542,6 +543,7 @@ async fn antispam() { server.spam_filter_analyze_headers(&mut spam_ctx).await; spam_ctx.result.tags.retain(|t| t.starts_with("X_HDR_")); server.spam_filter_analyze_recipient(&mut spam_ctx).await; + server.spam_filter_analyze_domain(&mut spam_ctx).await; server.spam_filter_analyze_rules(&mut spam_ctx).await; spam_ctx.result.tags.retain(|t| !t.starts_with("X_HDR_")); }