From 44ae796d9b1ad4a623865f5e193eb70c0455253d Mon Sep 17 00:00:00 2001 From: mdecimus Date: Tue, 10 Dec 2024 18:56:09 +0100 Subject: [PATCH] Port Spam filter to Rust - part 4 --- crates/common/src/addresses.rs | 4 + crates/common/src/config/spamfilter.rs | 42 ++- crates/common/src/enterprise/alerts.rs | 4 + crates/common/src/expr/eval.rs | 45 ++- crates/common/src/expr/functions/mod.rs | 1 + crates/common/src/expr/mod.rs | 2 + crates/common/src/expr/parser.rs | 4 + crates/common/src/expr/tokenizer.rs | 11 +- crates/common/src/listener/mod.rs | 4 + crates/jmap/src/api/http.rs | 4 + crates/smtp/src/inbound/session.rs | 4 + crates/smtp/src/queue/mod.rs | 12 + crates/spam-filter/src/analysis/domain.rs | 198 ++++++++++ crates/spam-filter/src/analysis/init.rs | 40 +- crates/spam-filter/src/analysis/ip.rs | 124 +++++++ crates/spam-filter/src/analysis/iprev.rs | 23 -- crates/spam-filter/src/analysis/mod.rs | 78 +++- crates/spam-filter/src/analysis/url.rs | 261 +++++++++++-- crates/spam-filter/src/lib.rs | 6 +- crates/spam-filter/src/modules/dnsbl.rs | 15 +- resources/config/spamfilter/scripts/rbl.sieve | 351 ------------------ tests/src/smtp/config.rs | 4 + 22 files changed, 760 insertions(+), 477 deletions(-) create mode 100644 crates/spam-filter/src/analysis/domain.rs create mode 100644 crates/spam-filter/src/analysis/ip.rs delete mode 100644 crates/spam-filter/src/analysis/iprev.rs delete mode 100644 resources/config/spamfilter/scripts/rbl.sieve diff --git a/crates/common/src/addresses.rs b/crates/common/src/addresses.rs index b6d05cf0..edb7ec37 100644 --- a/crates/common/src/addresses.rs +++ b/crates/common/src/addresses.rs @@ -172,6 +172,10 @@ impl ResolveVariable for Address<'_> { fn resolve_variable(&self, _: u32) -> crate::expr::Variable { Variable::from(self.0) } + + fn resolve_global(&self, _: &str) -> Variable<'_> { + Variable::Integer(0) + } } impl AddressMapping { diff --git a/crates/common/src/config/spamfilter.rs b/crates/common/src/config/spamfilter.rs index 376b0de6..c0573ebd 100644 --- a/crates/common/src/config/spamfilter.rs +++ b/crates/common/src/config/spamfilter.rs @@ -6,12 +6,19 @@ use std::time::Duration; +use ahash::AHashSet; +use mail_parser::HeaderName; use utils::{config::Config, glob::GlobSet}; -use super::{if_block::IfBlock, Expression}; +use super::if_block::IfBlock; #[derive(Debug, Clone, Default)] pub struct SpamFilterConfig { + pub max_rbl_ip_checks: usize, + pub max_rbl_domain_checks: usize, + pub max_rbl_email_checks: usize, + pub max_rbl_url_checks: usize, + pub list_dmarc_allow: GlobSet, pub list_spf_dkim_allow: GlobSet, pub list_freemail_providers: GlobSet, @@ -23,13 +30,26 @@ pub struct SpamFilterConfig { } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum Target { +pub enum Element { Url, Domain, Email, Ip, - Ipv4, - Ipv6, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum Location { + EnvelopeFrom, + EnvelopeTo, + DkimPassing, + Ehlo, + Header(HeaderName<'static>), + BodyText, + BodyHtml, + BodyRaw, + Message, + Attachment, + Tcp, } #[derive(Debug, Clone)] @@ -43,15 +63,17 @@ pub struct RemoteListConfig { pub max_entries: usize, // 100000 pub max_entry_size: usize, // 256 pub format: RemoteListFormat, - pub target: Target, + pub element: Element, + pub element_location: AHashSet, pub tag: String, } #[derive(Debug, Clone)] pub struct DnsblConfig { pub id: String, - pub zone: Expression, - pub target: Target, + pub zone: IfBlock, + pub element: Element, + pub element_location: AHashSet, pub tags: IfBlock, } @@ -70,3 +92,9 @@ impl SpamFilterConfig { SpamFilterConfig::default() } } + +impl From> for Location { + fn from(header: HeaderName<'static>) -> Self { + Location::Header(header) + } +} diff --git a/crates/common/src/enterprise/alerts.rs b/crates/common/src/enterprise/alerts.rs index 6cff45e2..dda7eea2 100644 --- a/crates/common/src/enterprise/alerts.rs +++ b/crates/common/src/enterprise/alerts.rs @@ -119,6 +119,10 @@ impl ResolveVariable for CollectorResolver { Variable::Integer(0) } } + + fn resolve_global(&self, _: &str) -> Variable<'_> { + Variable::Integer(0) + } } impl AlertContent { diff --git a/crates/common/src/expr/eval.rs b/crates/common/src/expr/eval.rs index 60ac4ba8..d9bcafcd 100644 --- a/crates/common/src/expr/eval.rs +++ b/crates/common/src/expr/eval.rs @@ -4,9 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{borrow::Cow, cmp::Ordering, fmt::Display}; +use std::{borrow::Cow, cmp::Ordering, fmt::Display, net::IpAddr}; use hyper::StatusCode; +use mail_auth::common::resolver::ToReverseName; use trc::EvalEvent; use crate::Server; @@ -14,6 +15,7 @@ use crate::Server; use super::{ functions::{ResolveVariable, FUNCTIONS}, if_block::IfBlock, + tokenizer::TokenMap, BinaryOperator, Constant, Expression, ExpressionItem, UnaryOperator, Variable, }; @@ -164,6 +166,9 @@ impl Expression { ExpressionItem::Variable(v) => { stack.push(resolver.resolve_variable(*v)); } + ExpressionItem::Global(v) => { + stack.push(resolver.resolve_global(v)); + } ExpressionItem::Constant(val) => { stack.push(Variable::from(val)); } @@ -682,19 +687,45 @@ impl<'x> TryFrom> for StatusCode { } impl<'x> ResolveVariable for &'x str { - fn resolve_variable(&self, variable: u32) -> Variable<'x> { - match variable { - 0 => Variable::String((*self).into()), - _ => Variable::Integer(0), - } + fn resolve_variable(&self, _: u32) -> Variable<'x> { + Variable::String((*self).into()) + } + + fn resolve_global(&self, _: &str) -> Variable<'_> { + Variable::Integer(0) } } impl ResolveVariable for Vec { + fn resolve_variable(&self, _: u32) -> Variable<'_> { + Variable::Array(self.iter().map(|v| Variable::String(v.into())).collect()) + } + + fn resolve_global(&self, _: &str) -> Variable<'_> { + Variable::Integer(0) + } +} + +impl ResolveVariable for IpAddr { fn resolve_variable(&self, variable: u32) -> Variable<'_> { match variable { - 0 => Variable::Array(self.iter().map(|v| Variable::String(v.into())).collect()), + 0 => Variable::String(self.to_string().into()), + 1 => Variable::String(self.to_reverse_name().into()), _ => Variable::Integer(0), } } + + fn resolve_global(&self, _: &str) -> Variable<'_> { + Variable::Integer(0) + } +} + +impl TokenMap { + pub fn new_ip() -> Self { + TokenMap::default().with_variables_map([("ip", 0), ("reverse_ip", 1)]) + } + + pub fn new_single(name: &'static str) -> Self { + TokenMap::default().with_variables_map([(name, 0)]) + } } diff --git a/crates/common/src/expr/functions/mod.rs b/crates/common/src/expr/functions/mod.rs index 11b17ec0..6fe274f8 100644 --- a/crates/common/src/expr/functions/mod.rs +++ b/crates/common/src/expr/functions/mod.rs @@ -16,6 +16,7 @@ pub mod text; pub trait ResolveVariable: Sync + Send { fn resolve_variable(&self, variable: u32) -> Variable<'_>; + fn resolve_global(&self, variable: &str) -> Variable<'_>; } impl<'x> Variable<'x> { diff --git a/crates/common/src/expr/mod.rs b/crates/common/src/expr/mod.rs index 0c190de3..ddd2f5f2 100644 --- a/crates/common/src/expr/mod.rs +++ b/crates/common/src/expr/mod.rs @@ -83,6 +83,7 @@ pub struct Expression { #[derive(Debug, Clone)] pub enum ExpressionItem { Variable(u32), + Global(String), Capture(u32), Constant(Constant), BinaryOperator(BinaryOperator), @@ -187,6 +188,7 @@ pub enum UnaryOperator { #[derive(Debug, Clone)] pub enum Token { Variable(u32), + Global(String), Capture(u32), Function { name: Cow<'static, str>, diff --git a/crates/common/src/expr/parser.rs b/crates/common/src/expr/parser.rs index 28fc0756..55214438 100644 --- a/crates/common/src/expr/parser.rs +++ b/crates/common/src/expr/parser.rs @@ -41,6 +41,10 @@ impl<'x> ExpressionParser<'x> { self.inc_arg_count(); self.output.push(ExpressionItem::Constant(c)) } + Token::Global(g) => { + self.inc_arg_count(); + self.output.push(ExpressionItem::Global(g)) + } Token::Capture(c) => { self.inc_arg_count(); self.output.push(ExpressionItem::Capture(c)) diff --git a/crates/common/src/expr/tokenizer.rs b/crates/common/src/expr/tokenizer.rs index 7e7a6e0a..59e774a2 100644 --- a/crates/common/src/expr/tokenizer.rs +++ b/crates/common/src/expr/tokenizer.rs @@ -297,8 +297,15 @@ impl<'x> Tokenizer<'x> { } } - if let Some(regex_capture) = buf.strip_prefix('$').and_then(|v| v.parse::().ok()) { - Ok(Token::Capture(regex_capture)) + if let Some(variable) = buf.strip_prefix('$').filter(|s| !s.is_empty()) { + if variable.chars().all(|c| c.is_ascii_digit()) { + Ok(variable + .parse::() + .map(Token::Capture) + .unwrap_or_else(|_| Token::Global(variable.into()))) + } else { + Ok(Token::Global(variable.into())) + } } else if let Some((idx, (name, _, num_args))) = FUNCTIONS .iter() .enumerate() diff --git a/crates/common/src/listener/mod.rs b/crates/common/src/listener/mod.rs index 6704a228..5f4ffecd 100644 --- a/crates/common/src/listener/mod.rs +++ b/crates/common/src/listener/mod.rs @@ -234,6 +234,10 @@ impl ResolveVariable for SessionData { _ => crate::expr::Variable::default(), } } + + fn resolve_global(&self, _: &str) -> Variable<'_> { + Variable::Integer(0) + } } impl Debug for TcpAcceptor { diff --git a/crates/jmap/src/api/http.rs b/crates/jmap/src/api/http.rs index 5c5ce0eb..0553dc22 100644 --- a/crates/jmap/src/api/http.rs +++ b/crates/jmap/src/api/http.rs @@ -794,6 +794,10 @@ impl ResolveVariable for HttpContext<'_> { _ => Variable::default(), } } + + fn resolve_global(&self, _: &str) -> Variable<'_> { + Variable::Integer(0) + } } pub async fn fetch_body( diff --git a/crates/smtp/src/inbound/session.rs b/crates/smtp/src/inbound/session.rs index 58653636..e43ad991 100644 --- a/crates/smtp/src/inbound/session.rs +++ b/crates/smtp/src/inbound/session.rs @@ -581,4 +581,8 @@ impl ResolveVariable for Session { _ => expr::Variable::default(), } } + + fn resolve_global(&self, _: &str) -> Variable<'_> { + Variable::Integer(0) + } } diff --git a/crates/smtp/src/queue/mod.rs b/crates/smtp/src/queue/mod.rs index 5849054c..fd8d9d8d 100644 --- a/crates/smtp/src/queue/mod.rs +++ b/crates/smtp/src/queue/mod.rs @@ -280,6 +280,10 @@ impl<'x> ResolveVariable for QueueEnvelope<'x> { _ => "".into(), } } + + fn resolve_global(&self, _: &str) -> Variable<'_> { + Variable::Integer(0) + } } impl ResolveVariable for Message { @@ -297,6 +301,10 @@ impl ResolveVariable for Message { _ => "".into(), } } + + fn resolve_global(&self, _: &str) -> Variable<'_> { + Variable::Integer(0) + } } pub struct RecipientDomain<'x>(&'x str); @@ -314,6 +322,10 @@ impl<'x> ResolveVariable for RecipientDomain<'x> { _ => "".into(), } } + + fn resolve_global(&self, _: &str) -> Variable<'_> { + Variable::Integer(0) + } } #[inline(always)] diff --git a/crates/spam-filter/src/analysis/domain.rs b/crates/spam-filter/src/analysis/domain.rs new file mode 100644 index 00000000..a50f6ce3 --- /dev/null +++ b/crates/spam-filter/src/analysis/domain.rs @@ -0,0 +1,198 @@ +use std::{collections::HashSet, future::Future}; + +use common::{ + config::spamfilter::{Element, Location}, + expr::{functions::ResolveVariable, Variable}, + Server, +}; +use mail_auth::DkimResult; +use mail_parser::HeaderName; +use nlp::tokenizers::types::TokenType; + +use crate::{modules::dnsbl::is_dnsbl, Email, Recipient, SpamFilterContext, TextPart}; + +use super::{is_trusted_domain, ElementLocation, SpamFilterResolver}; + +pub trait SpamFilterAnalyzeDomain: Sync + Send { + fn spam_filter_analyze_domain( + &self, + ctx: &mut SpamFilterContext<'_>, + ) -> impl Future + Send; +} + +impl SpamFilterAnalyzeDomain for Server { + async fn spam_filter_analyze_domain(&self, ctx: &mut SpamFilterContext<'_>) { + // Obtain email addresses and domains + let mut domains = HashSet::new(); + let mut emails = HashSet::new(); + + // Add DKIM domains + for dkim in ctx.input.dkim_result { + if dkim.result() == &DkimResult::Pass { + if let Some(domain) = dkim.signature().map(|s| &s.d) { + domains.insert(ElementLocation::new( + domain.to_lowercase(), + Location::DkimPassing, + )); + } + } + } + + // Add EHLO domain + domains.insert(ElementLocation::new( + ctx.output.ehlo_host.fqdn.to_string(), + Location::Ehlo, + )); + + // Add PTR + if let Some(ptr) = &ctx.output.iprev_ptr { + domains.insert(ElementLocation::new(ptr.clone(), Location::Tcp)); + } + + // Add From, Envelope From and Reply-To + emails.insert(ElementLocation::new( + ctx.output.from.clone(), + HeaderName::From, + )); + if let Some(reply_to) = &ctx.output.reply_to { + emails.insert(ElementLocation::new(reply_to.clone(), HeaderName::ReplyTo)); + } + emails.insert(ElementLocation::new( + Recipient { + email: ctx.output.env_from_addr.clone(), + name: None, + }, + Location::EnvelopeFrom, + )); + + // Add emails found in the message + for (part_id, part) in ctx.output.text_parts.iter().enumerate() { + let is_body = ctx.input.message.text_body.contains(&part_id) + || ctx.input.message.html_body.contains(&part_id); + match part { + TextPart::Plain { tokens, .. } => emails.extend(tokens.iter().filter_map(|t| { + if let TokenType::Email(email) = t { + Some(ElementLocation::new( + Recipient { + email: Email::new(email), + name: None, + }, + if is_body { + Location::BodyText + } else { + Location::Attachment + }, + )) + } else { + None + } + })), + TextPart::Html { tokens, .. } => emails.extend(tokens.iter().filter_map(|t| { + if let TokenType::Email(email) = t { + Some(ElementLocation::new( + Recipient { + email: Email::new(email), + name: None, + }, + if is_body { + Location::BodyHtml + } else { + Location::Attachment + }, + )) + } else { + None + } + })), + TextPart::None => (), + } + } + + // Validate email + for email in emails { + // Skip trusted domains + if is_trusted_domain( + self, + &email.element.email.domain_part.fqdn, + ctx.input.span_id, + ) + .await + { + continue; + } + + // Check Email DNSBL + if ctx.result.rbl_email_checks < self.core.spam.max_rbl_email_checks { + for dnsbl in &self.core.spam.dnsbls { + if dnsbl.element == Element::Email + && dnsbl.element_location.contains(&email.location) + { + if let Some(tag) = + is_dnsbl(self, dnsbl, SpamFilterResolver::new(ctx, &email.element)) + .await + { + ctx.result.add_tag(tag); + } + } + } + ctx.result.rbl_email_checks += 1; + } + + domains.insert(ElementLocation::new( + email.element.email.domain_part.fqdn, + email.location, + )); + } + + // Validate domains + for domain in domains { + // Skip trusted domains + if is_trusted_domain(self, &domain.element, ctx.input.span_id).await { + continue; + } + + // Check Domain DNSBL + if ctx.result.rbl_domain_checks < self.core.spam.max_rbl_domain_checks { + for dnsbl in &self.core.spam.dnsbls { + if dnsbl.element == Element::Domain + && dnsbl.element_location.contains(&domain.location) + { + if let Some(tag) = is_dnsbl( + self, + dnsbl, + SpamFilterResolver::new(ctx, &domain.element.as_str()), + ) + .await + { + ctx.result.add_tag(tag); + } + } + } + ctx.result.rbl_domain_checks += 1; + } + } + } +} + +pub const V_RCPT_EMAIL: u32 = 0; +pub const V_RCPT_NAME: u32 = 1; +pub const V_RCPT_LOCAL: u32 = 2; +pub const V_RCPT_DOMAIN: u32 = 3; +pub const V_RCPT_DOMAIN_SLD: u32 = 4; + +impl ResolveVariable for Recipient { + fn resolve_variable(&self, variable: u32) -> Variable<'_> { + match variable { + V_RCPT_EMAIL => Variable::String(self.email.address.as_str().into()), + V_RCPT_NAME => Variable::String(self.name.as_deref().unwrap_or_default().into()), + V_RCPT_LOCAL => Variable::String(self.email.local_part.as_str().into()), + V_RCPT_DOMAIN => Variable::String(self.email.domain_part.fqdn.as_str().into()), + V_RCPT_DOMAIN_SLD => Variable::String(self.email.domain_part.sld_or_default().into()), + _ => Variable::Integer(0), + } + } + + fn resolve_global(&self, _: &str) -> Variable<'_> { + Variable::Integer(0) + } +} diff --git a/crates/spam-filter/src/analysis/init.rs b/crates/spam-filter/src/analysis/init.rs index a3fb9ed6..8d49419f 100644 --- a/crates/spam-filter/src/analysis/init.rs +++ b/crates/spam-filter/src/analysis/init.rs @@ -1,11 +1,9 @@ -use std::collections::HashSet; - use common::Server; use mail_parser::{parsers::fields::thread::thread_name, HeaderName, PartType}; use nlp::tokenizers::types::{TokenType, TypesTokenizer}; use crate::{ - modules::html::{html_to_tokens, HtmlToken, HREF, SRC}, + modules::html::{html_to_tokens, HtmlToken}, Email, Hostname, Recipient, SpamFilterContext, SpamFilterInput, SpamFilterOutput, SpamFilterResult, TextPart, }; @@ -186,37 +184,6 @@ impl SpamFilterInit for Server { } text_parts.extend(text_parts_nested); - // Extract URLs - let mut urls: HashSet = - HashSet::from_iter(subject_tokens.iter().filter_map(|t| t.url_lowercase(false))); - for part in &text_parts { - match part { - TextPart::Plain { tokens, .. } => { - urls.extend(tokens.iter().filter_map(|t| t.url_lowercase(false))); - } - TextPart::Html { - html_tokens, - tokens, - .. - } => { - for token in html_tokens { - if let HtmlToken::StartTag { attributes, .. } = token { - for (attr, value) in attributes { - match value { - Some(value) if [HREF, SRC].contains(attr) => { - urls.insert(value.trim().to_lowercase()); - } - _ => {} - } - } - } - } - urls.extend(tokens.iter().filter_map(|t| t.url_lowercase(false))); - } - TextPart::None => {} - } - } - let env_from_addr = Email::new(input.env_from); SpamFilterContext { output: SpamFilterOutput { @@ -247,12 +214,9 @@ impl SpamFilterInit for Server { recipients_cc, recipients_bcc, text_parts, - urls, }, input, - result: SpamFilterResult { - tags: Default::default(), - }, + result: SpamFilterResult::default(), } } } diff --git a/crates/spam-filter/src/analysis/ip.rs b/crates/spam-filter/src/analysis/ip.rs new file mode 100644 index 00000000..d25a1f38 --- /dev/null +++ b/crates/spam-filter/src/analysis/ip.rs @@ -0,0 +1,124 @@ +use std::{future::Future, net::IpAddr}; + +use common::{ + config::spamfilter::{Element, Location}, + Server, +}; +use mail_auth::IprevResult; +use mail_parser::{HeaderName, HeaderValue, Host}; +use nlp::tokenizers::types::TokenType; +use store::ahash::AHashSet; + +use crate::{modules::dnsbl::is_dnsbl, SpamFilterContext, TextPart}; + +use super::{ElementLocation, SpamFilterResolver}; + +pub trait SpamFilterAnalyzeIpRev: Sync + Send { + fn spam_filter_analyze_ip( + &self, + ctx: &mut SpamFilterContext<'_>, + ) -> impl Future + Send; +} + +impl SpamFilterAnalyzeIpRev for Server { + async fn spam_filter_analyze_ip(&self, ctx: &mut SpamFilterContext<'_>) { + // IP Address RBL + let mut ips = + AHashSet::from_iter([ElementLocation::new(ctx.input.remote_ip, Location::Tcp)]); + + // Obtain IP addresses from Received headers + for header in ctx.input.message.headers() { + if let (HeaderName::Received, HeaderValue::Received(received)) = + (&header.name, &header.value) + { + if let Some(ip) = received.from_ip() { + ips.insert(ElementLocation::new(ip, HeaderName::Received)); + } + for host in [&received.from, &received.helo, &received.by] + .into_iter() + .flatten() + { + if let Host::IpAddr(ip) = host { + ips.insert(ElementLocation::new(*ip, HeaderName::Received)); + } + } + } + } + + // Obtain IP addresses from the message body + for (part_id, part) in ctx.output.text_parts.iter().enumerate() { + let is_body = ctx.input.message.text_body.contains(&part_id) + || ctx.input.message.html_body.contains(&part_id); + match part { + TextPart::Plain { tokens, .. } => ips.extend(tokens.iter().filter_map(|t| { + if let TokenType::IpAddr(ip) = t { + ip.parse::().ok().map(|ip| { + ElementLocation::new( + ip, + if is_body { + Location::BodyText + } else { + Location::Attachment + }, + ) + }) + } else { + None + } + })), + TextPart::Html { tokens, .. } => ips.extend(tokens.iter().filter_map(|t| { + if let TokenType::IpAddr(ip) = t { + ip.parse::().ok().map(|ip| { + ElementLocation::new( + ip, + if is_body { + Location::BodyHtml + } else { + Location::Attachment + }, + ) + }) + } else { + None + } + })), + TextPart::None => (), + } + } + + // Validate IP addresses + for ip in ips { + if ip.element.is_loopback() + || ip.element.is_multicast() + || ip.element.is_unspecified() + || self.is_ip_allowed(&ip.element) + { + continue; + } else if self.is_ip_blocked(&ip.element) { + ctx.result.add_tag("IP_BLOCKED"); + continue; + } + + for dnsbl in &self.core.spam.dnsbls { + if dnsbl.element == Element::Ip && dnsbl.element_location.contains(&ip.location) { + if let Some(tag) = + is_dnsbl(self, dnsbl, SpamFilterResolver::new(ctx, &ip.element)).await + { + ctx.result.add_tag(tag); + } + } + } + ctx.result.rbl_ip_checks += 1; + if ctx.result.rbl_ip_checks >= self.core.spam.max_rbl_ip_checks { + break; + } + } + + // Reverse DNS validation + match &ctx.input.iprev_result.result { + IprevResult::TempError(_) => ctx.result.add_tag("RDNS_DNSFAIL"), + IprevResult::Fail(_) | IprevResult::PermError(_) => ctx.result.add_tag("RDNS_DNSFAIL"), + IprevResult::Pass | IprevResult::None => (), + } + } +} diff --git a/crates/spam-filter/src/analysis/iprev.rs b/crates/spam-filter/src/analysis/iprev.rs deleted file mode 100644 index f3868735..00000000 --- a/crates/spam-filter/src/analysis/iprev.rs +++ /dev/null @@ -1,23 +0,0 @@ -use std::future::Future; - -use common::Server; -use mail_auth::IprevResult; - -use crate::SpamFilterContext; - -pub trait SpamFilterAnalyzeIpRev: Sync + Send { - fn spam_filter_analyze_iprev( - &self, - ctx: &mut SpamFilterContext<'_>, - ) -> impl Future + Send; -} - -impl SpamFilterAnalyzeIpRev for Server { - async fn spam_filter_analyze_iprev(&self, ctx: &mut SpamFilterContext<'_>) { - match &ctx.input.iprev_result.result { - IprevResult::TempError(_) => ctx.result.add_tag("RDNS_DNSFAIL"), - IprevResult::Fail(_) | IprevResult::PermError(_) => ctx.result.add_tag("RDNS_DNSFAIL"), - IprevResult::Pass | IprevResult::None => (), - } - } -} diff --git a/crates/spam-filter/src/analysis/mod.rs b/crates/spam-filter/src/analysis/mod.rs index 510226c0..ddb395b6 100644 --- a/crates/spam-filter/src/analysis/mod.rs +++ b/crates/spam-filter/src/analysis/mod.rs @@ -1,17 +1,26 @@ -use std::borrow::Cow; +use std::{ + borrow::Cow, + hash::{Hash, Hasher}, +}; +use common::{ + config::spamfilter::Location, + expr::{functions::ResolveVariable, Variable}, + Server, +}; use mail_parser::{parsers::MessageStream, Header}; -use crate::{Recipient, SpamFilterInput, SpamFilterOutput, SpamFilterResult}; +use crate::{Recipient, SpamFilterContext, SpamFilterInput, SpamFilterOutput, SpamFilterResult}; pub mod bounce; pub mod date; pub mod dmarc; +pub mod domain; pub mod ehlo; pub mod from; pub mod headers; pub mod init; -pub mod iprev; +pub mod ip; pub mod messageid; pub mod received; pub mod recipient; @@ -45,3 +54,66 @@ impl SpamFilterResult { self.tags.insert(tag.into()); } } + +pub(crate) struct SpamFilterResolver<'x, T: ResolveVariable> { + pub ctx: &'x SpamFilterContext<'x>, + pub item: &'x T, +} + +impl ResolveVariable for SpamFilterResolver<'_, T> { + fn resolve_variable(&self, variable: u32) -> common::expr::Variable<'_> { + self.item.resolve_variable(variable) + } + + fn resolve_global(&self, variable: &str) -> common::expr::Variable<'_> { + Variable::Integer(self.ctx.result.tags.contains(variable).into()) + } +} + +impl<'x, T: ResolveVariable> SpamFilterResolver<'x, T> { + pub fn new(ctx: &'x SpamFilterContext<'x>, item: &'x T) -> Self { + Self { ctx, item } + } +} + +pub(crate) struct ElementLocation { + pub element: T, + pub location: Location, +} + +impl Hash for ElementLocation { + fn hash(&self, state: &mut H) { + self.element.hash(state); + } +} + +impl PartialEq for ElementLocation { + fn eq(&self, other: &Self) -> bool { + self.element.eq(&other.element) + } +} + +impl Eq for ElementLocation {} + +impl ElementLocation { + pub fn new(element: T, location: impl Into) -> Self { + Self { + element, + location: location.into(), + } + } +} + +pub(crate) async fn is_trusted_domain(server: &Server, domain: &str, span_id: u64) -> bool { + if server.core.spam.list_trusted_domains.contains(domain) { + return true; + } + + match server.core.storage.directory.is_local_domain(domain).await { + Ok(result) => result, + Err(err) => { + trc::error!(err.span_id(span_id).caused_by(trc::location!())); + false + } + } +} diff --git a/crates/spam-filter/src/analysis/url.rs b/crates/spam-filter/src/analysis/url.rs index 1d2ce283..94b73dd4 100644 --- a/crates/spam-filter/src/analysis/url.rs +++ b/crates/spam-filter/src/analysis/url.rs @@ -1,22 +1,27 @@ +use std::collections::HashSet; use std::{borrow::Cow, future::Future, time::Duration}; +use common::config::spamfilter::{Element, Location}; +use common::expr::functions::ResolveVariable; +use common::expr::Variable; +use common::scripts::functions::unicode::CharUtils; use common::Server; -use common::{config::spamfilter::Target, scripts::functions::unicode::CharUtils}; -use hyper::{ - header::{HeaderName, LOCATION}, - Uri, -}; +use hyper::{header::LOCATION, Uri}; +use mail_parser::HeaderName; use nlp::tokenizers::types::TokenType; use reqwest::redirect::Policy; use unicode_security::MixedScript; use crate::modules::dnsbl::is_dnsbl; +use crate::modules::html::SRC; use crate::modules::remote_list::is_in_remote_list; use crate::{ modules::html::{HtmlToken, A, HREF}, Hostname, SpamFilterContext, TextPart, }; +use super::{is_trusted_domain, ElementLocation, SpamFilterResolver}; + pub trait SpamFilterAnalyzeUrl: Sync + Send { fn spam_filter_analyze_url( &self, @@ -26,10 +31,80 @@ pub trait SpamFilterAnalyzeUrl: Sync + Send { impl SpamFilterAnalyzeUrl for Server { async fn spam_filter_analyze_url(&self, ctx: &mut SpamFilterContext<'_>) { + // Extract URLs + let mut urls: HashSet> = HashSet::from_iter( + ctx.output + .subject_tokens + .iter() + .filter_map(|t| t.url_lowercase(false)) + .map(|url| ElementLocation::new(url, HeaderName::Subject)), + ); for (part_id, part) in ctx.output.text_parts.iter().enumerate() { - if ctx.input.message.text_body.contains(&part_id) - || ctx.input.message.html_body.contains(&part_id) - { + let is_body = ctx.input.message.text_body.contains(&part_id) + || ctx.input.message.html_body.contains(&part_id); + + match part { + TextPart::Plain { tokens, .. } => { + urls.extend( + tokens + .iter() + .filter_map(|t| t.url_lowercase(false)) + .map(|url| { + ElementLocation::new( + url, + if is_body { + Location::BodyText + } else { + Location::Attachment + }, + ) + }), + ); + } + TextPart::Html { + html_tokens, + tokens, + .. + } => { + for token in html_tokens { + if let HtmlToken::StartTag { attributes, .. } = token { + for (attr, value) in attributes { + match value { + Some(value) if [HREF, SRC].contains(attr) => { + urls.insert(ElementLocation::new( + value.trim().to_lowercase(), + if is_body { + Location::BodyHtml + } else { + Location::Attachment + }, + )); + } + _ => {} + } + } + } + } + urls.extend( + tokens + .iter() + .filter_map(|t| t.url_lowercase(false)) + .map(|url| { + ElementLocation::new( + url, + if is_body { + Location::BodyHtml + } else { + Location::Attachment + }, + ) + }), + ); + } + TextPart::None => {} + } + + if is_body { let is_single = match part { TextPart::Plain { tokens, .. } => is_single_url(tokens), TextPart::Html { @@ -42,13 +117,12 @@ impl SpamFilterAnalyzeUrl for Server { if is_single { ctx.result.add_tag("URL_ONLY"); - break; } } } - for url in &ctx.output.urls { - for ch in url.chars() { + for url in &urls { + for ch in url.element.chars() { if ch.is_zwsp() { ctx.result.add_tag("ZERO_WIDTH_SPACE_URL"); } @@ -59,12 +133,12 @@ impl SpamFilterAnalyzeUrl for Server { } // Skip non-URLs such as 'data:' and 'mailto:' - if !url.contains("://") { + if !url.element.contains("://") { continue; } // Parse url - let url_parsed = match url.parse::() { + let url_parsed = match url.element.parse::() { Ok(url) if url.host().is_some() => url, _ => { // URL could not be parsed @@ -76,25 +150,33 @@ impl SpamFilterAnalyzeUrl for Server { let host_sld = host.sld_or_default(); // Skip local and trusted domains - if self.core.spam.list_trusted_domains.contains(host_sld) - || self - .core - .storage - .directory - .is_local_domain(host_sld) - .await - .unwrap_or_default() - { + if is_trusted_domain(self, host_sld, ctx.input.span_id).await { continue; } - // Check for redirectors let mut redirected_urls = Vec::new(); - if host.ip.is_none() && self.core.spam.list_url_redirectors.contains(host_sld) { + if let Some(ip) = host.ip { + // Check IP DNSBL + if ctx.result.rbl_ip_checks < self.core.spam.max_rbl_ip_checks { + for dnsbl in &self.core.spam.dnsbls { + if dnsbl.element == Element::Ip + && dnsbl.element_location.contains(&url.location) + { + if let Some(tag) = + is_dnsbl(self, dnsbl, SpamFilterResolver::new(ctx, &ip)).await + { + ctx.result.add_tag(tag); + } + } + } + ctx.result.rbl_ip_checks += 1; + } + } else if self.core.spam.list_url_redirectors.contains(host_sld) { + // Check for redirectors ctx.result.add_tag("REDIRECTOR_URL"); let mut redirect_count = 0; - let mut url_redirect = Cow::Borrowed(url); + let mut url_redirect = Cow::Borrowed(url.element.as_str()); while redirect_count <= 0 { match http_get_header(url_redirect.as_ref(), LOCATION, Duration::from_secs(5)) @@ -114,12 +196,16 @@ impl SpamFilterAnalyzeUrl for Server { redirect_count += 1; continue; } else { - let location = location.to_lowercase(); - if !ctx.output.urls.contains(&location) { + let new_url = ElementLocation::new( + location.to_lowercase(), + url.location.clone(), + ); + if !urls.contains(&new_url) { redirected_urls.push(( - Cow::Owned(location), + Cow::Owned(new_url.element), location_parsed, host, + new_url.location, )); } } @@ -138,9 +224,14 @@ impl SpamFilterAnalyzeUrl for Server { } } - for (url, url_parsed, host) in [(Cow::Borrowed(url), url_parsed, host)] - .into_iter() - .chain(redirected_urls.into_iter()) + for (url, url_parsed, host, location) in [( + Cow::Borrowed(url.element.as_str()), + url_parsed, + host, + url.location.clone(), + )] + .into_iter() + .chain(redirected_urls.into_iter()) { let query = url_parsed .path_and_query() @@ -182,6 +273,26 @@ impl SpamFilterAnalyzeUrl for Server { // Onion URL ctx.result.add_tag("HAS_ONION_URI"); } + + // Check Domain DNSBL + if ctx.result.rbl_domain_checks < self.core.spam.max_rbl_domain_checks { + for dnsbl in &self.core.spam.dnsbls { + if matches!(dnsbl.element, Element::Domain) + && dnsbl.element_location.contains(&location) + { + if let Some(tag) = is_dnsbl( + self, + dnsbl, + SpamFilterResolver::new(ctx, &host.sld_or_default()), + ) + .await + { + ctx.result.add_tag(tag); + } + } + } + ctx.result.rbl_domain_checks += 1; + } } else { // URL is an ip address ctx.result.add_tag("R_SUSPICIOUS_URL"); @@ -207,22 +318,35 @@ impl SpamFilterAnalyzeUrl for Server { // Check remote lists for remote in &self.core.spam.remote_lists { - if matches!(remote.target, Target::Url) + if matches!(remote.element, Element::Url) + && remote.element_location.contains(&location) && is_in_remote_list(self, remote, url.as_ref(), ctx.input.span_id).await { ctx.result.add_tag(&remote.tag); } } - // Check DNSBL - for dnsbl in &self.core.spam.dnsbls { - if matches!(dnsbl.target, Target::Url) { - if let Some(tag) = - is_dnsbl(self, dnsbl, url.as_ref(), ctx.input.span_id).await + // Check URL DNSBL + if ctx.result.rbl_url_checks < self.core.spam.max_rbl_url_checks { + for dnsbl in &self.core.spam.dnsbls { + if matches!(dnsbl.element, Element::Url) + && dnsbl.element_location.contains(&location) { - ctx.result.add_tag(tag); + if let Some(tag) = is_dnsbl( + self, + dnsbl, + SpamFilterResolver::new( + ctx, + &UriHost::new(&url, &url_parsed, &host), + ), + ) + .await + { + ctx.result.add_tag(tag); + } } } + ctx.result.rbl_url_checks += 1; } } } @@ -231,7 +355,7 @@ impl SpamFilterAnalyzeUrl for Server { async fn http_get_header( url: &str, - header: HeaderName, + header: hyper::header::HeaderName, timeout: Duration, ) -> trc::Result> { reqwest::Client::builder() @@ -322,3 +446,62 @@ fn is_single_html_url>(html_tokens: &[HtmlToken], tokens: &[TokenT url_count == 1 } + +struct UriHost<'x> { + full_url: &'x str, + url: &'x Uri, + host: &'x Hostname, +} + +pub const V_URL_FULL: u32 = 0; +pub const V_URL_PATH_QUERY: u32 = 1; +pub const V_URL_PATH: u32 = 2; +pub const V_URL_QUERY: u32 = 3; +pub const V_URL_SCHEME: u32 = 4; +pub const V_URL_AUTHORITY: u32 = 5; +pub const V_URL_HOST: u32 = 6; +pub const V_URL_HOST_SLD: u32 = 7; +pub const V_URL_PORT: u32 = 8; + +impl ResolveVariable for UriHost<'_> { + fn resolve_variable(&self, variable: u32) -> Variable<'_> { + match variable { + V_URL_FULL => Variable::String(self.full_url.into()), + V_URL_PATH_QUERY => Variable::String( + self.url + .path_and_query() + .map(|p| p.as_str()) + .unwrap_or_default() + .into(), + ), + V_URL_PATH => Variable::String(self.url.path().into()), + V_URL_QUERY => Variable::String(self.url.query().unwrap_or_default().into()), + V_URL_SCHEME => Variable::String(self.url.scheme_str().unwrap_or_default().into()), + V_URL_AUTHORITY => Variable::String( + self.url + .authority() + .map(|a| a.as_str()) + .unwrap_or_default() + .into(), + ), + V_URL_HOST => Variable::String(self.host.fqdn.as_str().into()), + V_URL_HOST_SLD => Variable::String(self.host.sld_or_default().into()), + V_URL_PORT => Variable::Integer(self.url.port_u16().unwrap_or(0) as _), + _ => Variable::Integer(0), + } + } + + fn resolve_global(&self, _: &str) -> Variable<'_> { + Variable::Integer(0) + } +} + +impl<'x> UriHost<'x> { + pub fn new(full_url: &'x str, url: &'x Uri, host: &'x Hostname) -> Self { + Self { + full_url, + url, + host, + } + } +} diff --git a/crates/spam-filter/src/lib.rs b/crates/spam-filter/src/lib.rs index 09837740..aa488423 100644 --- a/crates/spam-filter/src/lib.rs +++ b/crates/spam-filter/src/lib.rs @@ -57,7 +57,6 @@ pub struct SpamFilterOutput<'x> { pub subject_tokens: Vec>, pub text_parts: Vec>, - pub urls: HashSet, } pub enum TextPart<'x> { @@ -73,8 +72,13 @@ pub enum TextPart<'x> { None, } +#[derive(Debug, Default)] pub struct SpamFilterResult { pub tags: AHashSet, + pub rbl_ip_checks: usize, + pub rbl_domain_checks: usize, + pub rbl_url_checks: usize, + pub rbl_email_checks: usize, } pub struct SpamFilterContext<'x> { diff --git a/crates/spam-filter/src/modules/dnsbl.rs b/crates/spam-filter/src/modules/dnsbl.rs index fb13906e..414ffde1 100644 --- a/crates/spam-filter/src/modules/dnsbl.rs +++ b/crates/spam-filter/src/modules/dnsbl.rs @@ -1,18 +1,19 @@ use std::time::Instant; -use common::{config::spamfilter::DnsblConfig, Server}; +use common::{config::spamfilter::DnsblConfig, expr::functions::ResolveVariable, Server}; use mail_auth::Error; use trc::SpamEvent; -pub async fn is_dnsbl( +use crate::analysis::SpamFilterResolver; + +pub(crate) async fn is_dnsbl( server: &Server, config: &DnsblConfig, - item: &str, - span_id: u64, + resolver: SpamFilterResolver<'_, impl ResolveVariable>, ) -> Option { let time = Instant::now(); let zone = server - .eval_expr::(&config.zone, &item, &config.id, span_id) + .eval_if::(&config.zone, &resolver, resolver.ctx.input.span_id) .await?; let todo = "use proper event error"; @@ -29,7 +30,9 @@ pub async fn is_dnsbl( Elapsed = time.elapsed() ); - server.eval_if(&config.tags, &result, span_id).await + server + .eval_if(&config.tags, &result, resolver.ctx.input.span_id) + .await } Err(Error::DnsRecordNotFound(_)) => { trc::event!( diff --git a/resources/config/spamfilter/scripts/rbl.sieve b/resources/config/spamfilter/scripts/rbl.sieve deleted file mode 100644 index d5810337..00000000 --- a/resources/config/spamfilter/scripts/rbl.sieve +++ /dev/null @@ -1,351 +0,0 @@ - -# Validate IP addresses -let "ip_addresses" "dedup(winnow([ env.remote_ip ] + header.received[*].rcvd.ip + header.received[*].rcvd.from.ip + header.received[*].rcvd.by.ip))"; -let "ip_addresses_len" "count(ip_addresses)"; -let "i" "0"; - -while "i < ip_addresses_len" { - let "ip_address" "ip_addresses[i]"; - let "is_from_addr" "i == 0"; - let "i" "i + 1"; - - if eval "ip_address == '127.0.0.1' || ip_address == '::1'" { - continue; - } - - # Do not check more than 10 IP addresses - if eval "i >= 10" { - break; - } - - let "ip_reverse" "ip_reverse_name(ip_address)"; - let "is_ip_v4" "len(ip_reverse) <= 15"; - - # Query SPAMHAUS - let "result" "rsplit_once(dns_query(ip_reverse + '.zen.spamhaus.org', 'ipv4')[0], '.')"; - if eval "result[0] == '127.0.0'" { - let "result" "result[1]"; - - if eval "result == 2" { - if eval "is_from_addr" { - let "t.RBL_SPAMHAUS_SBL" "1"; - } else { - let "t.RECEIVED_SPAMHAUS_SBL" "1"; - } - } elsif eval "result == 3" { - if eval "is_from_addr" { - let "t.RBL_SPAMHAUS_CSS" "1"; - } else { - let "t.RECEIVED_SPAMHAUS_CSS" "1"; - } - } elsif eval "result >= 4 && result <= 7" { - if eval "is_from_addr" { - let "t.RBL_SPAMHAUS_XBL" "1"; - } else { - let "t.RECEIVED_SPAMHAUS_XBL" "1"; - } - } elsif eval "result == 9" { - if eval "is_from_addr" { - let "t.RBL_SPAMHAUS_DROP" "1"; - } else { - let "t.RECEIVED_SPAMHAUS_PBL" "1"; - } - } elsif eval "result == 10 || result == 11" { - if eval "is_from_addr" { - let "t.RBL_SPAMHAUS_PBL" "1"; - } else { - let "t.RECEIVED_SPAMHAUS_PBL" "1"; - } - } elsif eval "result == 254" { - if eval "is_from_addr" { - let "t.RBL_SPAMHAUS_BLOCKED_OPENRESOLVER" "1"; - } else { - let "t.RECEIVED_SPAMHAUS_BLOCKED_OPENRESOLVER" "1"; - } - } elsif eval "result == 255" { - if eval "is_from_addr" { - let "t.RBL_SPAMHAUS_BLOCKED" "1"; - } else { - let "t.RECEIVED_SPAMHAUS_BLOCKED" "1"; - } - } else { - # Unrecognized result - let "t.RBL_SPAMHAUS" "1"; - } - } - - if eval "is_from_addr" { - # Query IP reputation at Mailspike - let "result" "rsplit_once(dns_query(ip_reverse + '.rep.mailspike.net', 'ipv4')[0], '.')"; - if eval "result[0] == '127.0.0'" { - let "result" "result[1]"; - - if eval "result == 10" { - let "t.RBL_MAILSPIKE_WORST" "1"; - } elsif eval "result == 11" { - let "t.RBL_MAILSPIKE_VERYBAD" "1"; - } elsif eval "result == 12" { - let "t.RBL_MAILSPIKE_BAD" "1"; - } elsif eval "result >= 13 && result <= 16" { - let "t.RWL_MAILSPIKE_NEUTRAL" "1"; - } elsif eval "result == 17" { - let "t.RWL_MAILSPIKE_POSSIBLE" "1"; - } elsif eval "result == 18" { - let "t.RWL_MAILSPIKE_GOOD" "1"; - } elsif eval "result == 19" { - let "t.RWL_MAILSPIKE_VERYGOOD" "1"; - } elsif eval "result == 20" { - let "t.RWL_MAILSPIKE_EXCELLENT" "1"; - } - } - - # Query SenderScore - if eval "dns_exists(ip_reverse + '.bl.score.senderscore.com', 'ipv4')" { - let "t.RBL_SENDERSCORE" "1"; - } - - # Query SpamEatingMonkey - if eval "is_ip_v4 && dns_exists(ip_reverse + '.bl.spameatingmonkey.net', 'ipv4')" { - let "t.RBL_SEM" "1"; - } elsif eval "!is_ip_v4 && dns_exists(ip_reverse + '.bl.ipv6.spameatingmonkey.net', 'ipv4')" { - let "t.RBL_SEM_IPV6" "1"; - } - - # Query VirusFree - if eval "dns_query(ip_reverse + '.bip.virusfree.cz', 'ipv4')[0] == '127.0.0.2'" { - let "t.RBL_VIRUSFREE_BOTNET" "1"; - } - - # Query NiX - if eval "dns_exists(ip_reverse + '.ix.dnsbl.manitu.net', 'ipv4')" { - let "t.RBL_NIXSPAM" "1"; - } - - # Query Spamcop - if eval "dns_exists(ip_reverse + '.bl.spamcop.net', 'ipv4')" { - let "t.RBL_SPAMCOP" "1"; - } - - # Query Barracuda - if eval "dns_exists(ip_reverse + '.b.barracudacentral.org', 'ipv4')" { - let "t.RBL_BARRACUDA" "1"; - } - } - - # Query Blocklist.de - if eval "dns_exists(ip_reverse + '.bl.blocklist.de', 'ipv4')" { - if eval "is_from_addr" { - let "t.RBL_BLOCKLISTDE" "1"; - } else { - let "t.RECEIVED_BLOCKLISTDE" "1"; - } - } - - # Query DNSWL - let "result" "rsplit_once(dns_query(ip_reverse + '.list.dnswl.org', 'ipv4')[0], '.')"; - if eval "starts_with(result[0], '127.')" { - let "result" "result[1]"; - - if eval "result == 0" { - let "t.RCVD_IN_DNSWL_NONE" "1"; - } elsif eval "result == 1" { - let "t.RCVD_IN_DNSWL_LOW" "1"; - } elsif eval "result == 2" { - let "t.RCVD_IN_DNSWL_MED" "1"; - } elsif eval "result == 3" { - let "t.RCVD_IN_DNSWL_HI" "1"; - } elsif eval "result == 255" { - let "t.DNSWL_BLOCKED" "1"; - } - } -} - -# Validate domain names -let "emails" "dedup(winnow(to_lowercase([from_addr, rto_addr, envelope.from] + tokenize(text_body, 'email'))))"; -let "emails_len" "count(emails)"; -let "domains" "dedup(winnow(to_lowercase([ env.helo_domain, env.iprev.ptr ] + email_part(emails, 'domain') + puny_decode(uri_part(urls, 'host')))))"; -let "domains_len" "count(domains)"; -let "i" "0"; - -while "i < domains_len" { - let "domain" "domains[i]"; - let "i" "i + 1"; - - # Skip invalid and local domain names - if eval "!contains(domain, '.') || - is_ip_addr(domain) || - is_local_domain(DOMAIN_DIRECTORY, domain_part(domain, 'sld')) || - key_exists('spam-allow', domain)" { - continue; - } - - # Do not check more than 10 domain names - if eval "i >= 10" { - break; - } - - # Query SpamHaus DBL - let "result" "rsplit_once(dns_query(domain + '.dbl.spamhaus.org', 'ipv4')[0], '.')"; - if eval "result[0] == '127.0.1'" { - let "result" "result[1]"; - - if eval "result == 2" { - let "t.DBL_SPAM" "1"; - } elsif eval "result == 4" { - let "t.DBL_PHISH" "1"; - } elsif eval "result == 5" { - let "t.DBL_MALWARE" "1"; - } elsif eval "result == 6" { - let "t.DBL_BOTNET" "1"; - } elsif eval "result == 102" { - let "t.DBL_ABUSE" "1"; - } elsif eval "result == 103" { - let "t.DBL_ABUSE_REDIR" "1"; - } elsif eval "result == 104" { - let "t.DBL_ABUSE_PHISH" "1"; - } elsif eval "result == 105" { - let "t.DBL_ABUSE_MALWARE" "1"; - } elsif eval "result == 106" { - let "t.DBL_ABUSE_BOTNET" "1"; - } elsif eval "result == 254" { - let "t.DBL_BLOCKED_OPENRESOLVER" "1"; - } elsif eval "result == 255" { - let "t.DBL_BLOCKED" "1"; - } - } - - # Query SURBL multi - let "result" "rsplit_once(dns_query(domain + '.multi.surbl.org', 'ipv4')[0], '.')"; - if eval "result[0] == '127.0.0'" { - let "result" "result[1]"; - - if eval "result == 128" { - let "t.CRACKED_SURBL" "1"; - } elsif eval "result == 64" { - let "t.ABUSE_SURBL" "1"; - } elsif eval "result == 16" { - let "t.MW_SURBL_MULTI" "1"; - } elsif eval "result == 8" { - let "t.PH_SURBL_MULTI" "1"; - } elsif eval "result == 1" { - let "t.SURBL_BLOCKED" "1"; - } - } - - # Query URIBL multi - let "result" "rsplit_once(dns_query(domain + '.multi.uribl.com', 'ipv4')[0], '.')"; - if eval "result[0] == '127.0.0'" { - let "result" "result[1]"; - - if eval "result == 1" { - let "t.URIBL_BLOCKED" "1"; - } elsif eval "result == 2" { - let "t.URIBL_BLACK" "1"; - } elsif eval "result == 4" { - let "t.URIBL_GREY" "1"; - } elsif eval "result == 8" { - let "t.URIBL_RED" "1"; - } - } - - # Query SpamEatingMonkey URIBL - if eval "dns_query(domain + '.uribl.spameatingmonkey.net', 'ipv4')[0] == '127.0.0.2'" { - let "t.SEM_URIBL" "1"; - } - - # Query SpamEatingMonkey FRESH15 - if eval "dns_query(domain + '.fresh15.spameatingmonkey.net', 'ipv4')[0] == '127.0.0.2'" { - let "t.SEM_URIBL_FRESH15" "1"; - } - -} - -# Check DKIM domains that passed validation -let "i" "count(env.dkim.domains)"; -while "i > 0" { - let "i" "i - 1"; - - # Query DNSWL - let "result" "rsplit_once(dns_query(env.dkim.domains[i] + '.dwl.dnswl.org', 'ipv4')[0], '.')"; - if eval "starts_with(result[0], '127.')" { - let "result" "result[1]"; - - if eval "result == 0" { - let "t.DWL_DNSWL_NONE" "1"; - } elsif eval "result == 1" { - let "t.DWL_DNSWL_LOW" "1"; - } elsif eval "result == 2" { - let "t.DWL_DNSWL_MED" "1"; - } elsif eval "result == 3" { - let "t.DWL_DNSWL_HI" "1"; - } elsif eval "result == 255" { - let "t.DWL_DNSWL_BLOCKED" "1"; - } - } -} - -# Validate email addresses -let "i" "0"; -while "i < emails_len" { - let "email" "emails[i]"; - let "i" "i + 1"; - - # Skip invalid and local e-mail addresses - if eval "!contains(email, '@') || is_local_domain(DOMAIN_DIRECTORY, domain_part(email_part(email, 'domain'), 'sld'))" { - continue; - } - - # Do not check more than 10 email addresses - if eval "i >= 10" { - break; - } - - # Query MSBL EBL - let "result" "rsplit_once(dns_query(hash(email, 'sha1') + '.ebl.msbl.org', 'ipv4')[0], '.')"; - if eval "result[1] == 2 || result[1] == 3" { - if eval "result[0] == '127.0.0'" { - let "t.MSBL_EBL" "1"; - } elsif eval "result[0] == '127.0.1'" { - let "t.MSBL_EBL_GREY" "1"; - } - } -} - - -# Validate URL hashes -let "i" "0"; -let "urls_len" "count(urls)"; -while "i < urls_len" { - let "url" "urls[i]"; - let "i" "i + 1"; - - # Do not check more than 10 URLs - if eval "i >= 10" { - break; - } - - # Skip URLs pointing to local or trusted domains - let "domain" "domain_part(uri_part(url, 'host'), 'sld')"; - if eval "is_local_domain(DOMAIN_DIRECTORY, domain) || - key_exists('spam-allow', domain)" { - continue; - } - - # Query SURBL HASHBL - let "result" "rsplit_once(dns_query(hash(url, 'md5') + '.hashbl.surbl.org', 'ipv4')[0], '.')"; - if eval "starts_with(result[0], '127.0.')" { - let "result" "result[1]"; - - if eval "result == 8" { - let "t.SURBL_HASHBL_PHISH" "1"; - } elsif eval "result == 16" { - let "t.SURBL_HASHBL_MALWARE" "1"; - } elsif eval "result == 64" { - let "t.SURBL_HASHBL_ABUSE" "1"; - } elsif eval "result == 128" { - let "t.SURBL_HASHBL_CRACKED" "1"; - } elsif eval "result[0] == '127.0.1'" { - let "t.SURBL_HASHBL_EMAIL" "1"; - } - } -} diff --git a/tests/src/smtp/config.rs b/tests/src/smtp/config.rs index 7811d6f8..1a9d17ea 100644 --- a/tests/src/smtp/config.rs +++ b/tests/src/smtp/config.rs @@ -509,6 +509,10 @@ impl ResolveVariable for TestEnvelope { _ => Default::default(), } } + + fn resolve_global(&self, _: &str) -> Variable<'_> { + Variable::Integer(0) + } } impl TestEnvelope {