From 8eda2251d8cd3029166ed3068471bb3cd088ba27 Mon Sep 17 00:00:00 2001 From: mdecimus Date: Tue, 31 Dec 2024 12:29:49 +0100 Subject: [PATCH] Spam filter config adjustments --- Cargo.lock | 1 + crates/common/Cargo.toml | 1 + crates/common/src/config/inner.rs | 14 +- crates/common/src/config/network.rs | 55 ++---- crates/common/src/config/smtp/session.rs | 2 +- crates/common/src/config/spamfilter.rs | 63 ++++++- crates/common/src/lib.rs | 4 +- crates/common/src/listener/asn.rs | 21 +-- crates/common/src/manager/mod.rs | 4 +- .../api/management/enterprise/telemetry.rs | 8 +- crates/jmap/src/api/management/spam.rs | 175 ++++++++++++------ crates/spam-filter/src/analysis/ip.rs | 7 +- crates/spam-filter/src/analysis/llm.rs | 18 +- crates/spam-filter/src/analysis/rules.rs | 7 +- crates/spam-filter/src/analysis/url.rs | 4 +- crates/spam-filter/src/modules/dnsbl.rs | 36 ++-- crates/spam-filter/src/modules/expression.rs | 49 ----- tests/src/smtp/inbound/asn.rs | 52 +++--- tests/src/smtp/mod.rs | 10 +- 19 files changed, 302 insertions(+), 229 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0d0aaf18..855bac7d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1231,6 +1231,7 @@ dependencies = [ "mail-send", "md5", "nlp", + "num_cpus", "opentelemetry", "opentelemetry-otlp", "opentelemetry-semantic-conventions", diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 0aca2f94..51c6ca85 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -66,6 +66,7 @@ biscuit = "0.7.0" rsa = "0.9.2" p256 = { version = "0.13", features = ["ecdh"] } p384 = { version = "0.13", features = ["ecdh"] } +num_cpus = "1.13.1" [target.'cfg(unix)'.dependencies] privdrop = "0.5.3" diff --git a/crates/common/src/config/inner.rs b/crates/common/src/config/inner.rs index f9115b6d..1431d21d 100644 --- a/crates/common/src/config/inner.rs +++ b/crates/common/src/config/inner.rs @@ -45,10 +45,10 @@ impl Data { // Parse capacities let shard_amount = config - .property::("cache.shard") - .unwrap_or(32) + .property::("limiter.shard") + .unwrap_or_else(|| (num_cpus::get() * 2) as u64) .next_power_of_two() as usize; - let capacity = config.property("cache.capacity").unwrap_or(100); + let capacity = config.property("limiter.capacity").unwrap_or(100); // Parse id generator let id_generator = config @@ -113,7 +113,7 @@ impl Caches { Caches { access_tokens: CacheWithTtl::from_config( config, - "access-tokens", + "access-token", MB_10, (std::mem::size_of::() + 255) as u64, ), @@ -125,8 +125,8 @@ impl Caches { ), permissions: Cache::from_config( config, - "permissions", - MB_10, + "permission", + MB_5, std::mem::size_of::() as u64, ), permissions_version: 0.into(), @@ -149,7 +149,7 @@ impl Caches { ), threads: Cache::from_config( config, - "threads", + "thread", MB_10, (std::mem::size_of::() + (500 * std::mem::size_of::())) as u64, ), diff --git a/crates/common/src/config/network.rs b/crates/common/src/config/network.rs index daf51d58..8cc1f2f0 100644 --- a/crates/common/src/config/network.rs +++ b/crates/common/src/config/network.rs @@ -39,7 +39,9 @@ pub enum AsnGeoLookupConfig { expires: Duration, timeout: Duration, max_size: usize, - resources: Vec, + headers: HeaderMap, + asn_resources: Vec, + geo_resources: Vec, }, Dns { zone_ipv4: String, @@ -53,12 +55,6 @@ pub enum AsnGeoLookupConfig { Disabled, } -#[derive(Clone)] -pub enum AsnGeoLookupResource { - Asn { url: String, headers: HeaderMap }, - Geo { url: String, headers: HeaderMap }, -} - #[derive(Clone)] pub struct FieldOrDefault { pub field: Option, @@ -196,50 +192,33 @@ impl AsnGeoLookupConfig { } .into(), "resource" => { - let mut resources = vec![]; + let asn_resources = config + .values("server.asn.urls.asn") + .map(|(_, v)| v.to_string()) + .collect::>(); + let geo_resources = config + .values("server.asn.urls.geo") + .map(|(_, v)| v.to_string()) + .collect::>(); - for id in config - .sub_keys("server.asn.resource", ".url") - .map(|k| k.to_string()) - .collect::>() - { - let id = id.as_str(); - let url = config - .value_require_non_empty(("server.asn.resource", id, "url"))? - .to_string(); - let headers = parse_http_headers(config, ("server.asn.resource", id)); - - resources.push( - match config.value_require(("server.asn.resource", id, "type"))? { - "asn" => AsnGeoLookupResource::Asn { url, headers }, - "geo" => AsnGeoLookupResource::Geo { url, headers }, - _ => { - config.new_build_error( - ("server.asn.resource", id), - "Invalid resource", - ); - continue; - } - }, - ); - } - - if resources.is_empty() { - config.new_build_error("server.asn.resource", "No resources found"); + if asn_resources.is_empty() && geo_resources.is_empty() { + config.new_build_error("server.asn.urls", "No resources found"); return None; } AsnGeoLookupConfig::Resource { + headers: parse_http_headers(config, "server.asn"), expires: config.property_or_default::("server.asn.expires", "1d")?, timeout: config.property_or_default::("server.asn.timeout", "5m")?, max_size: config .property("server.asn.max-size") .unwrap_or(100 * 1024 * 1024), - resources, + asn_resources, + geo_resources, } .into() } - "disabled" | "none" | "false" => AsnGeoLookupConfig::Disabled.into(), + "disable" | "disabled" | "none" | "false" => AsnGeoLookupConfig::Disabled.into(), _ => { config.new_build_error("server.asn.type", "Invalid value"); None diff --git a/crates/common/src/config/smtp/session.rs b/crates/common/src/config/smtp/session.rs index 464b5ae3..c49998d3 100644 --- a/crates/common/src/config/smtp/session.rs +++ b/crates/common/src/config/smtp/session.rs @@ -755,7 +755,7 @@ impl Default for SessionConfig { is_allowed: IfBlock::new::<()>( "session.mail.is-allowed", [], - "!is_empty(authenticated_as) || !key_exists('spam-block', sender_domain)", + "!is_empty(authenticated_as) || !key_exists('blocked-domains', sender_domain)", ), }, rcpt: Rcpt { diff --git a/crates/common/src/config/spamfilter.rs b/crates/common/src/config/spamfilter.rs index 1b2be048..98d41f69 100644 --- a/crates/common/src/config/spamfilter.rs +++ b/crates/common/src/config/spamfilter.rs @@ -4,17 +4,22 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{net::SocketAddr, time::Duration}; +use std::{ + net::{IpAddr, SocketAddr}, + time::Duration, +}; use ahash::AHashSet; +use mail_auth::common::resolver::ToReverseName; use nlp::bayes::BayesClassifier; use tokio::net::lookup_host; use utils::{ + cache::CacheItemWeight, config::{utils::ParseValue, Config}, glob::GlobMap, }; -use super::{if_block::IfBlock, tokenizer::TokenMap}; +use super::{functions::ResolveVariable, if_block::IfBlock, tokenizer::TokenMap, Variable}; #[derive(Debug, Clone, Default)] pub struct SpamFilterConfig { @@ -545,7 +550,7 @@ impl BayesConfig { .property_or_default("spam-filter.bayes.auto-learn.threshold.spam", "6.0") .unwrap_or(6.0), auto_learn_ham_threshold: config - .property_or_default("spam-filter.bayes.auto-learn.threshold.ham", "-2.0") + .property_or_default("spam-filter.bayes.auto-learn.threshold.ham", "-1.0") .unwrap_or(-2.0), score_spam: config .property_or_default("spam-filter.bayes.score.spam", "0.7") @@ -809,3 +814,55 @@ impl Element { } } } + +pub struct IpResolver { + ip: IpAddr, + ip_string: String, + reverse: String, + octets: Variable<'static>, +} + +impl ResolveVariable for IpResolver { + fn resolve_variable(&self, variable: u32) -> Variable<'_> { + match variable { + V_IP => Variable::String(self.ip_string.as_str().into()), + V_IP_REVERSE => Variable::String(self.reverse.as_str().into()), + V_IP_OCTETS => self.octets.clone(), + V_IP_IS_V4 => Variable::Integer(self.ip.is_ipv4() as _), + V_IP_IS_V6 => Variable::Integer(self.ip.is_ipv6() as _), + _ => Variable::Integer(0), + } + } + + fn resolve_global(&self, _: &str) -> Variable<'_> { + Variable::Integer(0) + } +} + +impl IpResolver { + pub fn new(ip: IpAddr) -> Self { + Self { + ip_string: ip.to_string(), + reverse: ip.to_reverse_name(), + octets: Variable::Array(match ip { + IpAddr::V4(ipv4_addr) => ipv4_addr + .octets() + .iter() + .map(|o| Variable::Integer(*o as _)) + .collect(), + IpAddr::V6(ipv6_addr) => ipv6_addr + .octets() + .iter() + .map(|o| Variable::Integer(*o as _)) + .collect(), + }), + ip, + } + } +} + +impl CacheItemWeight for IpResolver { + fn weight(&self) -> u64 { + (std::mem::size_of::() + self.ip_string.len() + self.reverse.len()) as u64 + } +} diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 145b2479..9411e696 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -23,7 +23,7 @@ use config::{ resolver::{Policy, Tlsa}, SmtpConfig, }, - spamfilter::SpamFilterConfig, + spamfilter::{IpResolver, SpamFilterConfig}, storage::Storage, telemetry::Metrics, }; @@ -152,7 +152,7 @@ pub struct Caches { pub dns_ipv6: CacheWithTtl>>, pub dns_tlsa: CacheWithTtl>, pub dbs_mta_sts: CacheWithTtl>, - pub dns_rbl: CacheWithTtl>>>, + pub dns_rbl: CacheWithTtl>>, } pub struct Ipc { diff --git a/crates/common/src/listener/asn.rs b/crates/common/src/listener/asn.rs index b0637344..9bb2def0 100644 --- a/crates/common/src/listener/asn.rs +++ b/crates/common/src/listener/asn.rs @@ -16,11 +16,7 @@ use mail_auth::common::resolver::ToReverseName; use store::write::now; use tokio::sync::Semaphore; -use crate::{ - config::network::{AsnGeoLookupConfig, AsnGeoLookupResource}, - manager::fetch_resource, - Server, -}; +use crate::{config::network::AsnGeoLookupConfig, manager::fetch_resource, Server}; pub struct AsnGeoLookupData { pub lock: Semaphore, @@ -153,18 +149,19 @@ impl Server { expires, timeout, max_size, - resources, + asn_resources, + geo_resources, + headers, } = &server.core.network.asn_geo_lookup { let mut asn_data = Data::new(); let mut country_data = Data::new(); - for lookup in resources { - let (url, headers, is_asn) = match lookup { - AsnGeoLookupResource::Asn { url, headers } => (url, headers, true), - AsnGeoLookupResource::Geo { url, headers } => (url, headers, false), - }; - + for (is_asn, url) in asn_resources + .iter() + .map(|url| (true, url)) + .chain(geo_resources.iter().map(|url| (false, url))) + { let time = Instant::now(); match fetch_resource(url, headers.clone().into(), *timeout, *max_size) .await diff --git a/crates/common/src/manager/mod.rs b/crates/common/src/manager/mod.rs index 90817dde..2fdfb689 100644 --- a/crates/common/src/manager/mod.rs +++ b/crates/common/src/manager/mod.rs @@ -30,10 +30,10 @@ pub const WEBADMIN_KEY: &[u8] = "STALWART_WEBADMIN".as_bytes(); impl ConfigManager { pub async fn fetch_resource(&self, resource_id: &str) -> Result, String> { if let Some(url) = self - .get(&format!("config.resource.{resource_id}")) + .get(&format!("{resource_id}.resource")) .await .map_err(|err| { - format!("Failed to fetch configuration key 'resource.{resource_id}': {err}",) + format!("Failed to fetch configuration key '{resource_id}.resource': {err}",) })? { fetch_resource(&url, None, Duration::from_secs(60), MAX_SIZE).await diff --git a/crates/jmap/src/api/management/enterprise/telemetry.rs b/crates/jmap/src/api/management/enterprise/telemetry.rs index 78a6f3ac..7bdfc97b 100644 --- a/crates/jmap/src/api/management/enterprise/telemetry.rs +++ b/crates/jmap/src/api/management/enterprise/telemetry.rs @@ -385,7 +385,13 @@ impl TelemetryApi for Server { .enterprise .as_ref() .and_then(|e| e.metrics_store.as_ref()) - .ok_or_else(|| manage::unsupported("No metrics store has been configured"))? + .ok_or_else(|| { + manage::error( + "No metrics store has been defined", + "You need to configure a metrics store in order to use this feature." + .into(), + ) + })? .store .query_metrics(after, before) .await?; diff --git a/crates/jmap/src/api/management/spam.rs b/crates/jmap/src/api/management/spam.rs index 9bae89e8..5c0dbd2e 100644 --- a/crates/jmap/src/api/management/spam.rs +++ b/crates/jmap/src/api/management/spam.rs @@ -4,15 +4,17 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{net::IpAddr, sync::Arc}; +use std::net::IpAddr; -use common::{auth::AccessToken, config::spamfilter::SpamFilterAction, Server}; +use common::{auth::AccessToken, config::spamfilter::SpamFilterAction, psl, Server}; use directory::{ backend::internal::manage::{self, ManageDirectory}, Permission, }; use hyper::Method; -use mail_auth::{ArcOutput, DkimOutput, IprevOutput}; +use mail_auth::{ + dmarc::verify::DmarcParameters, spf::verify::SpfParameters, AuthenticatedMessage, DmarcResult, +}; use mail_parser::{Message, MessageParser}; use serde::{Deserialize, Serialize}; use serde_json::json; @@ -29,10 +31,7 @@ use crate::api::{ HttpRequest, HttpResponse, JsonResponse, }; -use super::{ - decode_path_element, - troubleshoot::{AuthResult, DmarcPolicy}, -}; +use super::decode_path_element; pub trait ManageSpamHandler: Sync + Send { fn handle_manage_spam( @@ -50,27 +49,12 @@ pub trait ManageSpamHandler: Sync + Send { pub struct SpamClassifyRequest { pub message: String, - // Sender authentication - pub arc_result: AuthResult, - pub spf_ehlo_result: AuthResult, - pub spf_mail_from_result: AuthResult, - pub dkim_result: AuthResult, - pub dmarc_result: AuthResult, - pub dmarc_policy: DmarcPolicy, - pub iprev_result: AuthResult, - // Session details pub remote_ip: IpAddr, #[serde(default)] - pub remote_ip_ptr: Option, - #[serde(default)] - pub ehlo_domain: Option, + pub ehlo_domain: String, #[serde(default)] pub authenticated_as: Option, - #[serde(default)] - pub asn: Option, - #[serde(default)] - pub country: Option, // TLS #[serde(default)] @@ -114,12 +98,12 @@ impl ManageSpamHandler for Server { match (path.get(1).copied(), path.get(2).copied(), req.method()) { (Some("train"), Some(class @ ("ham" | "spam")), &Method::POST) => { let message = parse_message_or_err(body.as_deref().unwrap_or_default())?; - let input = if let Some(account) = path.get(3).copied() { + let input = if let Some(account) = path.get(3).copied().filter(|a| !a.is_empty()) { let account_id = self .store() .get_principal_id(decode_path_element(account).as_ref()) .await? - .ok_or_else(|| trc::ManageEvent::NotFound.into_err())?; + .ok_or_else(|| manage::not_found(account.to_string()))?; SpamFilterInput::from_account_message(&message, account_id, session.session_id) } else { SpamFilterInput::from_message(&message, session.session_id) @@ -141,48 +125,129 @@ impl ManageSpamHandler for Server { trc::EventType::Resource(trc::ResourceEvent::BadParameters).from_json_error(err) })?; - // Built classifier input + // Built spam filter input let message = parse_message_or_err(request.message.as_bytes())?; - let arc_result = ArcOutput::default().with_result(request.arc_result.into()); - let spf_ehlo_result = request.spf_ehlo_result.into(); - let spf_mail_from_result = request.spf_mail_from_result.into(); - let dkim_result = vec![match request.dkim_result { - AuthResult::Pass => DkimOutput::pass(), - AuthResult::Fail { details } => { - DkimOutput::fail(mail_auth::Error::Io(details.unwrap_or_default())) - } - AuthResult::Neutral { details } => { - DkimOutput::neutral(mail_auth::Error::Io(details.unwrap_or_default())) - } - AuthResult::TempError { details } => { - DkimOutput::temp_err(mail_auth::Error::Io(details.unwrap_or_default())) - } - AuthResult::PermError { details } => { - DkimOutput::perm_err(mail_auth::Error::Io(details.unwrap_or_default())) - } - _ => DkimOutput::neutral(mail_auth::Error::ParseError), - }]; - let dmarc_result = request.dmarc_result.into(); - let dmarc_policy = request.dmarc_policy.into(); - let iprev_result = IprevOutput { - result: request.iprev_result.into(), - ptr: request.remote_ip_ptr.map(|ptr| Arc::new(vec![ptr])), + + let remote_ip = request.remote_ip; + let ehlo_domain = request.ehlo_domain.to_lowercase(); + let mail_from = request.env_from.to_lowercase(); + let mail_from_domain = mail_from.rsplit_once('@').map(|(_, domain)| domain); + let local_host = self + .core + .storage + .config + .get("lookup.default.hostname") + .await + .unwrap_or_default() + .unwrap_or_else(|| "local.host".to_string()); + + let spf_ehlo_result = + self.core + .smtp + .resolvers + .dns + .verify_spf(self.inner.cache.build_auth_parameters( + SpfParameters::verify_ehlo(remote_ip, &ehlo_domain, &local_host), + )) + .await; + + let iprev_result = self + .core + .smtp + .resolvers + .dns + .verify_iprev(self.inner.cache.build_auth_parameters(remote_ip)) + .await; + + let spf_mail_from_result = if let Some(mail_from_domain) = mail_from_domain { + self.core + .smtp + .resolvers + .dns + .check_host(self.inner.cache.build_auth_parameters(SpfParameters::new( + remote_ip, + mail_from_domain, + &ehlo_domain, + &local_host, + &mail_from, + ))) + .await + } else { + self.core + .smtp + .resolvers + .dns + .check_host(self.inner.cache.build_auth_parameters(SpfParameters::new( + remote_ip, + &ehlo_domain, + &ehlo_domain, + &local_host, + &format!("postmaster@{ehlo_domain}"), + ))) + .await }; + + let auth_message = AuthenticatedMessage::from_parsed(&message, true); + + let dkim_output = self + .core + .smtp + .resolvers + .dns + .verify_dkim(self.inner.cache.build_auth_parameters(&auth_message)) + .await; + + let arc_output = self + .core + .smtp + .resolvers + .dns + .verify_arc(self.inner.cache.build_auth_parameters(&auth_message)) + .await; + + let dmarc_output = self + .core + .smtp + .resolvers + .dns + .verify_dmarc(self.inner.cache.build_auth_parameters(DmarcParameters { + message: &auth_message, + dkim_output: &dkim_output, + rfc5321_mail_from_domain: mail_from_domain.unwrap_or(ehlo_domain.as_str()), + spf_output: &spf_mail_from_result, + domain_suffix_fn: |domain| psl::domain_str(domain).unwrap_or(domain), + })) + .await; + let dmarc_pass = matches!(dmarc_output.spf_result(), DmarcResult::Pass) + || matches!(dmarc_output.dkim_result(), DmarcResult::Pass); + let dmarc_result = if dmarc_pass { + DmarcResult::Pass + } else if dmarc_output.spf_result() != &DmarcResult::None { + dmarc_output.spf_result().clone() + } else if dmarc_output.dkim_result() != &DmarcResult::None { + dmarc_output.dkim_result().clone() + } else { + DmarcResult::None + }; + let dmarc_policy = dmarc_output.policy(); + + let asn_geo = self.lookup_asn_country(remote_ip).await; + let input = SpamFilterInput { message: &message, span_id: session.session_id, - arc_result: Some(&arc_result), + arc_result: Some(&arc_output), spf_ehlo_result: Some(&spf_ehlo_result), spf_mail_from_result: Some(&spf_mail_from_result), - dkim_result: dkim_result.as_slice(), + dkim_result: dkim_output.as_slice(), dmarc_result: Some(&dmarc_result), dmarc_policy: Some(&dmarc_policy), iprev_result: Some(&iprev_result), remote_ip: request.remote_ip, - ehlo_domain: request.ehlo_domain.as_deref(), + ehlo_domain: Some(ehlo_domain.as_str()), authenticated_as: request.authenticated_as.as_deref(), - asn: request.asn, - country: request.country.as_deref(), + asn: asn_geo.asn.as_ref().map(|a| a.id), + country: asn_geo.country.as_ref().map(|c| c.as_str()), is_tls: request.is_tls, env_from: &request.env_from, env_from_flags: request.env_from_flags, diff --git a/crates/spam-filter/src/analysis/ip.rs b/crates/spam-filter/src/analysis/ip.rs index 185d7ab3..5e0e3ca0 100644 --- a/crates/spam-filter/src/analysis/ip.rs +++ b/crates/spam-filter/src/analysis/ip.rs @@ -7,7 +7,7 @@ use std::{future::Future, net::IpAddr}; use common::{ - config::spamfilter::{Element, Location}, + config::spamfilter::{Element, IpResolver, Location}, Server, }; use mail_auth::IprevResult; @@ -15,10 +15,7 @@ use mail_parser::{HeaderName, HeaderValue, Host}; use nlp::tokenizers::types::TokenType; use crate::{ - modules::{ - dnsbl::is_dnsbl, - expression::{IpResolver, SpamFilterResolver}, - }, + modules::{dnsbl::is_dnsbl, expression::SpamFilterResolver}, SpamFilterContext, TextPart, }; diff --git a/crates/spam-filter/src/analysis/llm.rs b/crates/spam-filter/src/analysis/llm.rs index bd7053a7..9a194daa 100644 --- a/crates/spam-filter/src/analysis/llm.rs +++ b/crates/spam-filter/src/analysis/llm.rs @@ -69,7 +69,20 @@ impl SpamFilterAnalyzeLlm for Server { confidence = Some(value); } } else if config.index_explanation.map_or(false, |i| i == idx) { - explanation = Some(value.replace('\n', " ")); + let explanation = explanation.get_or_insert_with(|| { + String::with_capacity(std::cmp::min(value.len(), 255)) + }); + + for value in value.chars() { + if !value.is_whitespace() { + explanation.push(value); + } else { + explanation.push(' '); + } + if explanation.len() == 255 { + break; + } + } } } } @@ -86,10 +99,9 @@ impl SpamFilterAnalyzeLlm for Server { _ => return, }; - if let (Some(header), Some(mut explanation)) = + if let (Some(header), Some(explanation)) = (&self.core.spam.headers.llm, explanation) { - explanation.truncate(512); ctx.result.header = format!("{header}: {category} ({explanation})\r\n",).into(); } diff --git a/crates/spam-filter/src/analysis/rules.rs b/crates/spam-filter/src/analysis/rules.rs index 500c9d8e..0c59278e 100644 --- a/crates/spam-filter/src/analysis/rules.rs +++ b/crates/spam-filter/src/analysis/rules.rs @@ -6,10 +6,13 @@ use std::future::Future; -use common::{config::spamfilter::Location, Server}; +use common::{ + config::spamfilter::{IpResolver, Location}, + Server, +}; use crate::{ - modules::expression::{EmailHeader, IpResolver, SpamFilterResolver, StringResolver}, + modules::expression::{EmailHeader, SpamFilterResolver, StringResolver}, SpamFilterContext, TextPart, }; diff --git a/crates/spam-filter/src/analysis/url.rs b/crates/spam-filter/src/analysis/url.rs index 4538daa3..b0e21b50 100644 --- a/crates/spam-filter/src/analysis/url.rs +++ b/crates/spam-filter/src/analysis/url.rs @@ -8,7 +8,7 @@ use std::collections::HashSet; use std::hash::{Hash, Hasher}; use std::{borrow::Cow, future::Future, time::Duration}; -use common::config::spamfilter::{Element, Location}; +use common::config::spamfilter::{Element, IpResolver, Location}; use common::scripts::functions::unicode::CharUtils; use common::Server; use hyper::{header::LOCATION, Uri}; @@ -17,7 +17,7 @@ use reqwest::redirect::Policy; use unicode_security::MixedScript; use crate::modules::dnsbl::is_dnsbl; -use crate::modules::expression::{IpResolver, SpamFilterResolver, StringResolver}; +use crate::modules::expression::{SpamFilterResolver, StringResolver}; use crate::modules::html::SRC; use crate::{ modules::html::{HtmlToken, A, HREF}, diff --git a/crates/spam-filter/src/modules/dnsbl.rs b/crates/spam-filter/src/modules/dnsbl.rs index a7ea9d12..2f52d104 100644 --- a/crates/spam-filter/src/modules/dnsbl.rs +++ b/crates/spam-filter/src/modules/dnsbl.rs @@ -6,15 +6,18 @@ use std::{ net::Ipv4Addr, + sync::Arc, time::{Duration, Instant}, }; -use common::{config::spamfilter::DnsBlServer, expr::functions::ResolveVariable, Server}; +use common::{ + config::spamfilter::{DnsBlServer, IpResolver}, + expr::functions::ResolveVariable, + Server, +}; use mail_auth::{common::resolver::IntoFqdn, Error}; use trc::SpamEvent; -use crate::modules::expression::IpResolver; - use super::expression::SpamFilterResolver; pub(crate) async fn is_dnsbl( @@ -77,13 +80,23 @@ pub(crate) async fn is_dnsbl( Elapsed = time.elapsed() ); + let entry = Arc::new(IpResolver::new( + result + .entry + .iter() + .copied() + .next() + .unwrap_or(Ipv4Addr::BROADCAST) + .into(), + )); + server.inner.cache.dns_rbl.insert_with_expiry( zone, - Some(result.entry.clone()), + Some(entry.clone()), result.expires, ); - result.entry + entry } Err(Error::DnsRecordNotFound(_)) => { trc::event!( @@ -118,18 +131,7 @@ pub(crate) async fn is_dnsbl( server .eval_if( &config.tags, - &SpamFilterResolver::new( - resolver.ctx, - &IpResolver::new( - result - .iter() - .copied() - .next() - .unwrap_or(Ipv4Addr::BROADCAST) - .into(), - ), - resolver.location, - ), + &SpamFilterResolver::new(resolver.ctx, result.as_ref(), resolver.location), resolver.ctx.input.span_id, ) .await diff --git a/crates/spam-filter/src/modules/expression.rs b/crates/spam-filter/src/modules/expression.rs index b7b23391..2e7f8a5e 100644 --- a/crates/spam-filter/src/modules/expression.rs +++ b/crates/spam-filter/src/modules/expression.rs @@ -4,13 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::net::IpAddr; - use common::{ config::spamfilter::*, expr::{functions::ResolveVariable, Variable}, }; -use mail_auth::common::resolver::ToReverseName; use mail_parser::{Header, HeaderValue}; use crate::{analysis::url::UrlParts, Recipient, SpamFilterContext, TextPart}; @@ -440,49 +437,3 @@ impl ResolveVariable for StringListResolver<'_> { Variable::Integer(0) } } - -pub struct IpResolver { - ip: IpAddr, - ip_string: String, - reverse: String, - octets: Variable<'static>, -} - -impl ResolveVariable for IpResolver { - fn resolve_variable(&self, variable: u32) -> Variable<'_> { - match variable { - V_IP => Variable::String(self.ip_string.as_str().into()), - V_IP_REVERSE => Variable::String(self.reverse.as_str().into()), - V_IP_OCTETS => self.octets.clone(), - V_IP_IS_V4 => Variable::Integer(self.ip.is_ipv4() as _), - V_IP_IS_V6 => Variable::Integer(self.ip.is_ipv6() as _), - _ => Variable::Integer(0), - } - } - - fn resolve_global(&self, _: &str) -> Variable<'_> { - Variable::Integer(0) - } -} - -impl IpResolver { - pub fn new(ip: IpAddr) -> Self { - Self { - ip_string: ip.to_string(), - reverse: ip.to_reverse_name(), - octets: Variable::Array(match ip { - IpAddr::V4(ipv4_addr) => ipv4_addr - .octets() - .iter() - .map(|o| Variable::Integer(*o as _)) - .collect(), - IpAddr::V6(ipv6_addr) => ipv6_addr - .octets() - .iter() - .map(|o| Variable::Integer(*o as _)) - .collect(), - }), - ip, - } - } -} diff --git a/tests/src/smtp/inbound/asn.rs b/tests/src/smtp/inbound/asn.rs index 75951cf8..9f47d05b 100644 --- a/tests/src/smtp/inbound/asn.rs +++ b/tests/src/smtp/inbound/asn.rs @@ -8,10 +8,7 @@ mod tests { use std::time::{Duration, Instant}; - use common::{ - config::network::{AsnGeoLookupConfig, AsnGeoLookupResource}, - Core, Server, - }; + use common::{config::network::AsnGeoLookupConfig, Core, Server}; #[tokio::test] #[ignore] @@ -55,31 +52,28 @@ mod tests { expires: Duration::from_secs(86400), timeout: Duration::from_secs(100), max_size: 100 * 1024 * 1024, - resources: vec![ - AsnGeoLookupResource::Asn { - //url: "file:///Users/me/code/playground/asn-ipv4.csv".to_string(), - url: "https://cdn.jsdelivr.net/npm/@ip-location-db/asn/asn-ipv4.csv".to_string(), - headers: Default::default(), - }, - AsnGeoLookupResource::Asn { - //url: "file:///Users/me/code/playground/asn-ipv6.csv".to_string(), - url: "https://cdn.jsdelivr.net/npm/@ip-location-db/asn/asn-ipv6.csv".to_string(), - headers: Default::default(), - }, - AsnGeoLookupResource::Geo { - //url: "file:///Users/me/code/playground/geolite2-geo-whois-asn-country-ipv4.csv" - // .to_string(), - url: "https://cdn.jsdelivr.net/npm/@ip-location-db/geolite2-geo-whois-asn-country/geolite2-geo-whois-asn-country-ipv4.csv" - .to_string(), - headers: Default::default(), - }, - AsnGeoLookupResource::Geo { - //url: "file:///Users/me/code/playground/geolite2-geo-whois-asn-country-ipv6.csv" - // .to_string(), - url: "https://cdn.jsdelivr.net/npm/@ip-location-db/geolite2-geo-whois-asn-country/geolite2-geo-whois-asn-country-ipv4.csv" - .to_string(), - headers: Default::default(), - }, + headers: Default::default(), + asn_resources: vec![ + //url: "file:///Users/me/code/playground/asn-ipv4.csv".to_string(), + //url: "file:///Users/me/code/playground/asn-ipv6.csv".to_string(), + "https://cdn.jsdelivr.net/npm/@ip-location-db/asn/asn-ipv4.csv".to_string(), + "https://cdn.jsdelivr.net/npm/@ip-location-db/asn/asn-ipv6.csv".to_string(), + ], + geo_resources: vec![ + //url: "file:///Users/me/code/playground/geolite2-geo-whois-asn-country-ipv4.csv" + // .to_string(), + //url: "file:///Users/me/code/playground/geolite2-geo-whois-asn-country-ipv6.csv" + // .to_string(), + concat!( + "https://cdn.jsdelivr.net/npm/@ip-location-db/geolite2-geo-whois-", + "asn-country/geolite2-geo-whois-asn-country-ipv4.csv" + ) + .to_string(), + concat!( + "https://cdn.jsdelivr.net/npm/@ip-location-db/geolite2-geo-whois-", + "asn-country/geolite2-geo-whois-asn-country-ipv6.csv" + ) + .to_string(), ], }; let server = Server { diff --git a/tests/src/smtp/mod.rs b/tests/src/smtp/mod.rs index 06946725..20e28ebf 100644 --- a/tests/src/smtp/mod.rs +++ b/tests/src/smtp/mod.rs @@ -14,6 +14,7 @@ use common::{ config::{ server::{Listeners, ServerProtocol}, smtp::resolver::Tlsa, + spamfilter::IpResolver, }, ipc::{QueueEvent, ReportingEvent}, manager::boot::{build_ipc, IpcReceivers}, @@ -315,7 +316,14 @@ impl DnsCache for Server { fn dnsbl_add(&self, name: &str, value: Vec, valid_until: std::time::Instant) { self.inner.cache.dns_rbl.insert_with_expiry( name.to_string(), - Some(Arc::new(value)), + Some(Arc::new(IpResolver::new( + value + .iter() + .copied() + .next() + .unwrap_or(Ipv4Addr::BROADCAST) + .into(), + ))), valid_until, ); }