From db7ae48c7719f7d732707436473975002326705b Mon Sep 17 00:00:00 2001 From: mdecimus Date: Fri, 6 Dec 2024 18:35:39 +0100 Subject: [PATCH] Port Spam filter to Rust - part 1 --- Cargo.lock | 25 ++- Cargo.toml | 1 + crates/common/src/config/mod.rs | 3 + crates/common/src/config/smtp/session.rs | 7 - crates/common/src/config/spamfilter.rs | 19 ++ crates/common/src/dns.rs | 63 ++++++ crates/common/src/lib.rs | 3 + crates/common/src/scripts/plugins/dns.rs | 77 +------ crates/common/src/scripts/plugins/mod.rs | 22 +- crates/imap-proto/Cargo.toml | 2 +- crates/spam-filter/Cargo.toml | 24 +++ crates/spam-filter/src/analysis/date.rs | 36 ++++ crates/spam-filter/src/analysis/dmarc.rs | 133 ++++++++++++ crates/spam-filter/src/analysis/ehlo.rs | 55 +++++ crates/spam-filter/src/analysis/headers.rs | 193 ++++++++++++++++++ crates/spam-filter/src/analysis/init.rs | 77 +++++++ crates/spam-filter/src/analysis/iprev.rs | 23 +++ crates/spam-filter/src/analysis/messageid.rs | 85 ++++++++ crates/spam-filter/src/analysis/mod.rs | 15 ++ crates/spam-filter/src/lib.rs | 96 +++++++++ .../src/modules}/bayes.rs | 0 crates/spam-filter/src/modules/mod.rs | 1 + .../src/modules}/pyzor.rs | 0 crates/spam-filter/src/modules/sanitize.rs | 38 ++++ crates/store/src/backend/memory/mod.rs | 53 +---- crates/trc/event-macro/Cargo.toml | 2 +- crates/utils/proc-macros/Cargo.toml | 2 +- crates/utils/src/glob.rs | 99 +++++++++ .../config/spamfilter/scripts/date.sieve | 19 -- .../config/spamfilter/scripts/dmarc.sieve | 91 --------- .../config/spamfilter/scripts/headers.sieve | 140 ------------- .../config/spamfilter/scripts/helo.sieve | 30 --- resources/config/spamfilter/scripts/ip.sieve | 8 - .../config/spamfilter/scripts/messageid.sieve | 68 ------ .../config/spamfilter/scripts/prelude.sieve | 43 ---- tests/Cargo.toml | 2 +- 36 files changed, 1010 insertions(+), 545 deletions(-) create mode 100644 crates/common/src/config/spamfilter.rs create mode 100644 crates/common/src/dns.rs create mode 100644 crates/spam-filter/Cargo.toml create mode 100644 crates/spam-filter/src/analysis/date.rs create mode 100644 crates/spam-filter/src/analysis/dmarc.rs create mode 100644 crates/spam-filter/src/analysis/ehlo.rs create mode 100644 crates/spam-filter/src/analysis/headers.rs create mode 100644 crates/spam-filter/src/analysis/init.rs create mode 100644 crates/spam-filter/src/analysis/iprev.rs create mode 100644 crates/spam-filter/src/analysis/messageid.rs create mode 100644 crates/spam-filter/src/analysis/mod.rs create mode 100644 crates/spam-filter/src/lib.rs rename crates/{common/src/scripts/plugins => spam-filter/src/modules}/bayes.rs (100%) create mode 100644 crates/spam-filter/src/modules/mod.rs rename crates/{common/src/scripts/plugins => spam-filter/src/modules}/pyzor.rs (100%) create mode 100644 crates/spam-filter/src/modules/sanitize.rs delete mode 100644 resources/config/spamfilter/scripts/date.sieve delete mode 100644 resources/config/spamfilter/scripts/dmarc.sieve delete mode 100644 resources/config/spamfilter/scripts/headers.sieve delete mode 100644 resources/config/spamfilter/scripts/helo.sieve delete mode 100644 resources/config/spamfilter/scripts/ip.sieve delete mode 100644 resources/config/spamfilter/scripts/messageid.sieve delete mode 100644 resources/config/spamfilter/scripts/prelude.sieve diff --git a/Cargo.lock b/Cargo.lock index 70ec9bd0..a8313c80 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2134,7 +2134,7 @@ dependencies = [ [[package]] name = "event_macro" -version = "0.1.0" +version = "0.10.7" dependencies = [ "proc-macro2", "quote", @@ -3254,7 +3254,7 @@ dependencies = [ [[package]] name = "imap_proto" -version = "0.1.0" +version = "0.10.7" dependencies = [ "ahash 0.8.11", "chrono", @@ -4890,7 +4890,7 @@ dependencies = [ [[package]] name = "proc_macros" -version = "0.1.0" +version = "0.10.7" dependencies = [ "proc-macro2", "quote", @@ -6442,6 +6442,23 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "spam-filter" +version = "0.10.7" +dependencies = [ + "common", + "mail-auth", + "mail-builder", + "mail-parser", + "mail-send", + "nlp", + "psl", + "store", + "tokio", + "trc", + "utils", +] + [[package]] name = "spin" version = "0.5.2" @@ -6712,7 +6729,7 @@ dependencies = [ [[package]] name = "tests" -version = "0.1.0" +version = "0.10.7" dependencies = [ "ahash 0.8.11", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 6a429e14..bd71ca41 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ members = [ "crates/smtp", "crates/managesieve", "crates/pop3", + "crates/spam-filter", "crates/nlp", "crates/store", "crates/directory", diff --git a/crates/common/src/config/mod.rs b/crates/common/src/config/mod.rs index 57032477..0068a17b 100644 --- a/crates/common/src/config/mod.rs +++ b/crates/common/src/config/mod.rs @@ -14,6 +14,7 @@ use hyper::{ HeaderMap, }; use ring::signature::{EcdsaKeyPair, RsaKeyPair}; +use spamfilter::SpamFilterConfig; use store::{BlobBackend, BlobStore, FtsStore, LookupStore, Store, Stores}; use telemetry::Metrics; use utils::config::{utils::AsKey, Config}; @@ -35,6 +36,7 @@ pub mod network; pub mod scripts; pub mod server; pub mod smtp; +pub mod spamfilter; pub mod storage; pub mod telemetry; @@ -181,6 +183,7 @@ impl Core { oauth: OAuthConfig::parse(config), acme: AcmeProviders::parse(config), metrics: Metrics::parse(config), + spam: SpamFilterConfig::parse(config), storage: Storage { data, blob, diff --git a/crates/common/src/config/smtp/session.rs b/crates/common/src/config/smtp/session.rs index a2c8e7d7..69c11635 100644 --- a/crates/common/src/config/smtp/session.rs +++ b/crates/common/src/config/smtp/session.rs @@ -799,14 +799,7 @@ impl Default for SessionConfig { subaddressing: AddressMapping::Enable, }, data: Data { - #[cfg(feature = "test_mode")] script: IfBlock::empty("session.data.script"), - #[cfg(not(feature = "test_mode"))] - script: IfBlock::new::<()>( - "session.data.script", - [("is_empty(authenticated_as)", "'spam-filter'")], - "'track-replies'", - ), pipe_commands: Default::default(), max_messages: IfBlock::new::<()>("session.data.limits.messages", [], "10"), max_message_size: IfBlock::new::<()>("session.data.limits.size", [], "104857600"), diff --git a/crates/common/src/config/spamfilter.rs b/crates/common/src/config/spamfilter.rs new file mode 100644 index 00000000..291f8393 --- /dev/null +++ b/crates/common/src/config/spamfilter.rs @@ -0,0 +1,19 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use utils::{config::Config, glob::GlobSet}; + +#[derive(Debug, Clone, Default)] +pub struct SpamFilterConfig { + pub list_dmarc_allow: GlobSet, + pub list_spf_dkim_allow: GlobSet, +} + +impl SpamFilterConfig { + pub fn parse(config: &mut Config) -> Self { + SpamFilterConfig::default() + } +} diff --git a/crates/common/src/dns.rs b/crates/common/src/dns.rs new file mode 100644 index 00000000..b65d7d87 --- /dev/null +++ b/crates/common/src/dns.rs @@ -0,0 +1,63 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::net::IpAddr; + +use mail_auth::{Error, IpLookupStrategy}; + +use crate::Core; + +impl Core { + pub async fn dns_exists_mx(&self, entry: &str) -> trc::Result { + match self.smtp.resolvers.dns.mx_lookup(entry).await { + Ok(result) => Ok(result.iter().any(|mx| !mx.exchanges.is_empty())), + Err(Error::DnsRecordNotFound(_)) => Ok(false), + Err(err) => Err(err.into()), + } + } + + pub async fn dns_exists_ip(&self, entry: &str) -> trc::Result { + match self + .smtp + .resolvers + .dns + .ip_lookup(entry, IpLookupStrategy::Ipv4thenIpv6, 10) + .await + { + Ok(result) => Ok(!result.is_empty()), + Err(Error::DnsRecordNotFound(_)) => Ok(false), + Err(err) => Err(err.into()), + } + } + + pub async fn dns_exists_ptr(&self, entry: &str) -> trc::Result { + if let Ok(addr) = entry.parse::() { + match self.smtp.resolvers.dns.ptr_lookup(addr).await { + Ok(result) => Ok(!result.is_empty()), + Err(Error::DnsRecordNotFound(_)) => Ok(false), + Err(err) => Err(err.into()), + } + } else { + Err(trc::EventType::Resource(trc::ResourceEvent::BadParameters).into_err()) + } + } + + pub async fn dns_exists_ipv4(&self, entry: &str) -> trc::Result { + match self.smtp.resolvers.dns.ipv4_lookup(entry).await { + Ok(result) => Ok(!result.is_empty()), + Err(Error::DnsRecordNotFound(_)) => Ok(false), + Err(err) => Err(err.into()), + } + } + + pub async fn dns_exists_ipv6(&self, entry: &str) -> trc::Result { + match self.smtp.resolvers.dns.ipv6_lookup(entry).await { + Ok(result) => Ok(!result.is_empty()), + Err(Error::DnsRecordNotFound(_)) => Ok(false), + Err(err) => Err(err.into()), + } + } +} diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 95c83fb1..731eabd7 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -20,6 +20,7 @@ use config::{ network::Network, scripts::{RemoteList, Scripting}, smtp::SmtpConfig, + spamfilter::SpamFilterConfig, storage::Storage, telemetry::Metrics, }; @@ -47,6 +48,7 @@ pub mod addresses; pub mod auth; pub mod config; pub mod core; +pub mod dns; #[cfg(feature = "enterprise")] pub mod enterprise; pub mod expr; @@ -205,6 +207,7 @@ pub struct Core { pub oauth: OAuthConfig, pub smtp: SmtpConfig, pub jmap: JmapConfig, + pub spam: SpamFilterConfig, pub imap: ImapConfig, pub metrics: Metrics, #[cfg(feature = "enterprise")] diff --git a/crates/common/src/scripts/plugins/dns.rs b/crates/common/src/scripts/plugins/dns.rs index 6e145ddd..f5f43870 100644 --- a/crates/common/src/scripts/plugins/dns.rs +++ b/crates/common/src/scripts/plugins/dns.rs @@ -6,7 +6,7 @@ use std::net::IpAddr; -use mail_auth::{Error, IpLookupStrategy}; +use mail_auth::IpLookupStrategy; use sieve::{runtime::Variable, FunctionMap}; use super::PluginContext; @@ -145,44 +145,12 @@ pub async fn exec_exists(ctx: PluginContext<'_>) -> trc::Result { let entry = ctx.arguments[0].to_string(); let record_type = ctx.arguments[1].to_string(); - Ok(if record_type.eq_ignore_ascii_case("ip") { - match ctx - .server - .core - .smtp - .resolvers - .dns - .ip_lookup(entry.as_ref(), IpLookupStrategy::Ipv4thenIpv6, 10) - .await - { - Ok(result) => i64::from(!result.is_empty()), - Err(Error::DnsRecordNotFound(_)) => 0, - Err(_) => -1, - } + let result = if record_type.eq_ignore_ascii_case("ip") { + ctx.server.core.dns_exists_ip(entry.as_ref()).await } else if record_type.eq_ignore_ascii_case("mx") { - match ctx - .server - .core - .smtp - .resolvers - .dns - .mx_lookup(entry.as_ref()) - .await - { - Ok(result) => i64::from(result.iter().any(|mx| !mx.exchanges.is_empty())), - Err(Error::DnsRecordNotFound(_)) => 0, - Err(_) => -1, - } + ctx.server.core.dns_exists_mx(entry.as_ref()).await } else if record_type.eq_ignore_ascii_case("ptr") { - if let Ok(addr) = entry.parse::() { - match ctx.server.core.smtp.resolvers.dns.ptr_lookup(addr).await { - Ok(result) => i64::from(!result.is_empty()), - Err(Error::DnsRecordNotFound(_)) => 0, - Err(_) => -1, - } - } else { - -1 - } + ctx.server.core.dns_exists_ptr(entry.as_ref()).await } else if record_type.eq_ignore_ascii_case("ipv4") { #[cfg(feature = "test_mode")] { @@ -191,37 +159,14 @@ pub async fn exec_exists(ctx: PluginContext<'_>) -> trc::Result { } } - match ctx - .server - .core - .smtp - .resolvers - .dns - .ipv4_lookup(entry.as_ref()) - .await - { - Ok(result) => i64::from(!result.is_empty()), - Err(Error::DnsRecordNotFound(_)) => 0, - Err(_) => -1, - } + ctx.server.core.dns_exists_ipv4(entry.as_ref()).await } else if record_type.eq_ignore_ascii_case("ipv6") { - match ctx - .server - .core - .smtp - .resolvers - .dns - .ipv6_lookup(entry.as_ref()) - .await - { - Ok(result) => i64::from(!result.is_empty()), - Err(Error::DnsRecordNotFound(_)) => 0, - Err(_) => -1, - } + ctx.server.core.dns_exists_ipv6(entry.as_ref()).await } else { - -1 - } - .into()) + return Ok((-1).into()); + }; + + Ok(result.map(i64::from).unwrap_or(-1).into()) } trait ShortError { diff --git a/crates/common/src/scripts/plugins/mod.rs b/crates/common/src/scripts/plugins/mod.rs index a1871395..c62248bf 100644 --- a/crates/common/src/scripts/plugins/mod.rs +++ b/crates/common/src/scripts/plugins/mod.rs @@ -4,14 +4,12 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -pub mod bayes; pub mod dns; pub mod exec; pub mod headers; pub mod http; pub mod llm_prompt; pub mod lookup; -pub mod pyzor; pub mod query; pub mod text; @@ -33,7 +31,7 @@ pub struct PluginContext<'x> { pub arguments: Vec, } -const PLUGINS_REGISTER: [RegisterPluginFnc; 19] = [ +const PLUGINS_REGISTER: [RegisterPluginFnc; 14] = [ query::register, exec::register, lookup::register, @@ -44,11 +42,6 @@ const PLUGINS_REGISTER: [RegisterPluginFnc; 19] = [ dns::register, dns::register_exists, http::register_header, - bayes::register_train, - bayes::register_untrain, - bayes::register_classify, - bayes::register_is_balanced, - pyzor::register, headers::register, text::register_tokenize, text::register_domain_part, @@ -98,15 +91,10 @@ impl Core { 7 => dns::exec(ctx).await, 8 => dns::exec_exists(ctx).await, 9 => http::exec_header(ctx).await, - 10 => bayes::exec_train(ctx).await, - 11 => bayes::exec_untrain(ctx).await, - 12 => bayes::exec_classify(ctx).await, - 13 => bayes::exec_is_balanced(ctx).await, - 14 => pyzor::exec(ctx).await, - 15 => headers::exec(ctx), - 16 => text::exec_tokenize(ctx), - 17 => text::exec_domain_part(ctx), - 18 => llm_prompt::exec(ctx).await, + 10 => headers::exec(ctx), + 11 => text::exec_tokenize(ctx), + 12 => text::exec_domain_part(ctx), + 13 => llm_prompt::exec(ctx).await, _ => unreachable!(), }; diff --git a/crates/imap-proto/Cargo.toml b/crates/imap-proto/Cargo.toml index 2ba5b823..878c5d5a 100644 --- a/crates/imap-proto/Cargo.toml +++ b/crates/imap-proto/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "imap_proto" -version = "0.1.0" +version = "0.10.7" edition = "2021" resolver = "2" diff --git a/crates/spam-filter/Cargo.toml b/crates/spam-filter/Cargo.toml new file mode 100644 index 00000000..1805f2c9 --- /dev/null +++ b/crates/spam-filter/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "spam-filter" +version = "0.10.7" +edition = "2021" +resolver = "2" + +[dependencies] +utils = { path = "../utils" } +nlp = { path = "../nlp" } +store = { path = "../store" } +trc = { path = "../trc" } +common = { path = "../common" } +mail-parser = { version = "0.9", features = ["full_encoding", "ludicrous_mode"] } +mail-builder = { version = "0.3", features = ["ludicrous_mode"] } +mail-auth = { version = "0.5" } +mail-send = { version = "0.4", default-features = false, features = ["cram-md5", "ring", "tls12"] } +psl = "2" + +[features] +test_mode = [] +enterprise = [] + +[dev-dependencies] +tokio = { version = "1.23", features = ["full"] } diff --git a/crates/spam-filter/src/analysis/date.rs b/crates/spam-filter/src/analysis/date.rs new file mode 100644 index 00000000..4788da52 --- /dev/null +++ b/crates/spam-filter/src/analysis/date.rs @@ -0,0 +1,36 @@ +use std::future::Future; + +use common::Core; +use store::write::now; + +use crate::SpamFilterContext; + +pub trait SpamFilterAnalyzeEhlo: Sync + Send { + fn spam_filter_analyze_date( + &self, + ctx: &mut SpamFilterContext<'_>, + ) -> impl Future + Send; +} + +impl SpamFilterAnalyzeEhlo for Core { + async fn spam_filter_analyze_date(&self, ctx: &mut SpamFilterContext<'_>) { + if let Some(date) = ctx.input.message.date() { + let date = date.to_timestamp(); + if date != 0 { + let date_diff = now() as i64 - date; + + if date_diff > 86400 { + // Older than a day + ctx.add_tag("DATE_IN_PAST"); + } else if -date_diff > 7200 { + //# More than 2 hours in the future + ctx.add_tag("DATE_IN_FUTURE"); + } + } else { + ctx.add_tag("INVALID_DATE"); + } + } else { + ctx.add_tag("MISSING_DATE"); + } + } +} diff --git a/crates/spam-filter/src/analysis/dmarc.rs b/crates/spam-filter/src/analysis/dmarc.rs new file mode 100644 index 00000000..9053e1cc --- /dev/null +++ b/crates/spam-filter/src/analysis/dmarc.rs @@ -0,0 +1,133 @@ +use std::future::Future; + +use common::Core; +use mail_auth::{ + common::verify::VerifySignature, dmarc::Policy, DkimResult, DmarcResult, SpfResult, +}; + +use crate::SpamFilterContext; + +pub trait SpamFilterAnalyzeEhlo: Sync + Send { + fn spam_filter_analyze_dmarc( + &self, + ctx: &mut SpamFilterContext<'_>, + ) -> impl Future + Send; +} + +impl SpamFilterAnalyzeEhlo for Core { + async fn spam_filter_analyze_dmarc(&self, ctx: &mut SpamFilterContext<'_>) { + ctx.add_tag(match ctx.input.spf_mail_from_result.result() { + SpfResult::Pass => "SPF_ALLOW", + SpfResult::Fail => "SPF_FAIL", + SpfResult::SoftFail => "SPF_SOFTFAIL", + SpfResult::Neutral => "SPF_NEUTRAL", + SpfResult::TempError => "SPF_DNSFAIL", + SpfResult::PermError => "SPF_PERMFAIL", + SpfResult::None => "SPF_NA", + }); + + ctx.add_tag( + match ctx + .input + .dkim_result + .iter() + .find(|r| matches!(r.result(), DkimResult::Pass)) + .or_else(|| ctx.input.dkim_result.first()) + .map(|r| r.result()) + .unwrap_or(&DkimResult::None) + { + DkimResult::Pass => "DKIM_ALLOW", + DkimResult::Fail(_) => "DKIM_REJECT", + DkimResult::PermError(_) => "DKIM_PERMFAIL", + DkimResult::TempError(_) => "DKIM_TEMPFAIL", + DkimResult::Neutral(_) | DkimResult::None => "DKIM_NA", + }, + ); + + ctx.add_tag(match ctx.input.arc_result.result() { + DkimResult::Pass => "ARC_ALLOW", + DkimResult::Fail(_) => "ARC_REJECT", + DkimResult::PermError(_) => "ARC_INVALID", + DkimResult::TempError(_) => "ARC_DNSFAIL", + DkimResult::Neutral(_) | DkimResult::None => "ARC_NA", + }); + + ctx.add_tag(match ctx.input.dmarc_result { + DmarcResult::Pass => "DMARC_POLICY_ALLOW", + DmarcResult::TempError(_) => "DMARC_DNSFAIL", + DmarcResult::PermError(_) => "DMARC_BAD_POLICY", + DmarcResult::None => "DMARC_NA", + DmarcResult::Fail(_) => match ctx.input.dmarc_policy { + Policy::Quarantine => "DMARC_POLICY_QUARANTINE", + Policy::Reject => "DMARC_POLICY_REJECT", + Policy::Unspecified | Policy::None => "DMARC_POLICY_SOFTFAIL", + }, + }); + + for header in ctx.input.message.headers() { + let header_name = header.name(); + if header_name.eq_ignore_ascii_case("DKIM-Signature") { + ctx.add_tag("DKIM_SIGNED"); + } else if header_name.eq_ignore_ascii_case("ARC-Seal") { + ctx.add_tag("ARC_SIGNED"); + } + } + + if self + .spam + .list_dmarc_allow + .contains(&ctx.output.from_addr.domain_part.fqdn) + { + if matches!(ctx.input.dmarc_result, DmarcResult::Pass) { + ctx.add_tag("ALLOWLIST_DMARC"); + } else { + ctx.add_tag("BLOCKLIST_DMARC"); + } + } else if self + .spam + .list_spf_dkim_allow + .contains(&ctx.output.from_addr.domain_part.fqdn) + { + let is_dkim_pass = matches!(ctx.input.arc_result.result(), DkimResult::Pass) + || ctx.input.dkim_result.iter().any(|r| { + matches!(r.result(), DkimResult::Pass) + && r.signature().map_or(false, |s| { + s.domain().to_lowercase() == ctx.output.from_addr.domain_part.fqdn + }) + }); + let is_spf_pass = matches!(ctx.input.spf_mail_from_result.result(), SpfResult::Pass); + + if is_dkim_pass && is_spf_pass { + ctx.add_tag("ALLOWLIST_SPF_DKIM"); + } else if is_dkim_pass { + ctx.add_tag("ALLOWLIST_DKIM"); + if !matches!( + ctx.input.spf_mail_from_result.result(), + SpfResult::TempError + ) { + ctx.add_tag("BLOCKLIST_SPF"); + } + } else if is_spf_pass { + ctx.add_tag("ALLOWLIST_SPF"); + if !ctx + .input + .dkim_result + .iter() + .any(|r| matches!(r.result(), DkimResult::TempError(_))) + { + ctx.add_tag("BLOCKLIST_DKIM"); + } + } else if !matches!( + ctx.input.spf_mail_from_result.result(), + SpfResult::TempError + ) && !ctx + .input + .dkim_result + .iter() + .any(|r| matches!(r.result(), DkimResult::TempError(_))) + { + ctx.add_tag("BLOCKLIST_SPF_DKIM"); + } + } + } +} diff --git a/crates/spam-filter/src/analysis/ehlo.rs b/crates/spam-filter/src/analysis/ehlo.rs new file mode 100644 index 00000000..d69465a6 --- /dev/null +++ b/crates/spam-filter/src/analysis/ehlo.rs @@ -0,0 +1,55 @@ +use std::future::Future; + +use common::Core; + +use crate::SpamFilterContext; + +pub trait SpamFilterAnalyzeEhlo: Sync + Send { + fn spam_filter_analyze_ehlo( + &self, + ctx: &mut SpamFilterContext<'_>, + ) -> impl Future + Send; +} + +impl SpamFilterAnalyzeEhlo for Core { + async fn spam_filter_analyze_ehlo(&self, ctx: &mut SpamFilterContext<'_>) { + if let Some(ehlo_ip) = ctx.output.ehlo_host.ip { + // Helo host is bare ip + ctx.add_tag("HELO_BAREIP"); + + if ehlo_ip != ctx.input.remote_ip { + // Helo A IP != hostname IP + ctx.add_tag("HELO_IP_A"); + } + } else if ctx.output.ehlo_host.sld.is_some() { + if ctx + .output + .iprev_ptr + .as_ref() + .map_or(false, |ptr| ptr != &ctx.output.ehlo_host.fqdn) + { + // Helo does not match reverse IP + ctx.add_tag("HELO_IPREV_MISMATCH"); + } + + if matches!( + ( + self.dns_exists_ip(&ctx.output.ehlo_host.fqdn).await, + self.dns_exists_mx(&ctx.output.ehlo_host.fqdn).await + ), + (Ok(false), Ok(false)) + ) { + // Helo no resolve to A or MX + ctx.add_tag("HELO_NORES_A_OR_MX"); + } + } else { + if ctx.output.ehlo_host.fqdn.contains("user") { + // Helo host contains 'user' + ctx.add_tag("RCVD_HELO_USER"); + } + + // Helo not FQDN + ctx.add_tag("HELO_NOT_FQDN"); + } + } +} diff --git a/crates/spam-filter/src/analysis/headers.rs b/crates/spam-filter/src/analysis/headers.rs new file mode 100644 index 00000000..71eceffd --- /dev/null +++ b/crates/spam-filter/src/analysis/headers.rs @@ -0,0 +1,193 @@ +use std::future::Future; + +use common::Core; +use mail_parser::HeaderName; +use store::ahash::AHashSet; + +use crate::SpamFilterContext; + +pub trait SpamFilterAnalyzeEhlo: Sync + Send { + fn spam_filter_analyze_headers( + &self, + ctx: &mut SpamFilterContext<'_>, + ) -> impl Future + Send; +} + +impl SpamFilterAnalyzeEhlo for Core { + async fn spam_filter_analyze_headers(&self, ctx: &mut SpamFilterContext<'_>) { + let mut list_score = 0.0; + let mut unique_headers = AHashSet::new(); + let raw_message = ctx.input.message.raw_message(); + + for header in ctx.input.message.headers() { + match &header.name { + HeaderName::ContentType + | HeaderName::ContentTransferEncoding + | HeaderName::Date + | HeaderName::From + | HeaderName::Sender + | HeaderName::To + | HeaderName::Cc + | HeaderName::Bcc + | HeaderName::ReplyTo + | HeaderName::Subject + | HeaderName::MessageId + | HeaderName::References + | HeaderName::InReplyTo => { + if !unique_headers.insert(header.name.clone()) { + ctx.add_tag("MULTIPLE_UNIQUE_HEADERS"); + } + + if !matches!(raw_message.get(header.offset_field), Some(b' ')) { + ctx.add_tag("HEADER_EMPTY_DELIMITER"); + } + } + HeaderName::ListArchive + | HeaderName::ListOwner + | HeaderName::ListHelp + | HeaderName::ListPost => { + list_score += 0.125; + } + HeaderName::ListId => { + list_score += 0.5125; + } + HeaderName::ListSubscribe => { + list_score += 0.25; + } + HeaderName::ListUnsubscribe => { + list_score += 0.25; + ctx.add_tag("HAS_LIST_UNSUB"); + } + HeaderName::Other(name) => { + let value = header + .value() + .as_text() + .unwrap_or_default() + .trim() + .to_lowercase(); + + if name.eq_ignore_ascii_case("Precedence") { + if value == "bulk" { + list_score += 0.25; + ctx.add_tag("PRECEDENCE_BULK"); + } else if value == "list" { + list_score += 0.25; + } + } else if name.eq_ignore_ascii_case("X-Loop") { + list_score += 0.125; + } else if name.eq_ignore_ascii_case("X-Priority") { + match value.parse::().unwrap_or(i32::MAX) { + 0 => { + ctx.add_tag("HAS_X_PRIO_ZERO"); + } + 1 => { + ctx.add_tag("HAS_X_PRIO_ONE"); + } + 2 => { + ctx.add_tag("HAS_X_PRIO_TWO"); + } + 3 | 4 => { + ctx.add_tag("HAS_X_PRIO_THREE"); + } + 4..=10000 => { + ctx.add_tag("HAS_X_PRIO_FIVE"); + } + _ => {} + } + } else if name.eq_ignore_ascii_case("X-Mailer") { + if name != "X-Mailer" { + ctx.add_tag("XM_CASE"); + } + if !value.is_empty() { + if !value.as_bytes().iter().any(|&b| b.is_ascii_digit()) { + ctx.add_tag("XM_UA_NO_VERSION"); + } + + if value.contains("phpmailer") { + ctx.add_tag("HAS_PHPMAILER_SIG"); + } + } + } else if name.eq_ignore_ascii_case("User-Agent") { + if !value.is_empty() + && !value.as_bytes().iter().any(|&b| b.is_ascii_digit()) + { + ctx.add_tag("XM_UA_NO_VERSION"); + } + } else if name.eq_ignore_ascii_case("Organization") + || name.eq_ignore_ascii_case("Organisation") + { + ctx.add_tag("HAS_ORG_HEADER"); + } else if name.eq_ignore_ascii_case("X-Originating-IP") { + ctx.add_tag("HAS_XOIP"); + } else if name.eq_ignore_ascii_case("X-KLMS-AntiSpam-Status") { + if value.contains("spam") { + ctx.add_tag("KLMS_SPAM"); + } + } else if name.eq_ignore_ascii_case("X-Spam") + || name.eq_ignore_ascii_case("X-Spam-Flag") + || name.eq_ignore_ascii_case("X-Spam-Status") + { + if value.contains("yes") || value.contains("true") || value.contains("spam") + { + ctx.add_tag("SPAM_FLAG"); + } + } else if name.eq_ignore_ascii_case("X-UI-Filterresults") + || name.eq_ignore_ascii_case("X-UI-Out-Filterresults") + { + if value.contains("junk") { + ctx.add_tag("UNITEDINTERNET_SPAM"); + } + } else if name.eq_ignore_ascii_case("X-PHP-Originating-Script") { + ctx.add_tag("HAS_X_POS"); + if value.contains("eval()") { + ctx.add_tag("X_PHP_EVAL"); + } + if value.contains("../") { + ctx.add_tag("HIDDEN_SOURCE_OBJ"); + } + } else if name.eq_ignore_ascii_case("X-PHP-Script") { + ctx.add_tag("HAS_X_PHP_SCRIPT"); + if value.contains("eval()") { + ctx.add_tag("X_PHP_EVAL"); + } + if value.contains("../") { + ctx.add_tag("HIDDEN_SOURCE_OBJ"); + } + if value.contains("sendmail.php") { + ctx.add_tag("PHP_XPS_PATTERN"); + } + } else if name.eq_ignore_ascii_case("X-Source") + || name.eq_ignore_ascii_case("X-Source-Args") + || name.eq_ignore_ascii_case("X-Source-Dir") + { + ctx.add_tag("HAS_X_SOURCE"); + if value.contains("'../") { + ctx.add_tag("HIDDEN_SOURCE_OBJ"); + } + } else if name.eq_ignore_ascii_case("X-Authenticated-Sender") { + if value.contains(": ") { + ctx.add_tag("HAS_X_AS"); + } + } else if name.eq_ignore_ascii_case("X-Get-Message-Sender-Via") { + if value.contains("authenticated_id:") { + ctx.add_tag("HAS_X_GMSV"); + } + } else if name.eq_ignore_ascii_case("X-AntiAbuse") { + ctx.add_tag("HAS_X_ANTIABUSE"); + } else if name.eq_ignore_ascii_case("X-Authentication-Warning") { + ctx.add_tag("HAS_XAW"); + } + } + _ => {} + } + } + + if list_score >= 1.0 { + ctx.add_tag("MAILLIST"); + } + + if unique_headers.is_empty() { + ctx.add_tag("MISSING_ESSENTIAL_HEADERS"); + } + } +} diff --git a/crates/spam-filter/src/analysis/init.rs b/crates/spam-filter/src/analysis/init.rs new file mode 100644 index 00000000..77eae653 --- /dev/null +++ b/crates/spam-filter/src/analysis/init.rs @@ -0,0 +1,77 @@ +use common::Core; +use mail_parser::{parsers::fields::thread::thread_name, HeaderName}; +use store::ahash::AHashSet; + +use crate::{Email, Hostname, SpamFilterContext, SpamFilterInput, SpamFilterOutput}; + +pub trait SpamFilterInit { + fn spam_filter_init<'x>(&self, input: SpamFilterInput<'x>) -> SpamFilterContext<'x>; +} + +impl SpamFilterInit for Core { + fn spam_filter_init<'x>(&self, input: SpamFilterInput<'x>) -> SpamFilterContext<'x> { + let subject = input.message.subject().unwrap_or_default().to_lowercase(); + let from = input.message.from().and_then(|f| f.first()); + let mut recipients = AHashSet::new(); + for header in input.message.headers() { + if matches!( + header.name, + HeaderName::To | HeaderName::Cc | HeaderName::Bcc + ) { + if let Some(addrs) = header.value().as_address() { + for addr in addrs.iter() { + if let Some(addr) = addr.address() { + recipients.insert(Email::new(addr)); + } + } + } + } + } + + let output = SpamFilterOutput { + tags: Default::default(), + ehlo_host: Hostname::new(input.ehlo_domain), + iprev_ptr: input + .iprev_result + .ptr + .as_ref() + .and_then(|ptr| ptr.first()) + .map(|ptr| ptr.strip_suffix('.').unwrap_or(ptr).to_lowercase()), + env_from_addr: Email::new(input.env_mail_from), + from_addr: Email::new(from.and_then(|f| f.address()).unwrap_or_default()), + from_name: from + .and_then(|f| f.name()) + .unwrap_or_default() + .to_lowercase(), + subject_thread: thread_name(&subject).to_string(), + subject, + recipients, + }; + + SpamFilterContext { output, input } + } +} + +/* + +use std::future::Future; + +use common::Core; + +use crate::SpamFilterContext; + +pub trait SpamFilterAnalyzeEhlo: Sync + Send { + fn spam_filter_analyze_ehlo( + &self, + ctx: &mut SpamFilterContext<'_>, + ) -> impl Future + Send; +} + +impl SpamFilterAnalyzeEhlo for Core { + async fn spam_filter_analyze_ehlo(&self, ctx: &mut SpamFilterContext<'_>) { + todo!() + } +} + + +*/ diff --git a/crates/spam-filter/src/analysis/iprev.rs b/crates/spam-filter/src/analysis/iprev.rs new file mode 100644 index 00000000..b59df277 --- /dev/null +++ b/crates/spam-filter/src/analysis/iprev.rs @@ -0,0 +1,23 @@ +use std::future::Future; + +use common::Core; +use mail_auth::IprevResult; + +use crate::SpamFilterContext; + +pub trait SpamFilterAnalyzeEhlo: Sync + Send { + fn spam_filter_analyze_iprev( + &self, + ctx: &mut SpamFilterContext<'_>, + ) -> impl Future + Send; +} + +impl SpamFilterAnalyzeEhlo for Core { + async fn spam_filter_analyze_iprev(&self, ctx: &mut SpamFilterContext<'_>) { + match &ctx.input.iprev_result.result { + IprevResult::TempError(_) => ctx.add_tag("RDNS_DNSFAIL"), + IprevResult::Fail(_) | IprevResult::PermError(_) => ctx.add_tag("RDNS_DNSFAIL"), + IprevResult::Pass | IprevResult::None => (), + } + } +} diff --git a/crates/spam-filter/src/analysis/messageid.rs b/crates/spam-filter/src/analysis/messageid.rs new file mode 100644 index 00000000..69fa2d75 --- /dev/null +++ b/crates/spam-filter/src/analysis/messageid.rs @@ -0,0 +1,85 @@ +use std::future::Future; + +use common::Core; +use mail_parser::HeaderName; + +use crate::{Hostname, SpamFilterContext}; + +pub trait SpamFilterAnalyzeEhlo: Sync + Send { + fn spam_filter_analyze_message_id( + &self, + ctx: &mut SpamFilterContext<'_>, + ) -> impl Future + Send; +} + +impl SpamFilterAnalyzeEhlo for Core { + async fn spam_filter_analyze_message_id(&self, ctx: &mut SpamFilterContext<'_>) { + let mid_raw = ctx + .input + .message + .header_raw(HeaderName::MessageId) + .unwrap_or_default() + .trim(); + + if !mid_raw.is_empty() { + let mid = ctx + .input + .message + .message_id() + .unwrap_or_default() + .to_lowercase(); + if let Some(mid_host) = mid.rsplit_once('@').map(|(_, host)| Hostname::new(host)) { + if mid_host.ip.is_some() { + if mid_host.fqdn.starts_with('[') { + ctx.add_tag("MID_RHS_IP_LITERAL"); + } else { + ctx.add_tag("MID_BARE_IP"); + } + } else if !mid_host.fqdn.contains('.') { + ctx.add_tag("MID_RHS_NOT_FQDN"); + } else if mid_host.fqdn.starts_with("www.") { + ctx.add_tag("MID_RHS_WWW"); + } + + if !mid_raw.is_ascii() || mid_raw.contains('(') || mid.starts_with('@') { + ctx.add_tag("INVALID_MSGID"); + } + + if mid_host.fqdn.len() > 255 { + ctx.add_tag("MID_RHS_TOO_LONG"); + } + + // From address present in Message-ID checks + for sender in [&ctx.output.from_addr, &ctx.output.env_from_addr] { + if !sender.address.is_empty() { + if mid.contains(&sender.address) { + ctx.output.tags.insert("MID_CONTAINS_FROM".to_string()); + } else if mid_host.fqdn == sender.domain_part.fqdn { + ctx.output.tags.insert("MID_RHS_MATCH_FROM".to_string()); + } else if matches!((&mid_host.sld, &sender.domain_part.sld), (Some(mid_sld), Some(sender_sld)) if mid_sld == sender_sld) + { + ctx.output.tags.insert("MID_RHS_MATCH_FROMTLD".to_string()); + } + } + } + + // To/Cc addresses present in Message-ID checks + for addr in &ctx.output.recipients { + if mid.contains(&addr.address) { + ctx.output.tags.insert("MID_CONTAINS_TO".to_string()); + } else if mid_host.fqdn == addr.domain_part.fqdn { + ctx.output.tags.insert("MID_RHS_MATCH_TO".to_string()); + } + } + } else { + ctx.add_tag("INVALID_MSGID"); + } + + if !mid_raw.starts_with('<') || !mid_raw.ends_with('>') { + ctx.add_tag("MID_MISSING_BRACKETS"); + } + } else { + ctx.add_tag("MISSING_MID"); + } + } +} diff --git a/crates/spam-filter/src/analysis/mod.rs b/crates/spam-filter/src/analysis/mod.rs new file mode 100644 index 00000000..055089b1 --- /dev/null +++ b/crates/spam-filter/src/analysis/mod.rs @@ -0,0 +1,15 @@ +use crate::SpamFilterContext; + +pub mod date; +pub mod dmarc; +pub mod ehlo; +pub mod headers; +pub mod init; +pub mod iprev; +pub mod messageid; + +impl SpamFilterContext<'_> { + pub fn add_tag(&mut self, tag: impl Into) { + self.output.tags.insert(tag.into()); + } +} diff --git a/crates/spam-filter/src/lib.rs b/crates/spam-filter/src/lib.rs new file mode 100644 index 00000000..9b6da274 --- /dev/null +++ b/crates/spam-filter/src/lib.rs @@ -0,0 +1,96 @@ +pub mod analysis; +pub mod modules; + +use std::hash::{Hash, Hasher}; +use std::net::IpAddr; + +use mail_auth::{dmarc::Policy, ArcOutput, DkimOutput, DmarcResult, IprevOutput, SpfOutput}; +use mail_parser::Message; +use store::ahash::AHashSet; + +pub struct SpamFilterInput<'x> { + pub message: &'x Message<'x>, + + // Sender authentication + pub arc_result: &'x ArcOutput<'x>, + pub spf_ehlo_result: &'x SpfOutput, + pub spf_mail_from_result: &'x SpfOutput, + pub dkim_result: &'x [DkimOutput<'x>], + pub dmarc_result: &'x DmarcResult, + pub dmarc_policy: &'x Policy, + pub iprev_result: &'x IprevOutput, + + // Session details + pub remote_ip: IpAddr, + pub ehlo_domain: &'x str, + pub authenticated_as: &'x str, + + // TLS + pub tls_version: &'x str, + pub tls_cipher: &'x str, + + // Envelope + pub env_mail_from: &'x str, + pub env_rcpt_to: &'x [&'x str], +} + +pub struct SpamFilterOutput { + pub tags: AHashSet, + pub ehlo_host: Hostname, + pub iprev_ptr: Option, + + pub env_from_addr: Email, + pub from_addr: Email, + pub from_name: String, + pub recipients: AHashSet, + + pub subject: String, + pub subject_thread: String, +} + +pub struct SpamFilterContext<'x> { + pub input: SpamFilterInput<'x>, + pub output: SpamFilterOutput, +} + +#[derive(Debug)] +pub struct Hostname { + pub fqdn: String, + pub ip: Option, + pub sld: Option, +} + +#[derive(Debug)] +pub struct Email { + pub address: String, + pub local_part: String, + pub domain_part: Hostname, +} + +impl PartialEq for Hostname { + fn eq(&self, other: &Self) -> bool { + self.fqdn.eq(&other.fqdn) + } +} + +impl Eq for Hostname {} + +impl PartialEq for Email { + fn eq(&self, other: &Self) -> bool { + self.address.eq(&other.address) + } +} + +impl Eq for Email {} + +impl Hash for Hostname { + fn hash(&self, state: &mut H) { + self.fqdn.hash(state) + } +} + +impl Hash for Email { + fn hash(&self, state: &mut H) { + self.address.hash(state) + } +} diff --git a/crates/common/src/scripts/plugins/bayes.rs b/crates/spam-filter/src/modules/bayes.rs similarity index 100% rename from crates/common/src/scripts/plugins/bayes.rs rename to crates/spam-filter/src/modules/bayes.rs diff --git a/crates/spam-filter/src/modules/mod.rs b/crates/spam-filter/src/modules/mod.rs new file mode 100644 index 00000000..d5e11042 --- /dev/null +++ b/crates/spam-filter/src/modules/mod.rs @@ -0,0 +1 @@ +pub mod sanitize; diff --git a/crates/common/src/scripts/plugins/pyzor.rs b/crates/spam-filter/src/modules/pyzor.rs similarity index 100% rename from crates/common/src/scripts/plugins/pyzor.rs rename to crates/spam-filter/src/modules/pyzor.rs diff --git a/crates/spam-filter/src/modules/sanitize.rs b/crates/spam-filter/src/modules/sanitize.rs new file mode 100644 index 00000000..107065e7 --- /dev/null +++ b/crates/spam-filter/src/modules/sanitize.rs @@ -0,0 +1,38 @@ +use std::net::IpAddr; + +use crate::{Email, Hostname}; + +impl Hostname { + pub fn new(host: &str) -> Self { + let fqdn = host.to_lowercase(); + let ip = fqdn + .strip_prefix('[') + .and_then(|ip| ip.strip_suffix(']')) + .unwrap_or(&fqdn) + .parse::() + .ok(); + + Hostname { + ip, + sld: if ip.is_none() { + psl::domain_str(&fqdn).map(str::to_string) + } else { + None + }, + fqdn, + } + } +} + +impl Email { + pub fn new(address: &str) -> Self { + let address = address.to_lowercase(); + let (local_part, domain) = address.rsplit_once('@').unwrap_or_default(); + + Email { + local_part: local_part.to_string(), + domain_part: Hostname::new(domain), + address, + } + } +} diff --git a/crates/store/src/backend/memory/mod.rs b/crates/store/src/backend/memory/mod.rs index 54d0f5ef..5426df6d 100644 --- a/crates/store/src/backend/memory/mod.rs +++ b/crates/store/src/backend/memory/mod.rs @@ -5,25 +5,11 @@ */ use ahash::AHashMap; -use utils::{config::Config, glob::GlobPattern}; +use utils::{config::Config, glob::GlobMap}; use crate::{LookupStore, Stores, Value}; -#[derive(Debug, Default)] -pub struct MemoryStore { - entries: AHashMap>, - globs: Vec<(GlobPattern, Value<'static>)>, -} - -impl MemoryStore { - pub fn get(&self, id: &str) -> Option<&Value<'static>> { - self.entries.get(id).or_else(|| { - self.globs - .iter() - .find_map(|(pattern, value)| pattern.matches(id).then_some(value)) - }) - } -} +pub type MemoryStore = GlobMap>; impl Stores { pub fn parse_memory_stores(&mut self, config: &mut Config) { @@ -35,24 +21,6 @@ impl Stores { .split_once('.') .filter(|(id, key)| !id.is_empty() && !key.is_empty()) { - // Detect if the key is a glob pattern - let mut last_ch = '\0'; - let mut has_escape = false; - let mut is_glob = false; - for ch in key.chars() { - match ch { - '\\' => { - has_escape = true; - } - '*' | '?' if last_ch != '\\' => { - is_glob = true; - } - _ => {} - } - - last_ch = ch; - } - // Detect value type let value = if !value.is_empty() { let mut has_integers = false; @@ -98,21 +66,10 @@ impl Stores { }; // Add entry - let store = lookups + lookups .entry(id.to_string()) - .or_insert_with(MemoryStore::default); - if is_glob { - store.globs.push((GlobPattern::compile(key, false), value)); - } else { - store.entries.insert( - if has_escape { - key.replace('\\', "") - } else { - key.to_string() - }, - value, - ); - } + .or_insert_with(MemoryStore::default) + .insert(key, value); } else { errors.push(key.to_string()); } diff --git a/crates/trc/event-macro/Cargo.toml b/crates/trc/event-macro/Cargo.toml index 46fe7980..ed2ebe34 100644 --- a/crates/trc/event-macro/Cargo.toml +++ b/crates/trc/event-macro/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "event_macro" -version = "0.1.0" +version = "0.10.7" edition = "2021" [lib] diff --git a/crates/utils/proc-macros/Cargo.toml b/crates/utils/proc-macros/Cargo.toml index 7f1060ea..80eedf3a 100644 --- a/crates/utils/proc-macros/Cargo.toml +++ b/crates/utils/proc-macros/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "proc_macros" -version = "0.1.0" +version = "0.10.7" edition = "2021" [lib] diff --git a/crates/utils/src/glob.rs b/crates/utils/src/glob.rs index 668fb21e..394d80f2 100644 --- a/crates/utils/src/glob.rs +++ b/crates/utils/src/glob.rs @@ -4,6 +4,8 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use ahash::{AHashMap, AHashSet}; + #[derive(Debug, Clone, PartialEq, Eq)] pub struct GlobPattern { pattern: Vec, @@ -61,6 +63,36 @@ impl GlobPattern { } } + pub fn try_compile(pattern: &str, to_lower: bool) -> Result { + // Detect if the key is a glob pattern + let mut last_ch = '\0'; + let mut has_escape = false; + let mut is_glob = false; + for ch in pattern.chars() { + match ch { + '\\' => { + has_escape = true; + } + '*' | '?' if last_ch != '\\' => { + is_glob = true; + } + _ => {} + } + + last_ch = ch; + } + + if is_glob { + Ok(GlobPattern::compile(pattern, to_lower)) + } else { + Err(if has_escape { + pattern.replace('\\', "") + } else { + pattern.to_string() + }) + } + } + // Credits: Algorithm ported from https://research.swtch.com/glob pub fn matches(&self, value: &str) -> bool { let value = if self.to_lower { @@ -108,3 +140,70 @@ impl GlobPattern { true } } + +#[derive(Debug, Clone, Default)] +pub struct GlobSet { + entries: AHashSet, + patterns: Vec, +} + +#[derive(Debug, Clone)] +pub struct GlobMap { + entries: AHashMap, + patterns: Vec<(GlobPattern, V)>, +} + +impl GlobSet { + pub fn new() -> Self { + GlobSet::default() + } + + pub fn insert(&mut self, pattern: &str) { + match GlobPattern::try_compile(pattern, false) { + Ok(glob) => { + self.patterns.push(glob); + } + Err(entry) => { + self.entries.insert(entry); + } + } + } + + pub fn contains(&self, key: &str) -> bool { + self.entries.contains(key) || self.patterns.iter().any(|pattern| pattern.matches(key)) + } +} + +impl GlobMap { + pub fn new() -> Self { + GlobMap { + entries: AHashMap::new(), + patterns: Vec::new(), + } + } + + pub fn insert(&mut self, pattern: &str, value: V) { + match GlobPattern::try_compile(pattern, false) { + Ok(glob) => { + self.patterns.push((glob, value)); + } + Err(entry) => { + self.entries.insert(entry, value); + } + } + } + + pub fn get(&self, key: &str) -> Option<&V> { + self.entries.get(key).or_else(|| { + self.patterns + .iter() + .find_map(|(pattern, value)| pattern.matches(key).then_some(value)) + }) + } +} + +impl Default for GlobMap { + fn default() -> Self { + GlobMap::new() + } +} diff --git a/resources/config/spamfilter/scripts/date.sieve b/resources/config/spamfilter/scripts/date.sieve deleted file mode 100644 index cf02dda3..00000000 --- a/resources/config/spamfilter/scripts/date.sieve +++ /dev/null @@ -1,19 +0,0 @@ -if eval "header.date.exists" { - let "date" "header.date.date"; - - if eval "date != 0" { - let "date_diff" "env.now - date"; - - if eval "date_diff > 86400" { - # Older than a day - let "t.DATE_IN_PAST" "1"; - } elsif eval "-date_diff > 7200" { - # More than 2 hours in the future - let "t.DATE_IN_FUTURE" "1"; - } - } else { - let "t.INVALID_DATE" "1"; - } -} else { - let "t.MISSING_DATE" "1"; -} diff --git a/resources/config/spamfilter/scripts/dmarc.sieve b/resources/config/spamfilter/scripts/dmarc.sieve deleted file mode 100644 index bd626d3d..00000000 --- a/resources/config/spamfilter/scripts/dmarc.sieve +++ /dev/null @@ -1,91 +0,0 @@ -if eval "env.spf.result == 'pass'" { - let "t.SPF_ALLOW" "1"; -} elsif eval "env.spf.result == 'fail'" { - let "t.SPF_FAIL" "1"; -} elsif eval "env.spf.result == 'softfail'" { - let "t.SPF_SOFTFAIL" "1"; -} elsif eval "env.spf.result == 'neutral'" { - let "t.SPF_NEUTRAL" "1"; -} elsif eval "env.spf.result == 'temperror'" { - let "t.SPF_DNSFAIL" "1"; -} elsif eval "env.spf.result == 'permerror'" { - let "t.SPF_PERMFAIL" "1"; -} else { - let "t.SPF_NA" "1"; -} - -if eval "env.dkim.result == 'pass'" { - let "t.DKIM_ALLOW" "1"; -} elsif eval "env.dkim.result == 'fail'" { - let "t.DKIM_REJECT" "1"; -} elsif eval "env.dkim.result == 'temperror'" { - let "t.DKIM_TEMPFAIL" "1"; -} elsif eval "env.dkim.result == 'permerror'" { - let "t.DKIM_PERMFAIL" "1"; -} else { - let "t.DKIM_NA" "1"; -} - -if eval "env.arc.result == 'pass'" { - let "t.ARC_ALLOW" "1"; -} elsif eval "env.arc.result == 'fail'" { - let "t.ARC_REJECT" "1"; -} elsif eval "env.arc.result == 'temperror'" { - let "t.ARC_DNSFAIL" "1"; -} elsif eval "env.arc.result == 'permerror'" { - let "t.ARC_INVALID" "1"; -} else { - let "t.ARC_NA" "1"; -} - -if eval "env.dmarc.result == 'pass'" { - let "t.DMARC_POLICY_ALLOW" "1"; -} elsif eval "env.dmarc.result == 'temperror'" { - let "t.DMARC_DNSFAIL" "1"; -} elsif eval "env.dmarc.result == 'permerror'" { - let "t.DMARC_BAD_POLICY" "1"; -} elsif eval "env.dmarc.result == 'fail'" { - if eval "env.dmarc.policy == 'quarantine'" { - let "t.DMARC_POLICY_QUARANTINE" "1"; - } elsif eval "env.dmarc.policy == 'reject'" { - let "t.DMARC_POLICY_REJECT" "1"; - } else { - let "t.DMARC_POLICY_SOFTFAIL" "1"; - } -} else { - let "t.DMARC_NA" "1"; -} - -if eval "header.DKIM-Signature.exists" { - let "t.DKIM_SIGNED" "1"; - if eval "header.ARC-Seal.exists" { - let "t.ARC_SIGNED" "1"; - } -} - -# Check allowlists -if eval "key_exists('spam-dmarc', from_domain)" { - if eval "t.DMARC_POLICY_ALLOW" { - let "t.ALLOWLIST_DMARC" "1"; - } else { - let "t.BLOCKLIST_DMARC" "1"; - } -} elsif eval "key_exists('spam-spdk', from_domain)" { - let "is_dkim_pass" "contains(env.dkim.domains, from_domain) || t.ARC_ALLOW"; - - if eval "is_dkim_pass && t.SPF_ALLOW" { - let "t.ALLOWLIST_SPF_DKIM" "1"; - } elsif eval "is_dkim_pass" { - let "t.ALLOWLIST_DKIM" "1"; - if eval "!t.SPF_DNSFAIL" { - let "t.BLOCKLIST_SPF" "1"; - } - } elsif eval "t.SPF_ALLOW" { - let "t.ALLOWLIST_SPF" "1"; - if eval "!t.DKIM_TEMPFAIL" { - let "t.BLOCKLIST_DKIM" "1"; - } - } elsif eval "!t.SPF_DNSFAIL && !t.DKIM_TEMPFAIL" { - let "t.BLOCKLIST_SPF_DKIM" "1"; - } -} diff --git a/resources/config/spamfilter/scripts/headers.sieve b/resources/config/spamfilter/scripts/headers.sieve deleted file mode 100644 index 81d4ea67..00000000 --- a/resources/config/spamfilter/scripts/headers.sieve +++ /dev/null @@ -1,140 +0,0 @@ -# Mailing list scores -let "ml_score" "count(header.List-Id:List-Archive:List-Owner:List-Help:List-Post:X-Loop:List-Subscribe:List-Unsubscribe[*].exists) * 0.125"; -if eval "ml_score < 1" { - if eval "header.List-Id.exists" { - let "ml_score" "ml_score + 0.50"; - } - if eval "header.List-Subscribe.exists && header.List-Unsubscribe.exists" { - let "ml_score" "ml_score + 0.25"; - } - if eval "header.Precedence.exists && (eq_ignore_case(header.Precedence, 'list') || eq_ignore_case(header.Precedence, 'bulk'))" { - let "ml_score" "ml_score + 0.25"; - } -} -if eval "ml_score >= 1" { - let "t.MAILLIST" "1"; -} - -# X-Priority -if eval "header.x-priority.exists" { - let "xp" "header.x-priority"; - if eval "xp == 0" { - let "t.HAS_X_PRIO_ZERO" "1"; - } elsif eval "xp == 1" { - let "t.HAS_X_PRIO_ONE" "1"; - } elsif eval "xp == 2" { - let "t.HAS_X_PRIO_TWO" "1"; - } elsif eval "xp <= 4" { - let "t.HAS_X_PRIO_THREE" "1"; - } elsif eval "xp >= 5" { - let "t.HAS_X_PRIO_FIVE" "1"; - } -} - -let "unique_header_names" "to_lowercase(header.Content-Type:Content-Transfer-Encoding:Date:From:Sender:Reply-To:To:Cc:Bcc:Message-ID:In-Reply-To:References:Subject[*].raw_name)"; -let "unique_header_names_len" "count(unique_header_names)"; -if eval "unique_header_names_len != count(dedup(unique_header_names))" { - let "t.MULTIPLE_UNIQUE_HEADERS" "1"; -} elsif eval "unique_header_names_len == 0" { - let "t.MISSING_ESSENTIAL_HEADERS" "1"; -} - -# Wrong case X-Mailer -if eval "header.x-mailer.exists && header.x-mailer.raw_name != 'X-Mailer'" { - let "t.XM_CASE" "1"; -} - -# Has organization header -if eval "header.organization:organisation.exists" { - let "t.HAS_ORG_HEADER" "1"; -} - -# Has X-Originating-IP header -if eval "header.X-Originating-IP.exists" { - let "t.HAS_XOIP" "1"; -} - -# Has List-Unsubscribe header -if eval "header.List-Unsubscribe.exists" { - let "t.HAS_LIST_UNSUB" "1"; -} - -# Missing version number in X-Mailer or User-Agent headers -if eval "(header.X-Mailer.exists && !has_digits(header.X-Mailer)) || (header.User-Agent.exists && !has_digits(header.User-Agent))" { - let "t.XM_UA_NO_VERSION" "1"; -} - -# Precedence is bulk -if eval "eq_ignore_case(header.Precedence, 'bulk')" { - let "t.PRECEDENCE_BULK" "1"; -} - -# Upstream SPAM filtering -if eval "contains_ignore_case(header.X-KLMS-AntiSpam-Status, 'spam')" { - # Kaspersky Security for Mail Server says this message is spam - let "t.KLMS_SPAM" "1"; -} -let "spam_hdr" "to_lowercase(header.X-Spam:X-Spam-Flag:X-Spam-Status)"; -if eval "contains(spam_hdr, 'yes') || contains(spam_hdr, 'true') || contains(spam_hdr, 'spam')" { - # Message was already marked as spam - let "t.SPAM_FLAG" "1"; -} -if eval "contains_ignore_case(header.X-UI-Filterresults:X-UI-Out-Filterresults, 'junk')" { - # United Internet says this message is spam - let "t.UNITEDINTERNET_SPAM" "1"; -} - -# Compromised hosts -if eval "header.X-PHP-Originating-Script.exists" { - let "t.HAS_X_POS" "1"; - if eval "contains(header.X-PHP-Originating-Script, 'eval()')" { - let "t.X_PHP_EVAL" "1"; - } - if eval "contains(header.X-PHP-Originating-Script, '../')" { - let "t.HIDDEN_SOURCE_OBJ" "1"; - } -} -if eval "header.X-PHP-Script.exists" { - let "t.HAS_X_PHP_SCRIPT" "1"; - if eval "contains(header.X-PHP-Script, 'eval()')" { - let "t.X_PHP_EVAL" "1"; - } - if eval "contains(header.X-PHP-Script, 'sendmail.php')" { - let "t.PHP_XPS_PATTERN" "1"; - } - if eval "contains(header.X-PHP-Script, '../')" { - let "t.HIDDEN_SOURCE_OBJ" "1"; - } -} -if eval "contains_ignore_case(header.X-Mailer, 'PHPMailer')" { - let "t.HAS_PHPMAILER_SIG" "1"; -} -if eval "header.X-Source:X-Source-Args:X-Source-Dir.exists" { - let "t.HAS_X_SOURCE" "1"; - if eval "contains(header.X-Source-Args, '../')" { - let "t.HIDDEN_SOURCE_OBJ" "1"; - } -} -if eval "contains(header.X-Authenticated-Sender, ': ')" { - let "t.HAS_X_AS" "1"; -} -if eval "contains(header.X-Get-Message-Sender-Via, 'authenticated_id:')" { - let "t.HAS_X_GMSV" "1"; -} -if eval "header.X-AntiAbuse.exists" { - let "t.HAS_X_ANTIABUSE" "1"; -} -if eval "header.X-Authentication-Warning.exists" { - let "t.HAS_XAW" "1"; -} - -# Check for empty delimiters in raw headers -let "raw_headers" "header.from:to:cc:subject:reply-to:date[*].raw"; -let "i" "count(raw_headers)"; -while "i > 0" { - let "i" "i - 1"; - if eval "!starts_with(raw_headers[i], ' ')" { - let "t.HEADER_EMPTY_DELIMITER" "1"; - break; - } -} diff --git a/resources/config/spamfilter/scripts/helo.sieve b/resources/config/spamfilter/scripts/helo.sieve deleted file mode 100644 index ac72dacc..00000000 --- a/resources/config/spamfilter/scripts/helo.sieve +++ /dev/null @@ -1,30 +0,0 @@ -if eval "!is_ip_addr(env.helo_domain)" { - let "helo" "env.helo_domain"; - - if eval "contains(helo, '.')" { - if eval "!is_empty(env.iprev.ptr) && !eq_ignore_case(helo, env.iprev.ptr)" { - # Helo does not match reverse IP - let "t.HELO_IPREV_MISMATCH" "1"; - } - if eval "!dns_exists(helo, 'ip') && !dns_exists(helo, 'mx')" { - # Helo no resolve to A or MX - let "t.HELO_NORES_A_OR_MX" "1"; - } - } else { - if eval "contains(helo, 'user')" { - # HELO contains 'user' - let "t.RCVD_HELO_USER" "1"; - } - - # Helo not FQDN - let "t.HELO_NOT_FQDN" "1"; - } -} else { - # Helo host is bare ip - let "t.HELO_BAREIP" "1"; - - if eval "env.helo_domain != env.remote_ip" { - # Helo A IP != hostname IP - let "t.HELO_IP_A" "1"; - } -} diff --git a/resources/config/spamfilter/scripts/ip.sieve b/resources/config/spamfilter/scripts/ip.sieve deleted file mode 100644 index cd676f66..00000000 --- a/resources/config/spamfilter/scripts/ip.sieve +++ /dev/null @@ -1,8 +0,0 @@ -# Reverse ip checks -if eval "env.iprev.result != ''" { - if eval "env.iprev.result == 'temperror'" { - let "t.RDNS_DNSFAIL" "1"; - } elsif eval "env.iprev.result == 'fail' || env.iprev.result == 'permerror'" { - let "t.RDNS_NONE" "1"; - } -} diff --git a/resources/config/spamfilter/scripts/messageid.sieve b/resources/config/spamfilter/scripts/messageid.sieve deleted file mode 100644 index a55f0ac7..00000000 --- a/resources/config/spamfilter/scripts/messageid.sieve +++ /dev/null @@ -1,68 +0,0 @@ -let "mid_raw" "trim(header.message-id.raw)"; - -if eval "!is_empty(mid_raw)" { - let "mid_lcase" "to_lowercase(header.message-id)"; - let "mid_rhs" "email_part(mid_lcase, 'domain')"; - - if eval "!is_empty(mid_rhs)" { - if eval "starts_with(mid_rhs, '[') && ends_with(mid_rhs, ']') && is_ip_addr(strip_suffix(strip_prefix(mid_rhs, '['), ']'))" { - let "t.MID_RHS_IP_LITERAL" "1"; - } elsif eval "is_ip_addr(mid_rhs)" { - let "t.MID_BARE_IP" "1"; - } elsif eval "!contains(mid_rhs, '.')" { - let "t.MID_RHS_NOT_FQDN" "1"; - } - - if eval "starts_with(mid_rhs, 'www.')" { - let "t.MID_RHS_WWW" "1"; - } - - if eval "!is_ascii(mid_raw) || contains(mid_raw, '(') || starts_with(mid_lcase, '@')" { - let "t.INVALID_MSGID" "1"; - } - - # From address present in Message-ID checks - let "sender" "from_addr"; - if eval "is_empty(sender)" { - let "sender" "envelope.from"; - } - if eval "!is_empty(sender)" { - if eval "contains(mid_lcase, sender)" { - let "t.MID_CONTAINS_FROM" "1"; - } else { - let "from_domain" "email_part(sender, 'domain')"; - let "mid_sld" "domain_part(mid_rhs, 'sld')"; - - if eval "mid_rhs == from_domain" { - let "t.MID_RHS_MATCH_FROM" "1"; - } elsif eval "!is_empty(mid_sld) && domain_part(from_domain, 'sld') == mid_sld" { - let "t.MID_RHS_MATCH_FROMTLD" "1"; - } - } - } - - # To/Cc addresses present in Message-ID checks - let "recipients_len" "count(recipients)"; - let "i" "0"; - - while "i < recipients_len" { - let "rcpt" "recipients[i]"; - let "i" "i + 1"; - if eval "contains(mid_lcase, rcpt)" { - let "t.MID_CONTAINS_TO" "1"; - } elsif eval "email_part(rcpt, 'domain') == mid_rhs" { - let "t.MID_RHS_MATCH_TO" "1"; - } - } - } else { - let "t.INVALID_MSGID" "1"; - } - - if eval "!starts_with(mid_raw, '<') || !contains(mid_raw, '>')" { - let "t.MID_MISSING_BRACKETS" "1"; - } - -} else { - let "t.MISSING_MID" "1"; -} - diff --git a/resources/config/spamfilter/scripts/prelude.sieve b/resources/config/spamfilter/scripts/prelude.sieve deleted file mode 100644 index 86367716..00000000 --- a/resources/config/spamfilter/scripts/prelude.sieve +++ /dev/null @@ -1,43 +0,0 @@ -# Convert body to plain text -let "text_body" "body.to_text"; - -# Obtain all URLs in the body -let "body_urls" "tokenize(text_body, 'uri')"; - -# Obtain all URLs in href and src attributes -let "html_body_urls" "html_attrs(body.html, '', ['href', 'src'])"; - -# Obtain all URLs in the subject, combine them with all other URLs and remove duplicates -let "urls" "dedup(tokenize(header.subject, 'uri') + body_urls + html_body_urls)"; - -# Obtain thread name and subject -let "subject_lc" "to_lowercase(header.subject)"; -let "subject_clean" "thread_name(header.subject)"; -let "body_and_subject" "subject_clean + ' ' + text_body"; - -# Obtain all recipients -let "recipients" "to_lowercase(header.to:cc:bcc[*].addr[*])"; -let "recipients_clean" "winnow(dedup(recipients))"; -let "recipients_to" "header.to[*].addr[*]"; -let "recipients_cc" "header.cc[*].addr[*]"; - -# Obtain From parts -let "from_name" "to_lowercase(trim(header.from.name))"; -let "from_addr" "to_lowercase(trim(header.from.addr))"; -let "from_local" "email_part(from_addr, 'local')"; -let "from_domain" "email_part(from_addr, 'domain')"; -let "from_domain_sld" "domain_part(from_domain, 'sld')"; - -# Obtain Reply-To address -let "rto_addr" "to_lowercase(header.reply-to.addr)"; - -# Obtain Envelope From parts -let "envfrom_local" "email_part(envelope.from, 'local')"; -let "envfrom_domain" "email_part(envelope.from, 'domain')"; -let "envfrom_domain_sld" "domain_part(envfrom_domain, 'sld')"; - -# Obtain HELO domain SLD -let "helo_domain_sld" "domain_part(env.helo_domain, 'sld')"; - -# Create score variable -let "score" "0.0"; diff --git a/tests/Cargo.toml b/tests/Cargo.toml index d687fc3e..f14069c7 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tests" -version = "0.1.0" +version = "0.10.7" edition = "2021" resolver = "2"