filip/Stalwart
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Port Spam filter to Rust - part 2

Browse Source
This commit is contained in:
mdecimus
2024-12-07 18:45:50 +01:00
parent db7ae48c77
commit 4453dc8f3d
19 changed files with 1074 additions and 598 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
crates/common/src/config/spamfilter.rs
View File

@@ -10,6 +10,8 @@ use utils::{config::Config, glob::GlobSet};
pub struct SpamFilterConfig {
pub list_dmarc_allow: GlobSet,
pub list_spf_dkim_allow: GlobSet,
pub list_freemail_providers: GlobSet,
pub list_disposable_providers: GlobSet,
}
impl SpamFilterConfig {

12
crates/common/src/scripts/functions/text.rs
View File

@@ -243,20 +243,24 @@ pub fn fn_levenshtein_distance<'x>(_: &'x Context<'x>, v: Vec<Variable>) -> Vari
let a = v[0].to_string();
let b = v[1].to_string();
levenshtein_distance(a.as_ref(), b.as_ref()).into()
}
pub fn levenshtein_distance(a: &str, b: &str) -> usize {
let mut result = 0;
/* Shortcut optimizations / degenerate cases. */
if a == b {
return result.into();
return result;
}
let length_a = a.chars().count();
let length_b = b.chars().count();
if length_a == 0 {
return length_b.into();
return length_b;
} else if length_b == 0 {
return length_a.into();
return length_a;
}
/* Initialize the vector.
@@ -297,7 +301,7 @@ pub fn fn_levenshtein_distance<'x>(_: &'x Context<'x>, v: Vec<Variable>) -> Vari
}
}
result.into()
result
}
pub fn fn_detect_language<'x>(_: &'x Context<'x>, v: Vec<Variable>) -> Variable {

1
crates/spam-filter/Cargo.toml
View File

@@ -10,6 +10,7 @@ nlp = { path = "../nlp" }
store = { path = "../store" }
trc = { path = "../trc" }
common = { path = "../common" }
smtp-proto = { version = "0.1", features = ["serde_support"] }
mail-parser = { version = "0.9", features = ["full_encoding", "ludicrous_mode"] }
mail-builder = { version = "0.3", features = ["ludicrous_mode"] }
mail-auth = { version = "0.5" }

12
crates/spam-filter/src/analysis/date.rs
View File

@@ -5,14 +5,14 @@ use store::write::now;
use crate::SpamFilterContext;
pub trait SpamFilterAnalyzeEhlo: Sync + Send {
pub trait SpamFilterAnalyzeDate: Sync + Send {
fn spam_filter_analyze_date(
&self,
ctx: &mut SpamFilterContext<'_>,
) -> impl Future<Output = ()> + Send;
}
impl SpamFilterAnalyzeEhlo for Core {
impl SpamFilterAnalyzeDate for Core {
async fn spam_filter_analyze_date(&self, ctx: &mut SpamFilterContext<'_>) {
if let Some(date) = ctx.input.message.date() {
let date = date.to_timestamp();
@@ -21,16 +21,16 @@ impl SpamFilterAnalyzeEhlo for Core {
if date_diff > 86400 {
// Older than a day
ctx.add_tag("DATE_IN_PAST");
ctx.result.add_tag("DATE_IN_PAST");
} else if -date_diff > 7200 {
//# More than 2 hours in the future
ctx.add_tag("DATE_IN_FUTURE");
ctx.result.add_tag("DATE_IN_FUTURE");
}
} else {
ctx.add_tag("INVALID_DATE");
ctx.result.add_tag("INVALID_DATE");
}
} else {
ctx.add_tag("MISSING_DATE");
ctx.result.add_tag("MISSING_DATE");
}
}
}

55
crates/spam-filter/src/analysis/dmarc.rs
View File

@@ -7,26 +7,27 @@ use mail_auth::{
use crate::SpamFilterContext;
pub trait SpamFilterAnalyzeEhlo: Sync + Send {
pub trait SpamFilterAnalyzeDmarc: Sync + Send {
fn spam_filter_analyze_dmarc(
&self,
ctx: &mut SpamFilterContext<'_>,
) -> impl Future<Output = ()> + Send;
}
impl SpamFilterAnalyzeEhlo for Core {
impl SpamFilterAnalyzeDmarc for Core {
async fn spam_filter_analyze_dmarc(&self, ctx: &mut SpamFilterContext<'_>) {
ctx.add_tag(match ctx.input.spf_mail_from_result.result() {
SpfResult::Pass => "SPF_ALLOW",
SpfResult::Fail => "SPF_FAIL",
SpfResult::SoftFail => "SPF_SOFTFAIL",
SpfResult::Neutral => "SPF_NEUTRAL",
SpfResult::TempError => "SPF_DNSFAIL",
SpfResult::PermError => "SPF_PERMFAIL",
SpfResult::None => "SPF_NA",
});
ctx.result
.add_tag(match ctx.input.spf_mail_from_result.result() {
SpfResult::Pass => "SPF_ALLOW",
SpfResult::Fail => "SPF_FAIL",
SpfResult::SoftFail => "SPF_SOFTFAIL",
SpfResult::Neutral => "SPF_NEUTRAL",
SpfResult::TempError => "SPF_DNSFAIL",
SpfResult::PermError => "SPF_PERMFAIL",
SpfResult::None => "SPF_NA",
});
ctx.add_tag(
ctx.result.add_tag(
match ctx
.input
.dkim_result
@@ -44,7 +45,7 @@ impl SpamFilterAnalyzeEhlo for Core {
},
);
ctx.add_tag(match ctx.input.arc_result.result() {
ctx.result.add_tag(match ctx.input.arc_result.result() {
DkimResult::Pass => "ARC_ALLOW",
DkimResult::Fail(_) => "ARC_REJECT",
DkimResult::PermError(_) => "ARC_INVALID",
@@ -52,7 +53,7 @@ impl SpamFilterAnalyzeEhlo for Core {
DkimResult::Neutral(_) | DkimResult::None => "ARC_NA",
});
ctx.add_tag(match ctx.input.dmarc_result {
ctx.result.add_tag(match ctx.input.dmarc_result {
DmarcResult::Pass => "DMARC_POLICY_ALLOW",
DmarcResult::TempError(_) => "DMARC_DNSFAIL",
DmarcResult::PermError(_) => "DMARC_BAD_POLICY",
@@ -67,55 +68,55 @@ impl SpamFilterAnalyzeEhlo for Core {
for header in ctx.input.message.headers() {
let header_name = header.name();
if header_name.eq_ignore_ascii_case("DKIM-Signature") {
ctx.add_tag("DKIM_SIGNED");
ctx.result.add_tag("DKIM_SIGNED");
} else if header_name.eq_ignore_ascii_case("ARC-Seal") {
ctx.add_tag("ARC_SIGNED");
ctx.result.add_tag("ARC_SIGNED");
}
}
if self
.spam
.list_dmarc_allow
.contains(&ctx.output.from_addr.domain_part.fqdn)
.contains(&ctx.output.from.email.domain_part.fqdn)
{
if matches!(ctx.input.dmarc_result, DmarcResult::Pass) {
ctx.add_tag("ALLOWLIST_DMARC");
ctx.result.add_tag("ALLOWLIST_DMARC");
} else {
ctx.add_tag("BLOCKLIST_DMARC");
ctx.result.add_tag("BLOCKLIST_DMARC");
}
} else if self
.spam
.list_spf_dkim_allow
.contains(&ctx.output.from_addr.domain_part.fqdn)
.contains(&ctx.output.from.email.domain_part.fqdn)
{
let is_dkim_pass = matches!(ctx.input.arc_result.result(), DkimResult::Pass)
|| ctx.input.dkim_result.iter().any(|r| {
matches!(r.result(), DkimResult::Pass)
&& r.signature().map_or(false, |s| {
s.domain().to_lowercase() == ctx.output.from_addr.domain_part.fqdn
s.domain().to_lowercase() == ctx.output.from.email.domain_part.fqdn
})
});
let is_spf_pass = matches!(ctx.input.spf_mail_from_result.result(), SpfResult::Pass);
if is_dkim_pass && is_spf_pass {
ctx.add_tag("ALLOWLIST_SPF_DKIM");
ctx.result.add_tag("ALLOWLIST_SPF_DKIM");
} else if is_dkim_pass {
ctx.add_tag("ALLOWLIST_DKIM");
ctx.result.add_tag("ALLOWLIST_DKIM");
if !matches!(
ctx.input.spf_mail_from_result.result(),
SpfResult::TempError
) {
ctx.add_tag("BLOCKLIST_SPF");
ctx.result.add_tag("BLOCKLIST_SPF");
}
} else if is_spf_pass {
ctx.add_tag("ALLOWLIST_SPF");
ctx.result.add_tag("ALLOWLIST_SPF");
if !ctx
.input
.dkim_result
.iter()
.any(|r| matches!(r.result(), DkimResult::TempError(_)))
{
ctx.add_tag("BLOCKLIST_DKIM");
ctx.result.add_tag("BLOCKLIST_DKIM");
}
} else if !matches!(
ctx.input.spf_mail_from_result.result(),
@@ -126,7 +127,7 @@ impl SpamFilterAnalyzeEhlo for Core {
.iter()
.any(|r| matches!(r.result(), DkimResult::TempError(_)))
{
ctx.add_tag("BLOCKLIST_SPF_DKIM");
ctx.result.add_tag("BLOCKLIST_SPF_DKIM");
}
}
}

12
crates/spam-filter/src/analysis/ehlo.rs
View File

@@ -15,11 +15,11 @@ impl SpamFilterAnalyzeEhlo for Core {
async fn spam_filter_analyze_ehlo(&self, ctx: &mut SpamFilterContext<'_>) {
if let Some(ehlo_ip) = ctx.output.ehlo_host.ip {
// Helo host is bare ip
ctx.add_tag("HELO_BAREIP");
ctx.result.add_tag("HELO_BAREIP");
if ehlo_ip != ctx.input.remote_ip {
// Helo A IP != hostname IP
ctx.add_tag("HELO_IP_A");
ctx.result.add_tag("HELO_IP_A");
}
} else if ctx.output.ehlo_host.sld.is_some() {
if ctx
@@ -29,7 +29,7 @@ impl SpamFilterAnalyzeEhlo for Core {
.map_or(false, |ptr| ptr != &ctx.output.ehlo_host.fqdn)
{
// Helo does not match reverse IP
ctx.add_tag("HELO_IPREV_MISMATCH");
ctx.result.add_tag("HELO_IPREV_MISMATCH");
}
if matches!(
@@ -40,16 +40,16 @@ impl SpamFilterAnalyzeEhlo for Core {
(Ok(false), Ok(false))
) {
// Helo no resolve to A or MX
ctx.add_tag("HELO_NORES_A_OR_MX");
ctx.result.add_tag("HELO_NORES_A_OR_MX");
}
} else {
if ctx.output.ehlo_host.fqdn.contains("user") {
// Helo host contains 'user'
ctx.add_tag("RCVD_HELO_USER");
ctx.result.add_tag("RCVD_HELO_USER");
}
// Helo not FQDN
ctx.add_tag("HELO_NOT_FQDN");
ctx.result.add_tag("HELO_NOT_FQDN");
}
}
}

310
crates/spam-filter/src/analysis/from.rs Normal file
View File

@@ -0,0 +1,310 @@
use std::future::Future;
use common::Core;
use mail_parser::HeaderName;
use smtp_proto::{MAIL_BODY_8BITMIME, MAIL_BODY_BINARYMIME, MAIL_SMTPUTF8};
use crate::{Email, SpamFilterContext};
pub trait SpamFilterAnalyzeFrom: Sync + Send {
fn spam_filter_analyze_from(
&self,
ctx: &mut SpamFilterContext<'_>,
) -> impl Future<Output = ()> + Send;
}
const SERVICE_ACCOUNTS: [&str; 9] = [
"www-data",
"anonymous",
"ftp",
"apache",
"nobody",
"guest",
"nginx",
"web",
"www",
];
pub(crate) const TITLES: [&str; 7] = ["mr. ", "mrs. ", "ms. ", "dr. ", "prof. ", "rev. ", "hon. "];
impl SpamFilterAnalyzeFrom for Core {
async fn spam_filter_analyze_from(&self, ctx: &mut SpamFilterContext<'_>) {
let mut from_count = 0;
let mut from_raw = b"".as_slice();
let mut crt = None;
let mut dnt = None;
let mut sender = None;
for header in ctx.input.message.headers() {
match &header.name {
HeaderName::From => {
from_count += 1;
from_raw = ctx
.input
.message
.raw_message()
.get(header.offset_start..header.offset_end)
.unwrap_or_default();
}
HeaderName::Sender => {
sender = header
.value()
.as_address()
.and_then(|addrs| addrs.first())
.and_then(|addr| addr.address())
.map(Email::new);
}
HeaderName::Other(name) => {
if name.eq_ignore_ascii_case("X-Confirm-Reading-To") {
crt = ctx
.input
.header_as_address(header)
.map(|s| s.to_lowercase());
} else if name.eq_ignore_ascii_case("Disposition-Notification-To") {
dnt = ctx
.input
.header_as_address(header)
.map(|s| s.to_lowercase());
}
}
_ => {}
}
}
match from_count {
0 => {
ctx.result.add_tag("MISSING_FROM");
}
1 => {}
_ => {
ctx.result.add_tag("MULTIPLE_FROM");
}
}
let env_from_empty = ctx.output.env_from_addr.address.is_empty();
let mut is_from_service_account = false;
let mut is_www_dot_domain = false;
let from_addr = &ctx.output.from.email;
let from_name = ctx.output.from.name.as_deref().unwrap_or_default();
if from_count > 0 {
// Validate address
let from_addr_is_valid = from_addr.is_valid();
if from_addr_is_valid {
if SERVICE_ACCOUNTS.contains(&from_addr.local_part.as_str()) {
is_from_service_account = true;
}
if from_addr.domain_part.fqdn.starts_with("www.") {
is_www_dot_domain = true;
}
if self
.spam
.list_freemail_providers
.contains(from_addr.domain_part.sld.as_deref().unwrap_or_default())
{
ctx.result.add_tag("FREEMAIL_FROM");
} else if self
.spam
.list_disposable_providers
.contains(from_addr.domain_part.sld.as_deref().unwrap_or_default())
{
ctx.result.add_tag("DISPOSABLE_FROM");
}
} else {
ctx.result.add_tag("FROM_INVALID");
}
// Validate from name
let from_name_trimmed = from_name.trim();
if from_name_trimmed.is_empty() {
ctx.result.add_tag("FROM_NO_DN");
} else if from_name_trimmed == from_addr.address {
ctx.result.add_tag("FROM_DN_EQ_ADDR");
} else {
let from_name_addr = Email::new(from_name_trimmed);
if from_addr_is_valid {
ctx.result.add_tag("FROM_HAS_DN");
}
if from_name_addr.is_valid() {
if (from_addr_is_valid
&& from_name_addr.domain_part.sld != from_addr.domain_part.sld)
|| (!env_from_empty
&& ctx.output.env_from_addr.domain_part.sld
!= from_name_addr.domain_part.sld)
|| (env_from_empty
&& ctx.output.ehlo_host.sld != from_name_addr.domain_part.sld)
{
ctx.result.add_tag("SPOOF_DISPLAY_NAME");
} else {
ctx.result.add_tag("FROM_NEQ_DISPLAY_NAME");
}
} else {
for title in TITLES {
if from_name.contains(title) {
ctx.result.add_tag("FROM_NAME_HAS_TITLE");
break;
}
}
if from_name.contains(" ") {
ctx.result.add_tag("FROM_NAME_EXCESS_SPACE");
}
}
}
// Check sender
if ctx.output.env_from_postmaster {
ctx.result.add_tag("FROM_BOUNCE");
}
if (!env_from_empty && ctx.output.env_from_addr.address == from_addr.address)
|| (!ctx.output.env_from_postmaster
&& from_addr_is_valid
&& from_addr.domain_part.sld == ctx.output.ehlo_host.sld)
{
ctx.result.add_tag("FROM_EQ_ENVFROM");
} else if from_addr_is_valid {
ctx.result.add_tag("FORGED_SENDER");
ctx.result.add_tag("FROM_NEQ_ENVFROM");
}
if from_addr.local_part.contains("+") {
ctx.result.add_tag("TAGGED_FROM");
}
// Validate FROM/TO relationship
if ctx.output.recipients_to.len() + ctx.output.recipients_cc.len() == 1 {
let rcpt = ctx
.output
.recipients_to
.first()
.or_else(|| ctx.output.recipients_cc.first())
.unwrap();
if rcpt.email.address == from_addr.address {
ctx.result.add_tag("TO_EQ_FROM");
} else if rcpt.email.domain_part.fqdn == from_addr.domain_part.fqdn {
ctx.result.add_tag("TO_DOM_EQ_FROM_DOM");
}
}
// Validate encoding
let from_raw_utf8 = std::str::from_utf8(from_raw);
if !from_raw.is_ascii() {
if (ctx.input.env_from_flags
& (MAIL_SMTPUTF8 | MAIL_BODY_8BITMIME | MAIL_BODY_BINARYMIME))
== 0
{
ctx.result.add_tag("FROM_NEEDS_ENCODING");
}
if from_raw_utf8.is_err() {
ctx.result.add_tag("INVALID_FROM_8BIT");
}
}
// Validate unnecessary encoding
let from_raw_utf8 = from_raw_utf8.unwrap_or_default();
if from_name.is_ascii()
&& from_addr.address.is_ascii()
&& from_raw_utf8.contains("=?")
&& from_raw_utf8.contains("?=")
{
if from_raw_utf8.contains("?q?") || from_raw_utf8.contains("?Q?") {
// From header is unnecessarily encoded in quoted-printable
ctx.result.add_tag("FROM_EXCESS_QP");
} else if from_raw_utf8.contains("?b?") || from_raw_utf8.contains("?B?") {
// From header is unnecessarily encoded in base64
ctx.result.add_tag("FROM_EXCESS_BASE64");
}
}
// Validate space in FROM
if !from_name.is_empty()
&& !from_addr.address.is_empty()
&& !from_raw_utf8.contains(" <")
{
ctx.result.add_tag("R_NO_SPACE_IN_FROM");
}
// Check whether read confirmation address is different to from address
if let Some(crt) = crt {
if crt != from_addr.address {
ctx.result.add_tag("HEADER_RCONFIRM_MISMATCH");
}
}
}
if !env_from_empty {
// Validate envelope address
if ctx.output.env_from_addr.is_valid() {
if SERVICE_ACCOUNTS.contains(&ctx.output.env_from_addr.local_part.as_str()) {
ctx.result.add_tag("ENVFROM_SERVICE_ACCT");
}
if self.spam.list_freemail_providers.contains(
ctx.output
.env_from_addr
.domain_part
.sld
.as_deref()
.unwrap_or_default(),
) {
ctx.result.add_tag("FREEMAIL_ENVFROM");
} else if self.spam.list_disposable_providers.contains(
ctx.output
.env_from_addr
.domain_part
.sld
.as_deref()
.unwrap_or_default(),
) {
ctx.result.add_tag("DISPOSABLE_ENVFROM");
}
// Mail from no resolve to A or MX
if matches!(
(
self.dns_exists_ip(&ctx.output.env_from_addr.domain_part.fqdn)
.await,
self.dns_exists_mx(&ctx.output.env_from_addr.domain_part.fqdn)
.await
),
(Ok(false), Ok(false))
) {
// Helo no resolve to A or MX
ctx.result.add_tag("FROMHOST_NORES_A_OR_MX");
}
} else {
ctx.result.add_tag("ENVFROM_INVALID");
}
// Check whether disposition notification address is different to return path
if let Some(dnt) = dnt {
if dnt != ctx.output.env_from_addr.address {
ctx.result.add_tag("HEADER_FORGED_MDN");
}
}
}
for addr in [
ctx.output.reply_to.as_ref().map(|s| &s.email),
sender.as_ref(),
]
.into_iter()
.flatten()
{
if !is_from_service_account && SERVICE_ACCOUNTS.contains(&addr.local_part.as_str()) {
is_from_service_account = true;
}
if !is_www_dot_domain && addr.domain_part.fqdn.starts_with("www.") {
is_www_dot_domain = true;
}
}
if is_from_service_account {
ctx.result.add_tag("FROM_SERVICE_ACCT");
}
if is_www_dot_domain {
ctx.result.add_tag("WWW_DOT_DOMAIN");
}
}
}

72
crates/spam-filter/src/analysis/headers.rs
View File

@@ -6,14 +6,14 @@ use store::ahash::AHashSet;
use crate::SpamFilterContext;
pub trait SpamFilterAnalyzeEhlo: Sync + Send {
pub trait SpamFilterAnalyzeHeaders: Sync + Send {
fn spam_filter_analyze_headers(
&self,
ctx: &mut SpamFilterContext<'_>,
) -> impl Future<Output = ()> + Send;
}
impl SpamFilterAnalyzeEhlo for Core {
impl SpamFilterAnalyzeHeaders for Core {
async fn spam_filter_analyze_headers(&self, ctx: &mut SpamFilterContext<'_>) {
let mut list_score = 0.0;
let mut unique_headers = AHashSet::new();
@@ -35,11 +35,11 @@ impl SpamFilterAnalyzeEhlo for Core {
| HeaderName::References
| HeaderName::InReplyTo => {
if !unique_headers.insert(header.name.clone()) {
ctx.add_tag("MULTIPLE_UNIQUE_HEADERS");
ctx.result.add_tag("MULTIPLE_UNIQUE_HEADERS");
}
if !matches!(raw_message.get(header.offset_field), Some(b' ')) {
ctx.add_tag("HEADER_EMPTY_DELIMITER");
if !matches!(raw_message.get(header.offset_start), Some(b' ')) {
ctx.result.add_tag("HEADER_EMPTY_DELIMITER");
}
}
HeaderName::ListArchive
@@ -56,7 +56,7 @@ impl SpamFilterAnalyzeEhlo for Core {
}
HeaderName::ListUnsubscribe => {
list_score += 0.25;
ctx.add_tag("HAS_LIST_UNSUB");
ctx.result.add_tag("HAS_LIST_UNSUB");
}
HeaderName::Other(name) => {
let value = header
@@ -69,7 +69,7 @@ impl SpamFilterAnalyzeEhlo for Core {
if name.eq_ignore_ascii_case("Precedence") {
if value == "bulk" {
list_score += 0.25;
ctx.add_tag("PRECEDENCE_BULK");
ctx.result.add_tag("PRECEDENCE_BULK");
} else if value == "list" {
list_score += 0.25;
}
@@ -78,50 +78,50 @@ impl SpamFilterAnalyzeEhlo for Core {
} else if name.eq_ignore_ascii_case("X-Priority") {
match value.parse::<i32>().unwrap_or(i32::MAX) {
0 => {
ctx.add_tag("HAS_X_PRIO_ZERO");
ctx.result.add_tag("HAS_X_PRIO_ZERO");
}
1 => {
ctx.add_tag("HAS_X_PRIO_ONE");
ctx.result.add_tag("HAS_X_PRIO_ONE");
}
2 => {
ctx.add_tag("HAS_X_PRIO_TWO");
ctx.result.add_tag("HAS_X_PRIO_TWO");
}
3 | 4 => {
ctx.add_tag("HAS_X_PRIO_THREE");
ctx.result.add_tag("HAS_X_PRIO_THREE");
}
4..=10000 => {
ctx.add_tag("HAS_X_PRIO_FIVE");
ctx.result.add_tag("HAS_X_PRIO_FIVE");
}
_ => {}
}
} else if name.eq_ignore_ascii_case("X-Mailer") {
if name != "X-Mailer" {
ctx.add_tag("XM_CASE");
ctx.result.add_tag("XM_CASE");
}
if !value.is_empty() {
if !value.as_bytes().iter().any(|&b| b.is_ascii_digit()) {
ctx.add_tag("XM_UA_NO_VERSION");
ctx.result.add_tag("XM_UA_NO_VERSION");
}
if value.contains("phpmailer") {
ctx.add_tag("HAS_PHPMAILER_SIG");
ctx.result.add_tag("HAS_PHPMAILER_SIG");
}
}
} else if name.eq_ignore_ascii_case("User-Agent") {
if !value.is_empty()
&& !value.as_bytes().iter().any(|&b| b.is_ascii_digit())
{
ctx.add_tag("XM_UA_NO_VERSION");
ctx.result.add_tag("XM_UA_NO_VERSION");
}
} else if name.eq_ignore_ascii_case("Organization")
|| name.eq_ignore_ascii_case("Organisation")
{
ctx.add_tag("HAS_ORG_HEADER");
ctx.result.add_tag("HAS_ORG_HEADER");
} else if name.eq_ignore_ascii_case("X-Originating-IP") {
ctx.add_tag("HAS_XOIP");
ctx.result.add_tag("HAS_XOIP");
} else if name.eq_ignore_ascii_case("X-KLMS-AntiSpam-Status") {
if value.contains("spam") {
ctx.add_tag("KLMS_SPAM");
ctx.result.add_tag("KLMS_SPAM");
}
} else if name.eq_ignore_ascii_case("X-Spam")
|| name.eq_ignore_ascii_case("X-Spam-Flag")
@@ -129,53 +129,53 @@ impl SpamFilterAnalyzeEhlo for Core {
{
if value.contains("yes") || value.contains("true") || value.contains("spam")
{
ctx.add_tag("SPAM_FLAG");
ctx.result.add_tag("SPAM_FLAG");
}
} else if name.eq_ignore_ascii_case("X-UI-Filterresults")
|| name.eq_ignore_ascii_case("X-UI-Out-Filterresults")
{
if value.contains("junk") {
ctx.add_tag("UNITEDINTERNET_SPAM");
ctx.result.add_tag("UNITEDINTERNET_SPAM");
}
} else if name.eq_ignore_ascii_case("X-PHP-Originating-Script") {
ctx.add_tag("HAS_X_POS");
ctx.result.add_tag("HAS_X_POS");
if value.contains("eval()") {
ctx.add_tag("X_PHP_EVAL");
ctx.result.add_tag("X_PHP_EVAL");
}
if value.contains("../") {
ctx.add_tag("HIDDEN_SOURCE_OBJ");
ctx.result.add_tag("HIDDEN_SOURCE_OBJ");
}
} else if name.eq_ignore_ascii_case("X-PHP-Script") {
ctx.add_tag("HAS_X_PHP_SCRIPT");
ctx.result.add_tag("HAS_X_PHP_SCRIPT");
if value.contains("eval()") {
ctx.add_tag("X_PHP_EVAL");
ctx.result.add_tag("X_PHP_EVAL");
}
if value.contains("../") {
ctx.add_tag("HIDDEN_SOURCE_OBJ");
ctx.result.add_tag("HIDDEN_SOURCE_OBJ");
}
if value.contains("sendmail.php") {
ctx.add_tag("PHP_XPS_PATTERN");
ctx.result.add_tag("PHP_XPS_PATTERN");
}
} else if name.eq_ignore_ascii_case("X-Source")
|| name.eq_ignore_ascii_case("X-Source-Args")
|| name.eq_ignore_ascii_case("X-Source-Dir")
{
ctx.add_tag("HAS_X_SOURCE");
ctx.result.add_tag("HAS_X_SOURCE");
if value.contains("'../") {
ctx.add_tag("HIDDEN_SOURCE_OBJ");
ctx.result.add_tag("HIDDEN_SOURCE_OBJ");
}
} else if name.eq_ignore_ascii_case("X-Authenticated-Sender") {
if value.contains(": ") {
ctx.add_tag("HAS_X_AS");
ctx.result.add_tag("HAS_X_AS");
}
} else if name.eq_ignore_ascii_case("X-Get-Message-Sender-Via") {
if value.contains("authenticated_id:") {
ctx.add_tag("HAS_X_GMSV");
ctx.result.add_tag("HAS_X_GMSV");
}
} else if name.eq_ignore_ascii_case("X-AntiAbuse") {
ctx.add_tag("HAS_X_ANTIABUSE");
ctx.result.add_tag("HAS_X_ANTIABUSE");
} else if name.eq_ignore_ascii_case("X-Authentication-Warning") {
ctx.add_tag("HAS_XAW");
ctx.result.add_tag("HAS_XAW");
}
}
_ => {}
@@ -183,11 +183,11 @@ impl SpamFilterAnalyzeEhlo for Core {
}
if list_score >= 1.0 {
ctx.add_tag("MAILLIST");
ctx.result.add_tag("MAILLIST");
}
if unique_headers.is_empty() {
ctx.add_tag("MISSING_ESSENTIAL_HEADERS");
ctx.result.add_tag("MISSING_ESSENTIAL_HEADERS");
}
}
}

137
crates/spam-filter/src/analysis/init.rs
View File

@@ -1,54 +1,115 @@
use common::Core;
use mail_parser::{parsers::fields::thread::thread_name, HeaderName};
use store::ahash::AHashSet;
use crate::{Email, Hostname, SpamFilterContext, SpamFilterInput, SpamFilterOutput};
use crate::{
Email, Hostname, Recipient, SpamFilterContext, SpamFilterInput, SpamFilterOutput,
SpamFilterResult,
};
pub trait SpamFilterInit {
fn spam_filter_init<'x>(&self, input: SpamFilterInput<'x>) -> SpamFilterContext<'x>;
}
const POSTMASTER_ADDRESSES: [&str; 3] = ["postmaster", "mailer-daemon", "root"];
impl SpamFilterInit for Core {
fn spam_filter_init<'x>(&self, input: SpamFilterInput<'x>) -> SpamFilterContext<'x> {
let subject = input.message.subject().unwrap_or_default().to_lowercase();
let from = input.message.from().and_then(|f| f.first());
let mut recipients = AHashSet::new();
let mut subject = String::new();
let mut from = None;
let mut reply_to = None;
let mut recipients_to = Vec::new();
let mut recipients_cc = Vec::new();
let mut recipients_bcc = Vec::new();
for header in input.message.headers() {
if matches!(
header.name,
HeaderName::To | HeaderName::Cc | HeaderName::Bcc
) {
if let Some(addrs) = header.value().as_address() {
for addr in addrs.iter() {
if let Some(addr) = addr.address() {
recipients.insert(Email::new(addr));
match &header.name {
HeaderName::To | HeaderName::Cc | HeaderName::Bcc => {
if let Some(addrs) = header.value().as_address() {
for addr in addrs.iter() {
let rcpt = Recipient {
email: Email::new(addr.address().unwrap_or_default()),
name: addr.name().and_then(|s| {
let s = s.trim();
if !s.is_empty() {
Some(s.to_lowercase())
} else {
None
}
}),
};
if header.name == HeaderName::To {
recipients_to.push(rcpt);
} else if header.name == HeaderName::Cc {
recipients_cc.push(rcpt);
} else {
recipients_bcc.push(rcpt);
}
}
}
}
HeaderName::ReplyTo => {
reply_to = header
.value()
.as_address()
.and_then(|addrs| addrs.first())
.and_then(|addr| {
Some(Recipient {
email: Email::new(addr.address()?),
name: addr.name().and_then(|s| {
let s = s.trim();
if !s.is_empty() {
Some(s.to_lowercase())
} else {
None
}
}),
})
});
}
HeaderName::Subject => {
subject = header.value().as_text().unwrap_or_default().to_lowercase();
}
HeaderName::From => {
from = header.value().as_address().and_then(|addrs| addrs.first());
}
_ => {}
}
}
let output = SpamFilterOutput {
tags: Default::default(),
ehlo_host: Hostname::new(input.ehlo_domain),
iprev_ptr: input
.iprev_result
.ptr
.as_ref()
.and_then(|ptr| ptr.first())
.map(|ptr| ptr.strip_suffix('.').unwrap_or(ptr).to_lowercase()),
env_from_addr: Email::new(input.env_mail_from),
from_addr: Email::new(from.and_then(|f| f.address()).unwrap_or_default()),
from_name: from
.and_then(|f| f.name())
.unwrap_or_default()
.to_lowercase(),
subject_thread: thread_name(&subject).to_string(),
subject,
recipients,
};
SpamFilterContext { output, input }
let env_from_addr = Email::new(input.env_from);
SpamFilterContext {
output: SpamFilterOutput {
ehlo_host: Hostname::new(input.ehlo_domain),
iprev_ptr: input
.iprev_result
.ptr
.as_ref()
.and_then(|ptr| ptr.first())
.map(|ptr| ptr.strip_suffix('.').unwrap_or(ptr).to_lowercase()),
env_from_postmaster: env_from_addr.address.is_empty()
|| POSTMASTER_ADDRESSES.contains(&env_from_addr.local_part.as_str()),
env_from_addr,
env_to_addr: input
.env_rcpt_to
.iter()
.map(|rcpt| Email::new(rcpt))
.collect(),
from: Recipient {
email: Email::new(from.and_then(|f| f.address()).unwrap_or_default()),
name: from.and_then(|f| f.name()).map(|s| s.to_lowercase()),
},
reply_to,
subject_thread: thread_name(&subject).to_string(),
subject,
recipients_to,
recipients_cc,
recipients_bcc,
},
input,
result: SpamFilterResult {
tags: Default::default(),
},
}
}
}
@@ -60,15 +121,15 @@ use common::Core;
use crate::SpamFilterContext;
pub trait SpamFilterAnalyzeEhlo: Sync + Send {
fn spam_filter_analyze_ehlo(
pub trait SpamFilterAnalyze!: Sync + Send {
fn spam_filter_analyze_*(
&self,
ctx: &mut SpamFilterContext<'_>,
) -> impl Future<Output = ()> + Send;
}
impl SpamFilterAnalyzeEhlo for Core {
async fn spam_filter_analyze_ehlo(&self, ctx: &mut SpamFilterContext<'_>) {
impl SpamFilterAnalyze! for Core {
async fn spam_filter_analyze_*(&self, ctx: &mut SpamFilterContext<'_>) {
todo!()
}
}

8
crates/spam-filter/src/analysis/iprev.rs
View File

@@ -5,18 +5,18 @@ use mail_auth::IprevResult;
use crate::SpamFilterContext;
pub trait SpamFilterAnalyzeEhlo: Sync + Send {
pub trait SpamFilterAnalyzeIpRev: Sync + Send {
fn spam_filter_analyze_iprev(
&self,
ctx: &mut SpamFilterContext<'_>,
) -> impl Future<Output = ()> + Send;
}
impl SpamFilterAnalyzeEhlo for Core {
impl SpamFilterAnalyzeIpRev for Core {
async fn spam_filter_analyze_iprev(&self, ctx: &mut SpamFilterContext<'_>) {
match &ctx.input.iprev_result.result {
IprevResult::TempError(_) => ctx.add_tag("RDNS_DNSFAIL"),
IprevResult::Fail(_) | IprevResult::PermError(_) => ctx.add_tag("RDNS_DNSFAIL"),
IprevResult::TempError(_) => ctx.result.add_tag("RDNS_DNSFAIL"),
IprevResult::Fail(_) | IprevResult::PermError(_) => ctx.result.add_tag("RDNS_DNSFAIL"),
IprevResult::Pass | IprevResult::None => (),
}
}

40
crates/spam-filter/src/analysis/messageid.rs
View File

@@ -5,14 +5,14 @@ use mail_parser::HeaderName;
use crate::{Hostname, SpamFilterContext};
pub trait SpamFilterAnalyzeEhlo: Sync + Send {
pub trait SpamFilterAnalyzeMid: Sync + Send {
fn spam_filter_analyze_message_id(
&self,
ctx: &mut SpamFilterContext<'_>,
) -> impl Future<Output = ()> + Send;
}
impl SpamFilterAnalyzeEhlo for Core {
impl SpamFilterAnalyzeMid for Core {
async fn spam_filter_analyze_message_id(&self, ctx: &mut SpamFilterContext<'_>) {
let mid_raw = ctx
.input
@@ -31,55 +31,55 @@ impl SpamFilterAnalyzeEhlo for Core {
if let Some(mid_host) = mid.rsplit_once('@').map(|(_, host)| Hostname::new(host)) {
if mid_host.ip.is_some() {
if mid_host.fqdn.starts_with('[') {
ctx.add_tag("MID_RHS_IP_LITERAL");
ctx.result.add_tag("MID_RHS_IP_LITERAL");
} else {
ctx.add_tag("MID_BARE_IP");
ctx.result.add_tag("MID_BARE_IP");
}
} else if !mid_host.fqdn.contains('.') {
ctx.add_tag("MID_RHS_NOT_FQDN");
ctx.result.add_tag("MID_RHS_NOT_FQDN");
} else if mid_host.fqdn.starts_with("www.") {
ctx.add_tag("MID_RHS_WWW");
ctx.result.add_tag("MID_RHS_WWW");
}
if !mid_raw.is_ascii() || mid_raw.contains('(') || mid.starts_with('@') {
ctx.add_tag("INVALID_MSGID");
ctx.result.add_tag("INVALID_MSGID");
}
if mid_host.fqdn.len() > 255 {
ctx.add_tag("MID_RHS_TOO_LONG");
ctx.result.add_tag("MID_RHS_TOO_LONG");
}
// From address present in Message-ID checks
for sender in [&ctx.output.from_addr, &ctx.output.env_from_addr] {
for sender in [&ctx.output.from.email, &ctx.output.env_from_addr] {
if !sender.address.is_empty() {
if mid.contains(&sender.address) {
ctx.output.tags.insert("MID_CONTAINS_FROM".to_string());
ctx.result.add_tag("MID_CONTAINS_FROM");
} else if mid_host.fqdn == sender.domain_part.fqdn {
ctx.output.tags.insert("MID_RHS_MATCH_FROM".to_string());
ctx.result.add_tag("MID_RHS_MATCH_FROM");
} else if matches!((&mid_host.sld, &sender.domain_part.sld), (Some(mid_sld), Some(sender_sld)) if mid_sld == sender_sld)
{
ctx.output.tags.insert("MID_RHS_MATCH_FROMTLD".to_string());
ctx.result.add_tag("MID_RHS_MATCH_FROMTLD");
}
}
}
// To/Cc addresses present in Message-ID checks
for addr in &ctx.output.recipients {
if mid.contains(&addr.address) {
ctx.output.tags.insert("MID_CONTAINS_TO".to_string());
} else if mid_host.fqdn == addr.domain_part.fqdn {
ctx.output.tags.insert("MID_RHS_MATCH_TO".to_string());
for rcpt in ctx.output.all_recipients() {
if mid.contains(&rcpt.email.address) {
ctx.result.add_tag("MID_CONTAINS_TO");
} else if mid_host.fqdn == rcpt.email.domain_part.fqdn {
ctx.result.add_tag("MID_RHS_MATCH_TO");
}
}
} else {
ctx.add_tag("INVALID_MSGID");
ctx.result.add_tag("INVALID_MSGID");
}
if !mid_raw.starts_with('<') || !mid_raw.ends_with('>') {
ctx.add_tag("MID_MISSING_BRACKETS");
ctx.result.add_tag("MID_MISSING_BRACKETS");
}
} else {
ctx.add_tag("MISSING_MID");
ctx.result.add_tag("MISSING_MID");
}
}
}

36
crates/spam-filter/src/analysis/mod.rs
View File

@@ -1,15 +1,43 @@
use crate::SpamFilterContext;
use std::borrow::Cow;
use mail_parser::{parsers::MessageStream, Header};
use crate::{Recipient, SpamFilterInput, SpamFilterOutput, SpamFilterResult};
pub mod date;
pub mod dmarc;
pub mod ehlo;
pub mod from;
pub mod headers;
pub mod init;
pub mod iprev;
pub mod messageid;
pub mod recipient;
pub mod replyto;
impl SpamFilterContext<'_> {
pub fn add_tag(&mut self, tag: impl Into<String>) {
self.output.tags.insert(tag.into());
impl SpamFilterInput<'_> {
pub fn header_as_address(&self, header: &Header<'_>) -> Option<Cow<'_, str>> {
self.message
.raw_message()
.get(header.offset_start..header.offset_end)
.map(|bytes| MessageStream::new(bytes).parse_address())
.and_then(|addr| addr.into_address())
.and_then(|addr| addr.into_list().into_iter().next())
.and_then(|addr| addr.address)
}
}
impl SpamFilterOutput {
pub fn all_recipients(&self) -> impl Iterator<Item = &Recipient> {
self.recipients_to
.iter()
.chain(self.recipients_cc.iter())
.chain(self.recipients_bcc.iter())
}
}
impl SpamFilterResult {
pub fn add_tag(&mut self, tag: impl Into<String>) {
self.tags.insert(tag.into());
}
}

301
crates/spam-filter/src/analysis/recipient.rs Normal file
View File

@@ -0,0 +1,301 @@
use std::future::Future;
use common::{scripts::functions::text::levenshtein_distance, Core};
use mail_parser::HeaderName;
use smtp_proto::{MAIL_BODY_8BITMIME, MAIL_BODY_BINARYMIME, MAIL_SMTPUTF8};
use store::ahash::HashSet;
use crate::SpamFilterContext;
pub trait SpamFilterAnalyzeRecipient: Sync + Send {
fn spam_filter_analyze_recipient(
&self,
ctx: &mut SpamFilterContext<'_>,
) -> impl Future<Output = ()> + Send;
}
impl SpamFilterAnalyzeRecipient for Core {
async fn spam_filter_analyze_recipient(&self, ctx: &mut SpamFilterContext<'_>) {
let mut to_raw = b"".as_slice();
let mut cc_raw = b"".as_slice();
let mut bcc_raw = b"".as_slice();
let mut has_list_unsubscribe = false;
let mut has_list_id = false;
for header in ctx.input.message.headers() {
match &header.name {
HeaderName::To | HeaderName::Cc | HeaderName::Bcc => {
let raw = ctx
.input
.message
.raw_message()
.get(header.offset_start..header.offset_end)
.unwrap_or_default();
match header.name {
HeaderName::To => to_raw = raw,
HeaderName::Cc => cc_raw = raw,
HeaderName::Bcc => bcc_raw = raw,
_ => unreachable!(),
}
}
HeaderName::ListUnsubscribe => {
has_list_unsubscribe = true;
}
HeaderName::ListId => {
has_list_id = true;
}
_ => {}
}
}
if to_raw.is_empty() {
ctx.result.add_tag("MISSING_TO");
}
let to_raw_utf8 = std::str::from_utf8(to_raw);
let cc_raw_utf8 = std::str::from_utf8(cc_raw);
let bcc_raw_utf8 = std::str::from_utf8(bcc_raw);
for (raw, raw_utf8, recipients) in [
(to_raw, &to_raw_utf8, &ctx.output.recipients_to),
(cc_raw, &cc_raw_utf8, &ctx.output.recipients_cc),
(bcc_raw, &bcc_raw_utf8, &ctx.output.recipients_bcc),
] {
if !raw.is_empty() {
// Validate non-ASCII characters in recipient headers
if !raw.is_ascii() {
if (ctx.input.env_from_flags
& (MAIL_SMTPUTF8 | MAIL_BODY_8BITMIME | MAIL_BODY_BINARYMIME))
== 0
{
ctx.result.add_tag("TO_NEEDS_ENCODING");
}
if raw_utf8.is_err() {
ctx.result.add_tag("INVALID_TO_8BIT");
}
}
// Validate unnecessary encoding in recipient headers
let raw_utf8 = raw_utf8.unwrap_or_default();
if recipients.iter().all(|rcpt| {
rcpt.name.as_ref().map_or(true, |name| name.is_ascii())
&& rcpt.email.address.is_ascii()
}) && raw_utf8.contains("=?")
&& raw_utf8.contains("?=")
{
if raw_utf8.contains("?q?") || raw_utf8.contains("?Q?") {
// To header is unnecessarily encoded in quoted-printable
ctx.result.add_tag("TO_EXCESS_QP");
} else if raw_utf8.contains("?b?") || raw_utf8.contains("?B?") {
// To header is unnecessarily encoded in base64
ctx.result.add_tag("TO_EXCESS_BASE64");
}
}
// Check for spaces in recipient addresses
for token in raw_utf8.split('<') {
if let Some((addr, _)) = token.split_once('>') {
if addr.starts_with(' ') || addr.ends_with(' ') {
ctx.result.add_tag("TO_WRAPPED_IN_SPACES");
break;
}
}
}
}
}
let unique_recipients = ctx
.output
.all_recipients()
.filter(|rcpt| !rcpt.email.address.is_empty())
.collect::<HashSet<_>>();
let rcpt_count = unique_recipients.len();
match unique_recipients.len() {
0 => {
ctx.result.add_tag("RCPT_COUNT_ZERO");
for raw in &[to_raw_utf8, cc_raw_utf8, bcc_raw_utf8] {
if matches!(raw, Ok(raw) if raw.to_ascii_lowercase().contains("undisclosed")) {
ctx.result.add_tag("R_UNDISC_RCPT");
break;
}
}
return;
}
1 => {
ctx.result.add_tag("RCPT_COUNT_ONE");
}
2 => {
ctx.result.add_tag("RCPT_COUNT_TWO");
}
3 => {
ctx.result.add_tag("RCPT_COUNT_THREE");
}
4 | 5 => {
ctx.result.add_tag("RCPT_COUNT_FIVE");
}
6 | 7 => {
ctx.result.add_tag("RCPT_COUNT_SEVEN");
}
8..=12 => {
ctx.result.add_tag("RCPT_COUNT_TWELVE");
}
13.. => {
ctx.result.add_tag("RCPT_COUNT_GT_50");
}
}
let mut to_dn_eq_addr_count = 0;
let mut to_dn_count = 0;
let mut to_match_envrcpt = 0;
let is_from_info = ctx.output.from.email.local_part == "info";
for rcpt in &unique_recipients {
// Validate name
if let Some(rcpt_name) = &rcpt.name {
if rcpt_name == &rcpt.email.address {
to_dn_eq_addr_count += 1;
} else {
to_dn_count += 1;
if ["recipient", "recipients"].contains(&rcpt_name.as_str()) {
ctx.result.add_tag("TO_DN_RECIPIENTS");
}
}
}
// Recipient is present in envelope
if ctx.output.env_to_addr.contains(&rcpt.email) {
to_match_envrcpt += 1;
}
// Check if the local part is present in the subject
if !rcpt.email.local_part.is_empty() {
if ctx.output.subject.contains(&rcpt.email.address) {
ctx.result.add_tag("RCPT_ADDR_IN_SUBJECT");
} else if rcpt.email.local_part.len() > 3
&& ctx.output.subject.contains(&rcpt.email.local_part)
{
ctx.result.add_tag("RCPT_LOCAL_IN_SUBJECT");
}
if rcpt.email.local_part.contains('+') {
ctx.result.add_tag("TAGGED_RCPT");
}
}
// Check if it is an into to info
if has_list_unsubscribe && is_from_info && rcpt.email.local_part == "info" {
ctx.result.add_tag("INFO_TO_INFO_LU");
}
// Check for freemail or disposable domains
if let Some(domain) = rcpt.email.domain_part.sld.as_deref() {
if self.spam.list_freemail_providers.contains(domain) {
if ctx
.output
.recipients_to
.iter()
.any(|r| r.email == rcpt.email)
{
ctx.result.add_tag("FREEMAIL_TO");
} else {
ctx.result.add_tag("FREEMAIL_CC");
}
} else if self.spam.list_disposable_providers.contains(domain) {
if ctx
.output
.recipients_to
.iter()
.any(|r| r.email == rcpt.email)
{
ctx.result.add_tag("DISPOSABLE_TO");
} else {
ctx.result.add_tag("DISPOSABLE_CC");
}
}
}
}
if to_dn_count == 0 && to_dn_eq_addr_count == 0 {
ctx.result.add_tag("TO_DN_NONE");
} else if to_dn_count == rcpt_count {
ctx.result.add_tag("TO_DN_ALL");
} else if to_dn_count > 0 {
ctx.result.add_tag("TO_DN_SOME");
}
if to_dn_eq_addr_count == rcpt_count {
ctx.result.add_tag("TO_DN_EQ_ADDR_ALL");
} else if to_dn_eq_addr_count > 0 {
ctx.result.add_tag("TO_DN_EQ_ADDR_SOME");
}
if to_match_envrcpt == rcpt_count {
ctx.result.add_tag("TO_MATCH_ENVRCPT_ALL");
} else {
if to_match_envrcpt > 0 {
ctx.result.add_tag("TO_MATCH_ENVRCPT_SOME");
}
if !has_list_id && !has_list_unsubscribe {
for env_rcpt in &ctx.output.env_to_addr {
if !unique_recipients.iter().any(|rcpt| rcpt.email == *env_rcpt)
&& env_rcpt != &ctx.output.env_from_addr
{
ctx.result.add_tag("FORGED_RECIPIENTS");
break;
}
}
}
}
// Message from bounce and over 1 recipient
if rcpt_count > 1 && ctx.output.env_from_postmaster {
ctx.result.add_tag("RCPT_BOUNCEMOREONE");
}
for rcpts in [&ctx.output.recipients_to, &ctx.output.recipients_cc] {
let mut is_sorted = false;
if rcpts.len() >= 6 {
// Check if the recipients list is sorted
let mut sorted = true;
for i in 1..rcpts.len() {
if rcpts[i - 1].email.address > rcpts[i].email.address {
sorted = false;
break;
}
}
if sorted {
ctx.result.add_tag("SORTED_RECIPS");
is_sorted = true;
}
}
if !is_sorted && rcpt_count >= 5 {
// Look for similar recipients
let mut hits = 0;
let mut combinations = 0;
for i in 0..rcpts.len() {
for j in i + 1..rcpts.len() {
let a = &rcpts[i].email;
let b = &rcpts[j].email;
if levenshtein_distance(&a.local_part, &b.local_part) < 3
|| (a.domain_part.fqdn != b.domain_part.fqdn
&& levenshtein_distance(&a.domain_part.fqdn, &b.domain_part.fqdn)
< 4)
{
hits += 1;
}
combinations += 1;
}
}
if hits as f64 / combinations as f64 > 0.65 {
ctx.result.add_tag("SUSPICIOUS_RECIPS");
}
}
}
}
}

153
crates/spam-filter/src/analysis/replyto.rs Normal file
View File

@@ -0,0 +1,153 @@
use std::future::Future;
use common::Core;
use mail_parser::HeaderName;
use crate::SpamFilterContext;
use super::from::TITLES;
pub trait SpamFilterAnalyzeReplyTo: Sync + Send {
fn spam_filter_analyze_reply_to(
&self,
ctx: &mut SpamFilterContext<'_>,
) -> impl Future<Output = ()> + Send;
}
impl SpamFilterAnalyzeReplyTo for Core {
async fn spam_filter_analyze_reply_to(&self, ctx: &mut SpamFilterContext<'_>) {
let mut reply_to_raw = b"".as_slice();
let mut is_from_list = false;
for header in ctx.input.message.headers() {
match &header.name {
HeaderName::ReplyTo => {
reply_to_raw = ctx
.input
.message
.raw_message()
.get(header.offset_start..header.offset_end)
.unwrap_or_default();
}
HeaderName::ListUnsubscribe | HeaderName::ListId => {
is_from_list = true;
}
HeaderName::Other(name) => {
if !is_from_list {
is_from_list = name.eq_ignore_ascii_case("X-To-Get-Off-This-List")
|| name.eq_ignore_ascii_case("X-List")
|| name.eq_ignore_ascii_case("Auto-Submitted");
}
}
_ => {}
}
}
if reply_to_raw.is_empty() {
return;
}
if let Some(reply_to) = &ctx.output.reply_to {
let reply_to_name = reply_to.name.as_deref().unwrap_or_default();
ctx.result.add_tag("HAS_REPLYTO");
if reply_to.email == ctx.output.from.email {
ctx.result.add_tag("REPLYTO_EQ_FROM");
} else {
if reply_to.email.domain_part.sld == ctx.output.from.email.domain_part.sld {
ctx.result.add_tag("REPLYTO_DOM_EQ_FROM_DOM");
} else {
if !is_from_list
&& ctx
.output
.all_recipients()
.any(|r| r.email == reply_to.email)
{
ctx.result.add_tag("REPLYTO_EQ_TO_ADDR");
} else {
ctx.result.add_tag("REPLYTO_DOM_NEQ_FROM_DOM");
}
if !(is_from_list
|| ctx
.output
.recipients_to
.iter()
.any(|r| r.email == ctx.output.from.email)
|| ctx
.output
.env_to_addr
.iter()
.any(|r| r.domain_part.sld == ctx.output.from.email.domain_part.sld)
|| ctx.output.env_to_addr.len() == 1
&& ctx.output.env_to_addr.contains(&ctx.output.from.email))
{
ctx.result.add_tag("SPOOF_REPLYTO");
}
}
if !reply_to_name.is_empty()
&& reply_to_name == ctx.output.from.name.as_deref().unwrap_or_default()
{
ctx.result.add_tag("REPLYTO_DN_EQ_FROM_DN");
}
}
if reply_to.email == ctx.output.env_from_addr {
ctx.result.add_tag("REPLYTO_ADDR_EQ_FROM");
}
let reply_to_sld = reply_to
.email
.domain_part
.sld
.as_deref()
.unwrap_or_default();
if self.spam.list_freemail_providers.contains(reply_to_sld) {
ctx.result.add_tag("FREEMAIL_REPLYTO");
let from_domain_sld = ctx
.output
.from
.email
.domain_part
.sld
.as_deref()
.unwrap_or_default();
if reply_to_sld != from_domain_sld
&& self.spam.list_freemail_providers.contains(from_domain_sld)
{
ctx.result.add_tag("FREEMAIL_REPLYTO_NEQ_FROM_DOM");
}
} else if self.spam.list_disposable_providers.contains(reply_to_sld) {
ctx.result.add_tag("DISPOSABLE_REPLYTO");
}
// Validate unnecessary encoding
let reply_to_raw_utf8 = std::str::from_utf8(reply_to_raw).unwrap_or_default();
if reply_to.email.address.is_ascii()
&& reply_to_name.is_ascii()
&& reply_to_raw_utf8.contains("=?")
&& reply_to_raw_utf8.contains("?=")
{
if reply_to_raw_utf8.contains("?q?") || reply_to_raw_utf8.contains("?Q?") {
// Reply-To header is unnecessarily encoded in quoted-printable
ctx.result.add_tag("REPLYTO_EXCESS_QP");
} else if reply_to_raw_utf8.contains("?b?") || reply_to_raw_utf8.contains("?B?") {
// Reply-To header is unnecessarily encoded in base64
ctx.result.add_tag("REPLYTO_EXCESS_BASE64");
}
}
// Validate reply-to name
for title in TITLES {
if reply_to_name.contains(title) {
ctx.result.add_tag("REPLYTO_EMAIL_HAS_TITLE");
break;
}
}
} else {
ctx.result.add_tag("REPLYTO_UNPARSABLE");
}
}
}

74
crates/spam-filter/src/lib.rs
View File

@@ -1,6 +1,7 @@
pub mod analysis;
pub mod modules;
use std::collections::HashSet;
use std::hash::{Hash, Hasher};
use std::net::IpAddr;
@@ -30,43 +31,58 @@ pub struct SpamFilterInput<'x> {
pub tls_cipher: &'x str,
// Envelope
pub env_mail_from: &'x str,
pub env_from: &'x str,
pub env_from_flags: u64,
pub env_rcpt_to: &'x [&'x str],
}
pub struct SpamFilterOutput {
pub tags: AHashSet<String>,
pub ehlo_host: Hostname,
pub iprev_ptr: Option<String>,
pub env_from_addr: Email,
pub from_addr: Email,
pub from_name: String,
pub recipients: AHashSet<Email>,
pub env_from_postmaster: bool,
pub env_to_addr: HashSet<Email>,
pub from: Recipient,
pub recipients_to: Vec<Recipient>,
pub recipients_cc: Vec<Recipient>,
pub recipients_bcc: Vec<Recipient>,
pub reply_to: Option<Recipient>,
pub subject: String,
pub subject_thread: String,
}
pub struct SpamFilterResult {
pub tags: AHashSet<String>,
}
pub struct SpamFilterContext<'x> {
pub input: SpamFilterInput<'x>,
pub output: SpamFilterOutput,
pub result: SpamFilterResult,
}
#[derive(Debug)]
#[derive(Debug, Clone)]
pub struct Hostname {
pub fqdn: String,
pub ip: Option<IpAddr>,
pub sld: Option<String>,
}
#[derive(Debug)]
#[derive(Debug, Clone)]
pub struct Email {
pub address: String,
pub local_part: String,
pub domain_part: Hostname,
}
#[derive(Debug, Clone)]
pub struct Recipient {
pub email: Email,
pub name: Option<String>,
}
impl PartialEq for Hostname {
fn eq(&self, other: &Self) -> bool {
self.fqdn.eq(&other.fqdn)
@@ -94,3 +110,47 @@ impl Hash for Email {
self.address.hash(state)
}
}
impl Email {
pub fn is_valid(&self) -> bool {
self.domain_part.sld.is_some() && !self.local_part.is_empty()
}
}
impl PartialEq for Recipient {
fn eq(&self, other: &Self) -> bool {
self.email.eq(&other.email)
}
}
impl Eq for Recipient {}
impl Hash for Recipient {
fn hash<H: Hasher>(&self, state: &mut H) {
self.email.hash(state)
}
}
impl PartialOrd for Email {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
Some(self.cmp(other))
}
}
impl PartialOrd for Recipient {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
Some(self.cmp(other))
}
}
impl Ord for Email {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
self.address.cmp(&other.address)
}
}
impl Ord for Recipient {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
self.email.cmp(&other.email)
}
}