filip/Stalwart
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Port Spam filter to Rust - part 1

Browse Source
This commit is contained in:
mdecimus
2024-12-06 18:35:39 +01:00
parent e86a9c1319
commit db7ae48c77
36 changed files with 1010 additions and 545 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
Cargo.lock generated
View File

@@ -2134,7 +2134,7 @@ dependencies = [
[[package]]
name = "event_macro"
version = "0.1.0"
version = "0.10.7"
dependencies = [
"proc-macro2",
"quote",
@@ -3254,7 +3254,7 @@ dependencies = [
[[package]]
name = "imap_proto"
version = "0.1.0"
version = "0.10.7"
dependencies = [
"ahash 0.8.11",
"chrono",
@@ -4890,7 +4890,7 @@ dependencies = [
[[package]]
name = "proc_macros"
version = "0.1.0"
version = "0.10.7"
dependencies = [
"proc-macro2",
"quote",
@@ -6442,6 +6442,23 @@ dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "spam-filter"
version = "0.10.7"
dependencies = [
"common",
"mail-auth",
"mail-builder",
"mail-parser",
"mail-send",
"nlp",
"psl",
"store",
"tokio",
"trc",
"utils",
]
[[package]]
name = "spin"
version = "0.5.2"
@@ -6712,7 +6729,7 @@ dependencies = [
[[package]]
name = "tests"
version = "0.1.0"
version = "0.10.7"
dependencies = [
"ahash 0.8.11",
"async-trait",

1
Cargo.toml
View File

@@ -9,6 +9,7 @@ members = [
"crates/smtp",
"crates/managesieve",
"crates/pop3",
"crates/spam-filter",
"crates/nlp",
"crates/store",
"crates/directory",

3
crates/common/src/config/mod.rs
View File

@@ -14,6 +14,7 @@ use hyper::{
HeaderMap,
};
use ring::signature::{EcdsaKeyPair, RsaKeyPair};
use spamfilter::SpamFilterConfig;
use store::{BlobBackend, BlobStore, FtsStore, LookupStore, Store, Stores};
use telemetry::Metrics;
use utils::config::{utils::AsKey, Config};
@@ -35,6 +36,7 @@ pub mod network;
pub mod scripts;
pub mod server;
pub mod smtp;
pub mod spamfilter;
pub mod storage;
pub mod telemetry;
@@ -181,6 +183,7 @@ impl Core {
oauth: OAuthConfig::parse(config),
acme: AcmeProviders::parse(config),
metrics: Metrics::parse(config),
spam: SpamFilterConfig::parse(config),
storage: Storage {
data,
blob,

7
crates/common/src/config/smtp/session.rs
View File

@@ -799,14 +799,7 @@ impl Default for SessionConfig {
subaddressing: AddressMapping::Enable,
},
data: Data {
#[cfg(feature = "test_mode")]
script: IfBlock::empty("session.data.script"),
#[cfg(not(feature = "test_mode"))]
script: IfBlock::new::<()>(
"session.data.script",
[("is_empty(authenticated_as)", "'spam-filter'")],
"'track-replies'",
),
pipe_commands: Default::default(),
max_messages: IfBlock::new::<()>("session.data.limits.messages", [], "10"),
max_message_size: IfBlock::new::<()>("session.data.limits.size", [], "104857600"),

19
crates/common/src/config/spamfilter.rs Normal file
View File

@@ -0,0 +1,19 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use utils::{config::Config, glob::GlobSet};
#[derive(Debug, Clone, Default)]
pub struct SpamFilterConfig {
pub list_dmarc_allow: GlobSet,
pub list_spf_dkim_allow: GlobSet,
}
impl SpamFilterConfig {
pub fn parse(config: &mut Config) -> Self {
SpamFilterConfig::default()
}
}

63
crates/common/src/dns.rs Normal file
View File

@@ -0,0 +1,63 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use std::net::IpAddr;
use mail_auth::{Error, IpLookupStrategy};
use crate::Core;
impl Core {
pub async fn dns_exists_mx(&self, entry: &str) -> trc::Result<bool> {
match self.smtp.resolvers.dns.mx_lookup(entry).await {
Ok(result) => Ok(result.iter().any(|mx| !mx.exchanges.is_empty())),
Err(Error::DnsRecordNotFound(_)) => Ok(false),
Err(err) => Err(err.into()),
}
}
pub async fn dns_exists_ip(&self, entry: &str) -> trc::Result<bool> {
match self
.smtp
.resolvers
.dns
.ip_lookup(entry, IpLookupStrategy::Ipv4thenIpv6, 10)
.await
{
Ok(result) => Ok(!result.is_empty()),
Err(Error::DnsRecordNotFound(_)) => Ok(false),
Err(err) => Err(err.into()),
}
}
pub async fn dns_exists_ptr(&self, entry: &str) -> trc::Result<bool> {
if let Ok(addr) = entry.parse::<IpAddr>() {
match self.smtp.resolvers.dns.ptr_lookup(addr).await {
Ok(result) => Ok(!result.is_empty()),
Err(Error::DnsRecordNotFound(_)) => Ok(false),
Err(err) => Err(err.into()),
}
} else {
Err(trc::EventType::Resource(trc::ResourceEvent::BadParameters).into_err())
}
}
pub async fn dns_exists_ipv4(&self, entry: &str) -> trc::Result<bool> {
match self.smtp.resolvers.dns.ipv4_lookup(entry).await {
Ok(result) => Ok(!result.is_empty()),
Err(Error::DnsRecordNotFound(_)) => Ok(false),
Err(err) => Err(err.into()),
}
}
pub async fn dns_exists_ipv6(&self, entry: &str) -> trc::Result<bool> {
match self.smtp.resolvers.dns.ipv6_lookup(entry).await {
Ok(result) => Ok(!result.is_empty()),
Err(Error::DnsRecordNotFound(_)) => Ok(false),
Err(err) => Err(err.into()),
}
}
}

3
crates/common/src/lib.rs
View File

@@ -20,6 +20,7 @@ use config::{
network::Network,
scripts::{RemoteList, Scripting},
smtp::SmtpConfig,
spamfilter::SpamFilterConfig,
storage::Storage,
telemetry::Metrics,
};
@@ -47,6 +48,7 @@ pub mod addresses;
pub mod auth;
pub mod config;
pub mod core;
pub mod dns;
#[cfg(feature = "enterprise")]
pub mod enterprise;
pub mod expr;
@@ -205,6 +207,7 @@ pub struct Core {
pub oauth: OAuthConfig,
pub smtp: SmtpConfig,
pub jmap: JmapConfig,
pub spam: SpamFilterConfig,
pub imap: ImapConfig,
pub metrics: Metrics,
#[cfg(feature = "enterprise")]

77
crates/common/src/scripts/plugins/dns.rs
View File

@@ -6,7 +6,7 @@
use std::net::IpAddr;
use mail_auth::{Error, IpLookupStrategy};
use mail_auth::IpLookupStrategy;
use sieve::{runtime::Variable, FunctionMap};
use super::PluginContext;
@@ -145,44 +145,12 @@ pub async fn exec_exists(ctx: PluginContext<'_>) -> trc::Result<Variable> {
let entry = ctx.arguments[0].to_string();
let record_type = ctx.arguments[1].to_string();
Ok(if record_type.eq_ignore_ascii_case("ip") {
match ctx
.server
.core
.smtp
.resolvers
.dns
.ip_lookup(entry.as_ref(), IpLookupStrategy::Ipv4thenIpv6, 10)
.await
{
Ok(result) => i64::from(!result.is_empty()),
Err(Error::DnsRecordNotFound(_)) => 0,
Err(_) => -1,
}
let result = if record_type.eq_ignore_ascii_case("ip") {
ctx.server.core.dns_exists_ip(entry.as_ref()).await
} else if record_type.eq_ignore_ascii_case("mx") {
match ctx
.server
.core
.smtp
.resolvers
.dns
.mx_lookup(entry.as_ref())
.await
{
Ok(result) => i64::from(result.iter().any(|mx| !mx.exchanges.is_empty())),
Err(Error::DnsRecordNotFound(_)) => 0,
Err(_) => -1,
}
ctx.server.core.dns_exists_mx(entry.as_ref()).await
} else if record_type.eq_ignore_ascii_case("ptr") {
if let Ok(addr) = entry.parse::<IpAddr>() {
match ctx.server.core.smtp.resolvers.dns.ptr_lookup(addr).await {
Ok(result) => i64::from(!result.is_empty()),
Err(Error::DnsRecordNotFound(_)) => 0,
Err(_) => -1,
}
} else {
-1
}
ctx.server.core.dns_exists_ptr(entry.as_ref()).await
} else if record_type.eq_ignore_ascii_case("ipv4") {
#[cfg(feature = "test_mode")]
{
@@ -191,37 +159,14 @@ pub async fn exec_exists(ctx: PluginContext<'_>) -> trc::Result<Variable> {
}
}
match ctx
.server
.core
.smtp
.resolvers
.dns
.ipv4_lookup(entry.as_ref())
.await
{
Ok(result) => i64::from(!result.is_empty()),
Err(Error::DnsRecordNotFound(_)) => 0,
Err(_) => -1,
}
ctx.server.core.dns_exists_ipv4(entry.as_ref()).await
} else if record_type.eq_ignore_ascii_case("ipv6") {
match ctx
.server
.core
.smtp
.resolvers
.dns
.ipv6_lookup(entry.as_ref())
.await
{
Ok(result) => i64::from(!result.is_empty()),
Err(Error::DnsRecordNotFound(_)) => 0,
Err(_) => -1,
}
ctx.server.core.dns_exists_ipv6(entry.as_ref()).await
} else {
-1
}
.into())
return Ok((-1).into());
};
Ok(result.map(i64::from).unwrap_or(-1).into())
}
trait ShortError {

22
crates/common/src/scripts/plugins/mod.rs
View File

@@ -4,14 +4,12 @@
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
pub mod bayes;
pub mod dns;
pub mod exec;
pub mod headers;
pub mod http;
pub mod llm_prompt;
pub mod lookup;
pub mod pyzor;
pub mod query;
pub mod text;
@@ -33,7 +31,7 @@ pub struct PluginContext<'x> {
pub arguments: Vec<Variable>,
}
const PLUGINS_REGISTER: [RegisterPluginFnc; 19] = [
const PLUGINS_REGISTER: [RegisterPluginFnc; 14] = [
query::register,
exec::register,
lookup::register,
@@ -44,11 +42,6 @@ const PLUGINS_REGISTER: [RegisterPluginFnc; 19] = [
dns::register,
dns::register_exists,
http::register_header,
bayes::register_train,
bayes::register_untrain,
bayes::register_classify,
bayes::register_is_balanced,
pyzor::register,
headers::register,
text::register_tokenize,
text::register_domain_part,
@@ -98,15 +91,10 @@ impl Core {
7 => dns::exec(ctx).await,
8 => dns::exec_exists(ctx).await,
9 => http::exec_header(ctx).await,
10 => bayes::exec_train(ctx).await,
11 => bayes::exec_untrain(ctx).await,
12 => bayes::exec_classify(ctx).await,
13 => bayes::exec_is_balanced(ctx).await,
14 => pyzor::exec(ctx).await,
15 => headers::exec(ctx),
16 => text::exec_tokenize(ctx),
17 => text::exec_domain_part(ctx),
18 => llm_prompt::exec(ctx).await,
10 => headers::exec(ctx),
11 => text::exec_tokenize(ctx),
12 => text::exec_domain_part(ctx),
13 => llm_prompt::exec(ctx).await,
_ => unreachable!(),
};

2
crates/imap-proto/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "imap_proto"
version = "0.1.0"
version = "0.10.7"
edition = "2021"
resolver = "2"

24
crates/spam-filter/Cargo.toml Normal file
View File

@@ -0,0 +1,24 @@
[package]
name = "spam-filter"
version = "0.10.7"
edition = "2021"
resolver = "2"
[dependencies]
utils = { path = "../utils" }
nlp = { path = "../nlp" }
store = { path = "../store" }
trc = { path = "../trc" }
common = { path = "../common" }
mail-parser = { version = "0.9", features = ["full_encoding", "ludicrous_mode"] }
mail-builder = { version = "0.3", features = ["ludicrous_mode"] }
mail-auth = { version = "0.5" }
mail-send = { version = "0.4", default-features = false, features = ["cram-md5", "ring", "tls12"] }
psl = "2"
[features]
test_mode = []
enterprise = []
[dev-dependencies]
tokio = { version = "1.23", features = ["full"] }

36
crates/spam-filter/src/analysis/date.rs Normal file
View File

@@ -0,0 +1,36 @@
use std::future::Future;
use common::Core;
use store::write::now;
use crate::SpamFilterContext;
pub trait SpamFilterAnalyzeEhlo: Sync + Send {
fn spam_filter_analyze_date(
&self,
ctx: &mut SpamFilterContext<'_>,
) -> impl Future<Output = ()> + Send;
}
impl SpamFilterAnalyzeEhlo for Core {
async fn spam_filter_analyze_date(&self, ctx: &mut SpamFilterContext<'_>) {
if let Some(date) = ctx.input.message.date() {
let date = date.to_timestamp();
if date != 0 {
let date_diff = now() as i64 - date;
if date_diff > 86400 {
// Older than a day
ctx.add_tag("DATE_IN_PAST");
} else if -date_diff > 7200 {
//# More than 2 hours in the future
ctx.add_tag("DATE_IN_FUTURE");
}
} else {
ctx.add_tag("INVALID_DATE");
}
} else {
ctx.add_tag("MISSING_DATE");
}
}
}

133
crates/spam-filter/src/analysis/dmarc.rs Normal file
View File

@@ -0,0 +1,133 @@
use std::future::Future;
use common::Core;
use mail_auth::{
common::verify::VerifySignature, dmarc::Policy, DkimResult, DmarcResult, SpfResult,
};
use crate::SpamFilterContext;
pub trait SpamFilterAnalyzeEhlo: Sync + Send {
fn spam_filter_analyze_dmarc(
&self,
ctx: &mut SpamFilterContext<'_>,
) -> impl Future<Output = ()> + Send;
}
impl SpamFilterAnalyzeEhlo for Core {
async fn spam_filter_analyze_dmarc(&self, ctx: &mut SpamFilterContext<'_>) {
ctx.add_tag(match ctx.input.spf_mail_from_result.result() {
SpfResult::Pass => "SPF_ALLOW",
SpfResult::Fail => "SPF_FAIL",
SpfResult::SoftFail => "SPF_SOFTFAIL",
SpfResult::Neutral => "SPF_NEUTRAL",
SpfResult::TempError => "SPF_DNSFAIL",
SpfResult::PermError => "SPF_PERMFAIL",
SpfResult::None => "SPF_NA",
});
ctx.add_tag(
match ctx
.input
.dkim_result
.iter()
.find(|r| matches!(r.result(), DkimResult::Pass))
.or_else(|| ctx.input.dkim_result.first())
.map(|r| r.result())
.unwrap_or(&DkimResult::None)
{
DkimResult::Pass => "DKIM_ALLOW",
DkimResult::Fail(_) => "DKIM_REJECT",
DkimResult::PermError(_) => "DKIM_PERMFAIL",
DkimResult::TempError(_) => "DKIM_TEMPFAIL",
DkimResult::Neutral(_) | DkimResult::None => "DKIM_NA",
},
);
ctx.add_tag(match ctx.input.arc_result.result() {
DkimResult::Pass => "ARC_ALLOW",
DkimResult::Fail(_) => "ARC_REJECT",
DkimResult::PermError(_) => "ARC_INVALID",
DkimResult::TempError(_) => "ARC_DNSFAIL",
DkimResult::Neutral(_) | DkimResult::None => "ARC_NA",
});
ctx.add_tag(match ctx.input.dmarc_result {
DmarcResult::Pass => "DMARC_POLICY_ALLOW",
DmarcResult::TempError(_) => "DMARC_DNSFAIL",
DmarcResult::PermError(_) => "DMARC_BAD_POLICY",
DmarcResult::None => "DMARC_NA",
DmarcResult::Fail(_) => match ctx.input.dmarc_policy {
Policy::Quarantine => "DMARC_POLICY_QUARANTINE",
Policy::Reject => "DMARC_POLICY_REJECT",
Policy::Unspecified | Policy::None => "DMARC_POLICY_SOFTFAIL",
},
});
for header in ctx.input.message.headers() {
let header_name = header.name();
if header_name.eq_ignore_ascii_case("DKIM-Signature") {
ctx.add_tag("DKIM_SIGNED");
} else if header_name.eq_ignore_ascii_case("ARC-Seal") {
ctx.add_tag("ARC_SIGNED");
}
}
if self
.spam
.list_dmarc_allow
.contains(&ctx.output.from_addr.domain_part.fqdn)
{
if matches!(ctx.input.dmarc_result, DmarcResult::Pass) {
ctx.add_tag("ALLOWLIST_DMARC");
} else {
ctx.add_tag("BLOCKLIST_DMARC");
}
} else if self
.spam
.list_spf_dkim_allow
.contains(&ctx.output.from_addr.domain_part.fqdn)
{
let is_dkim_pass = matches!(ctx.input.arc_result.result(), DkimResult::Pass)
|| ctx.input.dkim_result.iter().any(|r| {
matches!(r.result(), DkimResult::Pass)
&& r.signature().map_or(false, |s| {
s.domain().to_lowercase() == ctx.output.from_addr.domain_part.fqdn
})
});
let is_spf_pass = matches!(ctx.input.spf_mail_from_result.result(), SpfResult::Pass);
if is_dkim_pass && is_spf_pass {
ctx.add_tag("ALLOWLIST_SPF_DKIM");
} else if is_dkim_pass {
ctx.add_tag("ALLOWLIST_DKIM");
if !matches!(
ctx.input.spf_mail_from_result.result(),
SpfResult::TempError
) {
ctx.add_tag("BLOCKLIST_SPF");
}
} else if is_spf_pass {
ctx.add_tag("ALLOWLIST_SPF");
if !ctx
.input
.dkim_result
.iter()
.any(|r| matches!(r.result(), DkimResult::TempError(_)))
{
ctx.add_tag("BLOCKLIST_DKIM");
}
} else if !matches!(
ctx.input.spf_mail_from_result.result(),
SpfResult::TempError
) && !ctx
.input
.dkim_result
.iter()
.any(|r| matches!(r.result(), DkimResult::TempError(_)))
{
ctx.add_tag("BLOCKLIST_SPF_DKIM");
}
}
}
}

55
crates/spam-filter/src/analysis/ehlo.rs Normal file
View File

@@ -0,0 +1,55 @@
use std::future::Future;
use common::Core;
use crate::SpamFilterContext;
pub trait SpamFilterAnalyzeEhlo: Sync + Send {
fn spam_filter_analyze_ehlo(
&self,
ctx: &mut SpamFilterContext<'_>,
) -> impl Future<Output = ()> + Send;
}
impl SpamFilterAnalyzeEhlo for Core {
async fn spam_filter_analyze_ehlo(&self, ctx: &mut SpamFilterContext<'_>) {
if let Some(ehlo_ip) = ctx.output.ehlo_host.ip {
// Helo host is bare ip
ctx.add_tag("HELO_BAREIP");
if ehlo_ip != ctx.input.remote_ip {
// Helo A IP != hostname IP
ctx.add_tag("HELO_IP_A");
}
} else if ctx.output.ehlo_host.sld.is_some() {
if ctx
.output
.iprev_ptr
.as_ref()
.map_or(false, |ptr| ptr != &ctx.output.ehlo_host.fqdn)
{
// Helo does not match reverse IP
ctx.add_tag("HELO_IPREV_MISMATCH");
}
if matches!(
(
self.dns_exists_ip(&ctx.output.ehlo_host.fqdn).await,
self.dns_exists_mx(&ctx.output.ehlo_host.fqdn).await
),
(Ok(false), Ok(false))
) {
// Helo no resolve to A or MX
ctx.add_tag("HELO_NORES_A_OR_MX");
}
} else {
if ctx.output.ehlo_host.fqdn.contains("user") {
// Helo host contains 'user'
ctx.add_tag("RCVD_HELO_USER");
}
// Helo not FQDN
ctx.add_tag("HELO_NOT_FQDN");
}
}
}

193
crates/spam-filter/src/analysis/headers.rs Normal file
View File

@@ -0,0 +1,193 @@
use std::future::Future;
use common::Core;
use mail_parser::HeaderName;
use store::ahash::AHashSet;
use crate::SpamFilterContext;
pub trait SpamFilterAnalyzeEhlo: Sync + Send {
fn spam_filter_analyze_headers(
&self,
ctx: &mut SpamFilterContext<'_>,
) -> impl Future<Output = ()> + Send;
}
impl SpamFilterAnalyzeEhlo for Core {
async fn spam_filter_analyze_headers(&self, ctx: &mut SpamFilterContext<'_>) {
let mut list_score = 0.0;
let mut unique_headers = AHashSet::new();
let raw_message = ctx.input.message.raw_message();
for header in ctx.input.message.headers() {
match &header.name {
HeaderName::ContentType
| HeaderName::ContentTransferEncoding
| HeaderName::Date
| HeaderName::From
| HeaderName::Sender
| HeaderName::To
| HeaderName::Cc
| HeaderName::Bcc
| HeaderName::ReplyTo
| HeaderName::Subject
| HeaderName::MessageId
| HeaderName::References
| HeaderName::InReplyTo => {
if !unique_headers.insert(header.name.clone()) {
ctx.add_tag("MULTIPLE_UNIQUE_HEADERS");
}
if !matches!(raw_message.get(header.offset_field), Some(b' ')) {
ctx.add_tag("HEADER_EMPTY_DELIMITER");
}
}
HeaderName::ListArchive
| HeaderName::ListOwner
| HeaderName::ListHelp
| HeaderName::ListPost => {
list_score += 0.125;
}
HeaderName::ListId => {
list_score += 0.5125;
}
HeaderName::ListSubscribe => {
list_score += 0.25;
}
HeaderName::ListUnsubscribe => {
list_score += 0.25;
ctx.add_tag("HAS_LIST_UNSUB");
}
HeaderName::Other(name) => {
let value = header
.value()
.as_text()
.unwrap_or_default()
.trim()
.to_lowercase();
if name.eq_ignore_ascii_case("Precedence") {
if value == "bulk" {
list_score += 0.25;
ctx.add_tag("PRECEDENCE_BULK");
} else if value == "list" {
list_score += 0.25;
}
} else if name.eq_ignore_ascii_case("X-Loop") {
list_score += 0.125;
} else if name.eq_ignore_ascii_case("X-Priority") {
match value.parse::<i32>().unwrap_or(i32::MAX) {
0 => {
ctx.add_tag("HAS_X_PRIO_ZERO");
}
1 => {
ctx.add_tag("HAS_X_PRIO_ONE");
}
2 => {
ctx.add_tag("HAS_X_PRIO_TWO");
}
3 | 4 => {
ctx.add_tag("HAS_X_PRIO_THREE");
}
4..=10000 => {
ctx.add_tag("HAS_X_PRIO_FIVE");
}
_ => {}
}
} else if name.eq_ignore_ascii_case("X-Mailer") {
if name != "X-Mailer" {
ctx.add_tag("XM_CASE");
}
if !value.is_empty() {
if !value.as_bytes().iter().any(|&b| b.is_ascii_digit()) {
ctx.add_tag("XM_UA_NO_VERSION");
}
if value.contains("phpmailer") {
ctx.add_tag("HAS_PHPMAILER_SIG");
}
}
} else if name.eq_ignore_ascii_case("User-Agent") {
if !value.is_empty()
&& !value.as_bytes().iter().any(|&b| b.is_ascii_digit())
{
ctx.add_tag("XM_UA_NO_VERSION");
}
} else if name.eq_ignore_ascii_case("Organization")
|| name.eq_ignore_ascii_case("Organisation")
{
ctx.add_tag("HAS_ORG_HEADER");
} else if name.eq_ignore_ascii_case("X-Originating-IP") {
ctx.add_tag("HAS_XOIP");
} else if name.eq_ignore_ascii_case("X-KLMS-AntiSpam-Status") {
if value.contains("spam") {
ctx.add_tag("KLMS_SPAM");
}
} else if name.eq_ignore_ascii_case("X-Spam")
|| name.eq_ignore_ascii_case("X-Spam-Flag")
|| name.eq_ignore_ascii_case("X-Spam-Status")
{
if value.contains("yes") || value.contains("true") || value.contains("spam")
{
ctx.add_tag("SPAM_FLAG");
}
} else if name.eq_ignore_ascii_case("X-UI-Filterresults")
|| name.eq_ignore_ascii_case("X-UI-Out-Filterresults")
{
if value.contains("junk") {
ctx.add_tag("UNITEDINTERNET_SPAM");
}
} else if name.eq_ignore_ascii_case("X-PHP-Originating-Script") {
ctx.add_tag("HAS_X_POS");
if value.contains("eval()") {
ctx.add_tag("X_PHP_EVAL");
}
if value.contains("../") {
ctx.add_tag("HIDDEN_SOURCE_OBJ");
}
} else if name.eq_ignore_ascii_case("X-PHP-Script") {
ctx.add_tag("HAS_X_PHP_SCRIPT");
if value.contains("eval()") {
ctx.add_tag("X_PHP_EVAL");
}
if value.contains("../") {
ctx.add_tag("HIDDEN_SOURCE_OBJ");
}
if value.contains("sendmail.php") {
ctx.add_tag("PHP_XPS_PATTERN");
}
} else if name.eq_ignore_ascii_case("X-Source")
|| name.eq_ignore_ascii_case("X-Source-Args")
|| name.eq_ignore_ascii_case("X-Source-Dir")
{
ctx.add_tag("HAS_X_SOURCE");
if value.contains("'../") {
ctx.add_tag("HIDDEN_SOURCE_OBJ");
}
} else if name.eq_ignore_ascii_case("X-Authenticated-Sender") {
if value.contains(": ") {
ctx.add_tag("HAS_X_AS");
}
} else if name.eq_ignore_ascii_case("X-Get-Message-Sender-Via") {
if value.contains("authenticated_id:") {
ctx.add_tag("HAS_X_GMSV");
}
} else if name.eq_ignore_ascii_case("X-AntiAbuse") {
ctx.add_tag("HAS_X_ANTIABUSE");
} else if name.eq_ignore_ascii_case("X-Authentication-Warning") {
ctx.add_tag("HAS_XAW");
}
}
_ => {}
}
}
if list_score >= 1.0 {
ctx.add_tag("MAILLIST");
}
if unique_headers.is_empty() {
ctx.add_tag("MISSING_ESSENTIAL_HEADERS");
}
}
}

77
crates/spam-filter/src/analysis/init.rs Normal file
View File

@@ -0,0 +1,77 @@
use common::Core;
use mail_parser::{parsers::fields::thread::thread_name, HeaderName};
use store::ahash::AHashSet;
use crate::{Email, Hostname, SpamFilterContext, SpamFilterInput, SpamFilterOutput};
pub trait SpamFilterInit {
fn spam_filter_init<'x>(&self, input: SpamFilterInput<'x>) -> SpamFilterContext<'x>;
}
impl SpamFilterInit for Core {
fn spam_filter_init<'x>(&self, input: SpamFilterInput<'x>) -> SpamFilterContext<'x> {
let subject = input.message.subject().unwrap_or_default().to_lowercase();
let from = input.message.from().and_then(|f| f.first());
let mut recipients = AHashSet::new();
for header in input.message.headers() {
if matches!(
header.name,
HeaderName::To | HeaderName::Cc | HeaderName::Bcc
) {
if let Some(addrs) = header.value().as_address() {
for addr in addrs.iter() {
if let Some(addr) = addr.address() {
recipients.insert(Email::new(addr));
}
}
}
}
}
let output = SpamFilterOutput {
tags: Default::default(),
ehlo_host: Hostname::new(input.ehlo_domain),
iprev_ptr: input
.iprev_result
.ptr
.as_ref()
.and_then(|ptr| ptr.first())
.map(|ptr| ptr.strip_suffix('.').unwrap_or(ptr).to_lowercase()),
env_from_addr: Email::new(input.env_mail_from),
from_addr: Email::new(from.and_then(|f| f.address()).unwrap_or_default()),
from_name: from
.and_then(|f| f.name())
.unwrap_or_default()
.to_lowercase(),
subject_thread: thread_name(&subject).to_string(),
subject,
recipients,
};
SpamFilterContext { output, input }
}
}
/*
use std::future::Future;
use common::Core;
use crate::SpamFilterContext;
pub trait SpamFilterAnalyzeEhlo: Sync + Send {
fn spam_filter_analyze_ehlo(
&self,
ctx: &mut SpamFilterContext<'_>,
) -> impl Future<Output = ()> + Send;
}
impl SpamFilterAnalyzeEhlo for Core {
async fn spam_filter_analyze_ehlo(&self, ctx: &mut SpamFilterContext<'_>) {
todo!()
}
}
*/

23
crates/spam-filter/src/analysis/iprev.rs Normal file
View File

@@ -0,0 +1,23 @@
use std::future::Future;
use common::Core;
use mail_auth::IprevResult;
use crate::SpamFilterContext;
pub trait SpamFilterAnalyzeEhlo: Sync + Send {
fn spam_filter_analyze_iprev(
&self,
ctx: &mut SpamFilterContext<'_>,
) -> impl Future<Output = ()> + Send;
}
impl SpamFilterAnalyzeEhlo for Core {
async fn spam_filter_analyze_iprev(&self, ctx: &mut SpamFilterContext<'_>) {
match &ctx.input.iprev_result.result {
IprevResult::TempError(_) => ctx.add_tag("RDNS_DNSFAIL"),
IprevResult::Fail(_) | IprevResult::PermError(_) => ctx.add_tag("RDNS_DNSFAIL"),
IprevResult::Pass | IprevResult::None => (),
}
}
}

85
crates/spam-filter/src/analysis/messageid.rs Normal file
View File

@@ -0,0 +1,85 @@
use std::future::Future;
use common::Core;
use mail_parser::HeaderName;
use crate::{Hostname, SpamFilterContext};
pub trait SpamFilterAnalyzeEhlo: Sync + Send {
fn spam_filter_analyze_message_id(
&self,
ctx: &mut SpamFilterContext<'_>,
) -> impl Future<Output = ()> + Send;
}
impl SpamFilterAnalyzeEhlo for Core {
async fn spam_filter_analyze_message_id(&self, ctx: &mut SpamFilterContext<'_>) {
let mid_raw = ctx
.input
.message
.header_raw(HeaderName::MessageId)
.unwrap_or_default()
.trim();
if !mid_raw.is_empty() {
let mid = ctx
.input
.message
.message_id()
.unwrap_or_default()
.to_lowercase();
if let Some(mid_host) = mid.rsplit_once('@').map(|(_, host)| Hostname::new(host)) {
if mid_host.ip.is_some() {
if mid_host.fqdn.starts_with('[') {
ctx.add_tag("MID_RHS_IP_LITERAL");
} else {
ctx.add_tag("MID_BARE_IP");
}
} else if !mid_host.fqdn.contains('.') {
ctx.add_tag("MID_RHS_NOT_FQDN");
} else if mid_host.fqdn.starts_with("www.") {
ctx.add_tag("MID_RHS_WWW");
}
if !mid_raw.is_ascii() || mid_raw.contains('(') || mid.starts_with('@') {
ctx.add_tag("INVALID_MSGID");
}
if mid_host.fqdn.len() > 255 {
ctx.add_tag("MID_RHS_TOO_LONG");
}
// From address present in Message-ID checks
for sender in [&ctx.output.from_addr, &ctx.output.env_from_addr] {
if !sender.address.is_empty() {
if mid.contains(&sender.address) {
ctx.output.tags.insert("MID_CONTAINS_FROM".to_string());
} else if mid_host.fqdn == sender.domain_part.fqdn {
ctx.output.tags.insert("MID_RHS_MATCH_FROM".to_string());
} else if matches!((&mid_host.sld, &sender.domain_part.sld), (Some(mid_sld), Some(sender_sld)) if mid_sld == sender_sld)
{
ctx.output.tags.insert("MID_RHS_MATCH_FROMTLD".to_string());
}
}
}
// To/Cc addresses present in Message-ID checks
for addr in &ctx.output.recipients {
if mid.contains(&addr.address) {
ctx.output.tags.insert("MID_CONTAINS_TO".to_string());
} else if mid_host.fqdn == addr.domain_part.fqdn {
ctx.output.tags.insert("MID_RHS_MATCH_TO".to_string());
}
}
} else {
ctx.add_tag("INVALID_MSGID");
}
if !mid_raw.starts_with('<') || !mid_raw.ends_with('>') {
ctx.add_tag("MID_MISSING_BRACKETS");
}
} else {
ctx.add_tag("MISSING_MID");
}
}
}

15
crates/spam-filter/src/analysis/mod.rs Normal file
View File

@@ -0,0 +1,15 @@
use crate::SpamFilterContext;
pub mod date;
pub mod dmarc;
pub mod ehlo;
pub mod headers;
pub mod init;
pub mod iprev;
pub mod messageid;
impl SpamFilterContext<'_> {
pub fn add_tag(&mut self, tag: impl Into<String>) {
self.output.tags.insert(tag.into());
}
}

96
crates/spam-filter/src/lib.rs Normal file
View File

@@ -0,0 +1,96 @@
pub mod analysis;
pub mod modules;
use std::hash::{Hash, Hasher};
use std::net::IpAddr;
use mail_auth::{dmarc::Policy, ArcOutput, DkimOutput, DmarcResult, IprevOutput, SpfOutput};
use mail_parser::Message;
use store::ahash::AHashSet;
pub struct SpamFilterInput<'x> {
pub message: &'x Message<'x>,
// Sender authentication
pub arc_result: &'x ArcOutput<'x>,
pub spf_ehlo_result: &'x SpfOutput,
pub spf_mail_from_result: &'x SpfOutput,
pub dkim_result: &'x [DkimOutput<'x>],
pub dmarc_result: &'x DmarcResult,
pub dmarc_policy: &'x Policy,
pub iprev_result: &'x IprevOutput,
// Session details
pub remote_ip: IpAddr,
pub ehlo_domain: &'x str,
pub authenticated_as: &'x str,
// TLS
pub tls_version: &'x str,
pub tls_cipher: &'x str,
// Envelope
pub env_mail_from: &'x str,
pub env_rcpt_to: &'x [&'x str],
}
pub struct SpamFilterOutput {
pub tags: AHashSet<String>,
pub ehlo_host: Hostname,
pub iprev_ptr: Option<String>,
pub env_from_addr: Email,
pub from_addr: Email,
pub from_name: String,
pub recipients: AHashSet<Email>,
pub subject: String,
pub subject_thread: String,
}
pub struct SpamFilterContext<'x> {
pub input: SpamFilterInput<'x>,
pub output: SpamFilterOutput,
}
#[derive(Debug)]
pub struct Hostname {
pub fqdn: String,
pub ip: Option<IpAddr>,
pub sld: Option<String>,
}
#[derive(Debug)]
pub struct Email {
pub address: String,
pub local_part: String,
pub domain_part: Hostname,
}
impl PartialEq for Hostname {
fn eq(&self, other: &Self) -> bool {
self.fqdn.eq(&other.fqdn)
}
}
impl Eq for Hostname {}
impl PartialEq for Email {
fn eq(&self, other: &Self) -> bool {
self.address.eq(&other.address)
}
}
impl Eq for Email {}
impl Hash for Hostname {
fn hash<H: Hasher>(&self, state: &mut H) {
self.fqdn.hash(state)
}
}
impl Hash for Email {
fn hash<H: Hasher>(&self, state: &mut H) {
self.address.hash(state)
}
}

0
crates/common/src/scripts/plugins/bayes.rs → crates/spam-filter/src/modules/bayes.rs
View File

1
crates/spam-filter/src/modules/mod.rs Normal file
View File

@@ -0,0 +1 @@
pub mod sanitize;

0
crates/common/src/scripts/plugins/pyzor.rs → crates/spam-filter/src/modules/pyzor.rs
View File

38
crates/spam-filter/src/modules/sanitize.rs Normal file
View File

@@ -0,0 +1,38 @@
use std::net::IpAddr;
use crate::{Email, Hostname};
impl Hostname {
pub fn new(host: &str) -> Self {
let fqdn = host.to_lowercase();
let ip = fqdn
.strip_prefix('[')
.and_then(|ip| ip.strip_suffix(']'))
.unwrap_or(&fqdn)
.parse::<IpAddr>()
.ok();
Hostname {
ip,
sld: if ip.is_none() {
psl::domain_str(&fqdn).map(str::to_string)
} else {
None
},
fqdn,
}
}
}
impl Email {
pub fn new(address: &str) -> Self {
let address = address.to_lowercase();
let (local_part, domain) = address.rsplit_once('@').unwrap_or_default();
Email {
local_part: local_part.to_string(),
domain_part: Hostname::new(domain),
address,
}
}
}

53
crates/store/src/backend/memory/mod.rs
View File

@@ -5,25 +5,11 @@
*/
use ahash::AHashMap;
use utils::{config::Config, glob::GlobPattern};
use utils::{config::Config, glob::GlobMap};
use crate::{LookupStore, Stores, Value};
#[derive(Debug, Default)]
pub struct MemoryStore {
entries: AHashMap<String, Value<'static>>,
globs: Vec<(GlobPattern, Value<'static>)>,
}
impl MemoryStore {
pub fn get(&self, id: &str) -> Option<&Value<'static>> {
self.entries.get(id).or_else(|| {
self.globs
.iter()
.find_map(|(pattern, value)| pattern.matches(id).then_some(value))
})
}
}
pub type MemoryStore = GlobMap<Value<'static>>;
impl Stores {
pub fn parse_memory_stores(&mut self, config: &mut Config) {
@@ -35,24 +21,6 @@ impl Stores {
.split_once('.')
.filter(|(id, key)| !id.is_empty() && !key.is_empty())
{
// Detect if the key is a glob pattern
let mut last_ch = '\0';
let mut has_escape = false;
let mut is_glob = false;
for ch in key.chars() {
match ch {
'\\' => {
has_escape = true;
}
'*' | '?' if last_ch != '\\' => {
is_glob = true;
}
_ => {}
}
last_ch = ch;
}
// Detect value type
let value = if !value.is_empty() {
let mut has_integers = false;
@@ -98,21 +66,10 @@ impl Stores {
};
// Add entry
let store = lookups
lookups
.entry(id.to_string())
.or_insert_with(MemoryStore::default);
if is_glob {
store.globs.push((GlobPattern::compile(key, false), value));
} else {
store.entries.insert(
if has_escape {
key.replace('\\', "")
} else {
key.to_string()
},
value,
);
}
.or_insert_with(MemoryStore::default)
.insert(key, value);
} else {
errors.push(key.to_string());
}

2
crates/trc/event-macro/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "event_macro"
version = "0.1.0"
version = "0.10.7"
edition = "2021"
[lib]

2
crates/utils/proc-macros/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "proc_macros"
version = "0.1.0"
version = "0.10.7"
edition = "2021"
[lib]

99
crates/utils/src/glob.rs
View File

@@ -4,6 +4,8 @@
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use ahash::{AHashMap, AHashSet};
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct GlobPattern {
pattern: Vec<PatternChar>,
@@ -61,6 +63,36 @@ impl GlobPattern {
}
}
pub fn try_compile(pattern: &str, to_lower: bool) -> Result<Self, String> {
// Detect if the key is a glob pattern
let mut last_ch = '\0';
let mut has_escape = false;
let mut is_glob = false;
for ch in pattern.chars() {
match ch {
'\\' => {
has_escape = true;
}
'*' | '?' if last_ch != '\\' => {
is_glob = true;
}
_ => {}
}
last_ch = ch;
}
if is_glob {
Ok(GlobPattern::compile(pattern, to_lower))
} else {
Err(if has_escape {
pattern.replace('\\', "")
} else {
pattern.to_string()
})
}
}
// Credits: Algorithm ported from https://research.swtch.com/glob
pub fn matches(&self, value: &str) -> bool {
let value = if self.to_lower {
@@ -108,3 +140,70 @@ impl GlobPattern {
true
}
}
#[derive(Debug, Clone, Default)]
pub struct GlobSet {
entries: AHashSet<String>,
patterns: Vec<GlobPattern>,
}
#[derive(Debug, Clone)]
pub struct GlobMap<V> {
entries: AHashMap<String, V>,
patterns: Vec<(GlobPattern, V)>,
}
impl GlobSet {
pub fn new() -> Self {
GlobSet::default()
}
pub fn insert(&mut self, pattern: &str) {
match GlobPattern::try_compile(pattern, false) {
Ok(glob) => {
self.patterns.push(glob);
}
Err(entry) => {
self.entries.insert(entry);
}
}
}
pub fn contains(&self, key: &str) -> bool {
self.entries.contains(key) || self.patterns.iter().any(|pattern| pattern.matches(key))
}
}
impl<V> GlobMap<V> {
pub fn new() -> Self {
GlobMap {
entries: AHashMap::new(),
patterns: Vec::new(),
}
}
pub fn insert(&mut self, pattern: &str, value: V) {
match GlobPattern::try_compile(pattern, false) {
Ok(glob) => {
self.patterns.push((glob, value));
}
Err(entry) => {
self.entries.insert(entry, value);
}
}
}
pub fn get(&self, key: &str) -> Option<&V> {
self.entries.get(key).or_else(|| {
self.patterns
.iter()
.find_map(|(pattern, value)| pattern.matches(key).then_some(value))
})
}
}
impl<V> Default for GlobMap<V> {
fn default() -> Self {
GlobMap::new()
}
}

19
resources/config/spamfilter/scripts/date.sieve
View File

@@ -1,19 +0,0 @@
if eval "header.date.exists" {
let "date" "header.date.date";
if eval "date != 0" {
let "date_diff" "env.now - date";
if eval "date_diff > 86400" {
# Older than a day
let "t.DATE_IN_PAST" "1";
} elsif eval "-date_diff > 7200" {
# More than 2 hours in the future
let "t.DATE_IN_FUTURE" "1";
}
} else {
let "t.INVALID_DATE" "1";
}
} else {
let "t.MISSING_DATE" "1";
}

91
resources/config/spamfilter/scripts/dmarc.sieve
View File

@@ -1,91 +0,0 @@
if eval "env.spf.result == 'pass'" {
let "t.SPF_ALLOW" "1";
} elsif eval "env.spf.result == 'fail'" {
let "t.SPF_FAIL" "1";
} elsif eval "env.spf.result == 'softfail'" {
let "t.SPF_SOFTFAIL" "1";
} elsif eval "env.spf.result == 'neutral'" {
let "t.SPF_NEUTRAL" "1";
} elsif eval "env.spf.result == 'temperror'" {
let "t.SPF_DNSFAIL" "1";
} elsif eval "env.spf.result == 'permerror'" {
let "t.SPF_PERMFAIL" "1";
} else {
let "t.SPF_NA" "1";
}
if eval "env.dkim.result == 'pass'" {
let "t.DKIM_ALLOW" "1";
} elsif eval "env.dkim.result == 'fail'" {
let "t.DKIM_REJECT" "1";
} elsif eval "env.dkim.result == 'temperror'" {
let "t.DKIM_TEMPFAIL" "1";
} elsif eval "env.dkim.result == 'permerror'" {
let "t.DKIM_PERMFAIL" "1";
} else {
let "t.DKIM_NA" "1";
}
if eval "env.arc.result == 'pass'" {
let "t.ARC_ALLOW" "1";
} elsif eval "env.arc.result == 'fail'" {
let "t.ARC_REJECT" "1";
} elsif eval "env.arc.result == 'temperror'" {
let "t.ARC_DNSFAIL" "1";
} elsif eval "env.arc.result == 'permerror'" {
let "t.ARC_INVALID" "1";
} else {
let "t.ARC_NA" "1";
}
if eval "env.dmarc.result == 'pass'" {
let "t.DMARC_POLICY_ALLOW" "1";
} elsif eval "env.dmarc.result == 'temperror'" {
let "t.DMARC_DNSFAIL" "1";
} elsif eval "env.dmarc.result == 'permerror'" {
let "t.DMARC_BAD_POLICY" "1";
} elsif eval "env.dmarc.result == 'fail'" {
if eval "env.dmarc.policy == 'quarantine'" {
let "t.DMARC_POLICY_QUARANTINE" "1";
} elsif eval "env.dmarc.policy == 'reject'" {
let "t.DMARC_POLICY_REJECT" "1";
} else {
let "t.DMARC_POLICY_SOFTFAIL" "1";
}
} else {
let "t.DMARC_NA" "1";
}
if eval "header.DKIM-Signature.exists" {
let "t.DKIM_SIGNED" "1";
if eval "header.ARC-Seal.exists" {
let "t.ARC_SIGNED" "1";
}
}
# Check allowlists
if eval "key_exists('spam-dmarc', from_domain)" {
if eval "t.DMARC_POLICY_ALLOW" {
let "t.ALLOWLIST_DMARC" "1";
} else {
let "t.BLOCKLIST_DMARC" "1";
}
} elsif eval "key_exists('spam-spdk', from_domain)" {
let "is_dkim_pass" "contains(env.dkim.domains, from_domain) || t.ARC_ALLOW";
if eval "is_dkim_pass && t.SPF_ALLOW" {
let "t.ALLOWLIST_SPF_DKIM" "1";
} elsif eval "is_dkim_pass" {
let "t.ALLOWLIST_DKIM" "1";
if eval "!t.SPF_DNSFAIL" {
let "t.BLOCKLIST_SPF" "1";
}
} elsif eval "t.SPF_ALLOW" {
let "t.ALLOWLIST_SPF" "1";
if eval "!t.DKIM_TEMPFAIL" {
let "t.BLOCKLIST_DKIM" "1";
}
} elsif eval "!t.SPF_DNSFAIL && !t.DKIM_TEMPFAIL" {
let "t.BLOCKLIST_SPF_DKIM" "1";
}
}

140
resources/config/spamfilter/scripts/headers.sieve
View File

@@ -1,140 +0,0 @@
# Mailing list scores
let "ml_score" "count(header.List-Id:List-Archive:List-Owner:List-Help:List-Post:X-Loop:List-Subscribe:List-Unsubscribe[*].exists) * 0.125";
if eval "ml_score < 1" {
if eval "header.List-Id.exists" {
let "ml_score" "ml_score + 0.50";
}
if eval "header.List-Subscribe.exists && header.List-Unsubscribe.exists" {
let "ml_score" "ml_score + 0.25";
}
if eval "header.Precedence.exists && (eq_ignore_case(header.Precedence, 'list') || eq_ignore_case(header.Precedence, 'bulk'))" {
let "ml_score" "ml_score + 0.25";
}
}
if eval "ml_score >= 1" {
let "t.MAILLIST" "1";
}
# X-Priority
if eval "header.x-priority.exists" {
let "xp" "header.x-priority";
if eval "xp == 0" {
let "t.HAS_X_PRIO_ZERO" "1";
} elsif eval "xp == 1" {
let "t.HAS_X_PRIO_ONE" "1";
} elsif eval "xp == 2" {
let "t.HAS_X_PRIO_TWO" "1";
} elsif eval "xp <= 4" {
let "t.HAS_X_PRIO_THREE" "1";
} elsif eval "xp >= 5" {
let "t.HAS_X_PRIO_FIVE" "1";
}
}
let "unique_header_names" "to_lowercase(header.Content-Type:Content-Transfer-Encoding:Date:From:Sender:Reply-To:To:Cc:Bcc:Message-ID:In-Reply-To:References:Subject[*].raw_name)";
let "unique_header_names_len" "count(unique_header_names)";
if eval "unique_header_names_len != count(dedup(unique_header_names))" {
let "t.MULTIPLE_UNIQUE_HEADERS" "1";
} elsif eval "unique_header_names_len == 0" {
let "t.MISSING_ESSENTIAL_HEADERS" "1";
}
# Wrong case X-Mailer
if eval "header.x-mailer.exists && header.x-mailer.raw_name != 'X-Mailer'" {
let "t.XM_CASE" "1";
}
# Has organization header
if eval "header.organization:organisation.exists" {
let "t.HAS_ORG_HEADER" "1";
}
# Has X-Originating-IP header
if eval "header.X-Originating-IP.exists" {
let "t.HAS_XOIP" "1";
}
# Has List-Unsubscribe header
if eval "header.List-Unsubscribe.exists" {
let "t.HAS_LIST_UNSUB" "1";
}
# Missing version number in X-Mailer or User-Agent headers
if eval "(header.X-Mailer.exists && !has_digits(header.X-Mailer)) || (header.User-Agent.exists && !has_digits(header.User-Agent))" {
let "t.XM_UA_NO_VERSION" "1";
}
# Precedence is bulk
if eval "eq_ignore_case(header.Precedence, 'bulk')" {
let "t.PRECEDENCE_BULK" "1";
}
# Upstream SPAM filtering
if eval "contains_ignore_case(header.X-KLMS-AntiSpam-Status, 'spam')" {
# Kaspersky Security for Mail Server says this message is spam
let "t.KLMS_SPAM" "1";
}
let "spam_hdr" "to_lowercase(header.X-Spam:X-Spam-Flag:X-Spam-Status)";
if eval "contains(spam_hdr, 'yes') || contains(spam_hdr, 'true') || contains(spam_hdr, 'spam')" {
# Message was already marked as spam
let "t.SPAM_FLAG" "1";
}
if eval "contains_ignore_case(header.X-UI-Filterresults:X-UI-Out-Filterresults, 'junk')" {
# United Internet says this message is spam
let "t.UNITEDINTERNET_SPAM" "1";
}
# Compromised hosts
if eval "header.X-PHP-Originating-Script.exists" {
let "t.HAS_X_POS" "1";
if eval "contains(header.X-PHP-Originating-Script, 'eval()')" {
let "t.X_PHP_EVAL" "1";
}
if eval "contains(header.X-PHP-Originating-Script, '../')" {
let "t.HIDDEN_SOURCE_OBJ" "1";
}
}
if eval "header.X-PHP-Script.exists" {
let "t.HAS_X_PHP_SCRIPT" "1";
if eval "contains(header.X-PHP-Script, 'eval()')" {
let "t.X_PHP_EVAL" "1";
}
if eval "contains(header.X-PHP-Script, 'sendmail.php')" {
let "t.PHP_XPS_PATTERN" "1";
}
if eval "contains(header.X-PHP-Script, '../')" {
let "t.HIDDEN_SOURCE_OBJ" "1";
}
}
if eval "contains_ignore_case(header.X-Mailer, 'PHPMailer')" {
let "t.HAS_PHPMAILER_SIG" "1";
}
if eval "header.X-Source:X-Source-Args:X-Source-Dir.exists" {
let "t.HAS_X_SOURCE" "1";
if eval "contains(header.X-Source-Args, '../')" {
let "t.HIDDEN_SOURCE_OBJ" "1";
}
}
if eval "contains(header.X-Authenticated-Sender, ': ')" {
let "t.HAS_X_AS" "1";
}
if eval "contains(header.X-Get-Message-Sender-Via, 'authenticated_id:')" {
let "t.HAS_X_GMSV" "1";
}
if eval "header.X-AntiAbuse.exists" {
let "t.HAS_X_ANTIABUSE" "1";
}
if eval "header.X-Authentication-Warning.exists" {
let "t.HAS_XAW" "1";
}
# Check for empty delimiters in raw headers
let "raw_headers" "header.from:to:cc:subject:reply-to:date[*].raw";
let "i" "count(raw_headers)";
while "i > 0" {
let "i" "i - 1";
if eval "!starts_with(raw_headers[i], ' ')" {
let "t.HEADER_EMPTY_DELIMITER" "1";
break;
}
}

30
resources/config/spamfilter/scripts/helo.sieve
View File

@@ -1,30 +0,0 @@
if eval "!is_ip_addr(env.helo_domain)" {
let "helo" "env.helo_domain";
if eval "contains(helo, '.')" {
if eval "!is_empty(env.iprev.ptr) && !eq_ignore_case(helo, env.iprev.ptr)" {
# Helo does not match reverse IP
let "t.HELO_IPREV_MISMATCH" "1";
}
if eval "!dns_exists(helo, 'ip') && !dns_exists(helo, 'mx')" {
# Helo no resolve to A or MX
let "t.HELO_NORES_A_OR_MX" "1";
}
} else {
if eval "contains(helo, 'user')" {
# HELO contains 'user'
let "t.RCVD_HELO_USER" "1";
}
# Helo not FQDN
let "t.HELO_NOT_FQDN" "1";
}
} else {
# Helo host is bare ip
let "t.HELO_BAREIP" "1";
if eval "env.helo_domain != env.remote_ip" {
# Helo A IP != hostname IP
let "t.HELO_IP_A" "1";
}
}

8
resources/config/spamfilter/scripts/ip.sieve
View File

@@ -1,8 +0,0 @@
# Reverse ip checks
if eval "env.iprev.result != ''" {
if eval "env.iprev.result == 'temperror'" {
let "t.RDNS_DNSFAIL" "1";
} elsif eval "env.iprev.result == 'fail' || env.iprev.result == 'permerror'" {
let "t.RDNS_NONE" "1";
}
}

68
resources/config/spamfilter/scripts/messageid.sieve
View File

@@ -1,68 +0,0 @@
let "mid_raw" "trim(header.message-id.raw)";
if eval "!is_empty(mid_raw)" {
let "mid_lcase" "to_lowercase(header.message-id)";
let "mid_rhs" "email_part(mid_lcase, 'domain')";
if eval "!is_empty(mid_rhs)" {
if eval "starts_with(mid_rhs, '[') && ends_with(mid_rhs, ']') && is_ip_addr(strip_suffix(strip_prefix(mid_rhs, '['), ']'))" {
let "t.MID_RHS_IP_LITERAL" "1";
} elsif eval "is_ip_addr(mid_rhs)" {
let "t.MID_BARE_IP" "1";
} elsif eval "!contains(mid_rhs, '.')" {
let "t.MID_RHS_NOT_FQDN" "1";
}
if eval "starts_with(mid_rhs, 'www.')" {
let "t.MID_RHS_WWW" "1";
}
if eval "!is_ascii(mid_raw) || contains(mid_raw, '(') || starts_with(mid_lcase, '@')" {
let "t.INVALID_MSGID" "1";
}
# From address present in Message-ID checks
let "sender" "from_addr";
if eval "is_empty(sender)" {
let "sender" "envelope.from";
}
if eval "!is_empty(sender)" {
if eval "contains(mid_lcase, sender)" {
let "t.MID_CONTAINS_FROM" "1";
} else {
let "from_domain" "email_part(sender, 'domain')";
let "mid_sld" "domain_part(mid_rhs, 'sld')";
if eval "mid_rhs == from_domain" {
let "t.MID_RHS_MATCH_FROM" "1";
} elsif eval "!is_empty(mid_sld) && domain_part(from_domain, 'sld') == mid_sld" {
let "t.MID_RHS_MATCH_FROMTLD" "1";
}
}
}
# To/Cc addresses present in Message-ID checks
let "recipients_len" "count(recipients)";
let "i" "0";
while "i < recipients_len" {
let "rcpt" "recipients[i]";
let "i" "i + 1";
if eval "contains(mid_lcase, rcpt)" {
let "t.MID_CONTAINS_TO" "1";
} elsif eval "email_part(rcpt, 'domain') == mid_rhs" {
let "t.MID_RHS_MATCH_TO" "1";
}
}
} else {
let "t.INVALID_MSGID" "1";
}
if eval "!starts_with(mid_raw, '<') || !contains(mid_raw, '>')" {
let "t.MID_MISSING_BRACKETS" "1";
}
} else {
let "t.MISSING_MID" "1";
}

43
resources/config/spamfilter/scripts/prelude.sieve
View File

@@ -1,43 +0,0 @@
# Convert body to plain text
let "text_body" "body.to_text";
# Obtain all URLs in the body
let "body_urls" "tokenize(text_body, 'uri')";
# Obtain all URLs in href and src attributes
let "html_body_urls" "html_attrs(body.html, '', ['href', 'src'])";
# Obtain all URLs in the subject, combine them with all other URLs and remove duplicates
let "urls" "dedup(tokenize(header.subject, 'uri') + body_urls + html_body_urls)";
# Obtain thread name and subject
let "subject_lc" "to_lowercase(header.subject)";
let "subject_clean" "thread_name(header.subject)";
let "body_and_subject" "subject_clean + ' ' + text_body";
# Obtain all recipients
let "recipients" "to_lowercase(header.to:cc:bcc[*].addr[*])";
let "recipients_clean" "winnow(dedup(recipients))";
let "recipients_to" "header.to[*].addr[*]";
let "recipients_cc" "header.cc[*].addr[*]";
# Obtain From parts
let "from_name" "to_lowercase(trim(header.from.name))";
let "from_addr" "to_lowercase(trim(header.from.addr))";
let "from_local" "email_part(from_addr, 'local')";
let "from_domain" "email_part(from_addr, 'domain')";
let "from_domain_sld" "domain_part(from_domain, 'sld')";
# Obtain Reply-To address
let "rto_addr" "to_lowercase(header.reply-to.addr)";
# Obtain Envelope From parts
let "envfrom_local" "email_part(envelope.from, 'local')";
let "envfrom_domain" "email_part(envelope.from, 'domain')";
let "envfrom_domain_sld" "domain_part(envfrom_domain, 'sld')";
# Obtain HELO domain SLD
let "helo_domain_sld" "domain_part(env.helo_domain, 'sld')";
# Create score variable
let "score" "0.0";

2
tests/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "tests"
version = "0.1.0"
version = "0.10.7"
edition = "2021"
resolver = "2"