Antispam implementation - part 1

This commit is contained in:
mdecimus
2023-08-11 18:46:09 +02:00
parent 32a50ec34c
commit d4fe318812
78 changed files with 25682 additions and 0 deletions

View File

@@ -0,0 +1,9 @@
[package]
name = "antispam"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
fancy-regex = "0.11.0"

View File

@@ -0,0 +1,364 @@
use std::collections::{HashMap, HashSet};
use super::Token;
#[derive(Debug, Clone)]
pub struct MetaExpression {
pub tokens: Vec<Token>,
pub token_depth: Vec<u32>,
}
impl MetaExpression {
pub fn from_meta(meta: &str) -> Self {
let mut tokens = Vec::new();
let mut token_depth = Vec::new();
let mut seen_comp = false;
let mut buf = String::new();
let mut pc = 0;
let mut iter = meta.chars().peekable();
while let Some(ch) = iter.next() {
match ch {
'A'..='Z' | 'a'..='z' | '0'..='9' | '_' => {
buf.push(ch);
}
_ => {
if !buf.is_empty() {
let token = Token::from(buf);
buf = String::new();
if matches!(token, Token::Tag(_))
&& !seen_comp
&& matches!(
iter.clone()
.find(|t| { ['&', '|', '>', '<', '='].contains(t) }),
None | Some('&' | '|')
)
{
tokens.push(token);
tokens.push(Token::Gt);
tokens.push(Token::Number(0));
token_depth.extend_from_slice(&[pc, pc, pc]);
seen_comp = true;
} else {
token_depth.push(pc);
tokens.push(token);
}
}
match ch {
'&' => {
seen_comp = false;
if matches!(iter.next(), Some('&')) {
tokens.push(Token::And);
token_depth.push(pc);
} else {
eprintln!("Warning: Single & in meta expression {meta} at {pc}",);
}
}
'|' => {
seen_comp = false;
if matches!(iter.next(), Some('|')) {
tokens.push(Token::Or);
token_depth.push(pc);
} else {
eprintln!("Warning: Single | in meta expression {meta} at {pc}",);
}
}
'!' => {
seen_comp = false;
token_depth.push(pc);
tokens.push(Token::Not)
}
'=' => {
seen_comp = true;
token_depth.push(pc);
tokens.push(match iter.next() {
Some('=') => Token::Eq,
Some('>') => Token::Ge,
Some('<') => Token::Le,
_ => {
eprintln!(
"Warning: Single = in meta expression {meta} at {pc}",
);
Token::Eq
}
});
}
'>' => {
seen_comp = true;
token_depth.push(pc);
tokens.push(match iter.peek() {
Some('=') => {
iter.next();
Token::Ge
}
_ => Token::Gt,
})
}
'<' => {
seen_comp = true;
token_depth.push(pc);
tokens.push(match iter.peek() {
Some('=') => {
iter.next();
Token::Le
}
_ => Token::Lt,
})
}
'(' => {
token_depth.push(pc);
pc += 1;
tokens.push(Token::OpenParen)
}
')' => {
if pc > 0 {
pc -= 1;
} else {
eprintln!(
"Warning: Unmatched close parenthesis in meta expression {meta}"
);
}
token_depth.push(pc);
tokens.push(Token::CloseParen)
}
'+' => {
token_depth.push(pc);
tokens.push(Token::Add)
}
'*' => {
token_depth.push(pc);
tokens.push(Token::Multiply)
}
'/' => {
token_depth.push(pc);
tokens.push(Token::Divide)
}
' ' => {}
_ => {
eprintln!("Warning: Invalid character {ch} in meta expression {meta}");
break;
}
}
}
}
}
if pc > 0 {
eprintln!("Warning: Unmatched open parenthesis in meta expression {meta}");
}
if !buf.is_empty() {
token_depth.push(pc);
tokens.push(Token::from(buf));
if !seen_comp {
tokens.push(Token::Gt);
tokens.push(Token::Number(0));
token_depth.push(pc);
token_depth.push(pc);
}
}
MetaExpression {
tokens,
token_depth,
}
}
}
impl From<String> for Token {
fn from(value: String) -> Self {
if let Ok(value) = value.parse() {
Token::Number(value)
} else {
Token::Tag(value)
}
}
}
impl From<MetaExpression> for String {
fn from(meta: MetaExpression) -> Self {
let mut script = String::from("if ");
let mut tokens = meta.tokens.iter().zip(meta.token_depth.iter()).enumerate();
let mut expr_end = None;
// Find start and end of logical expressions
let mut logical_pos_start: HashMap<usize, Token> = HashMap::new();
let mut logical_pos_end: HashSet<usize> = HashSet::new();
let mut depth_starts: HashMap<u32, usize> = HashMap::new();
for (pos, (token, depth)) in tokens.clone() {
if !depth_starts.contains_key(depth) {
depth_starts.insert(*depth, pos);
}
if matches!(token, Token::And | Token::Or) {
let block_start = *depth_starts.get(depth).unwrap();
if let std::collections::hash_map::Entry::Vacant(e) =
logical_pos_start.entry(block_start)
{
e.insert(token.clone());
// Find end
let mut logical_end = usize::MAX;
for (p, (_, d)) in tokens.clone() {
if depth == d {
logical_end = p;
}
}
logical_pos_end.insert(logical_end);
}
}
}
while let Some((pos, (token, depth))) = tokens.next() {
// Add blocks
if let Some(token) = logical_pos_start.remove(&pos) {
match token {
Token::And => script.push_str("allof("),
Token::Or => script.push_str("anyof("),
_ => unreachable!(),
}
} else {
match token {
Token::And | Token::Or => script.push_str(", "),
Token::Not => script.push_str("not "),
_ => (),
}
}
// Find expression type
if expr_end.is_none() {
if let Some((
pos,
(token @ (Token::Eq | Token::Gt | Token::Lt | Token::Ge | Token::Le), _),
)) = tokens.clone().find(|(_, (t, d))| {
depth == *d
&& matches!(
t,
Token::Eq
| Token::Gt
| Token::Lt
| Token::Ge
| Token::Le
| Token::Not
| Token::And
| Token::Or
)
}) {
script.push_str("string :");
match token {
Token::Eq => script.push_str("eq"),
Token::Gt => script.push_str("gt"),
Token::Lt => script.push_str("lt"),
Token::Ge => script.push_str("ge"),
Token::Le => script.push_str("gt"),
_ => unreachable!(),
}
script.push_str(" \"");
// Find expression end
for (p, (token, d)) in tokens.clone() {
if p > pos {
if depth <= d {
if matches!(token, Token::And | Token::Or) {
expr_end = Some(p - 1);
break;
} else {
expr_end = Some(p);
}
} else {
break;
}
}
}
}
}
match token {
Token::Tag(tag) => {
script.push_str(tag);
}
Token::Number(number) => {
script.push_str(&number.to_string());
}
Token::And | Token::Or | Token::Not => {}
Token::Gt | Token::Lt | Token::Eq | Token::Ge | Token::Le => {
script.push_str("\" \"");
}
Token::OpenParen => {
script.push('(');
}
Token::CloseParen => {
script.push(')');
}
Token::Add => {
script.push_str(" + ");
}
Token::Multiply => {
script.push_str(" * ");
}
Token::Divide => {
script.push_str(" / ");
}
}
// Add end of expression
if expr_end == Some(pos) {
script.push_str("\"");
expr_end = None;
}
// Add end of logical block
if logical_pos_end.contains(&pos) {
script.push(')');
}
}
script
}
}
#[cfg(test)]
mod test {
use super::MetaExpression;
#[test]
fn parse_meta() {
for (expr, expected) in [
(
concat!(
"( ! HTML_IMAGE_ONLY_16 ) && ",
"( __LOWER_E > 20 ) && ",
"( __E_LIKE_LETTER > ( (__LOWER_E * 14 ) / 10) ) && ",
"( __E_LIKE_LETTER < ( 10 * __LOWER_E ) )"
),
"",
),
("(__DRUGS_ERECTILE1 || __DRUGS_ERECTILE2)", ""),
("(__HELO_DYNAMIC_IPADDR && !HELO_STATIC_HOST)", ""),
("__ML2 || __ML4", ""),
("(__AT_HOTMAIL_MSGID && (!__FROM_HOTMAIL_COM && !__FROM_MSN_COM && !__FROM_YAHOO_COM))", ""),
("(0)", ""),
("RAZOR2_CHECK + DCC_CHECK + PYZOR_CHECK > 1", ""),
/*(("", ""),
("", ""),
("", ""),
("", ""),
("", ""),
("", ""),
("", ""),*/
] {
let meta = MetaExpression::from_meta(expr);
//println!("{:?}", meta.tokens);
let result = String::from(meta);
//println!("{}", expected);
println!("{}", result);
/*assert_eq!(
result,
expected,
"failed for {expr}"
);*/
}
}
}

View File

@@ -0,0 +1,192 @@
use std::collections::HashMap;
pub mod meta;
pub mod spamassassin;
pub mod utils;
#[derive(Debug, Default)]
struct Rule {
name: String,
t: RuleType,
scores: Vec<f64>,
description: HashMap<String, String>,
priority: i32,
flags: Vec<TestFlag>,
}
#[derive(Debug, Default)]
enum RuleType {
Header {
matches: HeaderMatches,
header: Header,
if_unset: Option<String>,
pattern: String,
},
Body {
pattern: String,
raw: bool,
},
Full {
pattern: String,
},
Uri {
pattern: String,
},
Eval {
function: String,
params: Vec<String>,
},
Meta {
tokens: Vec<Token>,
},
#[default]
None,
}
impl RuleType {
pub fn pattern(&mut self) -> Option<&mut String> {
match self {
RuleType::Header { pattern, .. } => Some(pattern),
RuleType::Body { pattern, .. } => Some(pattern),
RuleType::Full { pattern, .. } => Some(pattern),
RuleType::Uri { pattern, .. } => Some(pattern),
_ => None,
}
}
}
#[derive(Debug)]
enum TestFlag {
Net,
Nice,
UserConf,
Learn,
NoAutoLearn,
Publish,
Multiple,
NoTrim,
DomainsOnly,
NoSubject,
AutoLearnBody,
A,
MaxHits(u32),
DnsBlockRule(String),
}
#[derive(Debug, Default)]
enum Header {
#[default]
All,
MessageId,
AllExternal,
EnvelopeFrom,
ToCc,
Name {
name: String,
part: Vec<HeaderPart>,
},
}
#[derive(Debug, Default)]
enum HeaderMatches {
#[default]
Matches,
NotMatches,
Exists,
}
#[derive(Debug, Default)]
enum HeaderPart {
Name,
Addr,
#[default]
Raw,
}
#[derive(Debug, PartialEq, Eq, Clone)]
pub enum Token {
Tag(String),
Number(u32),
And,
Or,
Not,
Gt,
Lt,
Eq,
Ge,
Le,
OpenParen,
CloseParen,
Add,
Multiply,
Divide,
}
impl Rule {
fn score(&self) -> f64 {
self.scores.last().copied().unwrap_or_else(|| {
if self.name.starts_with("__") {
0.0
} else if self.name.starts_with("T_") {
0.01
} else {
1.0
}
})
}
}
impl Ord for Rule {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
match self.priority.cmp(&other.priority) {
std::cmp::Ordering::Equal => match self.score().partial_cmp(&other.score()).unwrap() {
std::cmp::Ordering::Equal => other.name.cmp(&self.name),
x => x,
},
x => x,
}
}
}
impl PartialOrd for Rule {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
Some(self.cmp(other))
}
}
impl PartialEq for Rule {
fn eq(&self, other: &Self) -> bool {
self.name == other.name && self.priority == other.priority && self.scores == other.scores
}
}
impl Eq for Rule {}
pub trait UnwrapResult<T> {
fn unwrap_result(self, action: &str) -> T;
}
impl<T> UnwrapResult<T> for Option<T> {
fn unwrap_result(self, message: &str) -> T {
match self {
Some(result) => result,
None => {
eprintln!("Failed to {}", message);
std::process::exit(1);
}
}
}
}
impl<T, E: std::fmt::Display> UnwrapResult<T> for Result<T, E> {
fn unwrap_result(self, message: &str) -> T {
match self {
Ok(result) => result,
Err(err) => {
eprintln!("Failed to {}: {}", message, err);
std::process::exit(1);
}
}
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,82 @@
use std::collections::HashMap;
pub fn replace_tags(
pattern: &str,
tag_start: char,
tag_end: char,
patterns: &HashMap<String, String>,
) -> String {
//print!("replacing {} ", pattern);
let mut result = String::with_capacity(pattern.len());
let mut chars = fix_broken_regex(pattern.trim()).chars();
let mut tag_pre = String::new();
let mut tag_inter = String::new();
let mut tag_post = String::new();
let mut is_adjacent = false;
'outer: while let Some(ch) = chars.next() {
if ch == tag_start {
let mut buf = String::new();
for ch in chars.by_ref() {
if ch == tag_end {
break;
} else if ch.is_ascii_alphanumeric() || ch.is_ascii_whitespace() {
buf.push(ch);
} else {
result.push(tag_start);
result.push_str(&buf);
result.push(ch);
continue 'outer;
}
}
if let Some(pattern) = patterns.get(&buf) {
if buf.starts_with("pre ") {
tag_pre = pattern.to_string();
} else if buf.starts_with("inter ") {
tag_inter = pattern.to_string();
} else if buf.starts_with("post ") {
tag_post = pattern.to_string();
} else {
if !tag_pre.is_empty() {
result.push_str(&tag_pre);
}
if !tag_inter.is_empty() && is_adjacent {
result.push_str(&tag_inter);
}
result.push_str(pattern);
if !tag_post.is_empty() {
result.push_str(&tag_post);
}
is_adjacent = true;
}
} else {
eprintln!("Warning: Unknown tag {}", buf);
}
} else {
result.push(ch);
is_adjacent = false;
}
}
//println!("to {}", result);
result
}
pub fn fix_broken_regex(value: &str) -> &str {
match value {
r#"/[\042\223\224\262\263\271]{2}\S{0,16}[\042\223\224\262\263\271]{2}/"# => {
//r#"[\"\u{93}\u{94}\u{B2}\u{B3}\u{B9}]{2}\S{0,16}[\"\u{93}\u{94}\u{B2}\u{B3}\u{B9}]{2}"#
r#"/[\x22\x93\x94\xB2\xB3\xB9]{2}\S{0,16}[\x22\x93\x94\xB2\xB3\xB9]{2}/"#
}
r#"/\b_{0,3}d[_\W]?[i1!|l\xEC-\xEF][_\W]?d[_\W]?r[_\W][e3\xE8-\xEB[_\W]?xx?_{0,3}\b/i"# => {
r#"/\b_{0,3}d[_\W]?[i1!|l\xEC-\xEF][_\W]?d[_\W]?r[_\W][e3\xE8-\xEB][_\W]?xx?_{0,3}\b/i"#
}
r#"/<!--(?:\s{1,10}[-\w'"]{1,40}){100}/im"# => r#"/<!--(?:\s{1,10}[-\w'"]{1,40}){5}/im"#,
r#"/\015/"# => r#"/\x0D/"#,
r#"/[({[<][. ]*(?-i:\xbc\xba[. ]*\xc0\xce[. ]*)?(?-i:\xb1\xa4(?:[. ]*|[\x00-\x7f]{0,3})\xb0\xed|\xc1\xa4[. ]*\xba\xb8|\xc8\xab[. ]*\xba\xb8)[. ]*[)}\]>]/"# => {
r#"/[\(\{\[\<][. ]*(?-i:\xbc\xba[. ]*\xc0\xce[. ]*)?(?-i:\xb1\xa4(?:[. ]*|[\x00-\x7f]{0,3})\xb0\xed|\xc1\xa4[. ]*\xba\xb8|\xc8\xab[. ]*\xba\xb8)[. ]*[\)\}\]\>]/"#
}
_ => value,
}
}

View File

@@ -0,0 +1,65 @@
use std::path::PathBuf;
use import::spamassassin::import_spamassassin;
pub mod import;
fn main() {
import_spamassassin(
PathBuf::from("/Users/me/code/mail-server/resources/spamassassin"),
"cf".to_string(),
false,
false,
);
}
const _IGNORE: &str = r#"
[antispam]
required-score = 5
add-headers = ["X-Spam-Checker-Version: SpamAssassin _VERSION_ (_SUBVERSION_) on _HOSTNAME_",
"X-Spam-Flag: _YESNOCAPS_", "X-Spam-Level: _STARS(*)_",
"X-Spam-Status: _YESNO_, score=_SCORE_ required=_REQD_ tests=_TESTS_ autolearn=_AUTOLEARN_ version=_VERSION_"]
originating-ip-headers = ["X-Yahoo-Post-IP", "X-Originating-IP", "X-Apparently-From",
"X-SenderIP X-AOL-IP", "X-MS-Exchange-CrossTenant-OriginalAttributedTenantConnectingIp"]
rewrite-headers = ["Subject: [SPAM] _SUBJECT_"]
redirect-patterns = ["""m'/(?:index.php)?\?.*(?<=[?&])URL=(.*?)(?:$|[&\#])'i""",
"""m'^https?:/*(?:\w+\.)?google(?:\.\w{2,3}){1,2}/url\?.*?(?<=[?&])q=(.*?)(?:$|[&\#])'i""",
"""m'^https?:/*(?:\w+\.)?google(?:\.\w{2,3}){1,2}/search\?.*?(?<=[?&])q=[^&]*?(?<=%20|..[=+\s])(?:site|inurl):(.*?)(?:$|%20|[\s+&\#])'i""",
"""m'^https?:/*(?:\w+\.)?google(?:\.\w{2,3}){1,2}/search\?.*?(?<=[?&])q=[^&]*?(?<=%20|..[=+\s])(?:"|%22)(.*?)(?:$|%22|["\s+&\#])'i""",
"""m'^https?:/*(?:\w+\.)?google(?:\.\w{2,3}){1,2}/translate\?.*?(?<=[?&])u=(.*?)(?:$|[&\#])'i""",
"""m'^https?:/*(?:\w+\.)?google(?:\.\w{2,3}){1,2}/pagead/iclk\?.*?(?<=[?&])adurl=(.*?)(?:$|[&\#])'i""",
"""m'^https?:/*(?:\w+\.)?aol\.com/redir\.adp\?.*(?<=[?&])_url=(.*?)(?:$|[&\#])'i""",
"""m'^https?/*(?:\w+\.)?facebook\.com/l/;(.*)'i""",
"""/^http:\/\/chkpt\.zdnet\.com\/chkpt\/\w+\/(.*)$/i""",
"""/^http:\/\/www(?:\d+)?\.nate\.com\/r\/\w+\/(.*)$/i""",
"""/^http:\/\/.+\.gov\/(?:.*\/)?externalLink\.jhtml\?.*url=(.*?)(?:&.*)?$/i""",
"""/^http:\/\/redir\.internet\.com\/.+?\/.+?\/(.*)$/i""",
"""/^http:\/\/(?:.*?\.)?adtech\.de\/.*(?:;|\|)link=(.*?)(?:;|$)/i""",
"""m'^http.*?/redirect\.php\?.*(?<=[?&])goto=(.*?)(?:$|[&\#])'i""",
"""m'^https?:/*(?:[^/]+\.)?emf\d\.com/r\.cfm.*?&r=(.*)'i"""
]
[antispam.autolearn]
enable = true
ignore-headers = [ "X-ACL-Warn", "X-Alimail-AntiSpam", "X-Amavis-Modified", "X-Anti*", "X-aol-global-disposition",
"X-ASF-*", "X-Assp-Version", "X-Authority-Analysis", "X-Authvirus", "X-Auto-Response-Suppress", "X-AV-Do-Run",
"X-AV-Status", "X-avast-antispam", "X-Backend", "X-Barracuda*", "X-Bayes*", "X-BitDefender*", "X-BL", "X-Bogosity",
"X-Boxtrapper", "X-Brightmail-Tracker", "X-BTI-AntiSpam", "X-Bugzilla-Version", "X-CanIt*", "X-Clapf-spamicity",
"X-Cloud-Security", "X-CM-Score", "X-CMAE-*", "X-Company", "X-Coremail-Antispam", "X-CRM114-*", "X-CT-Spam",
"X-CTCH-*", "X-Drweb-SpamState", "X-DSPAM*", "X-eavas*", "X-Enigmail-Version", "X-Eset*", "X-Exchange-Antispam-Report",
"X-ExtloopSabreCommercials1", "X-EYOU-SPAMVALUE", "X-FB-OUTBOUND-SPAM", "X-FEAS-SBL", "X-FILTER-SCORE", "X-Forefront*",
"X-Fuglu*", "X-getmail-filter-classifier", "X-GFIME-MASPAM", "X-Gmane-NNTP-Posting-Host", "X-GMX-Anti*", "X-He-Spam",
"X-hMailServer-Spam", "X-IAS", "X-iGspam-global", "X-Injected-Via-Gmane", "X-Interia-Antivirus", "X-IP-Spam-Verdict",
"X-Ironport*", "X-Junk*", "X-KLMS-*", "X-KMail-*", "X-MailCleaner-*", "X-MailFoundry", "X-MDMailLookup-Result",
"X-ME-*", "X-MessageFilter", "X-Microsoft-Antispam", "X-Mlf-Version", "X-MXScan-*", "X-NAI-Spam-*", "X-NetStation-Status",
"X-OVH-SPAM*", "X-PerlMx-*", "X-PFSI-Info", "X-PMX-*", "X-Policy-Service", "X-policyd-weight", "X-PreRBLs",
"X-Probable-Spam", "X-PROLinux-SpamCheck", "X-Proofpoint-*", "x-purgate-*", "X-Qmail-Scanner-*", "X-Quarantine-ID",
"X-RSpam-Report", "X-SA-*", "X-Scanned-by", "X-SmarterMail-CustomSpamHeader", "X-Spam*", "X-SPF-Scan-By", "X-STA-*",
"X-StarScan-Version", "X-SurGATE-Result", "X-SWITCHham-Score", "X-UI-*", "X-Univie*", "X-Virus*", "X-VR-*",
"X-WatchGuard*", "X-Whitelist-Domain", "X-WUM-CCI", "X_CMAE_Category" ]
threshold.ham = 0.1
threshold.spam = 12.0
"#;