From 3fa5c769bd361a876f0342384a2f238f971fdd70 Mon Sep 17 00:00:00 2001 From: mdecimus Date: Fri, 1 Sep 2023 14:41:58 +0200 Subject: [PATCH] STARTLS optional fix + Sieve functions --- Cargo.lock | 32 +- crates/antispam/src/import/spamassassin.rs | 499 +++++++++++------- crates/smtp/Cargo.toml | 3 +- crates/smtp/src/config/scripts.rs | 12 +- crates/smtp/src/inbound/session.rs | 4 + crates/smtp/src/outbound/delivery.rs | 2 +- crates/smtp/src/scripts/exec.rs | 22 +- crates/smtp/src/scripts/functions.rs | 87 +++ crates/smtp/src/scripts/mod.rs | 1 + .../smtp/src/scripts/plugins/detect_lang.rs | 52 ++ crates/smtp/src/scripts/plugins/mod.rs | 6 +- tests/resources/smtp/sieve/functions.sieve | 24 + tests/src/smtp/outbound/mod.rs | 1 + tests/src/smtp/outbound/tls.rs | 107 ++++ 14 files changed, 641 insertions(+), 211 deletions(-) create mode 100644 crates/smtp/src/scripts/functions.rs create mode 100644 crates/smtp/src/scripts/plugins/detect_lang.rs create mode 100644 tests/resources/smtp/sieve/functions.sieve create mode 100644 tests/src/smtp/outbound/tls.rs diff --git a/Cargo.lock b/Cargo.lock index d6315d7a..7672fe2f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -777,9 +777,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.27" +version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f56b4c72906975ca04becb8a30e102dfecddd0c06181e3e95ddc444be28881f8" +checksum = "95ed24df0632f708f5f6d8082675bef2596f7084dee3dd55f632290bf35bfe0f" dependencies = [ "android-tzdata", "iana-time-zone", @@ -823,20 +823,19 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.1" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c8d502cbaec4595d2e7d5f61e318f05417bd2b66fdc3809498f0d3fdf0bea27" +checksum = "6a13b88d2c62ff462f88e4a121f17a82c1af05693a2f192b5c38d14de73c19f6" dependencies = [ "clap_builder", "clap_derive", - "once_cell", ] [[package]] name = "clap_builder" -version = "4.4.1" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5891c7bc0edb3e1c2204fc5e94009affabeb1821c9e5fdc3959536c5c0bb984d" +checksum = "2bb9faaa7c2ef94b2743a21f5a29e6f0010dff4caa69ac8e9d6cf8b6fa74da08" dependencies = [ "anstream", "anstyle", @@ -846,9 +845,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.4.0" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9fd1a5729c4548118d7d70ff234a44868d00489a4b6597b0b020918a0e91a1a" +checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873" dependencies = [ "heck", "proc-macro2", @@ -2724,7 +2723,7 @@ dependencies = [ [[package]] name = "mail-parser" version = "0.8.2" -source = "git+https://github.com/stalwartlabs/mail-parser#da62659eb609a8222edf7d1f57b28550a853921e" +source = "git+https://github.com/stalwartlabs/mail-parser#7c08078617ef9b355da445dbf88df64879eb8059" dependencies = [ "encoding_rs", "serde", @@ -4202,9 +4201,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.10" +version = "0.38.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed6248e1caa625eb708e266e06159f135e8c26f2bb7ceb72dc4b2766d0340964" +checksum = "c0c3dde1fc030af041adc40e79c0e7fbcf431dd24870053d187d7c66e4b87453" dependencies = [ "bitflags 2.4.0", "errno", @@ -4553,7 +4552,7 @@ checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" [[package]] name = "sieve-rs" version = "0.3.1" -source = "git+https://github.com/stalwartlabs/sieve#51558ab5dec44907ab7be65c492b519c327ff0f5" +source = "git+https://github.com/stalwartlabs/sieve#53ff94606cf4a3e03b8820280ef8fae0bfa500ec" dependencies = [ "ahash 0.8.3", "bincode", @@ -4641,7 +4640,8 @@ dependencies = [ "tokio-rustls 0.24.1", "tracing", "utils", - "webpki-roots 0.23.1", + "webpki-roots 0.25.2", + "whatlang", "x509-parser", ] @@ -5924,9 +5924,9 @@ dependencies = [ [[package]] name = "webpki" -version = "0.22.0" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" +checksum = "f0e74f82d49d545ad128049b7e88f6576df2da6b02e9ce565c6f533be576957e" dependencies = [ "ring", "untrusted", diff --git a/crates/antispam/src/import/spamassassin.rs b/crates/antispam/src/import/spamassassin.rs index 09caa925..7d694651 100644 --- a/crates/antispam/src/import/spamassassin.rs +++ b/crates/antispam/src/import/spamassassin.rs @@ -3,6 +3,7 @@ use std::{ fmt::{Display, Write}, fs::{self}, path::PathBuf, + str::FromStr, }; use super::{ @@ -915,6 +916,13 @@ pub fn import_spamassassin(path: PathBuf, extension: String, do_warn: bool) { } } + "ok_languages" => { + lists + .entry("ok_languages".to_string()) + .or_default() + .extend(params.split_whitespace().map(|v| v.to_string())); + } + "fns_check" | "fns_ignore_dkim" | "fns_ignore_headers" @@ -931,7 +939,6 @@ pub fn import_spamassassin(path: PathBuf, extension: String, do_warn: bool) { | "report_safe" | "require_version" | "required_score" - | "ok_languages" | "ok_locales" | "unsafe_report" | "add_header" @@ -1094,6 +1101,10 @@ pub fn import_spamassassin(path: PathBuf, extension: String, do_warn: bool) { "set \"score\" \"0.0\";\n", "set \"spam_score\" \"5.0\";\n", "set \"awl_factor\" \"0.5\";\n", + "set \"body\" \"${body.to_text}\";\n", + "set \"body_len\" \"${body.len()}\";\n", + "set \"thread_name\" \"${header.subject.thread_name()}\";\n", + "set \"sent_date\" \"${header.date.date}\";\n", "\n" )); @@ -1282,9 +1293,10 @@ impl Display for Rule { RuleType::Uri { pattern } => { write!(f, "if match_uri {:?}", pattern)?; } - RuleType::Eval { function, params } => match function.as_str() { - "check_from_in_auto_welcomelist" | "check_from_in_auto_whitelist" => { - f.write_str(concat!( + RuleType::Eval { function, params } => { + match function.as_str() { + "check_from_in_auto_welcomelist" | "check_from_in_auto_whitelist" => { + f.write_str(concat!( "query :use \"spam\" :set [\"awl_score\", \"awl_count\"] \"SELECT score, count FROM awl WHERE sender = ? AND ip = ?\" [\"${env.from}\", \"%{env.remote_ip}\"];\n", "if eval \"awl_count > 0\" {\n", "\tquery :use \"spam\" \"UPDATE awl SET score = score + ?, count = count + 1 WHERE sender = ? AND ip = ?\" [\"%{score}\", \"${env.from}\", \"%{env.remote_ip}\"];\n", @@ -1293,168 +1305,300 @@ impl Display for Rule { "\tquery :use \"spam\" \"INSERT INTO awl (score, count, sender, ip) VALUES (?, 1, ?, ?)\" [\"%{score}\", \"${env.from}\", \"%{env.remote_ip}\"];\n", "}\n\n", ))?; - return Ok(()); - } - "check_from_in_blacklist" - | "check_from_in_blocklist" - | "check_from_in_default_welcomelist" - | "check_from_in_default_whitelist" - | "check_from_in_welcomelist" - | "check_from_in_whitelist" - | "check_to_in_blacklist" - | "check_to_in_blocklist" - | "check_to_in_welcomelist" - | "check_to_in_whitelist" - | "check_subject_in_blacklist" - | "check_subject_in_blocklist" - | "check_subject_in_welcomelist" - | "check_subject_in_whitelist" - | "check_to_in_more_spam" - | "check_to_in_all_spam" => { - let mut parts = function.split('_').peekable(); - parts.next(); - let header = parts.next().unwrap(); - parts.next(); - let mut list = String::new(); + return Ok(()); + } + "check_from_in_blacklist" + | "check_from_in_blocklist" + | "check_from_in_default_welcomelist" + | "check_from_in_default_whitelist" + | "check_from_in_welcomelist" + | "check_from_in_whitelist" + | "check_to_in_blacklist" + | "check_to_in_blocklist" + | "check_to_in_welcomelist" + | "check_to_in_whitelist" + | "check_subject_in_blacklist" + | "check_subject_in_blocklist" + | "check_subject_in_welcomelist" + | "check_subject_in_whitelist" + | "check_to_in_more_spam" + | "check_to_in_all_spam" => { + let mut parts = function.split('_').peekable(); + parts.next(); + let header = parts.next().unwrap(); + parts.next(); + let mut list = String::new(); - for part in parts { - if !list.is_empty() { - list.push('_'); + for part in parts { + if !list.is_empty() { + list.push('_'); + } + list.push_str(match part { + "welcomelist" | "whitelist" => "welcome", + "blacklist" | "blocklist" => "block", + "more" | "all" | "spam" => part, + "default" => "def", + _ => unreachable!(), + }) } - list.push_str(match part { - "welcomelist" | "whitelist" => "welcome", - "blacklist" | "blocklist" => "block", - "more" | "all" | "spam" => part, - "default" => "def", - _ => unreachable!(), - }) - } - let fnc = if header == "subject" { - "header" - } else { - "address" - }; - write!(f, "if {fnc} :list \"{header}\" \"sa/list_{list}_{header}\"")?; - } - "check_from_in_list" | "check_replyto_in_list" => { - let mut header = function.split('_').nth(1).unwrap(); - if header == "replyto" { - header = "reply-to"; + let fnc = if header == "subject" { + "header" + } else { + "address" + }; + write!(f, "if {fnc} :list \"{header}\" \"sa/list_{list}_{header}\"")?; } - #[allow(clippy::print_in_format_impl)] - if let Some(list) = params.first() { + "check_from_in_list" | "check_replyto_in_list" => { + let mut header = function.split('_').nth(1).unwrap(); + if header == "replyto" { + header = "reply-to"; + } + #[allow(clippy::print_in_format_impl)] + if let Some(list) = params.first() { + write!( + f, + "if address :list \"{header}\" \"sa/list_{}\"", + list.to_lowercase() + )?; + } else { + eprintln!("Warning: Found invalid 'check_{header}_in_list' command without parameters."); + write!(f, "if false")?; + } + } + "check_for_spf_helo_fail" + | "check_for_spf_helo_neutral" + | "check_for_spf_helo_none" + | "check_for_spf_helo_pass" + | "check_for_spf_helo_permerror" + | "check_for_spf_helo_softfail" + | "check_for_spf_helo_temperror" + | "check_for_spf_neutral" + | "check_for_spf_none" + | "check_for_spf_fail" + | "check_for_spf_pass" + | "check_for_spf_permerror" + | "check_for_spf_softfail" + | "check_for_spf_temperror" => { + let mut parts = function.split('_').rev(); + let result = parts.next().unwrap(); + let spf = if parts.next().unwrap() == "helo" { + "spf_ehlo" + } else { + "spf" + }; + write!(f, "if string :is \"${{env.{spf}_result}}\" \"{result}\"")?; + } + "check_arc_signed" => { + f.write_str("if string :value \"ne\" \"${env.arc_result}\" \"none\"")?; + } + "check_arc_valid" => { + f.write_str("if string :is \"${env.arc_result}\" \"pass\"")?; + } + "check_dmarc_missing" => { + f.write_str("if string :is \"${env.dmarc_policy}\" \"none\"")?; + } + "check_dmarc_pass" => { + f.write_str("if string :is \"${env.dmarc_result}\" \"pass\"")?; + } + "check_dmarc_none" | "check_dmarc_quarantine" | "check_dmarc_reject" => { + let policy = function.split('_').nth(2).unwrap(); + write!(f, "if allof(string :is \"${{env.dmarc_result}}\" \"fail\", string :is \"${{env.dmarc_policy}}\" \"{policy}\")")?; + } + "check_dkim_adsp" + | "check_dkim_signall" + | "check_dkim_signsome" + | "check_dkim_valid_author_sig" + | "check_access_database" + | "check_body_8bits" => { + // ADSP is deprecated (see https://datatracker.ietf.org/doc/status-change-adsp-rfc5617-to-historic/) + // check_body_8bits: Not really useful + f.write_str("if false")?; + } + "check_dkim_dependable" => { + writeln!(f, "set :local \"{}\" \"1\";", self.name)?; + return Ok(()); + } + "check_dkim_signed" => { + f.write_str("if string :value \"ne\" \"${env.dkim_result}\" \"none\"")?; + } + "check_dkim_testing" => { + f.write_str("if header :contains \"DKIM-Signature\" \"t=y\"")?; + } + "check_dkim_valid" => { + if params.is_empty() { + f.write_str("if string :is \"${env.dkim_result}\" \"pass\"")?; + } else { + f.write_str("if allof(string :is \"${env.dkim_result}\" \"pass\", ")?; + if params.len() > 1 { + f.write_str("anyof(")?; + } + for (pos, param) in params.iter().enumerate() { + if pos > 0 { + f.write_str(", ")?; + } + write!(f, "envelope :domain :contains \"from\" {param}")?; + } + if params.len() > 1 { + f.write_str("))")?; + } else { + f.write_str(")")?; + } + } + } + "check_dkim_valid_envelopefrom" => { + f.write_str("if allof(string :is \"${env.dkim_result}\" \"pass\", string :is \"${envelope.from}\" \"${env.from}\")")?; + } + "check_for_def_dkim_welcomelist_from" + | "check_for_def_dkim_whitelist_from" + | "check_for_dkim_welcomelist_from" + | "check_for_dkim_whitelist_from" + | "check_for_def_spf_welcomelist_from" + | "check_for_def_spf_whitelist_from" + | "check_for_spf_welcomelist_from" + | "check_for_spf_whitelist_from" => { + let list = match (function.contains("dkim"), function.contains("def")) { + (true, true) => "def_dkim", + (true, false) => "dkim", + (false, true) => "def_spf", + (false, false) => "spf", + }; + write!(f, "if address :list \"from\" \"sa/list_{list}\"")?; + } + "check_for_missing_to_header" => { + write!(f, "if not exists \"to\"")?; + } + "check_for_to_in_subject" => { + f.write_str("foreveryline \"${header.to[*].addr[*]}\" {\n")?; + f.write_str("\tif string :contains \"${header.subject}\" \"${line}\"")?; + self.fmt_match(f, 2)?; + f.write_str("\t\tbreak;\n\t}\n}\n\n")?; + return Ok(()); + } + "check_blank_line_ratio" => { + let mut params = params.iter(); + + if let (Some(min), Some(max), Some(min_lines)) = ( + params.next().and_then(param_to_num::), + params.next().and_then(param_to_num::), + params.next().and_then(param_to_num::), + ) { + f.write_str(concat!( + "set \"body_lines\" \"0\";\n", + "set \"body_empty_lines\" \"0\";\n", + "foreveryline \"${body}\" {\n", + "\tset \"body_lines\" \"%{body_lines + 1}\";\n", + "\tif string :is \"${line}\" \"\" {\n", + "\t\tset \"body_empty_lines\" \"%{body_empty_lines + 1}\";\n", + "\t}\n", + "}\n" + ))?; + + write!( + f, + concat!( + "if eval \"body_lines >= {} && body_empty_lines / body_lines", + " >= {} && body_empty_lines / body_lines <= {}\"" + ), + min_lines, + min / 100.0, + max / 100.0 + )?; + } else { + panic!("Warning: Invalid check_blank_line_ratio"); + } + } + "check_language" => { + f.write_str(concat!( + "if not string :list \"all\" \"sa/allowed_languages\" {\n", + "\tdetect_lang \"lang\" \"${thread_name} ${body}\";\n", + "\tif not string :list \"${lang}\" \"sa/allowed_languages\"", + ))?; + self.fmt_match(f, 2)?; + f.write_str("\t}\n}\n\n")?; + return Ok(()); + } + "check_body_length" => { write!( f, - "if address :list \"{header}\" \"sa/list_{}\"", - list.to_lowercase() + "if eval \"body_len < {}\" ", + params + .iter() + .next() + .and_then(param_to_num::) + .expect("missing body length on check_body_length") )?; - } else { - eprintln!("Warning: Found invalid 'check_{header}_in_list' command without parameters."); - write!(f, "if false")?; } - } - "check_for_spf_helo_fail" - | "check_for_spf_helo_neutral" - | "check_for_spf_helo_none" - | "check_for_spf_helo_pass" - | "check_for_spf_helo_permerror" - | "check_for_spf_helo_softfail" - | "check_for_spf_helo_temperror" - | "check_for_spf_neutral" - | "check_for_spf_none" - | "check_for_spf_fail" - | "check_for_spf_pass" - | "check_for_spf_permerror" - | "check_for_spf_softfail" - | "check_for_spf_temperror" => { - let mut parts = function.split('_').rev(); - let result = parts.next().unwrap(); - let spf = if parts.next().unwrap() == "helo" { - "spf_ehlo" - } else { - "spf" - }; - write!(f, "if string :is \"${{env.{spf}_result}}\" \"{result}\"")?; - } - "check_arc_signed" => { - f.write_str("if string :value \"ne\" \"${env.arc_result}\" \"none\"")?; - } - "check_arc_valid" => { - f.write_str("if string :is \"${env.arc_result}\" \"pass\"")?; - } - "check_dmarc_missing" => { - f.write_str("if string :is \"${env.dmarc_policy}\" \"none\"")?; - } - "check_dmarc_pass" => { - f.write_str("if string :is \"${env.dmarc_result}\" \"pass\"")?; - } - "check_dmarc_none" | "check_dmarc_quarantine" | "check_dmarc_reject" => { - let policy = function.split('_').nth(2).unwrap(); - write!(f, "if allof(string :is \"${{env.dmarc_result}}\" \"fail\", string :is \"${{env.dmarc_policy}}\" \"{policy}\")")?; - } - "check_dkim_adsp" - | "check_dkim_signall" - | "check_dkim_signsome" - | "check_dkim_valid_author_sig" => { - // ADSP is deprecated (see https://datatracker.ietf.org/doc/status-change-adsp-rfc5617-to-historic/) - f.write_str("if false")?; - } - "check_dkim_dependable" => { - writeln!(f, "set :local \"{}\" \"1\";", self.name)?; - return Ok(()); - } - "check_dkim_signed" => { - f.write_str("if string :value \"ne\" \"${env.dkim_result}\" \"none\"")?; - } - "check_dkim_testing" => { - f.write_str("if header :contains \"DKIM-Signature\" \"t=y\"")?; - } - "check_dkim_valid" => { - f.write_str("if string :is \"${env.dkim_result}\" \"pass\"")?; - } - "check_dkim_valid_envelopefrom" => { - f.write_str("if allof(string :is \"${env.dkim_result}\" \"pass\", string :is \"${envelope.from}\" \"${env.from}\")")?; - } - "check_for_def_dkim_welcomelist_from" - | "check_for_def_dkim_whitelist_from" - | "check_for_dkim_welcomelist_from" - | "check_for_dkim_whitelist_from" - | "check_for_def_spf_welcomelist_from" - | "check_for_def_spf_whitelist_from" - | "check_for_spf_welcomelist_from" - | "check_for_spf_whitelist_from" => { - let list = match (function.contains("dkim"), function.contains("def")) { - (true, true) => "def_dkim", - (true, false) => "dkim", - (false, true) => "def_spf", - (false, false) => "spf", - }; - write!(f, "if address :list \"from\" \"sa/list_{list}\"")?; - } - "check_for_missing_to_header" => { - write!(f, "if not exists \"to\"")?; - } - "check_for_to_in_subject" => { - f.write_str("if address :list \"to\" \"${header.subject}\"")?; - } + "check_equal_from_domains" => { + f.write_str("if not string :is \"${envelope.from.base_domain()}\" \"${header.from.base_domain()}\"")?; + } + "check_for_no_rdns_dotcom_helo" => { + f.write_str(concat!("if not string :is \"${env.iprev_result}\" [\"pass\", \"\", \"temperror\"]"))?; + } + "helo_ip_mismatch" => { + f.write_str(concat!( + "if allof(not string :is \"${env.iprev_ptr}\" \"\", ", + "not string is \"${env.iprev_ptr}\" \"${env.helo_domain}\")" + ))?; + } + "subject_is_all_caps" => { + f.write_str("if eval \"thread_name.len() >= 10 && thread_name.word_count() > 1 && thread_name.is_uppercase()\"")?; + } + "check_for_shifted_date" => { + let mut params = params.iter(); + let mut range = [None; 2]; + for item in range.iter_mut() { + let param = params + .next() + .expect("missing parameter on check_for_shifted_date"); + if !param.contains("undef") { + *item = (param_to_num::(¶m) + .expect("failed to parse parameter on check_for_shifted_date") + * 3600) + .into(); + } + } - _ => { - write!(f, "if {function}")?; - for param in params { - f.write_str(" ")?; - if let Some(param) = - param.strip_prefix('\'').and_then(|v| v.strip_suffix('\'')) - { - write!(f, "\"{param}\"")?; - } else if param.starts_with('\"') { - f.write_str(param)?; - } else { - write!(f, "\"{param}\"")?; + f.write_str("if eval \"sent_date > 0 && ")?; + + match (range[0], range[1]) { + (Some(from), Some(to)) => { + write!( + f, + "sent_date - env.now >= {from} && sent_date - env.now < {to}", + )?; + } + (Some(from), None) => { + write!(f, "sent_date - env.now >= {from}",)?; + } + (None, Some(to)) => { + write!(f, "sent_date - env.now < {to}",)?; + } + (None, None) => { + panic!("missing parameters on check_for_shifted_date"); + } + } + + f.write_str("\"")?; + } + + _ => { + write!(f, "if {function}")?; + for param in params { + f.write_str(" ")?; + if let Some(param) = + param.strip_prefix('\'').and_then(|v| v.strip_suffix('\'')) + { + write!(f, "\"{param}\"")?; + } else if param.starts_with('\"') { + f.write_str(param)?; + } else { + write!(f, "\"{param}\"")?; + } } } } - }, + } RuleType::Meta { expr } => { write!(f, "if eval {:?}", expr.expr.trim())?; } @@ -1463,15 +1607,24 @@ impl Display for Rule { } } - writeln!(f, " {{\n\tset :local \"{}\" \"1\";", self.name)?; + self.fmt_match(f, 1)?; + f.write_str("}\n\n") + } +} + +impl Rule { + fn fmt_match(&self, f: &mut std::fmt::Formatter<'_>, depth: usize) -> std::fmt::Result { + let spaces = "\t".repeat(depth); + writeln!(f, " {{\n{spaces}set :local \"{}\" \"1\";", self.name)?; for (var_name, pos) in &self.captured_vars { - writeln!(f, "\tset :local \"{}\" \"${{{}}}\";", var_name, pos)?; + writeln!(f, "{spaces}set :local \"{}\" \"${{{}}}\";", var_name, pos)?; } let score = self.score(); if score != 0.0 { - f.write_str("\tset \"score\" \"%{score")?; + f.write_str(&spaces)?; + f.write_str("set \"score\" \"%{score")?; if score > 0.0 { f.write_str(" + ")?; score.fmt(f)?; @@ -1480,29 +1633,17 @@ impl Display for Rule { (-score).fmt(f)?; } f.write_str("}\";\n")?; - - /*if score > 0.0 { - if self.forward_score_neg != 0.0 { - write!( - f, - "if eval \"score >= spam_score && score - {:.4} >= spam_score\"", - -self.forward_score_neg - )?; - } else { - f.write_str("if eval \"score >= spam_score\"")?; - } - } else if self.forward_score_pos != 0.0 { - write!( - f, - "if eval \"score < spam_score && score + {:.4} < spam_score\"", - self.forward_score_pos - )?; - } else { - f.write_str("if eval \"score < spam_score\"")?; - } - f.write_str(" {\n\t\treturn;\n\t}\n")?;*/ } - f.write_str("}\n\n") + Ok(()) } } + +fn param_to_num(text: impl AsRef) -> Option { + let text = text.as_ref(); + text.strip_prefix('\"') + .and_then(|v| v.strip_suffix('\"')) + .unwrap_or(text) + .parse::() + .ok() +} diff --git a/crates/smtp/Cargo.toml b/crates/smtp/Cargo.toml index 7fef3a25..7061e8cd 100644 --- a/crates/smtp/Cargo.toml +++ b/crates/smtp/Cargo.toml @@ -25,7 +25,7 @@ rustls = "0.21.0" rustls-pemfile = "1.0" tokio = { version = "1.23", features = ["full"] } tokio-rustls = { version = "0.24.0"} -webpki-roots = { version = "0.23.0"} +webpki-roots = { version = "0.25"} hyper = { version = "1.0.0-rc.4", features = ["server", "http1", "http2"] } hyper-util = { git = "https://github.com/hyperium/hyper-util" } http-body-util = "0.1.0-rc.3" @@ -47,6 +47,7 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" num_cpus = "1.15.0" lazy_static = "1.4" +whatlang = "0.16" [features] test_mode = [] diff --git a/crates/smtp/src/config/scripts.rs b/crates/smtp/src/config/scripts.rs index ee8657c5..308a4d74 100644 --- a/crates/smtp/src/config/scripts.rs +++ b/crates/smtp/src/config/scripts.rs @@ -27,7 +27,7 @@ use sieve::{compiler::grammar::Capability, Compiler, Runtime}; use crate::{ core::{SieveConfig, SieveCore}, - scripts::plugins::RegisterSievePlugins, + scripts::{functions::register_functions, plugins::RegisterSievePlugins}, }; use utils::config::{utils::AsKey, Config}; @@ -39,6 +39,9 @@ pub trait ConfigSieve { impl ConfigSieve for Config { fn parse_sieve(&self, ctx: &mut ConfigContext) -> super::Result { + // Register functions + let mut fnc_map = register_functions(); + // Allocate compiler and runtime let compiler = Compiler::new() .with_max_string_size(52428800) @@ -49,7 +52,8 @@ impl ConfigSieve for Config { .with_max_local_variables(8192) .with_max_header_size(10240) .with_max_includes(10) - .register_plugins(); + .register_plugins() + .register_functions(&mut fnc_map); let mut runtime = Runtime::new() .without_capabilities([ @@ -65,10 +69,12 @@ impl ConfigSieve for Config { Capability::Duplicate, ]) .with_capability(Capability::Plugins) + .with_capability(Capability::ForEveryLine) .with_max_variable_size(102400) .with_max_header_size(10240) .with_valid_notification_uri("mailto") - .with_valid_ext_lists(ctx.directory.lookups.keys().map(|k| k.to_string())); + .with_valid_ext_lists(ctx.directory.lookups.keys().map(|k| k.to_string())) + .with_functions(&mut fnc_map); if let Some(value) = self.property("sieve.limits.redirects")? { runtime.set_max_redirects(value); diff --git a/crates/smtp/src/inbound/session.rs b/crates/smtp/src/inbound/session.rs index 43abbeb4..967b3827 100644 --- a/crates/smtp/src/inbound/session.rs +++ b/crates/smtp/src/inbound/session.rs @@ -137,6 +137,10 @@ impl Session { Request::StartTls => { if !self.stream.is_tls() { self.write(b"220 2.0.0 Ready to start TLS.\r\n").await?; + #[cfg(any(test, feature = "test_mode"))] + if self.data.helo_domain.contains("badtls") { + return Err(()); + } self.state = State::default(); return Ok(false); } else { diff --git a/crates/smtp/src/outbound/delivery.rs b/crates/smtp/src/outbound/delivery.rs index c5142ac7..a822e90d 100644 --- a/crates/smtp/src/outbound/delivery.rs +++ b/crates/smtp/src/outbound/delivery.rs @@ -903,7 +903,6 @@ impl DeliveryAttempt { }; // Update status for the current domain and continue with the next one - domain.disable_tls = disable_tls; domain .set_status(delivery_result, queue_config.retry.eval(&envelope).await); continue 'next_domain; @@ -911,6 +910,7 @@ impl DeliveryAttempt { } // Update status + domain.disable_tls = disable_tls; domain.set_status(last_status, queue_config.retry.eval(&envelope).await); } self.message.domains = domains; diff --git a/crates/smtp/src/scripts/exec.rs b/crates/smtp/src/scripts/exec.rs index 19c6edab..443b726c 100644 --- a/crates/smtp/src/scripts/exec.rs +++ b/crates/smtp/src/scripts/exec.rs @@ -21,7 +21,7 @@ * for more details. */ -use std::sync::Arc; +use std::{sync::Arc, time::SystemTime}; use sieve::{Envelope, Sieve}; use smtp_proto::{MAIL_BY_NOTIFY, MAIL_BY_RETURN, MAIL_RET_FULL, MAIL_RET_HDRS}; @@ -40,6 +40,12 @@ impl Session { .set_variable("remote_ip", self.data.remote_ip.to_string()) .set_variable("helo_domain", self.data.helo_domain.to_string()) .set_variable("authenticated_as", self.data.authenticated_as.clone()) + .set_variable( + "now", + SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .map_or(0, |d| d.as_secs()), + ) .set_variable( "spf_result", self.data @@ -55,15 +61,13 @@ impl Session { .as_ref() .map(|r| r.result().as_str()) .unwrap_or_default(), - ) - .set_variable( - "iprev_result", - self.data - .iprev - .as_ref() - .map(|r| r.result().as_str()) - .unwrap_or_default(), ); + if let Some(ip_rev) = &self.data.iprev { + params = params.set_variable("iprev_result", ip_rev.result().as_str()); + if let Some(ptr) = ip_rev.ptr.as_ref().and_then(|addrs| addrs.first()) { + params = params.set_variable("iprev_ptr", ptr.to_string()); + } + } if let Some(mail_from) = &self.data.mail_from { params diff --git a/crates/smtp/src/scripts/functions.rs b/crates/smtp/src/scripts/functions.rs new file mode 100644 index 00000000..a16fd10c --- /dev/null +++ b/crates/smtp/src/scripts/functions.rs @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use mail_parser::parsers::fields::thread::thread_name; +use sieve::{runtime::Variable, FunctionMap}; + +pub fn register_functions() -> FunctionMap { + FunctionMap::new() + .with_function("trim", |v| v.to_cow().trim().to_string().into()) + .with_function("len", |v| v.to_cow().len().into()) + .with_function("to_lowercase", |v| { + v.to_cow().to_lowercase().to_string().into() + }) + .with_function("to_uppercase", |v| { + v.to_cow().to_uppercase().to_string().into() + }) + .with_function("language", |v| { + whatlang::detect_lang(v.to_cow().as_ref()) + .map(|l| l.code()) + .unwrap_or("unknown") + .into() + }) + .with_function("domain", |v| { + v.to_cow() + .rsplit_once('@') + .map_or(Variable::default(), |(_, d)| d.trim().to_string().into()) + }) + .with_function("base_domain", |v| { + v.to_cow() + .rsplit_once('@') + .map_or(Variable::default(), |(_, d)| { + d.split('.') + .rev() + .take(2) + .fold(String::new(), |a, b| { + if a.is_empty() { + b.to_string() + } else { + format!("{}.{}", b, a) + } + }) + .into() + }) + }) + .with_function("thread_name", |v| { + thread_name(v.to_cow().as_ref()).to_string().into() + }) + .with_function("is_uppercase", |v| { + v.to_cow() + .as_ref() + .chars() + .filter(|c| c.is_alphabetic()) + .all(|c| c.is_uppercase()) + .into() + }) + .with_function("is_lowercase", |v| { + v.to_cow() + .as_ref() + .chars() + .filter(|c| c.is_alphabetic()) + .all(|c| c.is_lowercase()) + .into() + }) + .with_function("word_count", |v| { + v.to_cow().as_ref().split_whitespace().count().into() + }) +} diff --git a/crates/smtp/src/scripts/mod.rs b/crates/smtp/src/scripts/mod.rs index b58e6c34..c956e594 100644 --- a/crates/smtp/src/scripts/mod.rs +++ b/crates/smtp/src/scripts/mod.rs @@ -29,6 +29,7 @@ use sieve::{runtime::Variable, Envelope}; pub mod envelope; pub mod event_loop; pub mod exec; +pub mod functions; pub mod plugins; #[derive(Debug)] diff --git a/crates/smtp/src/scripts/plugins/detect_lang.rs b/crates/smtp/src/scripts/plugins/detect_lang.rs new file mode 100644 index 00000000..aa70819a --- /dev/null +++ b/crates/smtp/src/scripts/plugins/detect_lang.rs @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use sieve::{runtime::Variable, Compiler, Input, SetVariable}; + +use super::PluginContext; + +pub fn register(plugin_id: u32, compiler: &mut Compiler) { + compiler + .register_plugin("detect_lang") + .with_id(plugin_id) + .with_variable_argument() + .with_string_argument(); +} + +pub fn exec(ctx: PluginContext<'_>) -> Input { + let mut arguments = ctx.arguments.into_iter(); + + let name = arguments.next().and_then(|a| a.unwrap_variable()).unwrap(); + let text = arguments.next().and_then(|a| a.unwrap_string()).unwrap(); + + Input::Variables { + list: vec![SetVariable { + name, + value: Variable::StringRef( + whatlang::detect_lang(&text) + .map(|l| l.code()) + .unwrap_or("unknown"), + ), + }], + } +} diff --git a/crates/smtp/src/scripts/plugins/mod.rs b/crates/smtp/src/scripts/plugins/mod.rs index afafb49c..f4ea06a3 100644 --- a/crates/smtp/src/scripts/plugins/mod.rs +++ b/crates/smtp/src/scripts/plugins/mod.rs @@ -21,6 +21,7 @@ * for more details. */ +pub mod detect_lang; pub mod exec; pub mod query; @@ -43,8 +44,9 @@ pub struct PluginContext<'x> { pub arguments: Vec>, } -const PLUGINS_EXEC: [ExecPluginFnc; 2] = [query::exec, exec::exec]; -const PLUGINS_REGISTER: [RegisterPluginFnc; 2] = [query::register, exec::register]; +const PLUGINS_EXEC: [ExecPluginFnc; 3] = [query::exec, exec::exec, detect_lang::exec]; +const PLUGINS_REGISTER: [RegisterPluginFnc; 3] = + [query::register, exec::register, detect_lang::register]; pub trait RegisterSievePlugins { fn register_plugins(self) -> Self; diff --git a/tests/resources/smtp/sieve/functions.sieve b/tests/resources/smtp/sieve/functions.sieve new file mode 100644 index 00000000..ca27ba45 --- /dev/null +++ b/tests/resources/smtp/sieve/functions.sieve @@ -0,0 +1,24 @@ +require ["variables", "include", "vnd.stalwart.plugins", "reject"]; + +set "address1" "john@doe.example.org"; +set "address2" "jane@smith.example.org"; +set "address3" "john@example.org"; +set "address4" "jane@example.org"; +set "address5" "john@localhost"; +set "address6" "jane@localhost"; + +if not string :is "${address1.base_domain()}" "${address2.base_domain()}" { + reject "${address1.base_domain()} != ${address2.base_domain()}"; + stop; +} + +if not string :is "${address3.base_domain()}" "${address4.base_domain()}" { + reject "${address3.base_domain()} != ${address4.base_domain()}"; + stop; +} + +if not string :is "${address5.base_domain()}" "${address6.base_domain()}" { + reject "${address5.base_domain()} != ${address6.base_domain()}"; + stop; +} + diff --git a/tests/src/smtp/outbound/mod.rs b/tests/src/smtp/outbound/mod.rs index e72973f7..3bc3765b 100644 --- a/tests/src/smtp/outbound/mod.rs +++ b/tests/src/smtp/outbound/mod.rs @@ -36,6 +36,7 @@ pub mod lmtp; pub mod mta_sts; pub mod smtp; pub mod throttle; +pub mod tls; const SERVER: &str = " [server] diff --git a/tests/src/smtp/outbound/tls.rs b/tests/src/smtp/outbound/tls.rs new file mode 100644 index 00000000..6f52b73c --- /dev/null +++ b/tests/src/smtp/outbound/tls.rs @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; + +use mail_auth::MX; +use utils::config::ServerProtocol; + +use crate::smtp::{ + inbound::{TestMessage, TestQueueEvent}, + outbound::start_test_server, + session::{TestSession, VerifyResponse}, + TestConfig, TestSMTP, +}; +use smtp::{ + config::{IfBlock, RequireOptional}, + core::{Session, SMTP}, + queue::{manager::Queue, DeliveryAttempt}, +}; + +#[tokio::test] +#[serial_test::serial] +async fn starttls_optional() { + /*tracing::subscriber::set_global_default( + tracing_subscriber::FmtSubscriber::builder() + .with_max_level(tracing::Level::TRACE) + .finish(), + ) + .unwrap();*/ + + // Start test server + let mut core = SMTP::test(); + core.session.config.rcpt.relay = IfBlock::new(true); + let mut remote_qr = core.init_test_queue("smtp_starttls_remote"); + let _rx = start_test_server(core.into(), &[ServerProtocol::Smtp]); + + // Add mock DNS entries + let mut core = SMTP::test(); + core.queue.config.hostname = IfBlock::new("badtls.foobar.org".to_string()); + core.resolvers.dns.mx_add( + "foobar.org", + vec![MX { + exchanges: vec!["mx.foobar.org".to_string()], + preference: 10, + }], + Instant::now() + Duration::from_secs(10), + ); + core.resolvers.dns.ipv4_add( + "mx.foobar.org", + vec!["127.0.0.1".parse().unwrap()], + Instant::now() + Duration::from_secs(10), + ); + + // Retry on failed STARTTLS + let mut local_qr = core.init_test_queue("smtp_starttls_local"); + core.session.config.rcpt.relay = IfBlock::new(true); + core.queue.config.tls.start = IfBlock::new(RequireOptional::Optional); + + let core = Arc::new(core); + let mut queue = Queue::default(); + let mut session = Session::test(core.clone()); + session.data.remote_ip = "10.0.0.1".parse().unwrap(); + session.eval_session_params().await; + session.ehlo("mx.test.org").await; + session + .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") + .await; + DeliveryAttempt::from(local_qr.read_event().await.unwrap_message()) + .try_deliver(core.clone(), &mut queue) + .await; + let mut retry = local_qr.read_event().await.unwrap_retry(); + assert!(retry.inner.domains[0].disable_tls); + retry.inner.domains[0].retry.due = Instant::now(); + DeliveryAttempt::from(retry.inner) + .try_deliver(core.clone(), &mut queue) + .await; + local_qr.read_event().await.unwrap_done(); + remote_qr + .read_event() + .await + .unwrap_message() + .read_lines() + .assert_not_contains("using TLSv1.3 with cipher"); +}