Port Spam filter to Rust - part 5
This commit is contained in:
mdecimus
@@ -15,12 +15,16 @@ mail-parser = { version = "0.9", features = ["full_encoding", "ludicrous_mode"]
|
||||
mail-builder = { version = "0.3", features = ["ludicrous_mode"] }
|
||||
mail-auth = { version = "0.5" }
|
||||
mail-send = { version = "0.4", default-features = false, features = ["cram-md5", "ring", "tls12"] }
|
||||
tokio = { version = "1.23", features = ["net", "macros"] }
|
||||
psl = "2"
|
||||
hyper = { version = "1.0.1", features = ["server", "http1", "http2"] }
|
||||
idna = "1.0"
|
||||
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls-webpki-roots", "http2", "stream"]}
|
||||
decancer = "3.0.1"
|
||||
unicode-security = "0.1.0"
|
||||
infer = "0.16"
|
||||
sha1 = "0.10"
|
||||
sha2 = "0.10.6"
|
||||
|
||||
[features]
|
||||
test_mode = []
|
||||
|
||||
323
crates/spam-filter/src/analysis/html.rs
Normal file
323
crates/spam-filter/src/analysis/html.rs
Normal file
@@ -0,0 +1,323 @@
|
||||
use std::future::Future;
|
||||
|
||||
use common::Server;
|
||||
use hyper::Uri;
|
||||
use mail_parser::MimeHeaders;
|
||||
use nlp::tokenizers::types::{TokenType, TypesTokenizer};
|
||||
|
||||
use crate::{modules::html::*, Hostname, SpamFilterContext, TextPart};
|
||||
|
||||
pub trait SpamFilterAnalyzeHtml: Sync + Send {
|
||||
fn spam_filter_analyze_html(
|
||||
&self,
|
||||
ctx: &mut SpamFilterContext<'_>,
|
||||
) -> impl Future<Output = ()> + Send;
|
||||
}
|
||||
|
||||
struct Href {
|
||||
url_parsed: Option<Uri>,
|
||||
host: Option<Hostname>,
|
||||
}
|
||||
|
||||
impl SpamFilterAnalyzeHtml for Server {
|
||||
async fn spam_filter_analyze_html(&self, ctx: &mut SpamFilterContext<'_>) {
|
||||
// Message only has text/html MIME parts
|
||||
if ctx.input.message.content_type().map_or(false, |ct| {
|
||||
ct.ctype().eq_ignore_ascii_case("text")
|
||||
&& ct
|
||||
.subtype()
|
||||
.unwrap_or_default()
|
||||
.eq_ignore_ascii_case("html")
|
||||
}) {
|
||||
ctx.result.add_tag("MIME_HTML_ONLY");
|
||||
}
|
||||
let mut last_href: Option<Href> = None;
|
||||
let mut html_img_words = 0;
|
||||
let mut html_text_chars = 0;
|
||||
let mut in_head: i32 = 0;
|
||||
let mut in_body: i32 = 0;
|
||||
|
||||
for (part_id, part) in ctx.output.text_parts.iter().enumerate() {
|
||||
let is_body_part = ctx.input.message.text_body.contains(&part_id)
|
||||
|| ctx.input.message.html_body.contains(&part_id);
|
||||
|
||||
let (html_tokens, tokens) = if let TextPart::Html {
|
||||
html_tokens,
|
||||
tokens,
|
||||
..
|
||||
} = part
|
||||
{
|
||||
(html_tokens, tokens)
|
||||
} else {
|
||||
continue;
|
||||
};
|
||||
let mut has_link_to_img = false;
|
||||
|
||||
for token in html_tokens {
|
||||
match token {
|
||||
HtmlToken::StartTag {
|
||||
name,
|
||||
attributes,
|
||||
is_self_closing,
|
||||
} => match *name {
|
||||
A => {
|
||||
if let Some(attr) = attributes.iter().find_map(|(attr, value)| {
|
||||
if *attr == HREF {
|
||||
value.as_deref()
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}) {
|
||||
let url = attr.trim().to_lowercase();
|
||||
let url_parsed = url.parse::<Uri>().ok();
|
||||
let href = Href {
|
||||
host: url_parsed
|
||||
.as_ref()
|
||||
.and_then(|uri| uri.host().map(Hostname::new)),
|
||||
url_parsed,
|
||||
};
|
||||
|
||||
if is_body_part
|
||||
&& attr.starts_with("data:")
|
||||
&& attr.contains(";base64,")
|
||||
{
|
||||
// Has Data URI encoding
|
||||
ctx.result.add_tag("HAS_DATA_URI");
|
||||
if attr.contains("text/") {
|
||||
// Uses Data URI encoding to obfuscate plain or HTML in base64
|
||||
ctx.result.add_tag("DATA_URI_OBFU");
|
||||
}
|
||||
} else if href.host.as_ref().map_or(false, |h| h.ip.is_some()) {
|
||||
// HTML anchor points to an IP address
|
||||
ctx.result.add_tag("HTTP_TO_IP");
|
||||
}
|
||||
|
||||
if !*is_self_closing {
|
||||
last_href = Some(href);
|
||||
}
|
||||
}
|
||||
}
|
||||
IMG if is_body_part => {
|
||||
let mut img_width = 800;
|
||||
let mut img_height = 600;
|
||||
|
||||
for (attr, value) in attributes {
|
||||
if let Some(value) =
|
||||
value.as_deref().map(|v| v.trim()).filter(|v| !v.is_empty())
|
||||
{
|
||||
let dimension = match *attr {
|
||||
WIDTH => &mut img_width,
|
||||
HEIGHT => &mut img_height,
|
||||
SRC => {
|
||||
let src = value.to_ascii_lowercase();
|
||||
if src.starts_with("data:") && src.contains(";base64,")
|
||||
{
|
||||
// Has Data URI encoding
|
||||
ctx.result.add_tag("Has Data URI encoding");
|
||||
}
|
||||
continue;
|
||||
}
|
||||
_ => {
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if let Some(pct) = value.strip_suffix('%') {
|
||||
if let Ok(pct) = pct.trim().parse::<u64>() {
|
||||
*dimension = (*dimension * pct) / 100;
|
||||
}
|
||||
} else if let Ok(value) = value.parse::<u64>() {
|
||||
*dimension = value;
|
||||
}
|
||||
}
|
||||
}
|
||||
let dimensions = img_width + img_height;
|
||||
|
||||
if last_href.is_some() && dimensions >= 210 {
|
||||
has_link_to_img = true;
|
||||
}
|
||||
|
||||
if dimensions > 100 {
|
||||
// We assume that a single picture 100x200 contains approx 3 words of text
|
||||
html_img_words += dimensions / 100;
|
||||
}
|
||||
}
|
||||
META => {
|
||||
let mut has_equiv_refresh = false;
|
||||
let mut has_content_url = false;
|
||||
|
||||
for (attr, value) in attributes {
|
||||
if let Some(value) =
|
||||
value.as_deref().map(|v| v.trim()).filter(|v| !v.is_empty())
|
||||
{
|
||||
if *attr == HTTP_EQUIV {
|
||||
if value.eq_ignore_ascii_case("refresh") {
|
||||
has_equiv_refresh = true;
|
||||
}
|
||||
} else if *attr == CONTENT
|
||||
&& value.to_ascii_lowercase().contains("url=")
|
||||
{
|
||||
has_content_url = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if has_equiv_refresh && has_content_url {
|
||||
// HTML meta refresh tag
|
||||
ctx.result.add_tag("HTML_META_REFRESH_URL");
|
||||
}
|
||||
}
|
||||
LINK if is_body_part => {
|
||||
let mut has_rel_style = false;
|
||||
let mut has_href_css = false;
|
||||
|
||||
for (attr, value) in attributes {
|
||||
if let Some(value) =
|
||||
value.as_deref().map(|v| v.trim()).filter(|v| !v.is_empty())
|
||||
{
|
||||
if *attr == REL {
|
||||
if value.to_ascii_lowercase().contains("stylesheet") {
|
||||
has_rel_style = true;
|
||||
}
|
||||
} else if *attr == HREF
|
||||
&& value.to_ascii_lowercase().ends_with(".css")
|
||||
{
|
||||
has_href_css = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if has_rel_style || has_href_css {
|
||||
// Has external CSS
|
||||
ctx.result.add_tag("EXT_CSS");
|
||||
}
|
||||
}
|
||||
HEAD if !*is_self_closing => {
|
||||
in_head += 1;
|
||||
}
|
||||
BODY if !*is_self_closing => {
|
||||
in_body += 1;
|
||||
}
|
||||
_ => {}
|
||||
},
|
||||
HtmlToken::EndTag { name } => match *name {
|
||||
A => {
|
||||
last_href = None;
|
||||
}
|
||||
HEAD => {
|
||||
in_head -= 1;
|
||||
}
|
||||
BODY => {
|
||||
in_body -= 1;
|
||||
}
|
||||
_ => (),
|
||||
},
|
||||
HtmlToken::Text { text } if in_head > 0 => {
|
||||
if let Some((href_url, href_host)) = last_href
|
||||
.as_ref()
|
||||
.and_then(|href| Some((href.url_parsed.as_ref()?, href.host.as_ref()?)))
|
||||
{
|
||||
for token in TypesTokenizer::new(text.as_ref())
|
||||
.tokenize_numbers(false)
|
||||
.tokenize_urls(true)
|
||||
.tokenize_urls_without_scheme(true)
|
||||
.tokenize_emails(true)
|
||||
{
|
||||
let text_url = match token.word {
|
||||
TokenType::Url(url) => url.to_lowercase(),
|
||||
TokenType::UrlNoScheme(url) => {
|
||||
format!("http://{}", url.to_lowercase())
|
||||
}
|
||||
_ => continue,
|
||||
};
|
||||
let text_url_parsed =
|
||||
if let Ok(text_url_parsed) = text_url.parse::<Uri>() {
|
||||
text_url_parsed
|
||||
} else {
|
||||
continue;
|
||||
};
|
||||
|
||||
if href_url.scheme().map(|s| s.as_str()).unwrap_or_default()
|
||||
== "http"
|
||||
&& text_url_parsed
|
||||
.scheme()
|
||||
.map(|s| s.as_str())
|
||||
.unwrap_or_default()
|
||||
== "https"
|
||||
{
|
||||
// The anchor text contains a distinct scheme compared to the target URL
|
||||
ctx.result.add_tag("HTTP_TO_HTTPS");
|
||||
}
|
||||
|
||||
if let Some(text_url_host) = text_url_parsed.host() {
|
||||
let text_url_host = Hostname::new(text_url_host);
|
||||
|
||||
if text_url_host.sld_or_default() != href_host.sld_or_default()
|
||||
{
|
||||
// The anchor text contains a different domain than the target URL
|
||||
ctx.result.add_tag("PHISHING");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if is_body_part {
|
||||
html_text_chars += text.chars().filter(|t| t.is_alphanumeric()).count();
|
||||
}
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
|
||||
if is_body_part {
|
||||
if in_head != 0 || in_body != 0 {
|
||||
// HTML tags are not properly closed
|
||||
ctx.result.add_tag("HTML_UNBALANCED_TAG");
|
||||
}
|
||||
|
||||
if has_link_to_img {
|
||||
match html_text_chars {
|
||||
0..1024 => {
|
||||
ctx.result.add_tag("HTML_SHORT_LINK_IMG_1");
|
||||
}
|
||||
1024..1536 => {
|
||||
ctx.result.add_tag("HTML_SHORT_LINK_IMG_2");
|
||||
}
|
||||
1536..2048 => {
|
||||
ctx.result.add_tag("HTML_SHORT_LINK_IMG_3");
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
|
||||
let mut html_words = 0;
|
||||
let mut html_uris = 0;
|
||||
|
||||
for token in tokens {
|
||||
match token {
|
||||
TokenType::Alphabetic(_)
|
||||
| TokenType::Alphanumeric(_)
|
||||
| TokenType::Email(_) => {
|
||||
html_words += 1;
|
||||
}
|
||||
TokenType::Url(_) | TokenType::UrlNoScheme(_) => {
|
||||
html_uris += 1;
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
|
||||
if (!has_link_to_img || html_text_chars >= 2048)
|
||||
&& html_img_words as f64 / (html_words as f64 + html_img_words as f64) > 0.5
|
||||
{
|
||||
// Message contains more images than text
|
||||
ctx.result.add_tag("HTML_TEXT_IMG_RATIO");
|
||||
}
|
||||
|
||||
if html_uris > 0 && html_words == 0 {
|
||||
// Message only contains URIs in HTML
|
||||
ctx.result.add_tag("BODY_URI_ONLY");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -13,14 +13,14 @@ use crate::{modules::dnsbl::is_dnsbl, SpamFilterContext, TextPart};
|
||||
|
||||
use super::{ElementLocation, SpamFilterResolver};
|
||||
|
||||
pub trait SpamFilterAnalyzeIpRev: Sync + Send {
|
||||
pub trait SpamFilterAnalyzeIp: Sync + Send {
|
||||
fn spam_filter_analyze_ip(
|
||||
&self,
|
||||
ctx: &mut SpamFilterContext<'_>,
|
||||
) -> impl Future<Output = ()> + Send;
|
||||
}
|
||||
|
||||
impl SpamFilterAnalyzeIpRev for Server {
|
||||
impl SpamFilterAnalyzeIp for Server {
|
||||
async fn spam_filter_analyze_ip(&self, ctx: &mut SpamFilterContext<'_>) {
|
||||
// IP Address RBL
|
||||
let mut ips =
|
||||
|
||||
426
crates/spam-filter/src/analysis/mime.rs
Normal file
426
crates/spam-filter/src/analysis/mime.rs
Normal file
@@ -0,0 +1,426 @@
|
||||
use std::{collections::HashSet, future::Future, vec};
|
||||
|
||||
use common::{
|
||||
scripts::functions::{array::cosine_similarity, unicode::CharUtils},
|
||||
Server,
|
||||
};
|
||||
use hyper::Uri;
|
||||
use mail_parser::{HeaderName, MimeHeaders, PartType};
|
||||
use nlp::tokenizers::types::TokenType;
|
||||
use unicode_security::MixedScript;
|
||||
|
||||
use crate::{Hostname, SpamFilterContext, TextPart};
|
||||
|
||||
pub trait SpamFilterAnalyzeMime: Sync + Send {
|
||||
fn spam_filter_analyze_mime(
|
||||
&self,
|
||||
ctx: &mut SpamFilterContext<'_>,
|
||||
) -> impl Future<Output = ()> + Send;
|
||||
}
|
||||
|
||||
impl SpamFilterAnalyzeMime for Server {
|
||||
async fn spam_filter_analyze_mime(&self, ctx: &mut SpamFilterContext<'_>) {
|
||||
let mut has_mime_version = false;
|
||||
let mut has_ct = false;
|
||||
let mut has_cte = false;
|
||||
let mut had_cd = false;
|
||||
let mut is_plain_text = false;
|
||||
|
||||
for header in ctx.input.message.headers() {
|
||||
match &header.name {
|
||||
HeaderName::MimeVersion => {
|
||||
if ctx
|
||||
.input
|
||||
.message
|
||||
.raw_message()
|
||||
.get(header.offset_field..header.offset_start - 1)
|
||||
!= Some(b"MIME-Version")
|
||||
{
|
||||
ctx.result.add_tag("MV_CASE");
|
||||
}
|
||||
has_mime_version = true;
|
||||
}
|
||||
HeaderName::ContentType => {
|
||||
has_ct = true;
|
||||
is_plain_text = header.value().as_content_type().map_or(false, |ct| {
|
||||
ct.ctype().eq_ignore_ascii_case("text")
|
||||
&& ct
|
||||
.subtype()
|
||||
.unwrap_or_default()
|
||||
.eq_ignore_ascii_case("plain")
|
||||
});
|
||||
}
|
||||
HeaderName::ContentTransferEncoding => {
|
||||
has_cte = true;
|
||||
}
|
||||
HeaderName::ContentDisposition => {
|
||||
had_cd = true;
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
|
||||
if !has_mime_version && (has_ct || has_cte) {
|
||||
ctx.result.add_tag("MISSING_MIME_VERSION");
|
||||
}
|
||||
if has_ct && !is_plain_text && !has_cte && !had_cd {
|
||||
// Only Content-Type header without other MIME headers
|
||||
ctx.result.add_tag("MIME_HEADER_CTYPE_ONLY");
|
||||
}
|
||||
let raw_message = ctx.input.message.raw_message();
|
||||
|
||||
let mut has_text_part = false;
|
||||
let mut is_encrypted = false;
|
||||
let mut is_encrypted_smime = false;
|
||||
let mut is_encrypted_pgp = false;
|
||||
|
||||
let mut num_parts = 0;
|
||||
let mut num_parts_size = 0;
|
||||
|
||||
for (part_id, part) in ctx.input.message.parts.iter().enumerate() {
|
||||
let mut ct = None;
|
||||
let mut cd = None;
|
||||
let mut ct_type = String::new();
|
||||
let mut ct_subtype = String::new();
|
||||
let mut cte = String::new();
|
||||
let mut is_attachment = ctx.input.message.attachments.contains(&part_id);
|
||||
let mut has_content_id = false;
|
||||
|
||||
for header in part.headers() {
|
||||
match &header.name {
|
||||
HeaderName::ContentType => {
|
||||
if let Some(ct_) = header.value().as_content_type() {
|
||||
ct_type = ct_.ctype().to_ascii_lowercase();
|
||||
ct_subtype = ct_.subtype().unwrap_or_default().to_ascii_lowercase();
|
||||
ct = Some(ct_);
|
||||
}
|
||||
|
||||
if ct_type.is_empty() {
|
||||
// Content-Type header can't be parsed
|
||||
ctx.result.add_tag("BROKEN_CONTENT_TYPE");
|
||||
}
|
||||
|
||||
if raw_message
|
||||
.get(header.offset_start..header.offset_end)
|
||||
.and_then(|s| s.trim_ascii_end().last())
|
||||
== Some(&b';')
|
||||
{
|
||||
// Content-Type header ends with a semi-colon
|
||||
ctx.result.add_tag("CT_EXTRA_SEMI");
|
||||
}
|
||||
}
|
||||
HeaderName::ContentTransferEncoding => {
|
||||
let cte_ = header.value().as_text().unwrap_or_default();
|
||||
cte = cte_.to_ascii_lowercase();
|
||||
|
||||
if cte != cte_ {
|
||||
ctx.result.add_tag("CTE_CASE");
|
||||
}
|
||||
}
|
||||
HeaderName::ContentDisposition => {
|
||||
cd = header.value().as_content_type();
|
||||
}
|
||||
HeaderName::ContentId => {
|
||||
has_content_id = true;
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
|
||||
match ct_type.as_str() {
|
||||
"multipart" => {
|
||||
let part_ids = match &part.body {
|
||||
PartType::Multipart(parts) => parts.as_slice(),
|
||||
_ => &[],
|
||||
};
|
||||
|
||||
match ct_subtype.as_str() {
|
||||
"alternative" => {
|
||||
let mut has_plain_part = false;
|
||||
let mut has_html_part = false;
|
||||
|
||||
let mut text_part_words = vec![];
|
||||
let mut text_part_uris = 0;
|
||||
|
||||
let mut html_part_words = vec![];
|
||||
let mut html_part_uris = 0;
|
||||
|
||||
for text_part in part_ids.iter().map(|id| &ctx.output.text_parts[*id]) {
|
||||
match text_part {
|
||||
TextPart::Plain { tokens, .. } if !has_plain_part => {
|
||||
words_and_uris(
|
||||
tokens,
|
||||
&mut text_part_words,
|
||||
&mut text_part_uris,
|
||||
);
|
||||
has_plain_part = true;
|
||||
}
|
||||
TextPart::Html { tokens, .. } if !has_html_part => {
|
||||
words_and_uris(
|
||||
tokens,
|
||||
&mut html_part_words,
|
||||
&mut html_part_uris,
|
||||
);
|
||||
has_html_part = true;
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
|
||||
// Multipart message mostly text/html MIME
|
||||
if has_html_part {
|
||||
if !has_plain_part {
|
||||
ctx.result.add_tag("MIME_MA_MISSING_TEXT");
|
||||
}
|
||||
} else if has_plain_part {
|
||||
ctx.result.add_tag("MIME_MA_MISSING_HTML");
|
||||
}
|
||||
|
||||
// HTML and text parts are different
|
||||
if has_plain_part
|
||||
&& has_html_part
|
||||
&& (!text_part_words.is_empty() || !html_part_words.is_empty())
|
||||
&& cosine_similarity(&text_part_words, &html_part_words) < 0.95
|
||||
{
|
||||
ctx.result.add_tag("R_PARTS_DIFFER");
|
||||
}
|
||||
|
||||
// Odd URI count between parts
|
||||
if text_part_uris != html_part_uris {
|
||||
ctx.result.add_tag("URI_COUNT_ODD");
|
||||
}
|
||||
}
|
||||
"mixed" => {
|
||||
let mut num_text_parts = 0;
|
||||
let mut has_other_parts = false;
|
||||
|
||||
for (sub_part_id, sub_part) in part_ids
|
||||
.iter()
|
||||
.map(|id| (*id, &ctx.input.message.parts[*id]))
|
||||
{
|
||||
let ctype = sub_part
|
||||
.content_type()
|
||||
.map(|ct| ct.ctype())
|
||||
.unwrap_or_default();
|
||||
|
||||
if ctype.eq_ignore_ascii_case("text")
|
||||
&& !ctx.input.message.attachments.contains(&sub_part_id)
|
||||
{
|
||||
num_text_parts += 1;
|
||||
} else if !ctype.eq_ignore_ascii_case("multipart") {
|
||||
has_other_parts = true;
|
||||
}
|
||||
}
|
||||
|
||||
// Found multipart/mixed without non-textual part
|
||||
if !has_other_parts && num_text_parts < 3 {
|
||||
ctx.result.add_tag("CTYPE_MIXED_BOGUS");
|
||||
}
|
||||
}
|
||||
"encrypted" => {
|
||||
is_encrypted = true;
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
|
||||
continue;
|
||||
}
|
||||
"text" => {
|
||||
let mut is_7bit = false;
|
||||
match cte.as_str() {
|
||||
"" | "7bit" => {
|
||||
if raw_message
|
||||
.get(part.raw_body_offset()..part.raw_end_offset())
|
||||
.map_or(false, |bytes| !bytes.is_ascii())
|
||||
{
|
||||
// MIME text part claims to be ASCII but isn't
|
||||
ctx.result.add_tag("R_BAD_CTE_7BIT");
|
||||
}
|
||||
is_7bit = true;
|
||||
}
|
||||
"base64" => {
|
||||
if part.contents().is_ascii() {
|
||||
// Has text part encoded in base64 that does not contain any 8bit characters
|
||||
ctx.result.add_tag("MIME_BASE64_TEXT_BOGUS");
|
||||
} else {
|
||||
// Has text part encoded in base64
|
||||
ctx.result.add_tag("MIME_BASE64_TEXT");
|
||||
}
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
|
||||
if !is_7bit
|
||||
&& ct_subtype == "plain"
|
||||
&& ct
|
||||
.and_then(|ct| ct.attribute("charset"))
|
||||
.map_or(true, |c| c.is_empty())
|
||||
{
|
||||
// Charset header is missing
|
||||
ctx.result.add_tag("R_MISSING_CHARSET");
|
||||
}
|
||||
|
||||
match &part.body {
|
||||
PartType::Text(text) | PartType::Html(text)
|
||||
if ctx.input.message.text_body.contains(&part_id)
|
||||
|| ctx.input.message.html_body.contains(&part_id) =>
|
||||
{
|
||||
if !text.as_ref().is_single_script() {
|
||||
// Text part contains multiple scripts
|
||||
ctx.result.add_tag("R_MIXED_CHARSET");
|
||||
}
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
|
||||
has_text_part = true;
|
||||
}
|
||||
"application" => match ct_subtype.as_str() {
|
||||
"pkcs7-mime" => {
|
||||
ctx.result.add_tag("ENCRYPTED_SMIME");
|
||||
is_attachment = false;
|
||||
is_encrypted_smime = true;
|
||||
}
|
||||
"pkcs7-signature" => {
|
||||
ctx.result.add_tag("SIGNED_SMIME");
|
||||
is_attachment = false;
|
||||
}
|
||||
"pgp-encrypted" => {
|
||||
ctx.result.add_tag("ENCRYPTED_PGP");
|
||||
is_attachment = false;
|
||||
is_encrypted_pgp = true;
|
||||
}
|
||||
"pgp-signature" => {
|
||||
ctx.result.add_tag("SIGNED_PGP");
|
||||
is_attachment = false;
|
||||
}
|
||||
"octet-stream" => {
|
||||
if !is_encrypted
|
||||
&& !has_content_id
|
||||
&& cd.map_or(true, |cd| {
|
||||
cd.attribute("type")
|
||||
.unwrap_or_default()
|
||||
.to_ascii_lowercase()
|
||||
!= "attachment"
|
||||
&& !cd.has_attribute("filename")
|
||||
})
|
||||
{
|
||||
ctx.result.add_tag("CTYPE_MISSING_DISPOSITION");
|
||||
}
|
||||
}
|
||||
_ => (),
|
||||
},
|
||||
_ => (),
|
||||
}
|
||||
|
||||
num_parts += 1;
|
||||
num_parts_size += part.len();
|
||||
|
||||
let ct_full = format!("{ct_type}/{ct_subtype}");
|
||||
|
||||
if is_attachment {
|
||||
// Has a MIME attachment
|
||||
ctx.result.add_tag("HAS_ATTACHMENT");
|
||||
match &part.body {
|
||||
PartType::Binary(bytes) | PartType::InlineBinary(bytes) => {
|
||||
if let Some(t) = infer::get(bytes.as_ref()) {
|
||||
if t.mime_type() != ct_full {
|
||||
// Known content-type
|
||||
ctx.result.add_tag("MIME_GOOD");
|
||||
} else if ct_full != "application/octet-stream" {
|
||||
// Known bad content-type
|
||||
ctx.result.add_tag("MIME_BAD");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
|
||||
// Analyze attachment name
|
||||
if let Some(attach_name) = part.attachment_name() {
|
||||
if attach_name.chars().any(|c| c.is_obscured()) {
|
||||
// Attachment name contains zero-width space
|
||||
ctx.result.add_tag("MIME_BAD_UNICODE");
|
||||
}
|
||||
let attach_name = attach_name.trim().to_lowercase();
|
||||
if let Some((name, ext)) = attach_name.rsplit_once('.').and_then(|(name, ext)| {
|
||||
Some((name, self.core.spam.list_file_extensions.get(ext)?))
|
||||
}) {
|
||||
let sub_ext = name
|
||||
.rsplit_once('.')
|
||||
.and_then(|(_, ext)| self.core.spam.list_file_extensions.get(ext));
|
||||
|
||||
if ext.is_bad {
|
||||
// Attachment has a bad extension
|
||||
if sub_ext.map_or(false, |e| e.is_bad) {
|
||||
ctx.result.add_tag("MIME_DOUBLE_BAD_EXTENSION");
|
||||
} else {
|
||||
ctx.result.add_tag("MIME_BAD_EXTENSION");
|
||||
}
|
||||
}
|
||||
|
||||
if ext.is_archive && sub_ext.map_or(false, |e| e.is_archive) {
|
||||
// Archive in archive
|
||||
ctx.result.add_tag("MIME_ARCHIVE_IN_ARCHIVE");
|
||||
}
|
||||
|
||||
if !ext.known_types.is_empty()
|
||||
&& ct_full != "application/octet-stream"
|
||||
&& !ext.known_types.contains(&ct_full)
|
||||
{
|
||||
// Invalid attachment mime type
|
||||
ctx.result.add_tag("MIME_BAD_ATTACHMENT");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match num_parts_size {
|
||||
0 => {
|
||||
// Message contains no parts
|
||||
ctx.result.add_tag("COMPLETELY_EMPTY");
|
||||
}
|
||||
1..64 if num_parts == 1 => {
|
||||
// Message contains only one short part
|
||||
ctx.result.add_tag("SINGLE_SHORT_PART");
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
|
||||
if has_text_part && (is_encrypted_pgp || is_encrypted_smime) {
|
||||
// Message contains both text and encrypted parts
|
||||
ctx.result.add_tag("BOGUS_ENCRYPTED_AND_TEXT");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn words_and_uris<'x, T: AsRef<str>>(
|
||||
tokens: &'x [TokenType<T>],
|
||||
words: &mut Vec<&'x str>,
|
||||
uri_count: &mut usize,
|
||||
) {
|
||||
let mut uris = HashSet::new();
|
||||
|
||||
for token in tokens {
|
||||
match token {
|
||||
TokenType::Alphabetic(v) | TokenType::Alphanumeric(v) => {
|
||||
words.push(v.as_ref());
|
||||
}
|
||||
TokenType::Url(v) => {
|
||||
if let Some(host) = v
|
||||
.as_ref()
|
||||
.parse::<Uri>()
|
||||
.ok()
|
||||
.and_then(|uri| uri.host().map(Hostname::new))
|
||||
{
|
||||
uris.insert(host.sld.unwrap_or(host.fqdn));
|
||||
}
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
|
||||
*uri_count = uris.len();
|
||||
}
|
||||
@@ -19,9 +19,12 @@ pub mod domain;
|
||||
pub mod ehlo;
|
||||
pub mod from;
|
||||
pub mod headers;
|
||||
pub mod html;
|
||||
pub mod init;
|
||||
pub mod ip;
|
||||
pub mod messageid;
|
||||
pub mod mime;
|
||||
pub mod pyzor;
|
||||
pub mod received;
|
||||
pub mod recipient;
|
||||
pub mod replyto;
|
||||
|
||||
35
crates/spam-filter/src/analysis/pyzor.rs
Normal file
35
crates/spam-filter/src/analysis/pyzor.rs
Normal file
@@ -0,0 +1,35 @@
|
||||
use std::future::Future;
|
||||
|
||||
use common::Server;
|
||||
|
||||
use crate::{modules::pyzor::pyzor_check, SpamFilterContext};
|
||||
|
||||
pub trait SpamFilterAnalyzePyzor: Sync + Send {
|
||||
fn spam_filter_analyze_pyzor(
|
||||
&self,
|
||||
ctx: &mut SpamFilterContext<'_>,
|
||||
) -> impl Future<Output = ()> + Send;
|
||||
}
|
||||
|
||||
impl SpamFilterAnalyzePyzor for Server {
|
||||
async fn spam_filter_analyze_pyzor(&self, ctx: &mut SpamFilterContext<'_>) {
|
||||
if let Some(config) = &self.core.spam.pyzor {
|
||||
match pyzor_check(ctx.input.message, config).await {
|
||||
Ok(Some(result)) => {
|
||||
if result.code == 200
|
||||
&& result.count > config.min_count
|
||||
&& (result.wl_count < config.min_wl_count
|
||||
|| (result.wl_count as f64 / result.count as f64) < config.ratio)
|
||||
{
|
||||
ctx.result.add_tag("PYZOR");
|
||||
}
|
||||
let todo = "log time";
|
||||
}
|
||||
Ok(None) => {}
|
||||
Err(err) => {
|
||||
trc::error!(err.span_id(ctx.input.span_id));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -438,7 +438,7 @@ fn is_single_html_url<T: AsRef<str>>(html_tokens: &[HtmlToken], tokens: &[TokenT
|
||||
url_count = 0;
|
||||
|
||||
for token in html_tokens {
|
||||
if matches!(token, HtmlToken::StartTag { name, attributes } if *name == A && attributes.iter().any(|(k, _)| *k == HREF))
|
||||
if matches!(token, HtmlToken::StartTag { name, attributes, .. } if *name == A && attributes.iter().any(|(k, _)| *k == HREF))
|
||||
{
|
||||
url_count += 1;
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ pub enum HtmlToken {
|
||||
StartTag {
|
||||
name: u64,
|
||||
attributes: Vec<(u64, Option<String>)>,
|
||||
is_self_closing: bool,
|
||||
},
|
||||
EndTag {
|
||||
name: u64,
|
||||
@@ -18,10 +19,46 @@ pub enum HtmlToken {
|
||||
}
|
||||
|
||||
pub(crate) const A: u64 = b'a' as u64;
|
||||
pub(crate) const IMG: u64 = (b'i' as u64) | (b'm' as u64) << 8 | (b'g' as u64) << 16;
|
||||
pub(crate) const HEAD: u64 =
|
||||
(b'h' as u64) | (b'e' as u64) << 8 | (b'a' as u64) << 16 | (b'd' as u64) << 24;
|
||||
pub(crate) const BODY: u64 =
|
||||
(b'b' as u64) | (b'o' as u64) << 8 | (b'd' as u64) << 16 | (b'y' as u64) << 24;
|
||||
pub(crate) const META: u64 =
|
||||
(b'm' as u64) | (b'e' as u64) << 8 | (b't' as u64) << 16 | (b'a' as u64) << 24;
|
||||
pub(crate) const LINK: u64 =
|
||||
(b'l' as u64) | (b'i' as u64) << 8 | (b'n' as u64) << 16 | (b'k' as u64) << 24;
|
||||
|
||||
pub(crate) const HREF: u64 =
|
||||
(b'h' as u64) | (b'r' as u64) << 8 | (b'e' as u64) << 16 | (b'f' as u64) << 24;
|
||||
pub(crate) const SRC: u64 = (b's' as u64) | (b'r' as u64) << 8 | (b'c' as u64) << 16;
|
||||
pub(crate) const WIDTH: u64 = (b'w' as u64)
|
||||
| (b'i' as u64) << 8
|
||||
| (b'd' as u64) << 16
|
||||
| (b't' as u64) << 24
|
||||
| (b'h' as u64) << 32;
|
||||
pub(crate) const HEIGHT: u64 = (b'h' as u64)
|
||||
| (b'e' as u64) << 8
|
||||
| (b'i' as u64) << 16
|
||||
| (b'g' as u64) << 24
|
||||
| (b'h' as u64) << 32
|
||||
| (b't' as u64) << 40;
|
||||
pub(crate) const REL: u64 = (b'r' as u64) | (b'e' as u64) << 8 | (b'l' as u64) << 16;
|
||||
pub(crate) const CONTENT: u64 = (b'c' as u64)
|
||||
| (b'o' as u64) << 8
|
||||
| (b'n' as u64) << 16
|
||||
| (b't' as u64) << 24
|
||||
| (b'e' as u64) << 32
|
||||
| (b'n' as u64) << 40
|
||||
| (b't' as u64) << 48;
|
||||
pub(crate) const HTTP_EQUIV: u64 = (b'h' as u64)
|
||||
| (b't' as u64) << 8
|
||||
| (b't' as u64) << 16
|
||||
| (b'p' as u64) << 24
|
||||
| (b'-' as u64) << 32
|
||||
| (b'e' as u64) << 40
|
||||
| (b'q' as u64) << 48
|
||||
| (b'u' as u64) << 56;
|
||||
|
||||
pub fn html_to_tokens(input: &str) -> Vec<HtmlToken> {
|
||||
let input = input.as_bytes();
|
||||
@@ -106,6 +143,7 @@ pub fn html_to_tokens(input: &str) -> Vec<HtmlToken> {
|
||||
}
|
||||
|
||||
let mut in_quote = false;
|
||||
let mut is_self_closing = false;
|
||||
|
||||
let mut key: u64 = 0;
|
||||
let mut shift = 0;
|
||||
@@ -123,6 +161,9 @@ pub fn html_to_tokens(input: &str) -> Vec<HtmlToken> {
|
||||
key |= ((ch - b'A' + b'a') as u64) << shift;
|
||||
shift += 8;
|
||||
}
|
||||
b'/' if !in_quote => {
|
||||
is_self_closing = true;
|
||||
}
|
||||
b'>' if !in_quote => {
|
||||
if shift != 0 {
|
||||
if tag == 0 {
|
||||
@@ -205,6 +246,7 @@ pub fn html_to_tokens(input: &str) -> Vec<HtmlToken> {
|
||||
tags.push(HtmlToken::StartTag {
|
||||
name: tag,
|
||||
attributes,
|
||||
is_self_closing,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -292,7 +334,8 @@ mod tests {
|
||||
tokens,
|
||||
vec![HtmlToken::StartTag {
|
||||
name: 7760228,
|
||||
attributes: vec![]
|
||||
attributes: vec![],
|
||||
is_self_closing: false
|
||||
}]
|
||||
);
|
||||
}
|
||||
@@ -325,14 +368,16 @@ mod tests {
|
||||
vec![
|
||||
HtmlToken::StartTag {
|
||||
name: 7760228,
|
||||
attributes: vec![]
|
||||
attributes: vec![],
|
||||
is_self_closing: false
|
||||
},
|
||||
HtmlToken::Text {
|
||||
text: "Hello,".to_string()
|
||||
},
|
||||
HtmlToken::StartTag {
|
||||
name: 1851879539,
|
||||
attributes: vec![]
|
||||
attributes: vec![],
|
||||
is_self_closing: false
|
||||
},
|
||||
HtmlToken::Text {
|
||||
text: " \" world \"".to_string()
|
||||
@@ -358,15 +403,18 @@ mod tests {
|
||||
attributes: vec![
|
||||
(1701869940, Some("text".to_string())),
|
||||
(435761734006, Some("test".to_string()))
|
||||
]
|
||||
],
|
||||
is_self_closing: false
|
||||
},
|
||||
HtmlToken::StartTag {
|
||||
name: 111516266162547,
|
||||
attributes: vec![]
|
||||
attributes: vec![],
|
||||
is_self_closing: true
|
||||
},
|
||||
HtmlToken::StartTag {
|
||||
name: 6647407,
|
||||
attributes: vec![(1920234593, None)]
|
||||
attributes: vec![(1920234593, None)],
|
||||
is_self_closing: true
|
||||
},
|
||||
HtmlToken::StartTag {
|
||||
name: 97,
|
||||
@@ -374,7 +422,8 @@ mod tests {
|
||||
(98, Some("1".to_string())),
|
||||
(98, None),
|
||||
(99, Some("123".to_string()))
|
||||
]
|
||||
],
|
||||
is_self_closing: false
|
||||
}
|
||||
]
|
||||
);
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
pub mod dnsbl;
|
||||
pub mod html;
|
||||
pub mod pyzor;
|
||||
pub mod remote_list;
|
||||
pub mod sanitize;
|
||||
|
||||
@@ -4,16 +4,14 @@
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use sieve::{runtime::Variable, FunctionMap};
|
||||
|
||||
use super::PluginContext;
|
||||
|
||||
use std::{
|
||||
borrow::Cow,
|
||||
io::Write,
|
||||
net::SocketAddr,
|
||||
time::{Duration, SystemTime},
|
||||
};
|
||||
|
||||
use common::config::spamfilter::PyzorConfig;
|
||||
use mail_parser::{decoders::html::add_html_token, Message, PartType};
|
||||
use nlp::tokenizers::types::{TokenType, TypesTokenizer};
|
||||
use sha1::{Digest, Sha1};
|
||||
@@ -24,29 +22,27 @@ const ATOMIC_NUM_LINES: usize = 4;
|
||||
const DIGEST_SPEC: &[(usize, usize)] = &[(20, 3), (60, 3)];
|
||||
|
||||
#[derive(Default, Debug, PartialEq, Eq)]
|
||||
struct PyzorResponse {
|
||||
code: u32,
|
||||
count: u64,
|
||||
wl_count: u64,
|
||||
pub(crate) struct PyzorResponse {
|
||||
pub code: u32,
|
||||
pub count: u64,
|
||||
pub wl_count: u64,
|
||||
}
|
||||
|
||||
pub fn register(plugin_id: u32, fnc_map: &mut FunctionMap) {
|
||||
fnc_map.set_external_function("pyzor_check", plugin_id, 2);
|
||||
}
|
||||
|
||||
pub async fn exec(ctx: PluginContext<'_>) -> trc::Result<Variable> {
|
||||
pub(crate) async fn pyzor_check(
|
||||
message: &Message<'_>,
|
||||
config: &PyzorConfig,
|
||||
) -> trc::Result<Option<PyzorResponse>> {
|
||||
// Make sure there is at least one text part
|
||||
if !ctx
|
||||
.message
|
||||
if !message
|
||||
.parts
|
||||
.iter()
|
||||
.any(|p| matches!(p.body, PartType::Text(_) | PartType::Html(_)))
|
||||
{
|
||||
return Ok(Variable::default());
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
// Hash message
|
||||
let request = ctx.message.pyzor_check_message();
|
||||
let request = message.pyzor_check_message();
|
||||
|
||||
#[cfg(feature = "test_mode")]
|
||||
{
|
||||
@@ -74,35 +70,21 @@ pub async fn exec(ctx: PluginContext<'_>) -> trc::Result<Variable> {
|
||||
}
|
||||
}
|
||||
|
||||
let address = ctx.arguments[0].to_string();
|
||||
let timeout = Duration::from_secs((ctx.arguments[1].to_integer() as u64).clamp(5, 60));
|
||||
|
||||
// Send message to address
|
||||
pyzor_send_message(address.as_ref(), timeout, &request)
|
||||
pyzor_send_message(config.address, config.timeout, &request)
|
||||
.await
|
||||
.map(Into::into)
|
||||
.map_err(|err| {
|
||||
trc::SpamEvent::PyzorError
|
||||
.into_err()
|
||||
.ctx(trc::Key::Url, address.to_string())
|
||||
.ctx(trc::Key::Url, config.address.to_string())
|
||||
.reason(err)
|
||||
.details("Pyzor failed")
|
||||
})
|
||||
}
|
||||
|
||||
impl From<PyzorResponse> for Variable {
|
||||
fn from(response: PyzorResponse) -> Self {
|
||||
vec![
|
||||
Variable::from(response.code),
|
||||
Variable::from(response.count),
|
||||
Variable::from(response.wl_count),
|
||||
]
|
||||
.into()
|
||||
}
|
||||
}
|
||||
|
||||
async fn pyzor_send_message(
|
||||
addr: &str,
|
||||
addr: SocketAddr,
|
||||
timeout: Duration,
|
||||
message: &str,
|
||||
) -> std::io::Result<PyzorResponse> {
|
||||
@@ -451,7 +433,7 @@ mod test {
|
||||
async fn send_message() {
|
||||
assert_eq!(
|
||||
pyzor_send_message(
|
||||
"public.pyzor.org:24441",
|
||||
"public.pyzor.org:24441".parse().unwrap(),
|
||||
Duration::from_secs(10),
|
||||
concat!(
|
||||
"Op: check\n",
|
||||
|
||||
Reference in New Issue
Block a user