Port Spam filter to Rust - part 5
This commit is contained in:
mdecimus
@@ -1,9 +0,0 @@
|
||||
|
||||
set "triplet" "g:${env.remote_ip}.${envelope.from}.${envelope.to}";
|
||||
|
||||
if eval "!key_exists(SPAM_DB, triplet)" {
|
||||
# Greylist sender for 30 days
|
||||
eval "key_set(SPAM_DB, triplet, '', 2592000)";
|
||||
reject "422 4.2.2 Greylisted, please try again in a few moments.";
|
||||
stop;
|
||||
}
|
||||
@@ -1,148 +0,0 @@
|
||||
|
||||
# Message only has text/html MIME parts
|
||||
if eval "header.content-type == 'text/html'" {
|
||||
let "t.MIME_HTML_ONLY" "1";
|
||||
}
|
||||
|
||||
foreverypart {
|
||||
if eval "eq_ignore_case(header.content-type, 'text/html')" {
|
||||
# Tokenize HTML
|
||||
let "is_body_part" "is_body()";
|
||||
let "html_tokens" "tokenize(part.text, 'html')";
|
||||
let "html_tokens_len" "len(html_tokens)";
|
||||
let "html_char_count" "0";
|
||||
let "html_space_count" "0";
|
||||
let "html_img_words" "0";
|
||||
let "html_words" "0";
|
||||
let "has_link_to_img" "0";
|
||||
let "has_uri" "0";
|
||||
let "has_text" "0";
|
||||
let "in_head" "0";
|
||||
let "in_body" "0";
|
||||
let "in_anchor" "0";
|
||||
let "in_anchor_href_ip" "0";
|
||||
let "in_anchor_href" "";
|
||||
|
||||
let "i" "0";
|
||||
while "i < html_tokens_len" {
|
||||
let "token" "html_tokens[i]";
|
||||
let "i" "i + 1";
|
||||
|
||||
# Tokens starting with '_' are text nodes
|
||||
if eval "starts_with(token, '_')" {
|
||||
if eval "in_head == 0" {
|
||||
let "html_char_count" "html_char_count + count_chars(token)";
|
||||
let "html_space_count" "html_space_count + count_spaces(token)";
|
||||
|
||||
let "text" "to_lowercase(trim(strip_prefix(token, '_')))";
|
||||
let "html_words" "html_words + len(tokenize(text, 'words'))";
|
||||
|
||||
let "uris" "tokenize(text, 'uri')";
|
||||
|
||||
if eval "!is_empty(uris)" {
|
||||
let "has_uri" "1";
|
||||
let "uri" "uris[0]";
|
||||
|
||||
if eval "in_anchor && !is_empty(in_anchor_href)" {
|
||||
if eval "contains(text, '://') &&
|
||||
uri_part(uri, 'scheme') != uri_part(in_anchor_href, 'scheme')" {
|
||||
# The anchor text contains a distinct scheme compared to the target URL
|
||||
let "t.HTTP_TO_HTTPS" "1";
|
||||
}
|
||||
if eval "(!in_anchor_href_ip && (domain_part(uri_part(uri, 'host'), 'sld') != domain_part(uri_part(in_anchor_href, 'host'), 'sld'))) ||
|
||||
(in_anchor_href_ip && (uri_part(uri, 'host') != uri_part(in_anchor_href, 'host')))" {
|
||||
let "t.PHISHING" "1";
|
||||
}
|
||||
}
|
||||
} elsif eval "!is_empty(text)" {
|
||||
let "has_text" "1";
|
||||
}
|
||||
}
|
||||
} elsif eval "starts_with(token, '<img')" {
|
||||
if eval "is_body_part" {
|
||||
let "dimensions" "html_attr_size(token, 'width', 800) + html_attr_size(token, 'height', 600)";
|
||||
|
||||
if eval "in_anchor && dimensions >= 210" {
|
||||
let "has_link_to_img" "1";
|
||||
}
|
||||
if eval "dimensions > 100" {
|
||||
# We assume that a single picture 100x200 contains approx 3 words of text
|
||||
let "html_img_words" "html_img_words + dimensions / 100";
|
||||
}
|
||||
|
||||
let "img_src" "html_attr(token, 'src')";
|
||||
if eval "starts_with(img_src, 'data:') && contains(img_src, ';base64,')" {
|
||||
# Has Data URI encoding
|
||||
let "t.HAS_DATA_URI" "1";
|
||||
}
|
||||
}
|
||||
} elsif eval "starts_with(token, '<head')" {
|
||||
let "in_head" "in_head + 1";
|
||||
} elsif eval "starts_with(token, '</head')" {
|
||||
let "in_head" "in_head - 1";
|
||||
} elsif eval "starts_with(token, '<body')" {
|
||||
let "in_body" "in_body + 1";
|
||||
} elsif eval "starts_with(token, '</body')" {
|
||||
let "in_body" "in_body - 1";
|
||||
} elsif eval "starts_with(token, '<a ')" {
|
||||
let "in_anchor" "1";
|
||||
let "in_anchor_href_ip" "0";
|
||||
let "in_anchor_href" "to_lowercase(trim(html_attr(token, 'href')))";
|
||||
|
||||
if eval "is_body_part && starts_with(in_anchor_href, 'data:') && contains(in_anchor_href, ';base64,')" {
|
||||
# Has Data URI encoding
|
||||
let "t.HAS_DATA_URI" "1";
|
||||
if eval "contains(in_anchor_href, 'text/')" {
|
||||
# Uses Data URI encoding to obfuscate plain or HTML in base64
|
||||
let "t.DATA_URI_OBFU" "1";
|
||||
}
|
||||
} elsif eval "is_ip_addr(uri_part(in_anchor_href, 'host'))" {
|
||||
# HTML anchor points to an IP address
|
||||
let "t.HTTP_TO_IP" "1";
|
||||
let "in_anchor_href_ip" "1";
|
||||
}
|
||||
} elsif eval "in_anchor && starts_with(token, '</a')" {
|
||||
let "in_anchor" "0";
|
||||
} elsif eval "starts_with(token, '<meta ')" {
|
||||
if eval "eq_ignore_case(html_attr(token, 'http-equiv'), 'refresh') &&
|
||||
contains_ignore_case(html_attr(token, 'content'), 'url=')" {
|
||||
# HTML meta refresh tag
|
||||
let "t.HTML_META_REFRESH_URL" "1";
|
||||
}
|
||||
} elsif eval "starts_with(token, '<link') && is_body_part &&
|
||||
(contains_ignore_case(html_attr(token, 'rel'), 'stylesheet') ||
|
||||
contains_ignore_case(html_attr(token, 'href'), '.css') )" {
|
||||
let "t.EXT_CSS" "1";
|
||||
}
|
||||
}
|
||||
|
||||
if eval "is_body_part" {
|
||||
# Check for unbalanced tags
|
||||
if eval "in_head != 0 || in_body != 0" {
|
||||
let "t.HTML_UNBALANCED_TAG" "1";
|
||||
}
|
||||
|
||||
# Check for short HTML parts with a link to an image
|
||||
if eval "has_link_to_img" {
|
||||
if eval "html_char_count < 1024" {
|
||||
let "t.HTML_SHORT_LINK_IMG_1" "1";
|
||||
} elsif eval "html_char_count < 1536" {
|
||||
let "t.HTML_SHORT_LINK_IMG_2" "1";
|
||||
} elsif eval "html_char_count < 2048" {
|
||||
let "t.HTML_SHORT_LINK_IMG_3" "1";
|
||||
}
|
||||
}
|
||||
|
||||
if eval "(!has_link_to_img || html_char_count >= 2048) &&
|
||||
(html_img_words / (html_words + html_img_words) > 0.5)" {
|
||||
# Message contains more images than text
|
||||
let "t.HTML_TEXT_IMG_RATIO" "1";
|
||||
}
|
||||
|
||||
if eval "has_uri && !has_text" {
|
||||
let "t.BODY_URI_ONLY" "1";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,232 +0,0 @@
|
||||
if eval "!header.mime-version.exists" {
|
||||
if eval "header.content-type.exists || header.content-transfer-encoding.exists" {
|
||||
let "t.MISSING_MIME_VERSION" "1";
|
||||
}
|
||||
} elsif eval "header.mime-version.raw_name != 'MIME-Version'" {
|
||||
let "t.MV_CASE" "1";
|
||||
}
|
||||
|
||||
let "has_text_part" "0";
|
||||
let "is_encrypted" "0";
|
||||
let "parts_num" "0";
|
||||
let "parts_max_len" "0";
|
||||
|
||||
if eval "header.Content-Type.exists && !header.Content-Disposition:Content-Transfer-Encoding:MIME-Version.exists && !eq_ignore_case(header.Content-Type, 'text/plain')" {
|
||||
# Only Content-Type header without other MIME headers
|
||||
let "t.MIME_HEADER_CTYPE_ONLY" "1";
|
||||
}
|
||||
|
||||
foreverypart {
|
||||
let "content_type" "to_lowercase(header.content-type)";
|
||||
let "type" "to_lowercase(header.content-type.type)";
|
||||
let "subtype" "to_lowercase(header.content-type.subtype)";
|
||||
let "cte" "header.content-transfer-encoding";
|
||||
let "part_is_attachment" "is_attachment()";
|
||||
|
||||
if eval "cte != '' && !is_lowercase(cte)" {
|
||||
let "cte" "to_lowercase(cte)";
|
||||
let "t.CTE_CASE" "1";
|
||||
}
|
||||
|
||||
if eval "ends_with(header.content-type.raw, ';')" {
|
||||
# Content-Type header ends with a semi-colon
|
||||
let "t.CT_EXTRA_SEMI" "1";
|
||||
}
|
||||
|
||||
if eval "type == 'multipart'" {
|
||||
if eval "subtype == 'alternative'" {
|
||||
let "has_plain_part" "0";
|
||||
let "has_html_part" "0";
|
||||
|
||||
let "text_part_words" "";
|
||||
let "text_part_uris" "0";
|
||||
|
||||
let "html_part_words" "";
|
||||
let "html_part_uris" "0";
|
||||
|
||||
foreverypart {
|
||||
let "ma_ct" "to_lowercase(header.content-type)";
|
||||
|
||||
if eval "!has_plain_part && ma_ct == 'text/plain'" {
|
||||
let "text_part" "part.text";
|
||||
let "text_part_words" "tokenize(text_part, 'words')";
|
||||
let "text_part_uris" "count(dedup(uri_part(tokenize(text_part, 'uri_strict'), 'host')))";
|
||||
let "has_plain_part" "1";
|
||||
} elsif eval "!has_html_part && ma_ct == 'text/html'" {
|
||||
let "html_part" "html_to_text(part.text)";
|
||||
let "html_part_words" "tokenize(html_part, 'words')";
|
||||
let "html_part_uris" "count(dedup(uri_part(tokenize(part.text, 'uri_strict'), 'host')))";
|
||||
let "has_html_part" "1";
|
||||
}
|
||||
}
|
||||
|
||||
# Multipart message mostly text/html MIME
|
||||
if eval "has_html_part" {
|
||||
if eval "!has_plain_part" {
|
||||
let "t.MIME_MA_MISSING_TEXT" "1";
|
||||
}
|
||||
} elsif eval "has_plain_part" {
|
||||
let "t.MIME_MA_MISSING_HTML" "1";
|
||||
}
|
||||
|
||||
# HTML and text parts are different
|
||||
if eval "!t.R_PARTS_DIFFER && has_html_part && has_plain_part &&
|
||||
(!is_empty(text_part_words) || !is_empty(html_part_words)) &&
|
||||
cosine_similarity(text_part_words, html_part_words) < 0.95" {
|
||||
let "t.R_PARTS_DIFFER" "1";
|
||||
}
|
||||
|
||||
# Odd URI count between parts
|
||||
if eval "text_part_uris != html_part_uris" {
|
||||
set "t.URI_COUNT_ODD" "1";
|
||||
}
|
||||
} elsif eval "subtype == 'mixed'" {
|
||||
let "num_text_parts" "0";
|
||||
let "has_other_part" "0";
|
||||
|
||||
foreverypart {
|
||||
if eval "eq_ignore_case(header.content-type.type, 'text') && !is_attachment()" {
|
||||
let "num_text_parts" "num_text_parts + 1";
|
||||
} elsif eval "!eq_ignore_case(header.content-type.type, 'multipart')" {
|
||||
let "has_other_part" "1";
|
||||
}
|
||||
}
|
||||
|
||||
# Found multipart/mixed without non-textual part
|
||||
if eval "!has_other_part && num_text_parts < 3" {
|
||||
let "t.CTYPE_MIXED_BOGUS" "1";
|
||||
}
|
||||
} elsif eval "subtype == 'encrypted'" {
|
||||
set "is_encrypted" "1";
|
||||
}
|
||||
} else {
|
||||
if eval "type == 'text'" {
|
||||
# MIME text part claims to be ASCII but isn't
|
||||
if eval "cte == '' || cte == '7bit'" {
|
||||
if eval "!is_ascii(part.raw)" {
|
||||
let "t.R_BAD_CTE_7BIT" "1";
|
||||
}
|
||||
} else {
|
||||
if eval "cte == 'base64'" {
|
||||
if eval "is_ascii(part.text)" {
|
||||
# Has text part encoded in base64 that does not contain any 8bit characters
|
||||
let "t.MIME_BASE64_TEXT_BOGUS" "1";
|
||||
} else {
|
||||
# Has text part encoded in base64
|
||||
let "t.MIME_BASE64_TEXT" "1";
|
||||
}
|
||||
}
|
||||
|
||||
if eval "subtype == 'plain' && is_empty(header.content-type.attr.charset)" {
|
||||
# Charset header is missing
|
||||
let "t.R_MISSING_CHARSET" "1";
|
||||
}
|
||||
}
|
||||
let "has_text_part" "1";
|
||||
} elsif eval "type == 'application'" {
|
||||
if eval "subtype == 'pkcs7-mime'" {
|
||||
let "t.ENCRYPTED_SMIME" "1";
|
||||
let "part_is_attachment" "0";
|
||||
} elsif eval "subtype == 'pkcs7-signature'" {
|
||||
let "t.SIGNED_SMIME" "1";
|
||||
let "part_is_attachment" "0";
|
||||
} elsif eval "subtype == 'pgp-encrypted'" {
|
||||
let "t.ENCRYPTED_PGP" "1";
|
||||
let "part_is_attachment" "0";
|
||||
} elsif eval "subtype == 'pgp-signature'" {
|
||||
let "t.SIGNED_PGP" "1";
|
||||
let "part_is_attachment" "0";
|
||||
} elsif eval "subtype == 'octet-stream'" {
|
||||
if eval "!is_encrypted &&
|
||||
!header.content-id.exists &&
|
||||
(!header.content-disposition.exists ||
|
||||
(!eq_ignore_case(header.content-disposition.type, 'attachment') &&
|
||||
is_empty(header.content-disposition.attr.filename)))" {
|
||||
let "t.CTYPE_MISSING_DISPOSITION" "1";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Increase part count
|
||||
let "parts_num" "parts_num + 1";
|
||||
if eval "parts_num == 1" {
|
||||
let "parts_len" "mime_part_len()";
|
||||
if eval "parts_len > parts_max_len" {
|
||||
let "parts_max_len" "parts_len";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if eval "is_empty(type) && header.content-type.exists" {
|
||||
let "t.BROKEN_CONTENT_TYPE" "1";
|
||||
}
|
||||
|
||||
if eval "part_is_attachment" {
|
||||
# Has a MIME attachment
|
||||
let "t.HAS_ATTACHMENT" "1";
|
||||
|
||||
# Detect and compare mime type
|
||||
let "detected_mime_type" "detect_file_type('mime')";
|
||||
if eval "!is_empty(detected_mime_type)" {
|
||||
if eval "detected_mime_type == content_type" {
|
||||
# Known content-type
|
||||
let "t.MIME_GOOD" "1";
|
||||
} elsif eval "content_type != 'application/octet-stream'" {
|
||||
# Known bad content-type
|
||||
let "t.MIME_BAD" "1";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Analyze attachment name
|
||||
let "attach_name" "attachment_name()";
|
||||
if eval "!is_empty(attach_name)" {
|
||||
if eval "has_obscured(attach_name)" {
|
||||
let "t.MIME_BAD_UNICODE" "1";
|
||||
}
|
||||
let "name_parts" "rsplit(to_lowercase(attach_name), '.')";
|
||||
if eval "count(name_parts) > 1" {
|
||||
let "ext_type" "key_get('spam-mime', name_parts[0])";
|
||||
if eval "!is_empty(ext_type)" {
|
||||
let "ext_type_double" "key_get('spam-mime', name_parts[1])";
|
||||
if eval "contains(ext_type, 'BAD')" {
|
||||
# Bad extension
|
||||
if eval "contains(ext_type_double, 'BAD')" {
|
||||
let "t.MIME_DOUBLE_BAD_EXTENSION" "1";
|
||||
} else {
|
||||
let "t.MIME_BAD_EXTENSION" "1";
|
||||
}
|
||||
}
|
||||
if eval "contains(ext_type, 'AR') && contains(ext_type_double, 'AR')" {
|
||||
# Archive in archive
|
||||
let "t.MIME_ARCHIVE_IN_ARCHIVE" "1";
|
||||
}
|
||||
|
||||
if eval "contains(ext_type, '/') &&
|
||||
content_type != 'application/octet-stream' &&
|
||||
!contains(split(ext_type, '|'), content_type)" {
|
||||
# Invalid attachment mime type
|
||||
let "t.MIME_BAD_ATTACHMENT" "1";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
# Message contains both text and encrypted parts
|
||||
if eval "has_text_part && (t.ENCRYPTED_SMIME || t.ENCRYPTED_PGP)" {
|
||||
let "t.BOGUS_ENCRYPTED_AND_TEXT" "1";
|
||||
}
|
||||
|
||||
# Message contains only one short part
|
||||
if eval "parts_num == 1 && parts_max_len < 64" {
|
||||
let "t.SINGLE_SHORT_PART" "1";
|
||||
} elsif eval "parts_max_len == 0" {
|
||||
let "t.COMPLETELY_EMPTY" "1";
|
||||
}
|
||||
|
||||
# Check for mixed script in body
|
||||
if eval "!is_single_script(text_body)" {
|
||||
let "t.R_MIXED_CHARSET" "1";
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
# Check message hash against Pyzor on public.pyzor.org:24441 using a 5 second timeout
|
||||
let "pyzor_response" "pyzor_check('public.pyzor.org:24441', 5)";
|
||||
|
||||
if eval "!is_empty(pyzor_response) && pyzor_response[0] == 200" {
|
||||
let "count" "pyzor_response[1]";
|
||||
let "wl_count" "pyzor_response[2]";
|
||||
|
||||
if eval "count > 5 && (wl_count < 10 || wl_count / count < 0.2)" {
|
||||
let "t.PYZOR" "1";
|
||||
}
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
|
||||
|
||||
# Obtain thread name and subject
|
||||
let "contents" "thread_name(header.subject) + ' ' + body.to_text";
|
||||
|
||||
if eval "env.train == 'spam'" {
|
||||
eval "bayes_train(SPAM_DB, contents, true)";
|
||||
} elsif eval "env.train == 'ham'" {
|
||||
eval "bayes_train(SPAM_DB, contents, false)";
|
||||
} else {
|
||||
reject "Missing variable 'train'";
|
||||
}
|
||||
Reference in New Issue
Block a user