/* * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd * * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ use sieve::{runtime::Variable, FunctionMap}; use super::PluginContext; use std::{ borrow::Cow, io::Write, time::{Duration, SystemTime}, }; use mail_parser::{decoders::html::add_html_token, Message, PartType}; use nlp::tokenizers::types::{TokenType, TypesTokenizer}; use sha1::{Digest, Sha1}; use tokio::net::UdpSocket; const MIN_LINE_LENGTH: usize = 8; const ATOMIC_NUM_LINES: usize = 4; const DIGEST_SPEC: &[(usize, usize)] = &[(20, 3), (60, 3)]; #[derive(Default, Debug, PartialEq, Eq)] struct PyzorResponse { code: u32, count: u64, wl_count: u64, } pub fn register(plugin_id: u32, fnc_map: &mut FunctionMap) { fnc_map.set_external_function("pyzor_check", plugin_id, 2); } pub async fn exec(ctx: PluginContext<'_>) -> trc::Result { // Make sure there is at least one text part if !ctx .message .parts .iter() .any(|p| matches!(p.body, PartType::Text(_) | PartType::Html(_))) { return Ok(Variable::default()); } // Hash message let request = ctx.message.pyzor_check_message(); #[cfg(feature = "test_mode")] { if request.contains("b5b476f0b5ba6e1c038361d3ded5818dd39c90a2") { return Ok(PyzorResponse { code: 200, count: 1000, wl_count: 0, } .into()); } else if request.contains("d67d4b8bfc3860449e3418bb6017e2612f3e2a99") { return Ok(PyzorResponse { code: 200, count: 60, wl_count: 10, } .into()); } else if request.contains("81763547012b75e57a20d18ce0b93014208cdfdb") { return Ok(PyzorResponse { code: 200, count: 50, wl_count: 20, } .into()); } } let address = ctx.arguments[0].to_string(); let timeout = Duration::from_secs((ctx.arguments[1].to_integer() as u64).clamp(5, 60)); // Send message to address pyzor_send_message(address.as_ref(), timeout, &request) .await .map(Into::into) .map_err(|err| { trc::SpamEvent::PyzorError .into_err() .ctx(trc::Key::Url, address.to_string()) .reason(err) .details("Pyzor failed") }) } impl From for Variable { fn from(response: PyzorResponse) -> Self { vec![ Variable::from(response.code), Variable::from(response.count), Variable::from(response.wl_count), ] .into() } } async fn pyzor_send_message( addr: &str, timeout: Duration, message: &str, ) -> std::io::Result { let socket = UdpSocket::bind("0.0.0.0:0").await?; tokio::time::timeout(timeout, socket.send_to(message.as_bytes(), addr)).await??; let mut buffer = vec![0u8; 1024]; let (size, _) = tokio::time::timeout(timeout, socket.recv_from(&mut buffer)).await??; let raw_response = std::str::from_utf8(&buffer[..size]) .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidData, err))?; let mut response = PyzorResponse { code: u32::MAX, count: u64::MAX, wl_count: u64::MAX, }; for line in raw_response.lines() { if let Some((k, v)) = line.split_once(':') { if k.eq_ignore_ascii_case("code") { response.code = v.trim().parse().map_err(|_| { std::io::Error::new( std::io::ErrorKind::InvalidData, format!("Invalid line: {raw_response}"), ) })?; } else if k.eq_ignore_ascii_case("count") { response.count = v.trim().parse().map_err(|_| { std::io::Error::new( std::io::ErrorKind::InvalidData, format!("Invalid line: {raw_response}"), ) })?; } else if k.eq_ignore_ascii_case("wl-count") { response.wl_count = v.trim().parse().map_err(|_| { std::io::Error::new( std::io::ErrorKind::InvalidData, format!("Invalid line: {raw_response}"), ) })?; } } } if response.code != u32::MAX && response.count != u64::MAX && response.wl_count != u64::MAX { Ok(response) } else { Err(std::io::Error::new( std::io::ErrorKind::InvalidData, format!("Invalid response: {raw_response}"), )) } } trait PyzorDigest { fn pyzor_digest(&self, writer: W) -> W; } pub trait PyzorCheck { fn pyzor_check_message(&self) -> String; } impl PyzorDigest for Message<'_> { fn pyzor_digest(&self, writer: W) -> W { let parts = self .parts .iter() .filter_map(|part| match &part.body { PartType::Text(text) => Some(text.as_ref().into()), PartType::Html(html) => Some(html_to_text(html.as_ref()).into()), _ => None, }) .collect::>>(); pyzor_digest(writer, parts.iter().flat_map(|text| text.lines())) } } impl PyzorCheck for Message<'_> { fn pyzor_check_message(&self) -> String { let time = SystemTime::now() .duration_since(SystemTime::UNIX_EPOCH) .map_or(0, |d| d.as_secs()); pyzor_create_message( self, time, (time & 0xFFFF) as u16 ^ ((time >> 16) & 0xFFFF) as u16, ) } } fn pyzor_create_message(message: &Message<'_>, time: u64, thread: u16) -> String { // Hash message let hash = message.pyzor_digest(Sha1::new()).finalize(); // Hash key let mut hash_key = Sha1::new(); hash_key.update("anonymous:".as_bytes()); let hash_key = hash_key.finalize(); // Hash message let message = format!( "Op: check\nOp-Digest: {hash:x}\nThread: {thread}\nPV: 2.1\nUser: anonymous\nTime: {time}" ); let mut msg_hash = Sha1::new(); msg_hash.update(message.as_bytes()); let msg_hash = msg_hash.finalize(); // Sign let mut sig = Sha1::new(); sig.update(msg_hash); sig.update(format!(":{time}:{hash_key:x}")); let sig = sig.finalize(); format!("{message}\nSig: {sig:x}\n") } fn pyzor_digest<'x, I, W>(mut writer: W, lines: I) -> W where I: Iterator, W: Write, { let mut result = Vec::with_capacity(16); for line in lines { let mut clean_line = String::with_capacity(line.len()); let mut token_start = usize::MAX; let mut token_end = usize::MAX; let add_line = |line: &mut String, span: &str| { if !span.contains(char::from(0)) { if span.len() < 10 { line.push_str(span); } } else { let span = span.replace(char::from(0), ""); if span.len() < 10 { line.push_str(&span); } } }; for token in TypesTokenizer::new(line) { match token.word { TokenType::Alphabetic(_) | TokenType::Alphanumeric(_) | TokenType::Integer(_) | TokenType::Float(_) | TokenType::Other(_) | TokenType::Punctuation(_) => { if token_start == usize::MAX { token_start = token.from; } token_end = token.to; } TokenType::Space | TokenType::Url(_) | TokenType::UrlNoScheme(_) | TokenType::UrlNoHost(_) | TokenType::IpAddr(_) | TokenType::Email(_) => { if token_start != usize::MAX { add_line(&mut clean_line, &line[token_start..token_end]); token_start = usize::MAX; token_end = usize::MAX; } } } } if token_start != usize::MAX { add_line(&mut clean_line, &line[token_start..token_end]); } if clean_line.len() >= MIN_LINE_LENGTH { result.push(clean_line); } } if result.len() > ATOMIC_NUM_LINES { for (offset, length) in DIGEST_SPEC { for i in 0..*length { if let Some(line) = result.get((*offset * result.len() / 100) + i) { let _ = writer.write_all(line.as_bytes()); } } } } else { for line in result { let _ = writer.write_all(line.as_bytes()); } } writer } fn html_to_text(input: &str) -> String { let mut result = String::with_capacity(input.len()); let input = input.as_bytes(); let mut in_tag = false; let mut in_comment = false; let mut in_style = false; let mut in_script = false; let mut is_token_start = true; let mut is_after_space = false; let mut is_tag_close = false; let mut token_start = 0; let mut token_end = 0; let mut tag_token_pos = 0; let mut comment_pos = 0; for (pos, ch) in input.iter().enumerate() { if !in_comment { match ch { b'<' => { if !(in_tag || in_style || in_script || is_token_start) { add_html_token( &mut result, &input[token_start..token_end + 1], is_after_space, ); is_after_space = false; } tag_token_pos = 0; in_tag = true; is_token_start = true; is_tag_close = false; continue; } b'>' if in_tag => { if tag_token_pos == 1 { if let Some(tag) = input.get(token_start..token_end + 1) { if tag.eq_ignore_ascii_case(b"style") { in_style = !is_tag_close; } else if tag.eq_ignore_ascii_case(b"script") { in_script = !is_tag_close; } } } in_tag = false; is_token_start = true; is_after_space = !result.is_empty(); continue; } b'/' if in_tag => { if tag_token_pos == 0 { is_tag_close = true; } continue; } b'!' if in_tag && tag_token_pos == 0 => { if let Some(b"--") = input.get(pos + 1..pos + 3) { in_comment = true; continue; } } b' ' | b'\t' | b'\r' | b'\n' => { if !(in_tag || in_style || in_script) { if !is_token_start { add_html_token( &mut result, &input[token_start..token_end + 1], is_after_space, ); } is_after_space = true; } is_token_start = true; continue; } b'&' if !(in_tag || is_token_start || in_style || in_script) => { add_html_token( &mut result, &input[token_start..token_end + 1], is_after_space, ); is_token_start = true; is_after_space = false; } b';' if !(in_tag || is_token_start || in_style || in_script) => { add_html_token(&mut result, &input[token_start..pos + 1], is_after_space); is_token_start = true; is_after_space = false; continue; } _ => (), } if is_token_start { token_start = pos; is_token_start = false; if in_tag { tag_token_pos += 1; } } token_end = pos; } else { match ch { b'-' => comment_pos += 1, b'>' if comment_pos == 2 => { comment_pos = 0; in_comment = false; in_tag = false; is_token_start = true; } _ => comment_pos = 0, } } } if !(in_tag || is_token_start || in_style || in_script) { add_html_token( &mut result, &input[token_start..token_end + 1], is_after_space, ); } result.shrink_to_fit(); result } #[cfg(test)] mod test { use std::time::Duration; use mail_parser::MessageParser; use sha1::Digest; use sha1::Sha1; use super::pyzor_create_message; use super::pyzor_send_message; use super::{html_to_text, pyzor_digest, PyzorDigest}; use super::PyzorResponse; #[ignore] #[tokio::test] async fn send_message() { assert_eq!( pyzor_send_message( "public.pyzor.org:24441", Duration::from_secs(10), concat!( "Op: check\n", "Op-Digest: b2c27325a034c581df0c9ef37e4a0d63208a3e7e\n", "Thread: 49005\n", "PV: 2.1\n", "User: anonymous\n", "Time: 1697468672\n", "Sig: 9cf4571b85d3887fdd0d4f444fd0c164e0290722\n" ), ) .await .unwrap(), PyzorResponse { code: 200, count: 0, wl_count: 0 } ); } #[test] fn message_pyzor() { let message = pyzor_create_message( &MessageParser::new().parse(HTML_TEXT_STYLE_SCRIPT).unwrap(), 1697468672, 49005, ); assert_eq!( message, concat!( "Op: check\n", "Op-Digest: b2c27325a034c581df0c9ef37e4a0d63208a3e7e\n", "Thread: 49005\n", "PV: 2.1\n", "User: anonymous\n", "Time: 1697468672\n", "Sig: 9cf4571b85d3887fdd0d4f444fd0c164e0290722\n" ) ); } #[test] fn digest_pyzor() { // HTML stripping assert_eq!(html_to_text(HTML_RAW), HTML_RAW_STRIPED); // Token stripping for strip_me in [ "t@abc.com", "t1@abc.com", "t+a@abc.com", "t.a@abc.com", "0A2D3f%a#S", "3sddkf9jdkd9", "@@#@@@@@@@@@", "http://spammer.com/special-offers?buy=now", ] { assert_eq!( String::from_utf8(pyzor_digest( Vec::new(), format!("Test {strip_me} Test2").lines(), )) .unwrap(), "TestTest2" ); } // Test short lines assert_eq!( String::from_utf8(pyzor_digest( Vec::new(), concat!("This line is included\n", "not this\n", "This also").lines(), )) .unwrap(), "ThislineisincludedThisalso" ); // Test atomic assert_eq!( String::from_utf8(pyzor_digest( Vec::new(), "All this message\nShould be included\nIn the digest".lines(), )) .unwrap(), "AllthismessageShouldbeincludedInthedigest" ); // Test spec let mut text = String::new(); for i in 0..100 { text += format!("Line{i} test test test\n").as_str(); } let mut expected = String::new(); for i in [20, 21, 22, 60, 61, 62] { expected += format!("Line{i}testtesttest").as_str(); } assert_eq!( String::from_utf8(pyzor_digest(Vec::new(), text.lines(),)).unwrap(), expected ); // Test email parsing for (input, expected) in [ ( HTML_TEXT, concat!( "Emailspam,alsoknownasjunkemailorbulkemail,isasubset", "ofspaminvolvingnearlyidenticalmessagessenttonumerous", "byemail.Clickingonlinksinspamemailmaysendusersto", "byemail.Clickingonlinksinspamemailmaysendusersto", "phishingwebsitesorsitesthatarehostingmalware.", "Emailspam.Emailspam,alsoknownasjunkemailorbulkemail,", "isasubsetofspaminvolvingnearlyidenticalmessage", "ssenttonumerousbyemail.Clickingonlinksinspamemailmaysenduse", "rstophishingwebsitesorsitesthatarehostingmalware." ), ), (HTML_TEXT_STYLE_SCRIPT, "Thisisatest.Thisisatest."), (TEXT_ATTACHMENT, "Thisisatestmailing"), (TEXT_ATTACHMENT_W_NULL, "Thisisatestmailing"), (TEXT_ATTACHMENT_W_MULTIPLE_NULLS, "Thisisatestmailing"), (TEXT_ATTACHMENT_W_SUBJECT_NULL, "Thisisatestmailing"), (TEXT_ATTACHMENT_W_CONTENTTYPE_NULL, "Thisisatestmailing"), ] { assert_eq!( String::from_utf8( MessageParser::new() .parse(input) .unwrap() .pyzor_digest(Vec::new(),) ) .unwrap(), expected, "failed for {input}" ) } // Test SHA hash assert_eq!( format!( "{:x}", MessageParser::new() .parse(HTML_TEXT_STYLE_SCRIPT) .unwrap() .pyzor_digest(Sha1::new(),) .finalize() ), "b2c27325a034c581df0c9ef37e4a0d63208a3e7e", ) } const HTML_TEXT: &str = r#"MIME-Version: 1.0 Sender: chirila@gapps.spamexperts.com Received: by 10.216.157.70 with HTTP; Thu, 16 Jan 2014 00:43:31 -0800 (PST) Date: Thu, 16 Jan 2014 10:43:31 +0200 Delivered-To: chirila@gapps.spamexperts.com X-Google-Sender-Auth: ybCmONS9U9D6ZUfjx-9_tY-hF2Q Message-ID: Subject: Test From: Alexandru Chirila To: Alexandru Chirila Content-Type: multipart/alternative; boundary=001a11c25ff293069304f0126bfd --001a11c25ff293069304f0126bfd Content-Type: text/plain; charset=ISO-8859-1 Email spam. Email spam, also known as junk email or unsolicited bulk email, is a subset of electronic spam involving nearly identical messages sent to numerous recipients by email. Clicking on links in spam email may send users to phishing web sites or sites that are hosting malware. --001a11c25ff293069304f0126bfd Content-Type: text/html; charset=ISO-8859-1 Content-Transfer-Encoding: quoted-printable
Email spam.

Email spam, also= known as junk email or unsolicited bulk email, is a subset of electronic s= pam involving nearly identical messages sent to numerous recipients by emai= l. Clicking on links in spam email may send users to phishing web sites or = sites that are hosting malware.
--001a11c25ff293069304f0126bfd-- "#; const HTML_TEXT_STYLE_SCRIPT: &str = r#"MIME-Version: 1.0 Sender: chirila@gapps.spamexperts.com Received: by 10.216.157.70 with HTTP; Thu, 16 Jan 2014 00:43:31 -0800 (PST) Date: Thu, 16 Jan 2014 10:43:31 +0200 Delivered-To: chirila@gapps.spamexperts.com X-Google-Sender-Auth: ybCmONS9U9D6ZUfjx-9_tY-hF2Q Message-ID: Subject: Test From: Alexandru Chirila To: Alexandru Chirila Content-Type: multipart/alternative; boundary=001a11c25ff293069304f0126bfd --001a11c25ff293069304f0126bfd Content-Type: text/plain; charset=ISO-8859-1 This is a test. --001a11c25ff293069304f0126bfd Content-Type: text/html; charset=ISO-8859-1 Content-Transfer-Encoding: quoted-printable
This is a test.
--001a11c25ff293069304f0126bfd-- "#; const TEXT_ATTACHMENT: &str = r#"MIME-Version: 1.0 Received: by 10.76.127.40 with HTTP; Fri, 17 Jan 2014 02:21:43 -0800 (PST) Date: Fri, 17 Jan 2014 12:21:43 +0200 Delivered-To: chirila.s.alexandru@gmail.com Message-ID: Subject: Test From: Alexandru Chirila To: Alexandru Chirila Content-Type: multipart/mixed; boundary=f46d040a62c49bb1c804f027e8cc --f46d040a62c49bb1c804f027e8cc Content-Type: multipart/alternative; boundary=f46d040a62c49bb1c404f027e8ca --f46d040a62c49bb1c404f027e8ca Content-Type: text/plain; charset=ISO-8859-1 This is a test mailing --f46d040a62c49bb1c404f027e8ca-- --f46d040a62c49bb1c804f027e8cc Content-Type: image/png; name="tar.png" Content-Disposition: attachment; filename="tar.png" Content-Transfer-Encoding: base64 X-Attachment-Id: f_hqjas5ad0 iVBORw0KGgoAAAANSUhEUgAAAskAAADlCAAAAACErzVVAAAACXBIWXMAAAsTAAALEwEAmpwYAAAD QmCC --f46d040a62c49bb1c804f027e8cc--"#; const TEXT_ATTACHMENT_W_NULL: &str = "MIME-Version: 1.0 Received: by 10.76.127.40 with HTTP; Fri, 17 Jan 2014 02:21:43 -0800 (PST) Date: Fri, 17 Jan 2014 12:21:43 +0200 Delivered-To: chirila.s.alexandru@gmail.com Message-ID: Subject: Test From: Alexandru Chirila To: Alexandru Chirila Content-Type: multipart/mixed; boundary=f46d040a62c49bb1c804f027e8cc --f46d040a62c49bb1c804f027e8cc Content-Type: multipart/alternative; boundary=f46d040a62c49bb1c404f027e8ca --f46d040a62c49bb1c404f027e8ca Content-Type: text/plain; charset=ISO-8859-1 This is a test ma\0iling --f46d040a62c49bb1c804f027e8cc--"; const TEXT_ATTACHMENT_W_MULTIPLE_NULLS: &str = "MIME-Version: 1.0 Received: by 10.76.127.40 with HTTP; Fri, 17 Jan 2014 02:21:43 -0800 (PST) Date: Fri, 17 Jan 2014 12:21:43 +0200 Delivered-To: chirila.s.alexandru@gmail.com Message-ID: Subject: Test From: Alexandru Chirila To: Alexandru Chirila Content-Type: multipart/mixed; boundary=f46d040a62c49bb1c804f027e8cc --f46d040a62c49bb1c804f027e8cc Content-Type: multipart/alternative; boundary=f46d040a62c49bb1c404f027e8ca --f46d040a62c49bb1c404f027e8ca Content-Type: text/plain; charset=ISO-8859-1 This is a test ma\0\0\0iling --f46d040a62c49bb1c804f027e8cc--"; const TEXT_ATTACHMENT_W_SUBJECT_NULL: &str = "MIME-Version: 1.0 Received: by 10.76.127.40 with HTTP; Fri, 17 Jan 2014 02:21:43 -0800 (PST) Date: Fri, 17 Jan 2014 12:21:43 +0200 Delivered-To: chirila.s.alexandru@gmail.com Message-ID: Subject: Te\0\0\0st From: Alexandru Chirila To: Alexandru Chirila Content-Type: multipart/mixed; boundary=f46d040a62c49bb1c804f027e8cc --f46d040a62c49bb1c804f027e8cc Content-Type: multipart/alternative; boundary=f46d040a62c49bb1c404f027e8ca --f46d040a62c49bb1c404f027e8ca Content-Type: text/plain; charset=ISO-8859-1 This is a test mailing --f46d040a62c49bb1c804f027e8cc--"; const TEXT_ATTACHMENT_W_CONTENTTYPE_NULL: &str = "MIME-Version: 1.0 Received: by 10.76.127.40 with HTTP; Fri, 17 Jan 2014 02:21:43 -0800 (PST) Date: Fri, 17 Jan 2014 12:21:43 +0200 Delivered-To: chirila.s.alexandru@gmail.com Message-ID: Subject: Test From: Alexandru Chirila To: Alexandru Chirila Content-Type: multipart/mixed; boundary=f46d040a62c49bb1c804f027e8cc --f46d040a62c49bb1c804f027e8cc Content-Type: multipart/alternative; boundary=f46d040a62c49bb1c404f027e8ca --f46d040a62c49bb1c404f027e8ca Content-Type: text/plain; charset=\"iso-8859-1\0\0\0\" This is a test mailing --f46d040a62c49bb1c804f027e8cc--"; const HTML_RAW: &str = r#"Email spam

Email spam, also known as junk email or unsolicited bulk email (UBE), is a subset of electronic spam involving nearly identical messages sent to numerous recipients by email. Clicking on links in spam email may send users to phishing web sites or sites that are hosting malware."#; const HTML_RAW_STRIPED : &str = concat!("Email spam Email spam , also known as junk email or unsolicited bulk email ( UBE )," , " is a subset of electronic spam involving nearly identical messages sent to numerous recipients by email" , " . Clicking on links in spam email may send users to phishing web sites or sites that are hosting malware ."); }