Settings hot reloading - Part 2

crates/common/src/scripts/functions/array.rs

@@ -0,0 +1,150 @@
+/*
+ * Copyright (c) 2023 Stalwart Labs Ltd.
+ *
+ * This file is part of Stalwart Mail Server.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of
+ * the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ * in the LICENSE file at the top-level directory of this distribution.
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * You can be released from the requirements of the AGPLv3 license by
+ * purchasing a commercial license. Please contact licensing@stalw.art
+ * for more details.
+*/
+
+use std::collections::{HashMap, HashSet};
+
+use sieve::{runtime::Variable, Context};
+
+pub fn fn_count<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    match &v[0] {
+        Variable::Array(a) => a.len(),
+        v => {
+            if !v.is_empty() {
+                1
+            } else {
+                0
+            }
+        }
+    }
+    .into()
+}
+
+pub fn fn_sort<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    let is_asc = v[1].to_bool();
+    let mut arr = (*v[0].to_array()).clone();
+    if is_asc {
+        arr.sort_unstable_by(|a, b| b.cmp(a));
+    } else {
+        arr.sort_unstable();
+    }
+    arr.into()
+}
+
+pub fn fn_dedup<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    let arr = v[0].to_array();
+    let mut result = Vec::with_capacity(arr.len());
+
+    for item in arr.iter() {
+        if !result.contains(item) {
+            result.push(item.clone());
+        }
+    }
+
+    result.into()
+}
+
+pub fn fn_cosine_similarity<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    let mut word_freq: HashMap<Variable, [u32; 2]> = HashMap::new();
+
+    for (idx, var) in v.into_iter().enumerate() {
+        match var {
+            Variable::Array(l) => {
+                for item in l.iter() {
+                    word_freq.entry(item.clone()).or_insert([0, 0])[idx] += 1;
+                }
+            }
+            _ => {
+                for char in var.to_string().chars() {
+                    word_freq.entry(char.to_string().into()).or_insert([0, 0])[idx] += 1;
+                }
+            }
+        }
+    }
+
+    let mut dot_product = 0;
+    let mut magnitude_a = 0;
+    let mut magnitude_b = 0;
+
+    for (_word, count) in word_freq.iter() {
+        dot_product += count[0] * count[1];
+        magnitude_a += count[0] * count[0];
+        magnitude_b += count[1] * count[1];
+    }
+
+    if magnitude_a != 0 && magnitude_b != 0 {
+        dot_product as f64 / (magnitude_a as f64).sqrt() / (magnitude_b as f64).sqrt()
+    } else {
+        0.0
+    }
+    .into()
+}
+
+pub fn fn_jaccard_similarity<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    let mut word_freq = [HashSet::new(), HashSet::new()];
+
+    for (idx, var) in v.into_iter().enumerate() {
+        match var {
+            Variable::Array(l) => {
+                for item in l.iter() {
+                    word_freq[idx].insert(item.clone());
+                }
+            }
+            _ => {
+                for char in var.to_string().chars() {
+                    word_freq[idx].insert(char.to_string().into());
+                }
+            }
+        }
+    }
+
+    let intersection_size = word_freq[0].intersection(&word_freq[1]).count();
+    let union_size = word_freq[0].union(&word_freq[1]).count();
+
+    if union_size != 0 {
+        intersection_size as f64 / union_size as f64
+    } else {
+        0.0
+    }
+    .into()
+}
+
+pub fn fn_is_intersect<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    match (&v[0], &v[1]) {
+        (Variable::Array(a), Variable::Array(b)) => a.iter().any(|x| b.contains(x)),
+        (Variable::Array(a), item) | (item, Variable::Array(a)) => a.contains(item),
+        _ => false,
+    }
+    .into()
+}
+
+pub fn fn_winnow<'x>(_: &'x Context<'x, ()>, mut v: Vec<Variable>) -> Variable {
+    match v.remove(0) {
+        Variable::Array(a) => a
+            .iter()
+            .filter(|i| !i.is_empty())
+            .cloned()
+            .collect::<Vec<_>>()
+            .into(),
+        v => v,
+    }
+}

crates/common/src/scripts/functions/email.rs

@@ -0,0 +1,108 @@
+/*
+ * Copyright (c) 2023 Stalwart Labs Ltd.
+ *
+ * This file is part of Stalwart Mail Server.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of
+ * the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ * in the LICENSE file at the top-level directory of this distribution.
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * You can be released from the requirements of the AGPLv3 license by
+ * purchasing a commercial license. Please contact licensing@stalw.art
+ * for more details.
+*/
+
+use sieve::{runtime::Variable, Context};
+
+use super::ApplyString;
+
+pub fn fn_is_email<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    let mut last_ch = 0;
+    let mut in_quote = false;
+    let mut at_count = 0;
+    let mut dot_count = 0;
+    let mut lp_len = 0;
+    let mut value = 0;
+
+    for ch in v[0].to_string().bytes() {
+        match ch {
+            b'0'..=b'9'
+            | b'a'..=b'z'
+            | b'A'..=b'Z'
+            | b'!'
+            | b'#'
+            | b'$'
+            | b'%'
+            | b'&'
+            | b'\''
+            | b'*'
+            | b'+'
+            | b'-'
+            | b'/'
+            | b'='
+            | b'?'
+            | b'^'
+            | b'_'
+            | b'`'
+            | b'{'
+            | b'|'
+            | b'}'
+            | b'~'
+            | 0x7f..=u8::MAX => {
+                value += 1;
+            }
+            b'.' if !in_quote => {
+                if last_ch != b'.' && last_ch != b'@' && value != 0 {
+                    value += 1;
+                    if at_count == 1 {
+                        dot_count += 1;
+                    }
+                } else {
+                    return false.into();
+                }
+            }
+            b'@' if !in_quote => {
+                at_count += 1;
+                lp_len = value;
+                value = 0;
+            }
+            b'>' | b':' | b',' | b' ' if in_quote => {
+                value += 1;
+            }
+            b'\"' if !in_quote || last_ch != b'\\' => {
+                in_quote = !in_quote;
+            }
+            b'\\' if in_quote && last_ch != b'\\' => (),
+            _ => {
+                if !in_quote {
+                    return false.into();
+                }
+            }
+        }
+
+        last_ch = ch;
+    }
+
+    (at_count == 1 && dot_count > 0 && lp_len > 0 && value > 0).into()
+}
+
+pub fn fn_email_part<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    v[0].transform(|s| {
+        s.rsplit_once('@')
+            .map(|(u, d)| match v[1].to_string().as_ref() {
+                "local" => Variable::from(u.trim()),
+                "domain" => Variable::from(d.trim()),
+                _ => Variable::default(),
+            })
+            .unwrap_or_default()
+    })
+}

crates/common/src/scripts/functions/header.rs

@@ -0,0 +1,113 @@
+/*
+ * Copyright (c) 2023 Stalwart Labs Ltd.
+ *
+ * This file is part of Stalwart Mail Server.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of
+ * the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ * in the LICENSE file at the top-level directory of this distribution.
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * You can be released from the requirements of the AGPLv3 license by
+ * purchasing a commercial license. Please contact licensing@stalw.art
+ * for more details.
+*/
+
+use mail_parser::{parsers::fields::thread::thread_name, HeaderName, HeaderValue, MimeHeaders};
+use sieve::{compiler::ReceivedPart, runtime::Variable, Context};
+
+use super::ApplyString;
+
+pub fn fn_received_part<'x>(ctx: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    if let (Ok(part), Some(HeaderValue::Received(rcvd))) = (
+        ReceivedPart::try_from(v[1].to_string().as_ref()),
+        ctx.message()
+            .part(ctx.part())
+            .and_then(|p| {
+                p.headers
+                    .iter()
+                    .filter(|h| h.name == HeaderName::Received)
+                    .nth((v[0].to_integer() as usize).saturating_sub(1))
+            })
+            .map(|h| &h.value),
+    ) {
+        part.eval(rcvd).unwrap_or_default()
+    } else {
+        Variable::default()
+    }
+}
+
+pub fn fn_is_encoding_problem<'x>(ctx: &'x Context<'x, ()>, _: Vec<Variable>) -> Variable {
+    ctx.message()
+        .part(ctx.part())
+        .map(|p| p.is_encoding_problem)
+        .unwrap_or_default()
+        .into()
+}
+
+pub fn fn_is_attachment<'x>(ctx: &'x Context<'x, ()>, _: Vec<Variable>) -> Variable {
+    ctx.message().attachments.contains(&ctx.part()).into()
+}
+
+pub fn fn_is_body<'x>(ctx: &'x Context<'x, ()>, _: Vec<Variable>) -> Variable {
+    (ctx.message().text_body.contains(&ctx.part()) || ctx.message().html_body.contains(&ctx.part()))
+        .into()
+}
+
+pub fn fn_attachment_name<'x>(ctx: &'x Context<'x, ()>, _: Vec<Variable>) -> Variable {
+    ctx.message()
+        .part(ctx.part())
+        .and_then(|p| p.attachment_name())
+        .unwrap_or_default()
+        .into()
+}
+
+pub fn fn_mime_part_len<'x>(ctx: &'x Context<'x, ()>, _: Vec<Variable>) -> Variable {
+    ctx.message()
+        .part(ctx.part())
+        .map(|p| p.len())
+        .unwrap_or_default()
+        .into()
+}
+
+pub fn fn_thread_name<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    v[0].transform(|s| thread_name(s).into())
+}
+
+pub fn fn_is_header_utf8_valid<'x>(ctx: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    ctx.message()
+        .part(ctx.part())
+        .map(|p| {
+            let raw = ctx.message().raw_message();
+            let mut is_valid = true;
+            if let Some(header_name) = HeaderName::parse(v[0].to_string().as_ref()) {
+                for header in &p.headers {
+                    if header.name == header_name
+                        && raw
+                            .get(header.offset_start()..header.offset_end())
+                            .and_then(|raw| std::str::from_utf8(raw).ok())
+                            .is_none()
+                    {
+                        is_valid = false;
+                        break;
+                    }
+                }
+            } else {
+                is_valid = raw
+                    .get(p.raw_header_offset()..p.raw_body_offset())
+                    .and_then(|raw| std::str::from_utf8(raw).ok())
+                    .is_some();
+            }
+
+            Variable::from(is_valid)
+        })
+        .unwrap_or(Variable::Integer(1))
+}

crates/common/src/scripts/functions/html.rs

@@ -0,0 +1,439 @@
+/*
+ * Copyright (c) 2023 Stalwart Labs Ltd.
+ *
+ * This file is part of Stalwart Mail Server.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of
+ * the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ * in the LICENSE file at the top-level directory of this distribution.
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * You can be released from the requirements of the AGPLv3 license by
+ * purchasing a commercial license. Please contact licensing@stalw.art
+ * for more details.
+*/
+
+use std::borrow::Cow;
+
+use mail_parser::decoders::html::{add_html_token, html_to_text};
+use sieve::{runtime::Variable, Context};
+
+pub fn fn_html_to_text<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    html_to_text(v[0].to_string().as_ref()).into()
+}
+
+pub fn fn_html_has_tag<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    v[0].as_array()
+        .map(|arr| {
+            let token = v[1].to_string();
+            arr.iter().any(|v| {
+                v.to_string()
+                    .as_ref()
+                    .strip_prefix('<')
+                    .map_or(false, |tag| tag.starts_with(token.as_ref()))
+            })
+        })
+        .unwrap_or_default()
+        .into()
+}
+
+pub fn fn_html_attr_size<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    let t = v[0].to_string();
+    let mut dimension = None;
+
+    if let Some(value) = get_attribute(t.as_ref(), v[1].to_string().as_ref()) {
+        let value = value.trim();
+        if let Some(pct) = value.strip_suffix('%') {
+            if let Ok(pct) = pct.trim().parse::<u32>() {
+                dimension = ((v[2].to_integer() * pct as i64) / 100).into();
+            }
+        } else if let Ok(value) = value.parse::<u32>() {
+            dimension = (value as i64).into();
+        }
+    }
+
+    dimension.map(Variable::Integer).unwrap_or_default()
+}
+
+pub fn fn_html_attrs<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    html_attr_tokens(
+        v[0].to_string().as_ref(),
+        v[1].to_string().as_ref(),
+        v[2].to_string_array(),
+    )
+    .into()
+}
+
+pub fn fn_html_attr<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    get_attribute(v[0].to_string().as_ref(), v[1].to_string().as_ref())
+        .map(Variable::from)
+        .unwrap_or_default()
+}
+
+pub fn html_to_tokens(input: &str) -> Vec<Variable> {
+    let input = input.as_bytes();
+    let mut iter = input.iter().enumerate();
+    let mut tags = vec![];
+
+    let mut is_token_start = true;
+    let mut is_after_space = false;
+    let mut is_new_line = true;
+
+    let mut token_start = 0;
+    let mut token_end = 0;
+
+    let mut text = String::from("_");
+
+    while let Some((pos, &ch)) = iter.next() {
+        match ch {
+            b'<' => {
+                if !is_token_start {
+                    add_html_token(
+                        &mut text,
+                        &input[token_start..token_end + 1],
+                        is_after_space,
+                    );
+                    is_after_space = false;
+                    is_token_start = true;
+                }
+                if text.len() > 1 {
+                    tags.push(Variable::String(text.into()));
+                    text = String::from("_");
+                }
+
+                let mut tag = vec![b'<'];
+                if matches!(input.get(pos + 1..pos + 4), Some(b"!--")) {
+                    let mut last_ch: u8 = 0;
+                    for (_, &ch) in iter.by_ref() {
+                        match ch {
+                            b'>' if tag.len() > 3
+                                && matches!(tag.last(), Some(b'-'))
+                                && matches!(tag.get(tag.len() - 2), Some(b'-')) =>
+                            {
+                                break;
+                            }
+                            b' ' | b'\t' | b'\r' | b'\n' => {
+                                if last_ch != b' ' {
+                                    tag.push(b' ');
+                                } else {
+                                    last_ch = b' ';
+                                }
+                                continue;
+                            }
+                            _ => {
+                                tag.push(ch);
+                            }
+                        }
+                        last_ch = ch;
+                    }
+                } else {
+                    let mut in_quote = false;
+                    let mut last_ch = b' ';
+                    for (_, &ch) in iter.by_ref() {
+                        match ch {
+                            b'>' if !in_quote => {
+                                break;
+                            }
+                            b'"' => {
+                                in_quote = !in_quote;
+                                tag.push(b'"');
+                            }
+                            b' ' | b'\t' | b'\r' | b'\n' if !in_quote => {
+                                if last_ch != b' ' {
+                                    tag.push(b' ');
+                                    last_ch = b' ';
+                                }
+                                continue;
+                            }
+                            b'/' if !in_quote => {
+                                tag.push(b'/');
+                                last_ch = b' ';
+                                continue;
+                            }
+                            _ => {
+                                tag.push(if in_quote {
+                                    ch
+                                } else {
+                                    ch.to_ascii_lowercase()
+                                });
+                            }
+                        }
+                        last_ch = ch;
+                    }
+                }
+                tags.push(Variable::String(
+                    String::from_utf8(tag).unwrap_or_default().into(),
+                ));
+                continue;
+            }
+            b' ' | b'\t' | b'\r' | b'\n' => {
+                if !is_token_start {
+                    add_html_token(
+                        &mut text,
+                        &input[token_start..token_end + 1],
+                        is_after_space && !is_new_line,
+                    );
+                    is_new_line = false;
+                }
+                is_after_space = true;
+                is_token_start = true;
+                continue;
+            }
+            b'&' if !is_token_start => {
+                add_html_token(
+                    &mut text,
+                    &input[token_start..token_end + 1],
+                    is_after_space && !is_new_line,
+                );
+                is_new_line = false;
+                is_token_start = true;
+                is_after_space = false;
+            }
+            b';' if !is_token_start => {
+                add_html_token(
+                    &mut text,
+                    &input[token_start..pos + 1],
+                    is_after_space && !is_new_line,
+                );
+                is_token_start = true;
+                is_after_space = false;
+                is_new_line = false;
+                continue;
+            }
+            _ => (),
+        }
+
+        if is_token_start {
+            token_start = pos;
+            is_token_start = false;
+        }
+        token_end = pos;
+    }
+
+    if !is_token_start {
+        add_html_token(
+            &mut text,
+            &input[token_start..token_end + 1],
+            is_after_space && !is_new_line,
+        );
+    }
+    if text.len() > 1 {
+        tags.push(Variable::String(text.into()));
+    }
+
+    tags
+}
+
+pub fn html_attr_tokens(input: &str, tag: &str, attrs: Vec<Cow<str>>) -> Vec<Variable> {
+    let input = input.as_bytes();
+    let mut iter = input.iter().enumerate().peekable();
+    let mut tags = vec![];
+
+    while let Some((mut pos, &ch)) = iter.next() {
+        if ch == b'<' {
+            if !matches!(input.get(pos + 1..pos + 4), Some(b"!--")) {
+                let mut in_quote = false;
+                let mut last_ch_pos: usize = 0;
+
+                while matches!(iter.peek(), Some((_, &ch)) if ch.is_ascii_whitespace()) {
+                    pos += 1;
+                    iter.next();
+                }
+
+                let found_tag = tag.is_empty()
+                    || (matches!(input.get(pos + 1..pos + tag.len() + 1), Some(t) if t.eq_ignore_ascii_case(tag.as_bytes()))
+                        && matches!(input.get(pos + tag.len() + 1), Some(ch) if ch.is_ascii_whitespace()));
+
+                'outer: while let Some((pos, &ch)) = iter.next() {
+                    match ch {
+                        b'>' if !in_quote => {
+                            break;
+                        }
+                        b'"' => {
+                            in_quote = !in_quote;
+                        }
+                        b'=' if found_tag
+                            && !in_quote
+                            && attrs.iter().any(|attr| matches!(input.get(last_ch_pos.saturating_sub(attr.len()) + 1..last_ch_pos + 1), Some(a) if a.eq_ignore_ascii_case(attr.as_bytes())))
+                            && matches!(input.get(last_ch_pos + 1), Some(ch) if ch.is_ascii_whitespace() || *ch == b'=') =>
+                        {
+                            while matches!(iter.peek(), Some((_, &ch)) if ch.is_ascii_whitespace())
+                            {
+                                iter.next();
+                            }
+                            let mut tag = vec![];
+
+                            for (_, &ch) in iter.by_ref() {
+                                match ch {
+                                    b'>' if !in_quote => {
+                                        if !tag.is_empty() {
+                                            tags.push(Variable::String(
+                                                String::from_utf8(tag).unwrap_or_default().into(),
+                                            ));
+                                        }
+                                        break 'outer;
+                                    }
+                                    b'"' => {
+                                        if in_quote {
+                                            in_quote = false;
+                                            break;
+                                        } else {
+                                            in_quote = true;
+                                        }
+                                    }
+                                    b' ' | b'\t' | b'\r' | b'\n' if !in_quote => {
+                                        break;
+                                    }
+                                    _ => {
+                                        tag.push(ch);
+                                    }
+                                }
+                            }
+
+                            if !tag.is_empty() {
+                                tags.push(Variable::String(
+                                    String::from_utf8(tag).unwrap_or_default().into(),
+                                ));
+                            }
+                        }
+                        b' ' | b'\t' | b'\r' | b'\n' => {}
+                        _ => {
+                            last_ch_pos = pos;
+                        }
+                    }
+                }
+            } else {
+                let mut last_ch: u8 = 0;
+                let mut before_last_ch: u8 = 0;
+
+                for (_, &ch) in iter.by_ref() {
+                    if ch == b'>' && last_ch == b'-' && before_last_ch == b'-' {
+                        break;
+                    }
+                    before_last_ch = last_ch;
+                    last_ch = ch;
+                }
+            }
+        }
+    }
+
+    tags
+}
+
+pub fn html_img_area(arr: &[Variable]) -> u32 {
+    arr.iter()
+        .filter_map(|v| {
+            let t = v.to_string();
+            if t.starts_with("<img") {
+                let mut dimensions = [200u32, 200u32];
+
+                for (idx, attr) in ["width", "height"].into_iter().enumerate() {
+                    if let Some(value) = get_attribute(t.as_ref(), attr) {
+                        let value = value.trim();
+                        if let Some(pct) = value.strip_suffix('%') {
+                            if let Ok(pct) = pct.trim().parse::<u32>() {
+                                let size = if idx == 0 { 800 } else { 600 };
+                                dimensions[idx] = (size * pct) / 100;
+                            }
+                        } else if let Ok(value) = value.parse::<u32>() {
+                            dimensions[idx] = value;
+                        }
+                    }
+                }
+
+                Some(dimensions[0].saturating_mul(dimensions[1]))
+            } else {
+                None
+            }
+        })
+        .sum::<u32>()
+}
+
+pub fn get_attribute<'x>(tag: &'x str, attr_name: &str) -> Option<&'x str> {
+    let tag = tag.as_bytes();
+    let attr_name = attr_name.as_bytes();
+    let mut iter = tag.iter().enumerate().peekable();
+    let mut in_quote = false;
+    let mut start_pos = usize::MAX;
+    let mut end_pos = usize::MAX;
+
+    while let Some((pos, ch)) = iter.next() {
+        match ch {
+            b'=' if !in_quote => {
+                if start_pos != usize::MAX
+                    && end_pos != usize::MAX
+                    && tag
+                        .get(start_pos..end_pos + 1)
+                        .map_or(false, |name| name == attr_name)
+                {
+                    let mut token_start = 0;
+                    let mut token_end = 0;
+
+                    for (pos, ch) in iter.by_ref() {
+                        match ch {
+                            b'"' => {
+                                if !in_quote {
+                                    token_start = pos + 1;
+                                    in_quote = true;
+                                } else {
+                                    token_end = pos;
+                                    break;
+                                }
+                            }
+                            b' ' if !in_quote => {
+                                if token_start != 0 {
+                                    token_end = pos;
+                                    break;
+                                }
+                            }
+                            _ => {
+                                if token_start == 0 {
+                                    token_start = pos;
+                                }
+                            }
+                        }
+                    }
+
+                    return if token_start > 0 {
+                        if token_end == 0 {
+                            token_end = tag.len();
+                        }
+                        Some(std::str::from_utf8(&tag[token_start..token_end]).unwrap_or_default())
+                    } else {
+                        None
+                    };
+                } else {
+                    start_pos = usize::MAX;
+                    end_pos = usize::MAX;
+                }
+            }
+            b'"' => {
+                in_quote = !in_quote;
+            }
+            b' ' => {
+                if !in_quote && !matches!(iter.peek(), Some((_, b'='))) {
+                    start_pos = usize::MAX;
+                    end_pos = usize::MAX;
+                }
+            }
+            _ => {
+                if !in_quote {
+                    if start_pos == usize::MAX {
+                        start_pos = pos;
+                    }
+                    end_pos = pos;
+                }
+            }
+        }
+    }
+
+    None
+}

crates/common/src/scripts/functions/image.rs

@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2023 Stalwart Labs Ltd.
+ *
+ * This file is part of Stalwart Mail Server.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of
+ * the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ * in the LICENSE file at the top-level directory of this distribution.
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * You can be released from the requirements of the AGPLv3 license by
+ * purchasing a commercial license. Please contact licensing@stalw.art
+ * for more details.
+*/
+
+use sieve::{runtime::Variable, Context};
+
+pub fn fn_img_metadata<'x>(ctx: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    ctx.message()
+        .part(ctx.part())
+        .map(|p| p.contents())
+        .and_then(|bytes| {
+            let arg = v[1].to_string();
+            match arg.as_ref() {
+                "type" => imagesize::image_type(bytes).ok().map(|t| {
+                    Variable::from(match t {
+                        imagesize::ImageType::Aseprite => "aseprite",
+                        imagesize::ImageType::Avif => "avif",
+                        imagesize::ImageType::Bmp => "bmp",
+                        imagesize::ImageType::Dds => "dds",
+                        imagesize::ImageType::Exr => "exr",
+                        imagesize::ImageType::Farbfeld => "farbfeld",
+                        imagesize::ImageType::Gif => "gif",
+                        imagesize::ImageType::Hdr => "hdr",
+                        imagesize::ImageType::Heif => "heif",
+                        imagesize::ImageType::Ico => "ico",
+                        imagesize::ImageType::Jpeg => "jpeg",
+                        imagesize::ImageType::Jxl => "jxl",
+                        imagesize::ImageType::Ktx2 => "ktx2",
+                        imagesize::ImageType::Png => "png",
+                        imagesize::ImageType::Pnm => "pnm",
+                        imagesize::ImageType::Psd => "psd",
+                        imagesize::ImageType::Qoi => "qoi",
+                        imagesize::ImageType::Tga => "tga",
+                        imagesize::ImageType::Tiff => "tiff",
+                        imagesize::ImageType::Vtf => "vtf",
+                        imagesize::ImageType::Webp => "webp",
+                    })
+                }),
+                "width" => imagesize::blob_size(bytes)
+                    .ok()
+                    .map(|s| Variable::Integer(s.width as i64)),
+                "height" => imagesize::blob_size(bytes)
+                    .ok()
+                    .map(|s| Variable::Integer(s.height as i64)),
+                "area" => imagesize::blob_size(bytes)
+                    .ok()
+                    .map(|s| Variable::Integer(s.width.saturating_mul(s.height) as i64)),
+                "dimension" => imagesize::blob_size(bytes)
+                    .ok()
+                    .map(|s| Variable::Integer(s.width.saturating_add(s.height) as i64)),
+                _ => None,
+            }
+        })
+        .unwrap_or_default()
+}

crates/common/src/scripts/functions/misc.rs

@@ -0,0 +1,121 @@
+/*
+ * Copyright (c) 2023 Stalwart Labs Ltd.
+ *
+ * This file is part of Stalwart Mail Server.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of
+ * the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ * in the LICENSE file at the top-level directory of this distribution.
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * You can be released from the requirements of the AGPLv3 license by
+ * purchasing a commercial license. Please contact licensing@stalw.art
+ * for more details.
+*/
+
+use std::net::IpAddr;
+
+use mail_auth::common::resolver::ToReverseName;
+use sha1::Sha1;
+use sha2::{Sha256, Sha512};
+use sieve::{runtime::Variable, Context};
+
+use super::ApplyString;
+
+pub fn fn_is_empty<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    match &v[0] {
+        Variable::String(s) => s.is_empty(),
+        Variable::Integer(_) | Variable::Float(_) => false,
+        Variable::Array(a) => a.is_empty(),
+    }
+    .into()
+}
+
+pub fn fn_is_number<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    matches!(&v[0], Variable::Integer(_) | Variable::Float(_)).into()
+}
+
+pub fn fn_is_ip_addr<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    v[0].to_string().parse::<std::net::IpAddr>().is_ok().into()
+}
+
+pub fn fn_is_ipv4_addr<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    v[0].to_string()
+        .parse::<std::net::IpAddr>()
+        .map_or(false, |ip| matches!(ip, IpAddr::V4(_)))
+        .into()
+}
+
+pub fn fn_is_ipv6_addr<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    v[0].to_string()
+        .parse::<std::net::IpAddr>()
+        .map_or(false, |ip| matches!(ip, IpAddr::V6(_)))
+        .into()
+}
+
+pub fn fn_ip_reverse_name<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    v[0].to_string()
+        .parse::<std::net::IpAddr>()
+        .map(|ip| ip.to_reverse_name())
+        .unwrap_or_default()
+        .into()
+}
+
+pub fn fn_detect_file_type<'x>(ctx: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    ctx.message()
+        .part(ctx.part())
+        .and_then(|p| infer::get(p.contents()))
+        .map(|t| {
+            Variable::from(
+                if v[0].to_string() != "ext" {
+                    t.mime_type()
+                } else {
+                    t.extension()
+                }
+                .to_string(),
+            )
+        })
+        .unwrap_or_default()
+}
+
+pub fn fn_hash<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    use sha1::Digest;
+    let hash = v[1].to_string();
+
+    v[0].transform(|value| match hash.as_ref() {
+        "md5" => format!("{:x}", md5::compute(value.as_bytes())).into(),
+        "sha1" => {
+            let mut hasher = Sha1::new();
+            hasher.update(value.as_bytes());
+            format!("{:x}", hasher.finalize()).into()
+        }
+        "sha256" => {
+            let mut hasher = Sha256::new();
+            hasher.update(value.as_bytes());
+            format!("{:x}", hasher.finalize()).into()
+        }
+        "sha512" => {
+            let mut hasher = Sha512::new();
+            hasher.update(value.as_bytes());
+            format!("{:x}", hasher.finalize()).into()
+        }
+        _ => Variable::default(),
+    })
+}
+
+pub fn fn_is_var_names<'x>(ctx: &'x Context<'x, ()>, _: Vec<Variable>) -> Variable {
+    Variable::Array(
+        ctx.global_variable_names()
+            .map(|v| Variable::from(v.to_uppercase()))
+            .collect::<Vec<_>>()
+            .into(),
+    )
+}

crates/common/src/scripts/functions/mod.rs

@@ -0,0 +1,131 @@
+/*
+ * Copyright (c) 2023 Stalwart Labs Ltd.
+ *
+ * This file is part of Stalwart Mail Server.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of
+ * the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ * in the LICENSE file at the top-level directory of this distribution.
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * You can be released from the requirements of the AGPLv3 license by
+ * purchasing a commercial license. Please contact licensing@stalw.art
+ * for more details.
+*/
+
+mod array;
+mod email;
+mod header;
+pub mod html;
+mod image;
+mod misc;
+pub mod text;
+mod unicode;
+mod url;
+
+use sieve::{runtime::Variable, FunctionMap};
+
+use self::{
+    array::*, email::*, header::*, html::*, image::*, misc::*, text::*, unicode::*, url::*,
+};
+
+pub fn register_functions() -> FunctionMap<()> {
+    FunctionMap::new()
+        .with_function("trim", fn_trim)
+        .with_function("trim_start", fn_trim_start)
+        .with_function("trim_end", fn_trim_end)
+        .with_function("len", fn_len)
+        .with_function("count", fn_count)
+        .with_function("is_empty", fn_is_empty)
+        .with_function("is_number", fn_is_number)
+        .with_function("is_ascii", fn_is_ascii)
+        .with_function("to_lowercase", fn_to_lowercase)
+        .with_function("to_uppercase", fn_to_uppercase)
+        .with_function("detect_language", fn_detect_language)
+        .with_function("is_email", fn_is_email)
+        .with_function("thread_name", fn_thread_name)
+        .with_function("html_to_text", fn_html_to_text)
+        .with_function("is_uppercase", fn_is_uppercase)
+        .with_function("is_lowercase", fn_is_lowercase)
+        .with_function("has_digits", fn_has_digits)
+        .with_function("count_spaces", fn_count_spaces)
+        .with_function("count_uppercase", fn_count_uppercase)
+        .with_function("count_lowercase", fn_count_lowercase)
+        .with_function("count_chars", fn_count_chars)
+        .with_function("dedup", fn_dedup)
+        .with_function("lines", fn_lines)
+        .with_function("is_header_utf8_valid", fn_is_header_utf8_valid)
+        .with_function("img_metadata", fn_img_metadata)
+        .with_function("is_ip_addr", fn_is_ip_addr)
+        .with_function("is_ipv4_addr", fn_is_ipv4_addr)
+        .with_function("is_ipv6_addr", fn_is_ipv6_addr)
+        .with_function("ip_reverse_name", fn_ip_reverse_name)
+        .with_function("winnow", fn_winnow)
+        .with_function("has_zwsp", fn_has_zwsp)
+        .with_function("has_obscured", fn_has_obscured)
+        .with_function("is_single_script", fn_is_single_script)
+        .with_function("puny_decode", fn_puny_decode)
+        .with_function("unicode_skeleton", fn_unicode_skeleton)
+        .with_function("cure_text", fn_cure_text)
+        .with_function("detect_file_type", fn_detect_file_type)
+        .with_function_args("sort", fn_sort, 2)
+        .with_function_args("email_part", fn_email_part, 2)
+        .with_function_args("eq_ignore_case", fn_eq_ignore_case, 2)
+        .with_function_args("contains", fn_contains, 2)
+        .with_function_args("contains_ignore_case", fn_contains_ignore_case, 2)
+        .with_function_args("starts_with", fn_starts_with, 2)
+        .with_function_args("ends_with", fn_ends_with, 2)
+        .with_function_args("received_part", fn_received_part, 2)
+        .with_function_args("cosine_similarity", fn_cosine_similarity, 2)
+        .with_function_args("jaccard_similarity", fn_jaccard_similarity, 2)
+        .with_function_args("levenshtein_distance", fn_levenshtein_distance, 2)
+        .with_function_args("html_has_tag", fn_html_has_tag, 2)
+        .with_function_args("html_attr", fn_html_attr, 2)
+        .with_function_args("html_attrs", fn_html_attrs, 3)
+        .with_function_args("html_attr_size", fn_html_attr_size, 3)
+        .with_function_args("uri_part", fn_uri_part, 2)
+        .with_function_args("substring", fn_substring, 3)
+        .with_function_args("split", fn_split, 2)
+        .with_function_args("rsplit", fn_rsplit, 2)
+        .with_function_args("split_once", fn_split_once, 2)
+        .with_function_args("rsplit_once", fn_rsplit_once, 2)
+        .with_function_args("strip_prefix", fn_strip_prefix, 2)
+        .with_function_args("strip_suffix", fn_strip_suffix, 2)
+        .with_function_args("is_intersect", fn_is_intersect, 2)
+        .with_function_args("hash", fn_hash, 2)
+        .with_function_no_args("is_encoding_problem", fn_is_encoding_problem)
+        .with_function_no_args("is_attachment", fn_is_attachment)
+        .with_function_no_args("is_body", fn_is_body)
+        .with_function_no_args("var_names", fn_is_var_names)
+        .with_function_no_args("attachment_name", fn_attachment_name)
+        .with_function_no_args("mime_part_len", fn_mime_part_len)
+}
+
+pub trait ApplyString<'x> {
+    fn transform(&self, f: impl Fn(&'_ str) -> Variable) -> Variable;
+}
+
+impl<'x> ApplyString<'x> for Variable {
+    fn transform(&self, f: impl Fn(&'_ str) -> Variable) -> Variable {
+        match self {
+            Variable::String(s) => f(s),
+            Variable::Array(list) => list
+                .iter()
+                .map(|v| match v {
+                    Variable::String(s) => f(s),
+                    v => f(v.to_string().as_ref()),
+                })
+                .collect::<Vec<_>>()
+                .into(),
+            v => f(v.to_string().as_ref()),
+        }
+    }
+}

crates/common/src/scripts/functions/text.rs

@@ -0,0 +1,306 @@
+/*
+ * Copyright (c) 2023 Stalwart Labs Ltd.
+ *
+ * This file is part of Stalwart Mail Server.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of
+ * the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ * in the LICENSE file at the top-level directory of this distribution.
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * You can be released from the requirements of the AGPLv3 license by
+ * purchasing a commercial license. Please contact licensing@stalw.art
+ * for more details.
+*/
+
+use sieve::{runtime::Variable, Context};
+
+use super::ApplyString;
+
+pub fn fn_trim<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    v[0].transform(|s| Variable::from(s.trim()))
+}
+
+pub fn fn_trim_end<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    v[0].transform(|s| Variable::from(s.trim_end()))
+}
+
+pub fn fn_trim_start<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    v[0].transform(|s| Variable::from(s.trim_start()))
+}
+
+pub fn fn_len<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    match &v[0] {
+        Variable::String(s) => s.len(),
+        Variable::Array(a) => a.len(),
+        v => v.to_string().len(),
+    }
+    .into()
+}
+
+pub fn fn_to_lowercase<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    v[0].transform(|s| Variable::from(s.to_lowercase()))
+}
+
+pub fn fn_to_uppercase<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    v[0].transform(|s| Variable::from(s.to_uppercase()))
+}
+
+pub fn fn_is_uppercase<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    v[0].transform(|s| {
+        s.chars()
+            .filter(|c| c.is_alphabetic())
+            .all(|c| c.is_uppercase())
+            .into()
+    })
+}
+
+pub fn fn_is_lowercase<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    v[0].transform(|s| {
+        s.chars()
+            .filter(|c| c.is_alphabetic())
+            .all(|c| c.is_lowercase())
+            .into()
+    })
+}
+
+pub fn fn_has_digits<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    v[0].transform(|s| s.chars().any(|c| c.is_ascii_digit()).into())
+}
+
+pub fn tokenize_words(v: &Variable) -> Variable {
+    v.to_string()
+        .split_whitespace()
+        .filter(|word| word.chars().all(|c| c.is_alphanumeric()))
+        .map(|word| Variable::from(word.to_string()))
+        .collect::<Vec<_>>()
+        .into()
+}
+
+pub fn fn_count_spaces<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    v[0].to_string()
+        .as_ref()
+        .chars()
+        .filter(|c| c.is_whitespace())
+        .count()
+        .into()
+}
+
+pub fn fn_count_uppercase<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    v[0].to_string()
+        .as_ref()
+        .chars()
+        .filter(|c| c.is_alphabetic() && c.is_uppercase())
+        .count()
+        .into()
+}
+
+pub fn fn_count_lowercase<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    v[0].to_string()
+        .as_ref()
+        .chars()
+        .filter(|c| c.is_alphabetic() && c.is_lowercase())
+        .count()
+        .into()
+}
+
+pub fn fn_count_chars<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    v[0].to_string().as_ref().chars().count().into()
+}
+
+pub fn fn_eq_ignore_case<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    v[0].to_string()
+        .eq_ignore_ascii_case(v[1].to_string().as_ref())
+        .into()
+}
+
+pub fn fn_contains<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    match &v[0] {
+        Variable::String(s) => s.contains(v[1].to_string().as_ref()),
+        Variable::Array(arr) => arr.contains(&v[1]),
+        val => val.to_string().contains(v[1].to_string().as_ref()),
+    }
+    .into()
+}
+
+pub fn fn_contains_ignore_case<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    let needle = v[1].to_string();
+    match &v[0] {
+        Variable::String(s) => s.to_lowercase().contains(&needle.to_lowercase()),
+        Variable::Array(arr) => arr.iter().any(|v| match v {
+            Variable::String(s) => s.eq_ignore_ascii_case(needle.as_ref()),
+            _ => false,
+        }),
+        val => val.to_string().contains(needle.as_ref()),
+    }
+    .into()
+}
+
+pub fn fn_starts_with<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    v[0].to_string()
+        .starts_with(v[1].to_string().as_ref())
+        .into()
+}
+
+pub fn fn_ends_with<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    v[0].to_string().ends_with(v[1].to_string().as_ref()).into()
+}
+
+pub fn fn_lines<'x>(_: &'x Context<'x, ()>, mut v: Vec<Variable>) -> Variable {
+    match v.remove(0) {
+        Variable::String(s) => s
+            .lines()
+            .map(|s| Variable::from(s.to_string()))
+            .collect::<Vec<_>>()
+            .into(),
+        val => val,
+    }
+}
+
+pub fn fn_substring<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    v[0].to_string()
+        .chars()
+        .skip(v[1].to_usize())
+        .take(v[2].to_usize())
+        .collect::<String>()
+        .into()
+}
+
+pub fn fn_strip_prefix<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    let prefix = v[1].to_string();
+    v[0].transform(|s| {
+        s.strip_prefix(prefix.as_ref())
+            .map(Variable::from)
+            .unwrap_or_default()
+    })
+}
+
+pub fn fn_strip_suffix<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    let suffix = v[1].to_string();
+    v[0].transform(|s| {
+        s.strip_suffix(suffix.as_ref())
+            .map(Variable::from)
+            .unwrap_or_default()
+    })
+}
+
+pub fn fn_split<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    v[0].to_string()
+        .split(v[1].to_string().as_ref())
+        .map(|s| Variable::from(s.to_string()))
+        .collect::<Vec<_>>()
+        .into()
+}
+
+pub fn fn_rsplit<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    v[0].to_string()
+        .rsplit(v[1].to_string().as_ref())
+        .map(|s| Variable::from(s.to_string()))
+        .collect::<Vec<_>>()
+        .into()
+}
+
+pub fn fn_split_once<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    v[0].to_string()
+        .split_once(v[1].to_string().as_ref())
+        .map(|(a, b)| {
+            Variable::Array(
+                vec![Variable::from(a.to_string()), Variable::from(b.to_string())].into(),
+            )
+        })
+        .unwrap_or_default()
+}
+
+pub fn fn_rsplit_once<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    v[0].to_string()
+        .rsplit_once(v[1].to_string().as_ref())
+        .map(|(a, b)| {
+            Variable::Array(
+                vec![Variable::from(a.to_string()), Variable::from(b.to_string())].into(),
+            )
+        })
+        .unwrap_or_default()
+}
+
+/**
+ * `levenshtein-rs` - levenshtein
+ *
+ * MIT licensed.
+ *
+ * Copyright (c) 2016 Titus Wormer <tituswormer@gmail.com>
+ */
+pub fn fn_levenshtein_distance<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    let a = v[0].to_string();
+    let b = v[1].to_string();
+
+    let mut result = 0;
+
+    /* Shortcut optimizations / degenerate cases. */
+    if a == b {
+        return result.into();
+    }
+
+    let length_a = a.chars().count();
+    let length_b = b.chars().count();
+
+    if length_a == 0 {
+        return length_b.into();
+    } else if length_b == 0 {
+        return length_a.into();
+    }
+
+    /* Initialize the vector.
+     *
+     * This is why it’s fast, normally a matrix is used,
+     * here we use a single vector. */
+    let mut cache: Vec<usize> = (1..).take(length_a).collect();
+    let mut distance_a;
+    let mut distance_b;
+
+    /* Loop. */
+    for (index_b, code_b) in b.chars().enumerate() {
+        result = index_b;
+        distance_a = index_b;
+
+        for (index_a, code_a) in a.chars().enumerate() {
+            distance_b = if code_a == code_b {
+                distance_a
+            } else {
+                distance_a + 1
+            };
+
+            distance_a = cache[index_a];
+
+            result = if distance_a > result {
+                if distance_b > result {
+                    result + 1
+                } else {
+                    distance_b
+                }
+            } else if distance_b > distance_a {
+                distance_a + 1
+            } else {
+                distance_b
+            };
+
+            cache[index_a] = result;
+        }
+    }
+
+    result.into()
+}
+
+pub fn fn_detect_language<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    whatlang::detect_lang(v[0].to_string().as_ref())
+        .map(|l| l.code())
+        .unwrap_or("unknown")
+        .into()
+}

crates/common/src/scripts/functions/unicode.rs

@@ -0,0 +1,108 @@
+/*
+ * Copyright (c) 2023 Stalwart Labs Ltd.
+ *
+ * This file is part of Stalwart Mail Server.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of
+ * the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ * in the LICENSE file at the top-level directory of this distribution.
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * You can be released from the requirements of the AGPLv3 license by
+ * purchasing a commercial license. Please contact licensing@stalw.art
+ * for more details.
+*/
+
+use sieve::{runtime::Variable, Context};
+use unicode_security::MixedScript;
+
+pub fn fn_is_ascii<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    match &v[0] {
+        Variable::String(s) => s.chars().all(|c| c.is_ascii()),
+        Variable::Integer(_) | Variable::Float(_) => true,
+        Variable::Array(a) => a.iter().all(|v| match v {
+            Variable::String(s) => s.chars().all(|c| c.is_ascii()),
+            _ => true,
+        }),
+    }
+    .into()
+}
+
+pub fn fn_has_zwsp<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    match &v[0] {
+        Variable::String(s) => s.chars().any(|c| c.is_zwsp()),
+        Variable::Array(a) => a.iter().any(|v| match v {
+            Variable::String(s) => s.chars().any(|c| c.is_zwsp()),
+            _ => true,
+        }),
+        Variable::Integer(_) | Variable::Float(_) => false,
+    }
+    .into()
+}
+
+pub fn fn_has_obscured<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    match &v[0] {
+        Variable::String(s) => s.chars().any(|c| c.is_obscured()),
+        Variable::Array(a) => a.iter().any(|v| match v {
+            Variable::String(s) => s.chars().any(|c| c.is_obscured()),
+            _ => true,
+        }),
+        Variable::Integer(_) | Variable::Float(_) => false,
+    }
+    .into()
+}
+
+trait CharUtils {
+    fn is_zwsp(&self) -> bool;
+    fn is_obscured(&self) -> bool;
+}
+
+impl CharUtils for char {
+    fn is_zwsp(&self) -> bool {
+        matches!(
+            self,
+            '\u{200B}' | '\u{200C}' | '\u{200D}' | '\u{FEFF}' | '\u{00AD}'
+        )
+    }
+
+    fn is_obscured(&self) -> bool {
+        matches!(
+            self,
+            '\u{200B}'..='\u{200F}'
+                | '\u{2028}'..='\u{202F}'
+                | '\u{205F}'..='\u{206F}'
+                | '\u{FEFF}'
+        )
+    }
+}
+
+pub fn fn_cure_text<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    decancer::cure(v[0].to_string().as_ref(), decancer::Options::default())
+        .map(|s| s.into_str())
+        .unwrap_or_default()
+        .into()
+}
+
+pub fn fn_unicode_skeleton<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    unicode_security::skeleton(v[0].to_string().as_ref())
+        .collect::<String>()
+        .into()
+}
+
+pub fn fn_is_single_script<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    let text = v[0].to_string();
+    if !text.is_empty() {
+        text.as_ref().is_single_script()
+    } else {
+        true
+    }
+    .into()
+}

crates/common/src/scripts/functions/url.rs

@@ -0,0 +1,75 @@
+/*
+ * Copyright (c) 2023 Stalwart Labs Ltd.
+ *
+ * This file is part of Stalwart Mail Server.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of
+ * the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ * in the LICENSE file at the top-level directory of this distribution.
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * You can be released from the requirements of the AGPLv3 license by
+ * purchasing a commercial license. Please contact licensing@stalw.art
+ * for more details.
+*/
+
+use hyper::Uri;
+use sieve::{runtime::Variable, Context};
+
+use super::ApplyString;
+
+pub fn fn_uri_part<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    let part = v[1].to_string();
+    v[0].transform(|uri| {
+        uri.parse::<Uri>()
+            .ok()
+            .and_then(|uri| match part.as_ref() {
+                "scheme" => uri.scheme_str().map(|s| Variable::from(s.to_string())),
+                "host" => uri.host().map(|s| Variable::from(s.to_string())),
+                "scheme_host" => uri
+                    .scheme_str()
+                    .and_then(|s| (s, uri.host()?).into())
+                    .map(|(s, h)| Variable::from(format!("{}://{}", s, h))),
+                "path" => Variable::from(uri.path().to_string()).into(),
+                "port" => uri.port_u16().map(|port| Variable::Integer(port as i64)),
+                "query" => uri.query().map(|s| Variable::from(s.to_string())),
+                "path_query" => uri.path_and_query().map(|s| Variable::from(s.to_string())),
+                "authority" => uri.authority().map(|s| Variable::from(s.to_string())),
+                _ => None,
+            })
+            .unwrap_or_default()
+    })
+}
+
+pub fn fn_puny_decode<'x>(_: &'x Context<'x, ()>, v: Vec<Variable>) -> Variable {
+    v[0].transform(|domain| {
+        if domain.contains("xn--") {
+            let mut decoded = String::with_capacity(domain.len());
+            for part in domain.split('.') {
+                if !decoded.is_empty() {
+                    decoded.push('.');
+                }
+
+                if let Some(puny) = part
+                    .strip_prefix("xn--")
+                    .and_then(idna::punycode::decode_to_string)
+                {
+                    decoded.push_str(&puny);
+                } else {
+                    decoded.push_str(part);
+                }
+            }
+            decoded.into()
+        } else {
+            domain.into()
+        }
+    })
+}

crates/common/src/scripts/mod.rs

@@ -0,0 +1,50 @@
+use std::sync::Arc;
+
+use sieve::{runtime::Variable, Envelope};
+use store::Value;
+
+use crate::IntoString;
+
+pub mod functions;
+pub mod plugins;
+
+#[derive(Debug)]
+pub enum ScriptModification {
+    SetEnvelope {
+        name: Envelope,
+        value: String,
+    },
+    AddHeader {
+        name: Arc<String>,
+        value: Arc<String>,
+    },
+}
+
+pub fn into_sieve_value(value: Value) -> Variable {
+    match value {
+        Value::Integer(v) => Variable::Integer(v),
+        Value::Bool(v) => Variable::Integer(i64::from(v)),
+        Value::Float(v) => Variable::Float(v),
+        Value::Text(v) => Variable::String(v.into_owned().into()),
+        Value::Blob(v) => Variable::String(v.into_owned().into_string().into()),
+        Value::Null => Variable::default(),
+    }
+}
+
+pub fn into_store_value(value: Variable) -> Value<'static> {
+    match value {
+        Variable::String(v) => Value::Text(v.to_string().into()),
+        Variable::Integer(v) => Value::Integer(v),
+        Variable::Float(v) => Value::Float(v),
+        v => Value::Text(v.to_string().into_owned().into()),
+    }
+}
+
+pub fn to_store_value(value: &Variable) -> Value<'static> {
+    match value {
+        Variable::String(v) => Value::Text(v.to_string().into()),
+        Variable::Integer(v) => Value::Integer(*v),
+        Variable::Float(v) => Value::Float(*v),
+        v => Value::Text(v.to_string().into_owned().into()),
+    }
+}

crates/common/src/scripts/plugins/bayes.rs

@@ -0,0 +1,362 @@
+/*
+ * Copyright (c) 2023 Stalwart Labs Ltd.
+ *
+ * This file is part of Stalwart Mail Server.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of
+ * the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ * in the LICENSE file at the top-level directory of this distribution.
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * You can be released from the requirements of the AGPLv3 license by
+ * purchasing a commercial license. Please contact licensing@stalw.art
+ * for more details.
+*/
+
+use nlp::{
+    bayes::{
+        cache::BayesTokenCache, tokenize::BayesTokenizer, BayesClassifier, BayesModel, TokenHash,
+        Weights,
+    },
+    tokenizers::osb::{OsbToken, OsbTokenizer},
+};
+use sieve::{runtime::Variable, FunctionMap};
+use store::{write::key::KeySerializer, LookupStore, U64_LEN};
+use tokio::runtime::Handle;
+
+use super::PluginContext;
+
+pub fn register_train(plugin_id: u32, fnc_map: &mut FunctionMap<()>) {
+    fnc_map.set_external_function("bayes_train", plugin_id, 3);
+}
+
+pub fn register_untrain(plugin_id: u32, fnc_map: &mut FunctionMap<()>) {
+    fnc_map.set_external_function("bayes_untrain", plugin_id, 3);
+}
+
+pub fn register_classify(plugin_id: u32, fnc_map: &mut FunctionMap<()>) {
+    fnc_map.set_external_function("bayes_classify", plugin_id, 3);
+}
+
+pub fn register_is_balanced(plugin_id: u32, fnc_map: &mut FunctionMap<()>) {
+    fnc_map.set_external_function("bayes_is_balanced", plugin_id, 3);
+}
+
+pub fn exec_train(ctx: PluginContext<'_>) -> Variable {
+    train(ctx, true)
+}
+
+pub fn exec_untrain(ctx: PluginContext<'_>) -> Variable {
+    train(ctx, false)
+}
+
+fn train(ctx: PluginContext<'_>, is_train: bool) -> Variable {
+    let span: &tracing::Span = ctx.span;
+    let store = match &ctx.arguments[0] {
+        Variable::String(v) if !v.is_empty() => ctx.core.storage.lookups.get(v.as_ref()),
+        _ => Some(&ctx.core.storage.lookup),
+    };
+
+    let store = if let Some(store) = store {
+        store
+    } else {
+        tracing::warn!(
+            parent: span,
+            context = "sieve:bayes_train",
+            event = "failed",
+            reason = "Unknown store id",
+            lookup_store = ctx.arguments[0].to_string().as_ref(),
+        );
+        return false.into();
+    };
+    let text = ctx.arguments[1].to_string();
+    let is_spam = ctx.arguments[2].to_bool();
+    if text.is_empty() {
+        return false.into();
+    }
+    let handle = ctx.handle;
+
+    // Train the model
+    let mut model = BayesModel::default();
+    model.train(
+        OsbTokenizer::new(
+            BayesTokenizer::new(text.as_ref(), &ctx.core.smtp.resolvers.psl),
+            5,
+        ),
+        is_spam,
+    );
+    if model.weights.is_empty() {
+        return false.into();
+    }
+
+    tracing::debug!(
+        parent: span,
+        context = "sieve:bayes_train",
+        event = "train",
+        is_spam = is_spam,
+        num_tokens = model.weights.len(),
+    );
+
+    // Update weight and invalidate cache
+    let bayes_cache = &ctx.core.sieve.bayes_cache;
+    if is_train {
+        for (hash, weights) in model.weights {
+            if handle
+                .block_on(
+                    store.counter_incr(
+                        KeySerializer::new(U64_LEN)
+                            .write(hash.h1)
+                            .write(hash.h2)
+                            .finalize(),
+                        weights.into(),
+                        None,
+                        false,
+                    ),
+                )
+                .is_err()
+            {
+                return false.into();
+            }
+            bayes_cache.invalidate(&hash);
+        }
+
+        // Update training counts
+        let weights = if is_spam {
+            Weights { spam: 1, ham: 0 }
+        } else {
+            Weights { spam: 0, ham: 1 }
+        };
+        if handle
+            .block_on(
+                store.counter_incr(
+                    KeySerializer::new(U64_LEN)
+                        .write(0u64)
+                        .write(0u64)
+                        .finalize(),
+                    weights.into(),
+                    None,
+                    false,
+                ),
+            )
+            .is_err()
+        {
+            return false.into();
+        }
+    } else {
+        //TODO: Implement untrain
+        return false.into();
+    }
+
+    bayes_cache.invalidate(&TokenHash::default());
+
+    true.into()
+}
+
+pub fn exec_classify(ctx: PluginContext<'_>) -> Variable {
+    let span = ctx.span;
+    let store = match &ctx.arguments[0] {
+        Variable::String(v) if !v.is_empty() => ctx.core.storage.lookups.get(v.as_ref()),
+        _ => Some(&ctx.core.storage.lookup),
+    };
+    let store = if let Some(store) = store {
+        store
+    } else {
+        tracing::warn!(
+            parent: span,
+            context = "sieve:bayes_classify",
+            event = "failed",
+            reason = "Unknown store id",
+            lookup_id = ctx.arguments[0].to_string().as_ref(),
+        );
+        return Variable::default();
+    };
+    let text = ctx.arguments[1].to_string();
+    if text.is_empty() {
+        return Variable::default();
+    }
+
+    // Create classifier from defaults
+    let mut classifier = BayesClassifier::default();
+    if let Some(params) = ctx.arguments[2].as_array() {
+        if let Some(Variable::Integer(value)) = params.first() {
+            classifier.min_token_hits = *value as u32;
+        }
+        if let Some(Variable::Integer(value)) = params.get(1) {
+            classifier.min_tokens = *value as u32;
+        }
+        if let Some(Variable::Float(value)) = params.get(2) {
+            classifier.min_prob_strength = *value;
+        }
+        if let Some(Variable::Integer(value)) = params.get(3) {
+            classifier.min_learns = *value as u32;
+        }
+    }
+
+    let handle = ctx.handle;
+
+    // Obtain training counts
+    let bayes_cache = &ctx.core.sieve.bayes_cache;
+    let (spam_learns, ham_learns) =
+        if let Some(weights) = bayes_cache.get_or_update(TokenHash::default(), handle, store) {
+            (weights.spam, weights.ham)
+        } else {
+            tracing::warn!(
+                parent: span,
+                context = "sieve:classify",
+                event = "failed",
+                reason = "Failed to obtain training counts",
+            );
+            return Variable::default();
+        };
+
+    // Make sure we have enough training data
+    if spam_learns < classifier.min_learns || ham_learns < classifier.min_learns {
+        tracing::debug!(
+            parent: span,
+            context = "sieve:bayes_classify",
+            event = "skip-classify",
+            reason = "Not enough training data",
+            spam_learns = %spam_learns,
+            ham_learns = %ham_learns);
+        return Variable::default();
+    }
+
+    // Classify the text
+    classifier
+        .classify(
+            OsbTokenizer::<_, TokenHash>::new(
+                BayesTokenizer::new(text.as_ref(), &ctx.core.smtp.resolvers.psl),
+                5,
+            )
+            .filter_map(|t| {
+                OsbToken {
+                    inner: bayes_cache.get_or_update(t.inner, handle, store)?,
+                    idx: t.idx,
+                }
+                .into()
+            }),
+            ham_learns,
+            spam_learns,
+        )
+        .map(Variable::from)
+        .unwrap_or_default()
+}
+
+pub fn exec_is_balanced(ctx: PluginContext<'_>) -> Variable {
+    let min_balance = match &ctx.arguments[2] {
+        Variable::Float(n) => *n,
+        Variable::Integer(n) => *n as f64,
+        _ => 0.0,
+    };
+
+    if min_balance == 0.0 {
+        return true.into();
+    }
+
+    let span = ctx.span;
+    let store = match &ctx.arguments[0] {
+        Variable::String(v) if !v.is_empty() => ctx.core.storage.lookups.get(v.as_ref()),
+        _ => Some(&ctx.core.storage.lookup),
+    };
+    let store = if let Some(store) = store {
+        store
+    } else {
+        tracing::warn!(
+            parent: span,
+            context = "sieve:bayes_is_balanced",
+            event = "failed",
+            reason = "Unknown store id",
+            lookup_id = ctx.arguments[0].to_string().as_ref(),
+        );
+        return Variable::default();
+    };
+    let learn_spam = ctx.arguments[1].to_bool();
+
+    // Obtain training counts
+    let handle = ctx.handle;
+    let bayes_cache = &ctx.core.sieve.bayes_cache;
+    let (spam_learns, ham_learns) =
+        if let Some(weights) = bayes_cache.get_or_update(TokenHash::default(), handle, store) {
+            (weights.spam as f64, weights.ham as f64)
+        } else {
+            tracing::warn!(
+                parent: span,
+                context = "sieve:bayes_is_balanced",
+                event = "failed",
+                reason = "Failed to obtain training counts",
+            );
+            return Variable::default();
+        };
+
+    let result = if spam_learns > 0.0 || ham_learns > 0.0 {
+        if learn_spam {
+            (spam_learns / (ham_learns + 1.0)) <= 1.0 / min_balance
+        } else {
+            (ham_learns / (spam_learns + 1.0)) <= 1.0 / min_balance
+        }
+    } else {
+        true
+    };
+
+    tracing::debug!(
+        parent: span,
+        context = "sieve:bayes_is_balanced",
+        event = "result",
+        is_balanced = %result,
+        learn_spam = %learn_spam,
+        min_balance = %min_balance,
+        spam_learns = %spam_learns,
+        ham_learns = %ham_learns);
+
+    result.into()
+}
+
+trait LookupOrInsert {
+    fn get_or_update(
+        &self,
+        hash: TokenHash,
+        handle: &Handle,
+        get_token: &LookupStore,
+    ) -> Option<Weights>;
+}
+
+impl LookupOrInsert for BayesTokenCache {
+    fn get_or_update(
+        &self,
+        hash: TokenHash,
+        handle: &Handle,
+        get_token: &LookupStore,
+    ) -> Option<Weights> {
+        if let Some(weights) = self.get(&hash) {
+            weights.unwrap_or_default().into()
+        } else if let Ok(num) = handle.block_on(
+            get_token.counter_get(
+                KeySerializer::new(U64_LEN)
+                    .write(hash.h1)
+                    .write(hash.h2)
+                    .finalize(),
+            ),
+        ) {
+            if num != 0 {
+                let weights = Weights::from(num);
+                self.insert_positive(hash, weights);
+                weights
+            } else {
+                self.insert_negative(hash);
+                Weights::default()
+            }
+            .into()
+        } else {
+            // Something went wrong
+            None
+        }
+    }
+}

crates/common/src/scripts/plugins/dns.rs

@@ -0,0 +1,221 @@
+/*
+ * Copyright (c) 2023 Stalwart Labs Ltd.
+ *
+ * This file is part of Stalwart Mail Server.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of
+ * the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ * in the LICENSE file at the top-level directory of this distribution.
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * You can be released from the requirements of the AGPLv3 license by
+ * purchasing a commercial license. Please contact licensing@stalw.art
+ * for more details.
+*/
+
+use std::net::IpAddr;
+
+use mail_auth::{Error, IpLookupStrategy};
+use sieve::{runtime::Variable, FunctionMap};
+
+use super::PluginContext;
+
+pub fn register(plugin_id: u32, fnc_map: &mut FunctionMap<()>) {
+    fnc_map.set_external_function("dns_query", plugin_id, 2);
+}
+
+pub fn register_exists(plugin_id: u32, fnc_map: &mut FunctionMap<()>) {
+    fnc_map.set_external_function("dns_exists", plugin_id, 2);
+}
+
+pub fn exec(ctx: PluginContext<'_>) -> Variable {
+    let entry = ctx.arguments[0].to_string();
+    let record_type = ctx.arguments[1].to_string();
+
+    if record_type.eq_ignore_ascii_case("ip") {
+        match ctx.handle.block_on(ctx.core.smtp.resolvers.dns.ip_lookup(
+            entry.as_ref(),
+            IpLookupStrategy::Ipv4thenIpv6,
+            10,
+        )) {
+            Ok(result) => result
+                .iter()
+                .map(|ip| Variable::from(ip.to_string()))
+                .collect::<Vec<_>>()
+                .into(),
+            Err(err) => err.short_error().into(),
+        }
+    } else if record_type.eq_ignore_ascii_case("mx") {
+        match ctx
+            .handle
+            .block_on(ctx.core.smtp.resolvers.dns.mx_lookup(entry.as_ref()))
+        {
+            Ok(result) => result
+                .iter()
+                .flat_map(|mx| {
+                    mx.exchanges
+                        .iter()
+                        .map(|host| Variable::from(format!("{} {}", mx.preference, host)))
+                })
+                .collect::<Vec<_>>()
+                .into(),
+            Err(err) => err.short_error().into(),
+        }
+    } else if record_type.eq_ignore_ascii_case("txt") {
+        #[cfg(feature = "test_mode")]
+        {
+            if entry.contains("origin") {
+                return Variable::from("23028|US|arin|2002-01-04".to_string());
+            }
+        }
+
+        match ctx
+            .handle
+            .block_on(ctx.core.smtp.resolvers.dns.txt_raw_lookup(entry.as_ref()))
+        {
+            Ok(result) => Variable::from(String::from_utf8(result).unwrap_or_default()),
+            Err(err) => err.short_error().into(),
+        }
+    } else if record_type.eq_ignore_ascii_case("ptr") {
+        if let Ok(addr) = entry.parse::<IpAddr>() {
+            match ctx
+                .handle
+                .block_on(ctx.core.smtp.resolvers.dns.ptr_lookup(addr))
+            {
+                Ok(result) => result
+                    .iter()
+                    .map(|host| Variable::from(host.to_string()))
+                    .collect::<Vec<_>>()
+                    .into(),
+                Err(err) => err.short_error().into(),
+            }
+        } else {
+            Variable::default()
+        }
+    } else if record_type.eq_ignore_ascii_case("ipv4") {
+        #[cfg(feature = "test_mode")]
+        {
+            if entry.contains(".168.192.") {
+                let parts = entry.split('.').collect::<Vec<_>>();
+                return vec![Variable::from(format!("127.0.{}.{}", parts[1], parts[0]))].into();
+            }
+        }
+
+        match ctx
+            .handle
+            .block_on(ctx.core.smtp.resolvers.dns.ipv4_lookup(entry.as_ref()))
+        {
+            Ok(result) => result
+                .iter()
+                .map(|ip| Variable::from(ip.to_string()))
+                .collect::<Vec<_>>()
+                .into(),
+            Err(err) => err.short_error().into(),
+        }
+    } else if record_type.eq_ignore_ascii_case("ipv6") {
+        match ctx
+            .handle
+            .block_on(ctx.core.smtp.resolvers.dns.ipv6_lookup(entry.as_ref()))
+        {
+            Ok(result) => result
+                .iter()
+                .map(|ip| Variable::from(ip.to_string()))
+                .collect::<Vec<_>>()
+                .into(),
+            Err(err) => err.short_error().into(),
+        }
+    } else {
+        Variable::default()
+    }
+}
+
+pub fn exec_exists(ctx: PluginContext<'_>) -> Variable {
+    let entry = ctx.arguments[0].to_string();
+    let record_type = ctx.arguments[1].to_string();
+
+    if record_type.eq_ignore_ascii_case("ip") {
+        match ctx.handle.block_on(ctx.core.smtp.resolvers.dns.ip_lookup(
+            entry.as_ref(),
+            IpLookupStrategy::Ipv4thenIpv6,
+            10,
+        )) {
+            Ok(result) => i64::from(!result.is_empty()),
+            Err(Error::DnsRecordNotFound(_)) => 0,
+            Err(_) => -1,
+        }
+    } else if record_type.eq_ignore_ascii_case("mx") {
+        match ctx
+            .handle
+            .block_on(ctx.core.smtp.resolvers.dns.mx_lookup(entry.as_ref()))
+        {
+            Ok(result) => i64::from(result.iter().any(|mx| !mx.exchanges.is_empty())),
+            Err(Error::DnsRecordNotFound(_)) => 0,
+            Err(_) => -1,
+        }
+    } else if record_type.eq_ignore_ascii_case("ptr") {
+        if let Ok(addr) = entry.parse::<IpAddr>() {
+            match ctx
+                .handle
+                .block_on(ctx.core.smtp.resolvers.dns.ptr_lookup(addr))
+            {
+                Ok(result) => i64::from(!result.is_empty()),
+                Err(Error::DnsRecordNotFound(_)) => 0,
+                Err(_) => -1,
+            }
+        } else {
+            -1
+        }
+    } else if record_type.eq_ignore_ascii_case("ipv4") {
+        #[cfg(feature = "test_mode")]
+        {
+            if entry.starts_with("2.0.168.192.") {
+                return 1.into();
+            }
+        }
+
+        match ctx
+            .handle
+            .block_on(ctx.core.smtp.resolvers.dns.ipv4_lookup(entry.as_ref()))
+        {
+            Ok(result) => i64::from(!result.is_empty()),
+            Err(Error::DnsRecordNotFound(_)) => 0,
+            Err(_) => -1,
+        }
+    } else if record_type.eq_ignore_ascii_case("ipv6") {
+        match ctx
+            .handle
+            .block_on(ctx.core.smtp.resolvers.dns.ipv6_lookup(entry.as_ref()))
+        {
+            Ok(result) => i64::from(!result.is_empty()),
+            Err(Error::DnsRecordNotFound(_)) => 0,
+            Err(_) => -1,
+        }
+    } else {
+        -1
+    }
+    .into()
+}
+
+trait ShortError {
+    fn short_error(&self) -> &'static str;
+}
+
+impl ShortError for mail_auth::Error {
+    fn short_error(&self) -> &'static str {
+        match self {
+            mail_auth::Error::DnsError(_) => "temp_fail",
+            mail_auth::Error::DnsRecordNotFound(_) => "not_found",
+            mail_auth::Error::Io(_) => "io_error",
+            mail_auth::Error::InvalidRecordType => "invalid_record",
+            _ => "unknown_error",
+        }
+    }
+}

crates/common/src/scripts/plugins/exec.rs

@@ -0,0 +1,62 @@
+/*
+ * Copyright (c) 2023 Stalwart Labs Ltd.
+ *
+ * This file is part of Stalwart Mail Server.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of
+ * the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ * in the LICENSE file at the top-level directory of this distribution.
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * You can be released from the requirements of the AGPLv3 license by
+ * purchasing a commercial license. Please contact licensing@stalw.art
+ * for more details.
+*/
+
+use std::process::Command;
+
+use sieve::{runtime::Variable, FunctionMap};
+
+use super::PluginContext;
+
+pub fn register(plugin_id: u32, fnc_map: &mut FunctionMap<()>) {
+    fnc_map.set_external_function("exec", plugin_id, 2);
+}
+
+pub fn exec(ctx: PluginContext<'_>) -> Variable {
+    let span = ctx.span;
+    let mut arguments = ctx.arguments.into_iter();
+    match Command::new(
+        arguments
+            .next()
+            .map(|a| a.to_string().into_owned())
+            .unwrap_or_default(),
+    )
+    .args(
+        arguments
+            .next()
+            .map(|a| a.into_string_array())
+            .unwrap_or_default(),
+    )
+    .output()
+    {
+        Ok(result) => result.status.success().into(),
+        Err(err) => {
+            tracing::warn!(
+                parent: span,
+                context = "sieve",
+                event = "execute-failed",
+                reason = %err,
+            );
+            false.into()
+        }
+    }
+}

crates/common/src/scripts/plugins/headers.rs

@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2023 Stalwart Labs Ltd.
+ *
+ * This file is part of Stalwart Mail Server.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of
+ * the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ * in the LICENSE file at the top-level directory of this distribution.
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * You can be released from the requirements of the AGPLv3 license by
+ * purchasing a commercial license. Please contact licensing@stalw.art
+ * for more details.
+*/
+
+use sieve::{runtime::Variable, FunctionMap};
+
+use crate::scripts::ScriptModification;
+
+use super::PluginContext;
+
+pub fn register(plugin_id: u32, fnc_map: &mut FunctionMap<()>) {
+    fnc_map.set_external_function("add_header", plugin_id, 2);
+}
+
+pub fn exec(ctx: PluginContext<'_>) -> Variable {
+    if let (Variable::String(name), Variable::String(value)) =
+        (&ctx.arguments[0], &ctx.arguments[1])
+    {
+        ctx.modifications.push(ScriptModification::AddHeader {
+            name: name.clone(),
+            value: value.clone(),
+        });
+        true
+    } else {
+        false
+    }
+    .into()
+}

crates/common/src/scripts/plugins/http.rs

@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2023 Stalwart Labs Ltd.
+ *
+ * This file is part of Stalwart Mail Server.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of
+ * the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ * in the LICENSE file at the top-level directory of this distribution.
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * You can be released from the requirements of the AGPLv3 license by
+ * purchasing a commercial license. Please contact licensing@stalw.art
+ * for more details.
+*/
+
+use std::time::Duration;
+
+use reqwest::redirect::Policy;
+use sieve::{runtime::Variable, FunctionMap};
+
+use super::PluginContext;
+
+pub fn register_header(plugin_id: u32, fnc_map: &mut FunctionMap<()>) {
+    fnc_map.set_external_function("http_header", plugin_id, 4);
+}
+
+pub fn exec_header(ctx: PluginContext<'_>) -> Variable {
+    let url = ctx.arguments[0].to_string();
+    let header = ctx.arguments[1].to_string();
+    let agent = ctx.arguments[2].to_string();
+    let timeout = ctx.arguments[3].to_string().parse::<u64>().unwrap_or(5000);
+
+    #[cfg(feature = "test_mode")]
+    if url.contains("redirect.") {
+        return Variable::from(url.split_once("/?").unwrap().1.to_string());
+    }
+
+    if let Ok(client) = reqwest::Client::builder()
+        .user_agent(agent.as_ref())
+        .timeout(Duration::from_millis(timeout))
+        .redirect(Policy::none())
+        .danger_accept_invalid_certs(true)
+        .build()
+    {
+        let _enter = ctx.handle.enter();
+        ctx.handle
+            .block_on(client.get(url.as_ref()).send())
+            .ok()
+            .and_then(|response| {
+                response
+                    .headers()
+                    .get(header.as_ref())
+                    .and_then(|h| h.to_str().ok())
+                    .map(|h| Variable::from(h.to_string()))
+            })
+            .unwrap_or_default()
+    } else {
+        false.into()
+    }
+}

crates/common/src/scripts/plugins/lookup.rs

@@ -0,0 +1,433 @@
+/*
+ * Copyright (c) 2023 Stalwart Labs Ltd.
+ *
+ * This file is part of Stalwart Mail Server.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of
+ * the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ * in the LICENSE file at the top-level directory of this distribution.
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * You can be released from the requirements of the AGPLv3 license by
+ * purchasing a commercial license. Please contact licensing@stalw.art
+ * for more details.
+*/
+
+use std::{
+    collections::HashSet,
+    io::{BufRead, BufReader},
+    time::{Duration, Instant},
+};
+
+use mail_auth::flate2;
+use sieve::{runtime::Variable, FunctionMap};
+use store::{Deserialize, Value};
+
+use crate::{config::scripts::RemoteList, scripts::into_sieve_value, USER_AGENT};
+
+use super::PluginContext;
+
+pub fn register(plugin_id: u32, fnc_map: &mut FunctionMap<()>) {
+    fnc_map.set_external_function("key_exists", plugin_id, 2);
+}
+
+pub fn register_get(plugin_id: u32, fnc_map: &mut FunctionMap<()>) {
+    fnc_map.set_external_function("key_get", plugin_id, 2);
+}
+
+pub fn register_set(plugin_id: u32, fnc_map: &mut FunctionMap<()>) {
+    fnc_map.set_external_function("key_set", plugin_id, 4);
+}
+
+pub fn register_remote(plugin_id: u32, fnc_map: &mut FunctionMap<()>) {
+    fnc_map.set_external_function("key_exists_http", plugin_id, 3);
+}
+
+pub fn register_local_domain(plugin_id: u32, fnc_map: &mut FunctionMap<()>) {
+    fnc_map.set_external_function("is_local_domain", plugin_id, 2);
+}
+
+pub fn exec(ctx: PluginContext<'_>) -> Variable {
+    let store = match &ctx.arguments[0] {
+        Variable::String(v) if !v.is_empty() => ctx.core.storage.lookups.get(v.as_ref()),
+        _ => Some(&ctx.core.storage.lookup),
+    };
+
+    if let Some(store) = store {
+        match &ctx.arguments[1] {
+            Variable::Array(items) => {
+                for item in items.iter() {
+                    if !item.is_empty()
+                        && ctx
+                            .handle
+                            .block_on(store.key_exists(item.to_string().into_owned().into_bytes()))
+                            .unwrap_or(false)
+                    {
+                        return true.into();
+                    }
+                }
+                false
+            }
+            v if !v.is_empty() => ctx
+                .handle
+                .block_on(store.key_exists(v.to_string().into_owned().into_bytes()))
+                .unwrap_or(false),
+            _ => false,
+        }
+    } else {
+        tracing::warn!(
+            parent: ctx.span,
+            context = "sieve:lookup",
+            event = "failed",
+            reason = "Unknown lookup id",
+            lookup_id = ctx.arguments[0].to_string().as_ref(),
+        );
+        false
+    }
+    .into()
+}
+
+pub fn exec_get(ctx: PluginContext<'_>) -> Variable {
+    let store = match &ctx.arguments[0] {
+        Variable::String(v) if !v.is_empty() => ctx.core.storage.lookups.get(v.as_ref()),
+        _ => Some(&ctx.core.storage.lookup),
+    };
+
+    if let Some(store) = store {
+        ctx.handle
+            .block_on(
+                store.key_get::<VariableWrapper>(
+                    ctx.arguments[1].to_string().into_owned().into_bytes(),
+                ),
+            )
+            .unwrap_or_default()
+            .map(|v| v.into_inner())
+            .unwrap_or_default()
+    } else {
+        tracing::warn!(
+            parent: ctx.span,
+            context = "sieve:key_get",
+            event = "failed",
+            reason = "Unknown store or lookup id",
+            lookup_id = ctx.arguments[0].to_string().as_ref(),
+        );
+        Variable::default()
+    }
+}
+
+pub fn exec_set(ctx: PluginContext<'_>) -> Variable {
+    let store = match &ctx.arguments[0] {
+        Variable::String(v) if !v.is_empty() => ctx.core.storage.lookups.get(v.as_ref()),
+        _ => Some(&ctx.core.storage.lookup),
+    };
+
+    if let Some(store) = store {
+        let expires = match &ctx.arguments[3] {
+            Variable::Integer(v) => Some(*v as u64),
+            Variable::Float(v) => Some(*v as u64),
+            _ => None,
+        };
+
+        ctx.handle
+            .block_on(store.key_set(
+                ctx.arguments[1].to_string().into_owned().into_bytes(),
+                if !ctx.arguments[2].is_empty() {
+                    bincode::serialize(&ctx.arguments[2]).unwrap_or_default()
+                } else {
+                    vec![]
+                },
+                expires,
+            ))
+            .is_ok()
+            .into()
+    } else {
+        tracing::warn!(
+            parent: ctx.span,
+            context = "sieve:key_set",
+            event = "failed",
+            reason = "Unknown store id",
+            store_id = ctx.arguments[0].to_string().as_ref(),
+        );
+        Variable::default()
+    }
+}
+
+pub fn exec_remote(ctx: PluginContext<'_>) -> Variable {
+    let resource = ctx.arguments[0].to_string();
+    let item = ctx.arguments[1].to_string();
+
+    #[cfg(feature = "test_mode")]
+    {
+        if (resource.contains("open") && item.contains("open"))
+            || (resource.contains("tank") && item.contains("tank"))
+        {
+            return true.into();
+        }
+    }
+
+    if resource.is_empty() || item.is_empty() {
+        return false.into();
+    }
+
+    const TIMEOUT: Duration = Duration::from_secs(45);
+    const RETRY: Duration = Duration::from_secs(3600);
+    const MAX_ENTRY_SIZE: usize = 256;
+    const MAX_ENTRIES: usize = 100000;
+
+    match ctx.core.sieve.remote_lists.read().get(resource.as_ref()) {
+        Some(remote_list) if remote_list.expires < Instant::now() => {
+            return remote_list.entries.contains(item.as_ref()).into()
+        }
+        _ => {}
+    }
+
+    enum Format {
+        List,
+        Csv {
+            column: u32,
+            separator: char,
+            skip_first: bool,
+        },
+    }
+
+    // Obtain parameters
+    let mut format = Format::List;
+    let mut expires = Duration::from_secs(12 * 3600);
+
+    if let Some(arr) = ctx.arguments[2].as_array() {
+        // Obtain expiration
+        match arr.first() {
+            Some(Variable::Integer(v)) if *v > 0 => {
+                expires = Duration::from_secs(*v as u64);
+            }
+            Some(Variable::Float(v)) if *v > 0.0 => {
+                expires = Duration::from_secs(*v as u64);
+            }
+            _ => (),
+        }
+
+        // Obtain list type
+        if matches!(arr.get(1), Some(Variable::String(list_type)) if list_type.eq_ignore_ascii_case("csv"))
+        {
+            format = Format::Csv {
+                column: arr.get(2).map(|v| v.to_integer()).unwrap_or_default() as u32,
+                separator: arr
+                    .get(3)
+                    .and_then(|v| v.to_string().chars().next())
+                    .unwrap_or(','),
+                skip_first: arr.get(4).map_or(false, |v| v.to_bool()),
+            };
+        }
+    }
+
+    // Lock remote list for writing
+    let mut _lock = ctx.core.sieve.remote_lists.write();
+    let list = _lock
+        .entry(resource.to_string())
+        .or_insert_with(|| RemoteList {
+            entries: HashSet::new(),
+            expires: Instant::now(),
+        });
+
+    // Make sure that the list is still expired
+    if list.expires > Instant::now() {
+        return list.entries.contains(item.as_ref()).into();
+    }
+
+    let _enter = ctx.handle.enter();
+    match ctx
+        .handle
+        .block_on(
+            reqwest::Client::builder()
+                .timeout(TIMEOUT)
+                .user_agent(USER_AGENT)
+                .build()
+                .unwrap_or_default()
+                .get(resource.as_ref())
+                .send(),
+        )
+        .and_then(|r| {
+            if r.status().is_success() {
+                ctx.handle.block_on(r.bytes()).map(Ok)
+            } else {
+                Ok(Err(r))
+            }
+        }) {
+        Ok(Ok(bytes)) => {
+            let reader: Box<dyn std::io::Read> = if resource.ends_with(".gz") {
+                Box::new(flate2::read::GzDecoder::new(&bytes[..]))
+            } else {
+                Box::new(&bytes[..])
+            };
+
+            for (pos, line) in BufReader::new(reader).lines().enumerate() {
+                match line {
+                    Ok(line_) => {
+                        // Clear list once the first entry has been successfully fetched, decompressed and UTF8-decoded
+                        if pos == 0 {
+                            list.entries.clear();
+                        }
+
+                        match &format {
+                            Format::List => {
+                                let line = line_.trim();
+                                if !line.is_empty() {
+                                    list.entries.insert(line.to_string());
+                                }
+                            }
+                            Format::Csv {
+                                column,
+                                separator,
+                                skip_first,
+                            } if pos > 0 || !*skip_first => {
+                                let mut in_quote = false;
+                                let mut col_num = 0;
+                                let mut entry = String::new();
+
+                                for ch in line_.chars() {
+                                    if ch != '"' {
+                                        if ch == *separator && !in_quote {
+                                            if col_num == *column {
+                                                break;
+                                            } else {
+                                                col_num += 1;
+                                            }
+                                        } else if col_num == *column {
+                                            entry.push(ch);
+                                            if entry.len() > MAX_ENTRY_SIZE {
+                                                break;
+                                            }
+                                        }
+                                    } else {
+                                        in_quote = !in_quote;
+                                    }
+                                }
+
+                                if !entry.is_empty() {
+                                    list.entries.insert(entry);
+                                }
+                            }
+                            _ => (),
+                        }
+                    }
+                    Err(err) => {
+                        tracing::warn!(
+                            parent: ctx.span,
+                            context = "sieve:key_exists_http",
+                            event = "failed",
+                            resource = resource.as_ref(),
+                            reason = %err,
+                        );
+                        break;
+                    }
+                }
+
+                if list.entries.len() == MAX_ENTRIES {
+                    break;
+                }
+            }
+
+            tracing::debug!(
+                parent: ctx.span,
+                context = "sieve:key_exists_http",
+                event = "fetch",
+                resource = resource.as_ref(),
+                num_entries = list.entries.len(),
+            );
+
+            // Update expiration
+            list.expires = Instant::now() + expires;
+            return list.entries.contains(item.as_ref()).into();
+        }
+        Ok(Err(response)) => {
+            tracing::warn!(
+                parent: ctx.span,
+                context = "sieve:key_exists_http",
+                event = "failed",
+                resource = resource.as_ref(),
+                status = %response.status(),
+            );
+        }
+        Err(err) => {
+            tracing::warn!(
+                parent: ctx.span,
+                context = "sieve:key_exists_http",
+                event = "failed",
+                resource = resource.as_ref(),
+                reason = %err,
+            );
+        }
+    }
+
+    // Something went wrong, try again in one hour
+    list.expires = Instant::now() + RETRY;
+    false.into()
+}
+
+pub fn exec_local_domain(ctx: PluginContext<'_>) -> Variable {
+    let domain = ctx.arguments[0].to_string();
+
+    if !domain.is_empty() {
+        let directory = match &ctx.arguments[0] {
+            Variable::String(v) if !v.is_empty() => ctx.core.storage.directories.get(v.as_ref()),
+            _ => Some(&ctx.core.storage.directory),
+        };
+
+        if let Some(directory) = directory {
+            return ctx
+                .handle
+                .block_on(directory.is_local_domain(domain.as_ref()))
+                .unwrap_or_default()
+                .into();
+        } else {
+            tracing::warn!(
+                parent: ctx.span,
+                context = "sieve:is_local_domain",
+                event = "failed",
+                reason = "Unknown directory",
+                lookup_id = ctx.arguments[0].to_string().as_ref(),
+            );
+        }
+    }
+
+    Variable::default()
+}
+
+#[derive(Debug, PartialEq, Eq)]
+pub struct VariableWrapper(Variable);
+
+impl Deserialize for VariableWrapper {
+    fn deserialize(bytes: &[u8]) -> store::Result<Self> {
+        Ok(VariableWrapper(
+            bincode::deserialize::<Variable>(bytes).unwrap_or_else(|_| {
+                Variable::String(String::from_utf8_lossy(bytes).into_owned().into())
+            }),
+        ))
+    }
+}
+
+impl From<i64> for VariableWrapper {
+    fn from(value: i64) -> Self {
+        VariableWrapper(value.into())
+    }
+}
+
+impl VariableWrapper {
+    pub fn into_inner(self) -> Variable {
+        self.0
+    }
+}
+
+impl From<Value<'static>> for VariableWrapper {
+    fn from(value: Value<'static>) -> Self {
+        VariableWrapper(into_sieve_value(value))
+    }
+}

crates/common/src/scripts/plugins/mod.rs

@@ -0,0 +1,132 @@
+/*
+ * Copyright (c) 2023 Stalwart Labs Ltd.
+ *
+ * This file is part of Stalwart Mail Server.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of
+ * the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ * in the LICENSE file at the top-level directory of this distribution.
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * You can be released from the requirements of the AGPLv3 license by
+ * purchasing a commercial license. Please contact licensing@stalw.art
+ * for more details.
+*/
+
+pub mod bayes;
+pub mod dns;
+pub mod exec;
+pub mod headers;
+pub mod http;
+pub mod lookup;
+pub mod pyzor;
+pub mod query;
+pub mod text;
+
+use mail_parser::Message;
+use sieve::{runtime::Variable, FunctionMap, Input};
+use tokio::runtime::Handle;
+
+use crate::Core;
+
+use super::ScriptModification;
+
+type RegisterPluginFnc = fn(u32, &mut FunctionMap<()>) -> ();
+type ExecPluginFnc = fn(PluginContext<'_>) -> Variable;
+
+pub struct PluginContext<'x> {
+    pub span: &'x tracing::Span,
+    pub handle: &'x Handle,
+    pub core: &'x Core,
+    pub message: &'x Message<'x>,
+    pub modifications: &'x mut Vec<ScriptModification>,
+    pub arguments: Vec<Variable>,
+}
+
+const PLUGINS_EXEC: [ExecPluginFnc; 18] = [
+    query::exec,
+    exec::exec,
+    lookup::exec,
+    lookup::exec_get,
+    lookup::exec_set,
+    lookup::exec_remote,
+    lookup::exec_local_domain,
+    dns::exec,
+    dns::exec_exists,
+    http::exec_header,
+    bayes::exec_train,
+    bayes::exec_untrain,
+    bayes::exec_classify,
+    bayes::exec_is_balanced,
+    pyzor::exec,
+    headers::exec,
+    text::exec_tokenize,
+    text::exec_domain_part,
+];
+const PLUGINS_REGISTER: [RegisterPluginFnc; 18] = [
+    query::register,
+    exec::register,
+    lookup::register,
+    lookup::register_get,
+    lookup::register_set,
+    lookup::register_remote,
+    lookup::register_local_domain,
+    dns::register,
+    dns::register_exists,
+    http::register_header,
+    bayes::register_train,
+    bayes::register_untrain,
+    bayes::register_classify,
+    bayes::register_is_balanced,
+    pyzor::register,
+    headers::register,
+    text::register_tokenize,
+    text::register_domain_part,
+];
+
+pub trait RegisterSievePlugins {
+    fn register_plugins(self) -> Self;
+}
+
+impl RegisterSievePlugins for FunctionMap<()> {
+    fn register_plugins(mut self) -> Self {
+        #[cfg(feature = "test_mode")]
+        {
+            self.set_external_function("print", PLUGINS_EXEC.len() as u32, 1)
+        }
+
+        for (i, fnc) in PLUGINS_REGISTER.iter().enumerate() {
+            fnc(i as u32, &mut self);
+        }
+        self
+    }
+}
+
+impl Core {
+    pub fn run_plugin_blocking(&self, id: u32, ctx: PluginContext<'_>) -> Input {
+        #[cfg(feature = "test_mode")]
+        if id == PLUGINS_EXEC.len() as u32 {
+            return test_print(ctx);
+        }
+
+        PLUGINS_EXEC
+            .get(id as usize)
+            .map(|fnc| fnc(ctx))
+            .unwrap_or_default()
+            .into()
+    }
+}
+
+#[cfg(feature = "test_mode")]
+pub fn test_print(ctx: PluginContext<'_>) -> Input {
+    println!("{}", ctx.arguments[0].to_string());
+    Input::True
+}

crates/common/src/scripts/plugins/pyzor.rs

@@ -0,0 +1,834 @@
+/*
+ * Copyright (c) 2023 Stalwart Labs Ltd.
+ *
+ * This file is part of Stalwart Mail Server.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of
+ * the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ * in the LICENSE file at the top-level directory of this distribution.
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * You can be released from the requirements of the AGPLv3 license by
+ * purchasing a commercial license. Please contact licensing@stalw.art
+ * for more details.
+*/
+
+use sieve::{runtime::Variable, FunctionMap};
+
+use super::PluginContext;
+
+use std::{
+    borrow::Cow,
+    io::Write,
+    time::{Duration, SystemTime},
+};
+
+use mail_parser::{decoders::html::add_html_token, Message, PartType};
+use nlp::tokenizers::types::{TokenType, TypesTokenizer};
+use sha1::{Digest, Sha1};
+use tokio::net::UdpSocket;
+use utils::suffixlist::PublicSuffix;
+
+const MIN_LINE_LENGTH: usize = 8;
+const ATOMIC_NUM_LINES: usize = 4;
+const DIGEST_SPEC: &[(usize, usize)] = &[(20, 3), (60, 3)];
+
+#[derive(Default, Debug, PartialEq, Eq)]
+struct PyzorResponse {
+    code: u32,
+    count: u64,
+    wl_count: u64,
+}
+
+pub fn register(plugin_id: u32, fnc_map: &mut FunctionMap<()>) {
+    fnc_map.set_external_function("pyzor_check", plugin_id, 2);
+}
+
+pub fn exec(ctx: PluginContext<'_>) -> Variable {
+    // Make sure there is at least one text part
+    if !ctx
+        .message
+        .parts
+        .iter()
+        .any(|p| matches!(p.body, PartType::Text(_) | PartType::Html(_)))
+    {
+        return Variable::default();
+    }
+
+    // Hash message
+    let request = ctx
+        .message
+        .pyzor_check_message(&ctx.core.smtp.resolvers.psl);
+
+    #[cfg(feature = "test_mode")]
+    {
+        if request.contains("b5b476f0b5ba6e1c038361d3ded5818dd39c90a2") {
+            return PyzorResponse {
+                code: 200,
+                count: 1000,
+                wl_count: 0,
+            }
+            .into();
+        } else if request.contains("d67d4b8bfc3860449e3418bb6017e2612f3e2a99") {
+            return PyzorResponse {
+                code: 200,
+                count: 60,
+                wl_count: 10,
+            }
+            .into();
+        } else if request.contains("81763547012b75e57a20d18ce0b93014208cdfdb") {
+            return PyzorResponse {
+                code: 200,
+                count: 50,
+                wl_count: 20,
+            }
+            .into();
+        }
+    }
+
+    let span = ctx.span;
+    let address = ctx.arguments[0].to_string();
+    let timeout = Duration::from_secs(std::cmp::max(
+        std::cmp::min(ctx.arguments[1].to_integer() as u64, 60),
+        5,
+    ));
+    // Send message to address
+    match ctx
+        .handle
+        .block_on(pyzor_send_message(address.as_ref(), timeout, &request))
+    {
+        Ok(response) => response.into(),
+        Err(err) => {
+            tracing::debug!(
+                parent: span,
+                context = "sieve:pyzor_check",
+                event = "failed",
+                reason = %err,
+            );
+            Variable::default()
+        }
+    }
+}
+
+impl From<PyzorResponse> for Variable {
+    fn from(response: PyzorResponse) -> Self {
+        vec![
+            Variable::from(response.code),
+            Variable::from(response.count),
+            Variable::from(response.wl_count),
+        ]
+        .into()
+    }
+}
+
+async fn pyzor_send_message(
+    addr: &str,
+    timeout: Duration,
+    message: &str,
+) -> std::io::Result<PyzorResponse> {
+    let socket = UdpSocket::bind("0.0.0.0:0").await?;
+    tokio::time::timeout(timeout, socket.send_to(message.as_bytes(), addr)).await??;
+
+    let mut buffer = vec![0u8; 1024];
+    let (size, _) = tokio::time::timeout(timeout, socket.recv_from(&mut buffer)).await??;
+
+    let raw_response = std::str::from_utf8(&buffer[..size])
+        .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidData, err))?;
+    let mut response = PyzorResponse {
+        code: u32::MAX,
+        count: u64::MAX,
+        wl_count: u64::MAX,
+    };
+
+    for line in raw_response.lines() {
+        if let Some((k, v)) = line.split_once(':') {
+            if k.eq_ignore_ascii_case("code") {
+                response.code = v.trim().parse().map_err(|_| {
+                    std::io::Error::new(
+                        std::io::ErrorKind::InvalidData,
+                        format!("Invalid line: {raw_response}"),
+                    )
+                })?;
+            } else if k.eq_ignore_ascii_case("count") {
+                response.count = v.trim().parse().map_err(|_| {
+                    std::io::Error::new(
+                        std::io::ErrorKind::InvalidData,
+                        format!("Invalid line: {raw_response}"),
+                    )
+                })?;
+            } else if k.eq_ignore_ascii_case("wl-count") {
+                response.wl_count = v.trim().parse().map_err(|_| {
+                    std::io::Error::new(
+                        std::io::ErrorKind::InvalidData,
+                        format!("Invalid line: {raw_response}"),
+                    )
+                })?;
+            }
+        }
+    }
+
+    if response.code != u32::MAX && response.count != u64::MAX && response.wl_count != u64::MAX {
+        Ok(response)
+    } else {
+        Err(std::io::Error::new(
+            std::io::ErrorKind::InvalidData,
+            format!("Invalid response: {raw_response}"),
+        ))
+    }
+}
+
+trait PyzorDigest<W: Write> {
+    fn pyzor_digest(&self, writer: W, psl: &PublicSuffix) -> W;
+}
+
+pub trait PyzorCheck {
+    fn pyzor_check_message(&self, psl: &PublicSuffix) -> String;
+}
+
+impl<'x, W: Write> PyzorDigest<W> for Message<'x> {
+    fn pyzor_digest(&self, writer: W, psl: &PublicSuffix) -> W {
+        let parts = self
+            .parts
+            .iter()
+            .filter_map(|part| match &part.body {
+                PartType::Text(text) => Some(text.as_ref().into()),
+                PartType::Html(html) => Some(html_to_text(html.as_ref()).into()),
+                _ => None,
+            })
+            .collect::<Vec<Cow<str>>>();
+
+        pyzor_digest(writer, parts.iter().flat_map(|text| text.lines()), psl)
+    }
+}
+
+impl<'x> PyzorCheck for Message<'x> {
+    fn pyzor_check_message(&self, psl: &PublicSuffix) -> String {
+        let time = SystemTime::now()
+            .duration_since(SystemTime::UNIX_EPOCH)
+            .map_or(0, |d| d.as_secs());
+
+        pyzor_create_message(
+            self,
+            psl,
+            time,
+            (time & 0xFFFF) as u16 ^ ((time >> 16) & 0xFFFF) as u16,
+        )
+    }
+}
+
+fn pyzor_create_message(
+    message: &Message<'_>,
+    psl: &PublicSuffix,
+    time: u64,
+    thread: u16,
+) -> String {
+    // Hash message
+    let hash = message.pyzor_digest(Sha1::new(), psl).finalize();
+    // Hash key
+    let mut hash_key = Sha1::new();
+    hash_key.update("anonymous:".as_bytes());
+    let hash_key = hash_key.finalize();
+
+    // Hash message
+    let message = format!(
+        "Op: check\nOp-Digest: {hash:x}\nThread: {thread}\nPV: 2.1\nUser: anonymous\nTime: {time}"
+    );
+    let mut msg_hash = Sha1::new();
+    msg_hash.update(message.as_bytes());
+    let msg_hash = msg_hash.finalize();
+
+    // Sign
+    let mut sig = Sha1::new();
+    sig.update(msg_hash);
+    sig.update(&format!(":{time}:{hash_key:x}"));
+    let sig = sig.finalize();
+
+    format!("{message}\nSig: {sig:x}\n")
+}
+
+fn pyzor_digest<'x, I, W>(mut writer: W, lines: I, psl: &PublicSuffix) -> W
+where
+    I: Iterator<Item = &'x str>,
+    W: Write,
+{
+    let mut result = Vec::with_capacity(16);
+
+    for line in lines {
+        let mut clean_line = String::with_capacity(line.len());
+        let mut token_start = usize::MAX;
+        let mut token_end = usize::MAX;
+
+        let add_line = |line: &mut String, span: &str| {
+            if !span.contains(char::from(0)) {
+                if span.len() < 10 {
+                    line.push_str(span);
+                }
+            } else {
+                let span = span.replace(char::from(0), "");
+                if span.len() < 10 {
+                    line.push_str(&span);
+                }
+            }
+        };
+
+        for token in TypesTokenizer::new(line, psl) {
+            match token.word {
+                TokenType::Alphabetic(_)
+                | TokenType::Alphanumeric(_)
+                | TokenType::Integer(_)
+                | TokenType::Float(_)
+                | TokenType::Other(_)
+                | TokenType::Punctuation(_) => {
+                    if token_start == usize::MAX {
+                        token_start = token.from;
+                    }
+                    token_end = token.to;
+                }
+                TokenType::Space
+                | TokenType::Url(_)
+                | TokenType::UrlNoScheme(_)
+                | TokenType::UrlNoHost(_)
+                | TokenType::Email(_) => {
+                    if token_start != usize::MAX {
+                        add_line(&mut clean_line, &line[token_start..token_end]);
+                        token_start = usize::MAX;
+                        token_end = usize::MAX;
+                    }
+                }
+            }
+        }
+
+        if token_start != usize::MAX {
+            add_line(&mut clean_line, &line[token_start..token_end]);
+        }
+
+        if clean_line.len() >= MIN_LINE_LENGTH {
+            result.push(clean_line);
+        }
+    }
+
+    if result.len() > ATOMIC_NUM_LINES {
+        for (offset, length) in DIGEST_SPEC {
+            for i in 0..*length {
+                if let Some(line) = result.get((*offset * result.len() / 100) + i) {
+                    let _ = writer.write_all(line.as_bytes());
+                }
+            }
+        }
+    } else {
+        for line in result {
+            let _ = writer.write_all(line.as_bytes());
+        }
+    }
+
+    writer
+}
+
+fn html_to_text(input: &str) -> String {
+    let mut result = String::with_capacity(input.len());
+    let input = input.as_bytes();
+
+    let mut in_tag = false;
+    let mut in_comment = false;
+    let mut in_style = false;
+    let mut in_script = false;
+
+    let mut is_token_start = true;
+    let mut is_after_space = false;
+    let mut is_tag_close = false;
+
+    let mut token_start = 0;
+    let mut token_end = 0;
+
+    let mut tag_token_pos = 0;
+    let mut comment_pos = 0;
+
+    for (pos, ch) in input.iter().enumerate() {
+        if !in_comment {
+            match ch {
+                b'<' => {
+                    if !(in_tag || in_style || in_script || is_token_start) {
+                        add_html_token(
+                            &mut result,
+                            &input[token_start..token_end + 1],
+                            is_after_space,
+                        );
+                        is_after_space = false;
+                    }
+
+                    tag_token_pos = 0;
+                    in_tag = true;
+                    is_token_start = true;
+                    is_tag_close = false;
+                    continue;
+                }
+                b'>' if in_tag => {
+                    if tag_token_pos == 1 {
+                        if let Some(tag) = input.get(token_start..token_end + 1) {
+                            if tag.eq_ignore_ascii_case(b"style") {
+                                in_style = !is_tag_close;
+                            } else if tag.eq_ignore_ascii_case(b"script") {
+                                in_script = !is_tag_close;
+                            }
+                        }
+                    }
+
+                    in_tag = false;
+                    is_token_start = true;
+                    is_after_space = !result.is_empty();
+
+                    continue;
+                }
+                b'/' if in_tag => {
+                    if tag_token_pos == 0 {
+                        is_tag_close = true;
+                    }
+                    continue;
+                }
+                b'!' if in_tag && tag_token_pos == 0 => {
+                    if let Some(b"--") = input.get(pos + 1..pos + 3) {
+                        in_comment = true;
+                        continue;
+                    }
+                }
+                b' ' | b'\t' | b'\r' | b'\n' => {
+                    if !(in_tag || in_style || in_script) {
+                        if !is_token_start {
+                            add_html_token(
+                                &mut result,
+                                &input[token_start..token_end + 1],
+                                is_after_space,
+                            );
+                        }
+                        is_after_space = true;
+                    }
+
+                    is_token_start = true;
+                    continue;
+                }
+                b'&' if !(in_tag || is_token_start || in_style || in_script) => {
+                    add_html_token(
+                        &mut result,
+                        &input[token_start..token_end + 1],
+                        is_after_space,
+                    );
+                    is_token_start = true;
+                    is_after_space = false;
+                }
+                b';' if !(in_tag || is_token_start || in_style || in_script) => {
+                    add_html_token(&mut result, &input[token_start..pos + 1], is_after_space);
+                    is_token_start = true;
+                    is_after_space = false;
+                    continue;
+                }
+                _ => (),
+            }
+            if is_token_start {
+                token_start = pos;
+                is_token_start = false;
+                if in_tag {
+                    tag_token_pos += 1;
+                }
+            }
+            token_end = pos;
+        } else {
+            match ch {
+                b'-' => comment_pos += 1,
+                b'>' if comment_pos == 2 => {
+                    comment_pos = 0;
+                    in_comment = false;
+                    in_tag = false;
+                    is_token_start = true;
+                }
+                _ => comment_pos = 0,
+            }
+        }
+    }
+
+    if !(in_tag || is_token_start || in_style || in_script) {
+        add_html_token(
+            &mut result,
+            &input[token_start..token_end + 1],
+            is_after_space,
+        );
+    }
+
+    result.shrink_to_fit();
+    result
+}
+
+#[cfg(test)]
+mod test {
+    use std::time::Duration;
+
+    use mail_parser::MessageParser;
+    use sha1::Digest;
+    use sha1::Sha1;
+    use utils::suffixlist::PublicSuffix;
+
+    use super::pyzor_create_message;
+    use super::pyzor_send_message;
+    use super::{html_to_text, pyzor_digest, PyzorDigest};
+
+    use super::PyzorResponse;
+
+    #[ignore]
+    #[tokio::test]
+    async fn send_message() {
+        assert_eq!(
+            pyzor_send_message(
+                "public.pyzor.org:24441",
+                Duration::from_secs(10),
+                concat!(
+                    "Op: check\n",
+                    "Op-Digest: b2c27325a034c581df0c9ef37e4a0d63208a3e7e\n",
+                    "Thread: 49005\n",
+                    "PV: 2.1\n",
+                    "User: anonymous\n",
+                    "Time: 1697468672\n",
+                    "Sig: 9cf4571b85d3887fdd0d4f444fd0c164e0290722\n"
+                ),
+            )
+            .await
+            .unwrap(),
+            PyzorResponse {
+                code: 200,
+                count: 0,
+                wl_count: 0
+            }
+        );
+    }
+
+    #[test]
+    fn message_pyzor() {
+        let mut psl = PublicSuffix::default();
+        psl.suffixes.insert("com".to_string());
+        let message = pyzor_create_message(
+            &MessageParser::new().parse(HTML_TEXT_STYLE_SCRIPT).unwrap(),
+            &psl,
+            1697468672,
+            49005,
+        );
+
+        assert_eq!(
+            message,
+            concat!(
+                "Op: check\n",
+                "Op-Digest: b2c27325a034c581df0c9ef37e4a0d63208a3e7e\n",
+                "Thread: 49005\n",
+                "PV: 2.1\n",
+                "User: anonymous\n",
+                "Time: 1697468672\n",
+                "Sig: 9cf4571b85d3887fdd0d4f444fd0c164e0290722\n"
+            )
+        );
+    }
+
+    #[test]
+    fn digest_pyzor() {
+        let mut psl = PublicSuffix::default();
+        psl.suffixes.insert("com".to_string());
+
+        // HTML stripping
+        assert_eq!(html_to_text(HTML_RAW), HTML_RAW_STRIPED);
+
+        // Token stripping
+        for strip_me in [
+            "t@abc.com",
+            "t1@abc.com",
+            "t+a@abc.com",
+            "t.a@abc.com",
+            "0A2D3f%a#S",
+            "3sddkf9jdkd9",
+            "@@#@@@@@@@@@",
+            "http://spammer.com/special-offers?buy=now",
+        ] {
+            assert_eq!(
+                String::from_utf8(pyzor_digest(
+                    Vec::new(),
+                    format!("Test {strip_me} Test2").lines(),
+                    &psl
+                ))
+                .unwrap(),
+                "TestTest2"
+            );
+        }
+
+        // Test short lines
+        assert_eq!(
+            String::from_utf8(pyzor_digest(
+                Vec::new(),
+                concat!("This line is included\n", "not this\n", "This also").lines(),
+                &psl
+            ))
+            .unwrap(),
+            "ThislineisincludedThisalso"
+        );
+
+        // Test atomic
+        assert_eq!(
+            String::from_utf8(pyzor_digest(
+                Vec::new(),
+                "All this message\nShould be included\nIn the digest".lines(),
+                &psl
+            ))
+            .unwrap(),
+            "AllthismessageShouldbeincludedInthedigest"
+        );
+
+        // Test spec
+        let mut text = String::new();
+        for i in 0..100 {
+            text += &format!("Line{i} test test test\n");
+        }
+        let mut expected = String::new();
+        for i in [20, 21, 22, 60, 61, 62] {
+            expected += &format!("Line{i}testtesttest");
+        }
+        assert_eq!(
+            String::from_utf8(pyzor_digest(Vec::new(), text.lines(), &psl)).unwrap(),
+            expected
+        );
+
+        // Test email parsing
+        for (input, expected) in [
+            (
+                HTML_TEXT,
+                concat!(
+                    "Emailspam,alsoknownasjunkemailorbulkemail,isasubset",
+                    "ofspaminvolvingnearlyidenticalmessagessenttonumerous",
+                    "byemail.Clickingonlinksinspamemailmaysendusersto",
+                    "byemail.Clickingonlinksinspamemailmaysendusersto",
+                    "phishingwebsitesorsitesthatarehostingmalware.",
+                    "Emailspam.Emailspam,alsoknownasjunkemailorbulkemail,",
+                    "isasubsetofspaminvolvingnearlyidenticalmessage",
+                    "ssenttonumerousbyemail.Clickingonlinksinspamemailmaysenduse",
+                    "rstophishingwebsitesorsitesthatarehostingmalware."
+                ),
+            ),
+            (HTML_TEXT_STYLE_SCRIPT, "Thisisatest.Thisisatest."),
+            (TEXT_ATTACHMENT, "Thisisatestmailing"),
+            (TEXT_ATTACHMENT_W_NULL, "Thisisatestmailing"),
+            (TEXT_ATTACHMENT_W_MULTIPLE_NULLS, "Thisisatestmailing"),
+            (TEXT_ATTACHMENT_W_SUBJECT_NULL, "Thisisatestmailing"),
+            (TEXT_ATTACHMENT_W_CONTENTTYPE_NULL, "Thisisatestmailing"),
+        ] {
+            assert_eq!(
+                String::from_utf8(
+                    MessageParser::new()
+                        .parse(input)
+                        .unwrap()
+                        .pyzor_digest(Vec::new(), &psl)
+                )
+                .unwrap(),
+                expected,
+                "failed for {input}"
+            )
+        }
+
+        // Test SHA hash
+        assert_eq!(
+            format!(
+                "{:x}",
+                MessageParser::new()
+                    .parse(HTML_TEXT_STYLE_SCRIPT)
+                    .unwrap()
+                    .pyzor_digest(Sha1::new(), &psl)
+                    .finalize()
+            ),
+            "b2c27325a034c581df0c9ef37e4a0d63208a3e7e",
+        )
+    }
+
+    const HTML_TEXT: &str = r#"MIME-Version: 1.0
+Sender: chirila@gapps.spamexperts.com
+Received: by 10.216.157.70 with HTTP; Thu, 16 Jan 2014 00:43:31 -0800 (PST)
+Date: Thu, 16 Jan 2014 10:43:31 +0200
+Delivered-To: chirila@gapps.spamexperts.com
+X-Google-Sender-Auth: ybCmONS9U9D6ZUfjx-9_tY-hF2Q
+Message-ID: <CAK-mJS8sE-V6qtspzzZ+bZ1eSUE_FNMt3K-5kBOG-z3NMgU_Rg@mail.gmail.com>
+Subject: Test
+From: Alexandru Chirila <chirila@spamexperts.com>
+To: Alexandru Chirila <chirila@gapps.spamexperts.com>
+Content-Type: multipart/alternative; boundary=001a11c25ff293069304f0126bfd
+
+--001a11c25ff293069304f0126bfd
+Content-Type: text/plain; charset=ISO-8859-1
+
+Email spam.
+
+Email spam, also known as junk email or unsolicited bulk email, is a subset
+of electronic spam involving nearly identical messages sent to numerous
+recipients by email. Clicking on links in spam email may send users to
+phishing web sites or sites that are hosting malware.
+
+--001a11c25ff293069304f0126bfd
+Content-Type: text/html; charset=ISO-8859-1
+Content-Transfer-Encoding: quoted-printable
+
+<div dir=3D"ltr"><div>Email spam.</div><div><br></div><div>Email spam, also=
+ known as junk email or unsolicited bulk email, is a subset of electronic s=
+pam involving nearly identical messages sent to numerous recipients by emai=
+l. Clicking on links in spam email may send users to phishing web sites or =
+sites that are hosting malware.</div>
+</div>
+
+--001a11c25ff293069304f0126bfd--
+"#;
+
+    const HTML_TEXT_STYLE_SCRIPT: &str = r#"MIME-Version: 1.0
+Sender: chirila@gapps.spamexperts.com
+Received: by 10.216.157.70 with HTTP; Thu, 16 Jan 2014 00:43:31 -0800 (PST)
+Date: Thu, 16 Jan 2014 10:43:31 +0200
+Delivered-To: chirila@gapps.spamexperts.com
+X-Google-Sender-Auth: ybCmONS9U9D6ZUfjx-9_tY-hF2Q
+Message-ID: <CAK-mJS8sE-V6qtspzzZ+bZ1eSUE_FNMt3K-5kBOG-z3NMgU_Rg@mail.gmail.com>
+Subject: Test
+From: Alexandru Chirila <chirila@spamexperts.com>
+To: Alexandru Chirila <chirila@gapps.spamexperts.com>
+Content-Type: multipart/alternative; boundary=001a11c25ff293069304f0126bfd
+
+--001a11c25ff293069304f0126bfd
+Content-Type: text/plain; charset=ISO-8859-1
+
+This is a test.
+
+--001a11c25ff293069304f0126bfd
+Content-Type: text/html; charset=ISO-8859-1
+Content-Transfer-Encoding: quoted-printable
+
+<div dir=3D"ltr">
+<style> This is my style.</style>
+<script> This is my script.</script>
+<div>This is a test.</div>
+</div>
+
+--001a11c25ff293069304f0126bfd--
+"#;
+
+    const TEXT_ATTACHMENT: &str = r#"MIME-Version: 1.0
+Received: by 10.76.127.40 with HTTP; Fri, 17 Jan 2014 02:21:43 -0800 (PST)
+Date: Fri, 17 Jan 2014 12:21:43 +0200
+Delivered-To: chirila.s.alexandru@gmail.com
+Message-ID: <CALTHOsuHFaaatiXJKU=LdDCo4NmD_h49yvG2RDsWw17D0-NXJg@mail.gmail.com>
+Subject: Test
+From: Alexandru Chirila <chirila.s.alexandru@gmail.com>
+To: Alexandru Chirila <chirila.s.alexandru@gmail.com>
+Content-Type: multipart/mixed; boundary=f46d040a62c49bb1c804f027e8cc
+
+--f46d040a62c49bb1c804f027e8cc
+Content-Type: multipart/alternative; boundary=f46d040a62c49bb1c404f027e8ca
+
+--f46d040a62c49bb1c404f027e8ca
+Content-Type: text/plain; charset=ISO-8859-1
+
+This is a test mailing
+
+--f46d040a62c49bb1c404f027e8ca--
+--f46d040a62c49bb1c804f027e8cc
+Content-Type: image/png; name="tar.png"
+Content-Disposition: attachment; filename="tar.png"
+Content-Transfer-Encoding: base64
+X-Attachment-Id: f_hqjas5ad0
+
+iVBORw0KGgoAAAANSUhEUgAAAskAAADlCAAAAACErzVVAAAACXBIWXMAAAsTAAALEwEAmpwYAAAD
+QmCC
+--f46d040a62c49bb1c804f027e8cc--"#;
+
+    const TEXT_ATTACHMENT_W_NULL: &str = "MIME-Version: 1.0
+Received: by 10.76.127.40 with HTTP; Fri, 17 Jan 2014 02:21:43 -0800 (PST)
+Date: Fri, 17 Jan 2014 12:21:43 +0200
+Delivered-To: chirila.s.alexandru@gmail.com
+Message-ID: <CALTHOsuHFaaatiXJKU=LdDCo4NmD_h49yvG2RDsWw17D0-NXJg@mail.gmail.com>
+Subject: Test
+From: Alexandru Chirila <chirila.s.alexandru@gmail.com>
+To: Alexandru Chirila <chirila.s.alexandru@gmail.com>
+Content-Type: multipart/mixed; boundary=f46d040a62c49bb1c804f027e8cc
+
+--f46d040a62c49bb1c804f027e8cc
+Content-Type: multipart/alternative; boundary=f46d040a62c49bb1c404f027e8ca
+
+--f46d040a62c49bb1c404f027e8ca
+Content-Type: text/plain; charset=ISO-8859-1
+
+This is a test ma\0iling
+--f46d040a62c49bb1c804f027e8cc--";
+
+    const TEXT_ATTACHMENT_W_MULTIPLE_NULLS: &str = "MIME-Version: 1.0
+Received: by 10.76.127.40 with HTTP; Fri, 17 Jan 2014 02:21:43 -0800 (PST)
+Date: Fri, 17 Jan 2014 12:21:43 +0200
+Delivered-To: chirila.s.alexandru@gmail.com
+Message-ID: <CALTHOsuHFaaatiXJKU=LdDCo4NmD_h49yvG2RDsWw17D0-NXJg@mail.gmail.com>
+Subject: Test
+From: Alexandru Chirila <chirila.s.alexandru@gmail.com>
+To: Alexandru Chirila <chirila.s.alexandru@gmail.com>
+Content-Type: multipart/mixed; boundary=f46d040a62c49bb1c804f027e8cc
+
+--f46d040a62c49bb1c804f027e8cc
+Content-Type: multipart/alternative; boundary=f46d040a62c49bb1c404f027e8ca
+
+--f46d040a62c49bb1c404f027e8ca
+Content-Type: text/plain; charset=ISO-8859-1
+
+This is a test ma\0\0\0iling
+--f46d040a62c49bb1c804f027e8cc--";
+
+    const TEXT_ATTACHMENT_W_SUBJECT_NULL: &str = "MIME-Version: 1.0
+Received: by 10.76.127.40 with HTTP; Fri, 17 Jan 2014 02:21:43 -0800 (PST)
+Date: Fri, 17 Jan 2014 12:21:43 +0200
+Delivered-To: chirila.s.alexandru@gmail.com
+Message-ID: <CALTHOsuHFaaatiXJKU=LdDCo4NmD_h49yvG2RDsWw17D0-NXJg@mail.gmail.com>
+Subject: Te\0\0\0st
+From: Alexandru Chirila <chirila.s.alexandru@gmail.com>
+To: Alexandru Chirila <chirila.s.alexandru@gmail.com>
+Content-Type: multipart/mixed; boundary=f46d040a62c49bb1c804f027e8cc
+
+--f46d040a62c49bb1c804f027e8cc
+Content-Type: multipart/alternative; boundary=f46d040a62c49bb1c404f027e8ca
+
+--f46d040a62c49bb1c404f027e8ca
+Content-Type: text/plain; charset=ISO-8859-1
+
+This is a test mailing
+--f46d040a62c49bb1c804f027e8cc--";
+
+    const TEXT_ATTACHMENT_W_CONTENTTYPE_NULL: &str = "MIME-Version: 1.0
+Received: by 10.76.127.40 with HTTP; Fri, 17 Jan 2014 02:21:43 -0800 (PST)
+Date: Fri, 17 Jan 2014 12:21:43 +0200
+Delivered-To: chirila.s.alexandru@gmail.com
+Message-ID: <CALTHOsuHFaaatiXJKU=LdDCo4NmD_h49yvG2RDsWw17D0-NXJg@mail.gmail.com>
+Subject: Test
+From: Alexandru Chirila <chirila.s.alexandru@gmail.com>
+To: Alexandru Chirila <chirila.s.alexandru@gmail.com>
+Content-Type: multipart/mixed; boundary=f46d040a62c49bb1c804f027e8cc
+
+--f46d040a62c49bb1c804f027e8cc
+Content-Type: multipart/alternative; boundary=f46d040a62c49bb1c404f027e8ca
+
+--f46d040a62c49bb1c404f027e8ca
+Content-Type: text/plain; charset=\"iso-8859-1\0\0\0\"
+
+This is a test mailing
+--f46d040a62c49bb1c804f027e8cc--";
+
+    const HTML_RAW: &str = r#"<html><head><title>Email spam</title></head><body>
+<p><b>Email spam</b>, also known as <b>junk email</b> 
+or <b>unsolicited bulk email</b> (<i>UBE</i>), is a subset of 
+<a href="/wiki/Spam_(electronic)" title="Spam (electronic)">electronic spam</a> 
+involving nearly identical messages sent to numerous recipients by <a href="/wiki/Email" title="Email">
+email</a>. Clicking on <a href="/wiki/Html_email#Security_vulnerabilities" title="Html email" class="mw-redirect">
+links in spam email</a> may send users to <a href="/wiki/Phishing" title="Phishing">phishing</a> 
+web sites or sites that are hosting <a href="/wiki/Malware" title="Malware">malware</a>.</body></html>"#;
+
+    const HTML_RAW_STRIPED : &str = concat!("Email spam Email spam , also known as junk email or unsolicited bulk email ( UBE )," ,
+                    " is a subset of electronic spam involving nearly identical messages sent to numerous recipients by email" ,
+                    " . Clicking on links in spam email may send users to phishing web sites or sites that are hosting malware .");
+}

crates/common/src/scripts/plugins/query.rs

@@ -0,0 +1,124 @@
+/*
+ * Copyright (c) 2023 Stalwart Labs Ltd.
+ *
+ * This file is part of Stalwart Mail Server.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of
+ * the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ * in the LICENSE file at the top-level store of this distribution.
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * You can be released from the requirements of the AGPLv3 license by
+ * purchasing a commercial license. Please contact licensing@stalw.art
+ * for more details.
+*/
+
+use std::cmp::Ordering;
+
+use crate::scripts::{into_sieve_value, to_store_value};
+use sieve::{runtime::Variable, FunctionMap};
+use store::{Rows, Value};
+
+use super::PluginContext;
+
+pub fn register(plugin_id: u32, fnc_map: &mut FunctionMap<()>) {
+    fnc_map.set_external_function("query", plugin_id, 3);
+}
+
+pub fn exec(ctx: PluginContext<'_>) -> Variable {
+    let span = ctx.span;
+
+    // Obtain store name
+    let store = match &ctx.arguments[0] {
+        Variable::String(v) if !v.is_empty() => ctx.core.storage.lookups.get(v.as_ref()),
+        _ => Some(&ctx.core.storage.lookup),
+    };
+
+    let store = if let Some(store) = store {
+        store
+    } else {
+        tracing::warn!(
+            parent: span,
+            context = "sieve:query",
+            event = "failed",
+            reason = "Unknown store",
+            store = ctx.arguments[0].to_string().as_ref(),
+        );
+        return false.into();
+    };
+
+    // Obtain query string
+    let query = ctx.arguments[1].to_string();
+    if query.is_empty() {
+        tracing::warn!(
+            parent: span,
+            context = "sieve:query",
+            event = "invalid",
+            reason = "Empty query string",
+        );
+        return false.into();
+    }
+
+    // Obtain arguments
+    let arguments = match &ctx.arguments[2] {
+        Variable::Array(l) => l.iter().map(to_store_value).collect(),
+        v => vec![to_store_value(v)],
+    };
+
+    // Run query
+    if query
+        .as_bytes()
+        .get(..6)
+        .map_or(false, |q| q.eq_ignore_ascii_case(b"SELECT"))
+    {
+        if let Ok(mut rows) = ctx.handle.block_on(store.query::<Rows>(&query, arguments)) {
+            match rows.rows.len().cmp(&1) {
+                Ordering::Equal => {
+                    let mut row = rows.rows.pop().unwrap().values;
+                    match row.len().cmp(&1) {
+                        Ordering::Equal if !matches!(row.first(), Some(Value::Null)) => {
+                            row.pop().map(into_sieve_value).unwrap()
+                        }
+                        Ordering::Less => Variable::default(),
+                        _ => Variable::Array(
+                            row.into_iter()
+                                .map(into_sieve_value)
+                                .collect::<Vec<_>>()
+                                .into(),
+                        ),
+                    }
+                }
+                Ordering::Less => Variable::default(),
+                Ordering::Greater => rows
+                    .rows
+                    .into_iter()
+                    .map(|r| {
+                        Variable::Array(
+                            r.values
+                                .into_iter()
+                                .map(into_sieve_value)
+                                .collect::<Vec<_>>()
+                                .into(),
+                        )
+                    })
+                    .collect::<Vec<_>>()
+                    .into(),
+            }
+        } else {
+            false.into()
+        }
+    } else {
+        ctx.handle
+            .block_on(store.query::<usize>(&query, arguments))
+            .is_ok()
+            .into()
+    }
+}

crates/common/src/scripts/plugins/text.rs

@@ -0,0 +1,128 @@
+/*
+ * Copyright (c) 2023 Stalwart Labs Ltd.
+ *
+ * This file is part of Stalwart Mail Server.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of
+ * the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ * in the LICENSE file at the top-level store of this distribution.
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * You can be released from the requirements of the AGPLv3 license by
+ * purchasing a commercial license. Please contact licensing@stalw.art
+ * for more details.
+*/
+
+use nlp::tokenizers::types::{TokenType, TypesTokenizer};
+use sieve::{runtime::Variable, FunctionMap};
+
+use crate::scripts::functions::{html::html_to_tokens, text::tokenize_words, ApplyString};
+
+use super::PluginContext;
+
+#[derive(PartialEq, Eq, Clone, Copy)]
+enum MatchPart {
+    Sld,
+    Tld,
+    Host,
+}
+
+pub fn register_tokenize(plugin_id: u32, fnc_map: &mut FunctionMap<()>) {
+    fnc_map.set_external_function("tokenize", plugin_id, 2);
+}
+
+pub fn register_domain_part(plugin_id: u32, fnc_map: &mut FunctionMap<()>) {
+    fnc_map.set_external_function("domain_part", plugin_id, 2);
+}
+
+pub fn exec_tokenize(ctx: PluginContext<'_>) -> Variable {
+    let mut v = ctx.arguments;
+    let (urls, urls_without_scheme, emails) = match v[1].to_string().as_ref() {
+        "html" => return html_to_tokens(v[0].to_string().as_ref()).into(),
+        "words" => return tokenize_words(&v[0]),
+        "uri" | "url" => (true, true, true),
+        "uri_strict" | "url_strict" => (true, false, false),
+        "email" => (false, false, true),
+        _ => return Variable::default(),
+    };
+
+    match v.remove(0) {
+        v @ (Variable::String(_) | Variable::Array(_)) => {
+            TypesTokenizer::new(v.to_string().as_ref(), &ctx.core.smtp.resolvers.psl)
+                .tokenize_numbers(false)
+                .tokenize_urls(urls)
+                .tokenize_urls_without_scheme(urls_without_scheme)
+                .tokenize_emails(emails)
+                .filter_map(|t| match t.word {
+                    TokenType::Url(text) if urls => Variable::from(text.to_string()).into(),
+                    TokenType::UrlNoScheme(text) if urls_without_scheme => {
+                        Variable::from(format!("https://{text}")).into()
+                    }
+                    TokenType::Email(text) if emails => Variable::from(text.to_string()).into(),
+                    _ => None,
+                })
+                .collect::<Vec<_>>()
+                .into()
+        }
+        v => v,
+    }
+}
+
+pub fn exec_domain_part(ctx: PluginContext<'_>) -> Variable {
+    let v = ctx.arguments;
+    let match_part = match v[1].to_string().as_ref() {
+        "sld" => MatchPart::Sld,
+        "tld" => MatchPart::Tld,
+        "host" => MatchPart::Host,
+        _ => return Variable::default(),
+    };
+
+    v[0].transform(|domain| {
+        let d = domain.trim().to_lowercase();
+        let mut seen_dot = false;
+        for (pos, ch) in d.as_bytes().iter().enumerate().rev() {
+            if *ch == b'.' {
+                if seen_dot {
+                    let maybe_domain =
+                        std::str::from_utf8(&d.as_bytes()[pos + 1..]).unwrap_or_default();
+                    if !ctx.core.smtp.resolvers.psl.contains(maybe_domain) {
+                        return if match_part == MatchPart::Sld {
+                            maybe_domain
+                        } else {
+                            std::str::from_utf8(&d.as_bytes()[..pos]).unwrap_or_default()
+                        }
+                        .to_string()
+                        .into();
+                    }
+                } else if match_part == MatchPart::Tld {
+                    return std::str::from_utf8(&d.as_bytes()[pos + 1..])
+                        .unwrap_or_default()
+                        .to_string()
+                        .into();
+                } else {
+                    seen_dot = true;
+                }
+            }
+        }
+
+        if seen_dot {
+            if match_part == MatchPart::Sld {
+                d.into()
+            } else {
+                Variable::default()
+            }
+        } else if match_part == MatchPart::Host {
+            d.into()
+        } else {
+            Variable::default()
+        }
+    })
+}