Principal update API changes
This commit is contained in:
mdecimus
@@ -1,277 +0,0 @@
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
pub mod spamassassin;
|
||||
pub mod tokenizer;
|
||||
pub mod utils;
|
||||
|
||||
#[derive(Debug, Default, Clone)]
|
||||
struct Rule {
|
||||
name: String,
|
||||
t: RuleType,
|
||||
scores: Vec<f64>,
|
||||
captured_vars: Vec<(String, usize)>,
|
||||
required_vars: HashSet<String>,
|
||||
description: HashMap<String, String>,
|
||||
priority: i32,
|
||||
flags: Vec<TestFlag>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone)]
|
||||
enum RuleType {
|
||||
Header {
|
||||
matches: HeaderMatches,
|
||||
header: Header,
|
||||
part: HeaderPart,
|
||||
pattern: String,
|
||||
},
|
||||
Body {
|
||||
pattern: String,
|
||||
raw: bool,
|
||||
},
|
||||
Full {
|
||||
pattern: String,
|
||||
},
|
||||
Uri {
|
||||
pattern: String,
|
||||
},
|
||||
Eval {
|
||||
function: String,
|
||||
params: Vec<String>,
|
||||
},
|
||||
Meta {
|
||||
expr: MetaExpression,
|
||||
},
|
||||
|
||||
#[default]
|
||||
None,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct MetaExpression {
|
||||
pub tokens: Vec<Token>,
|
||||
pub expr: String,
|
||||
}
|
||||
|
||||
impl RuleType {
|
||||
pub fn pattern(&mut self) -> Option<&mut String> {
|
||||
match self {
|
||||
RuleType::Header { pattern, .. } => Some(pattern),
|
||||
RuleType::Body { pattern, .. } => Some(pattern),
|
||||
RuleType::Full { pattern, .. } => Some(pattern),
|
||||
RuleType::Uri { pattern, .. } => Some(pattern),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq, Clone)]
|
||||
enum TestFlag {
|
||||
Net,
|
||||
Nice,
|
||||
UserConf,
|
||||
Learn,
|
||||
NoAutoLearn,
|
||||
Publish,
|
||||
NoPublish,
|
||||
Multiple,
|
||||
NoTrim,
|
||||
DomainsOnly,
|
||||
NoSubject,
|
||||
AutoLearnBody,
|
||||
A,
|
||||
MaxHits(u32),
|
||||
DnsBlockRule(String),
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, PartialEq, Eq, PartialOrd, Ord, Clone, Hash)]
|
||||
enum Header {
|
||||
#[default]
|
||||
All,
|
||||
MessageId,
|
||||
EnvelopeFrom,
|
||||
ToCc,
|
||||
Received(ReceivedPart),
|
||||
Name(String),
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Hash)]
|
||||
enum ReceivedPart {
|
||||
From,
|
||||
FromIp,
|
||||
FromIpRev,
|
||||
By,
|
||||
For,
|
||||
Ident,
|
||||
Id,
|
||||
Protocol,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone, Copy)]
|
||||
enum HeaderMatches {
|
||||
#[default]
|
||||
Matches,
|
||||
NotMatches,
|
||||
Exists,
|
||||
NotExists,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Hash)]
|
||||
enum HeaderPart {
|
||||
Name,
|
||||
Addr,
|
||||
Raw,
|
||||
#[default]
|
||||
Default,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq, Clone)]
|
||||
pub enum Token {
|
||||
Tag(String),
|
||||
Number(u32),
|
||||
Logical(Logical),
|
||||
Comparator(Comparator),
|
||||
Operation(Operation),
|
||||
|
||||
OpenParen,
|
||||
CloseParen,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq, Clone)]
|
||||
pub enum Logical {
|
||||
And,
|
||||
Or,
|
||||
Not,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq, Clone)]
|
||||
pub enum Comparator {
|
||||
Gt,
|
||||
Lt,
|
||||
Eq,
|
||||
Ge,
|
||||
Le,
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq, Clone)]
|
||||
pub enum Operation {
|
||||
Add,
|
||||
Multiply,
|
||||
Divide,
|
||||
Subtract,
|
||||
And,
|
||||
Or,
|
||||
Not,
|
||||
}
|
||||
|
||||
impl Rule {
|
||||
fn score(&self) -> f64 {
|
||||
self.scores.last().copied().unwrap_or_else(|| {
|
||||
if self.is_subrule() {
|
||||
0.0
|
||||
} else if self.name.starts_with("T_") {
|
||||
0.01
|
||||
} else {
|
||||
1.0
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn is_subrule(&self) -> bool {
|
||||
self.name.starts_with("__")
|
||||
}
|
||||
}
|
||||
|
||||
impl Ord for Rule {
|
||||
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
|
||||
match self.priority.cmp(&other.priority) {
|
||||
std::cmp::Ordering::Equal => {
|
||||
match other
|
||||
.score()
|
||||
.abs()
|
||||
.partial_cmp(&self.score().abs())
|
||||
.unwrap()
|
||||
{
|
||||
std::cmp::Ordering::Equal => other.name.cmp(&self.name),
|
||||
x => x,
|
||||
}
|
||||
}
|
||||
x => x,
|
||||
}
|
||||
/*let this_score = self.score();
|
||||
let other_score = other.score();
|
||||
|
||||
let this_is_negative = this_score < 0.0;
|
||||
let other_is_negative = other_score < 0.0;
|
||||
|
||||
if this_is_negative != other_is_negative {
|
||||
if this_is_negative {
|
||||
std::cmp::Ordering::Less
|
||||
} else {
|
||||
std::cmp::Ordering::Greater
|
||||
}
|
||||
} else {
|
||||
let this_priority = if this_score != 0.0 {
|
||||
self.priority
|
||||
} else {
|
||||
9000
|
||||
};
|
||||
let other_priority = if other_score != 0.0 {
|
||||
other.priority
|
||||
} else {
|
||||
9000
|
||||
};
|
||||
|
||||
match this_priority.cmp(&other_priority) {
|
||||
std::cmp::Ordering::Equal => {
|
||||
match other_score.abs().partial_cmp(&this_score.abs()).unwrap() {
|
||||
std::cmp::Ordering::Equal => other.name.cmp(&self.name),
|
||||
x => x,
|
||||
}
|
||||
}
|
||||
x => x,
|
||||
}
|
||||
}*/
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialOrd for Rule {
|
||||
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
|
||||
Some(self.cmp(other))
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialEq for Rule {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
self.name == other.name && self.priority == other.priority && self.scores == other.scores
|
||||
}
|
||||
}
|
||||
|
||||
impl Eq for Rule {}
|
||||
|
||||
pub trait UnwrapResult<T> {
|
||||
fn unwrap_result(self, action: &str) -> T;
|
||||
}
|
||||
|
||||
impl<T> UnwrapResult<T> for Option<T> {
|
||||
fn unwrap_result(self, message: &str) -> T {
|
||||
match self {
|
||||
Some(result) => result,
|
||||
None => {
|
||||
eprintln!("Failed to {}", message);
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T, E: std::fmt::Display> UnwrapResult<T> for Result<T, E> {
|
||||
fn unwrap_result(self, message: &str) -> T {
|
||||
match self {
|
||||
Ok(result) => result,
|
||||
Err(err) => {
|
||||
eprintln!("Failed to {}: {}", message, err);
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,151 +0,0 @@
|
||||
use super::{Comparator, Logical, Operation, Token};
|
||||
|
||||
pub struct Tokenizer<'x> {
|
||||
expr: &'x str,
|
||||
iter: std::iter::Peekable<std::str::Chars<'x>>,
|
||||
buf: String,
|
||||
depth: u32,
|
||||
comparator_depth: u32,
|
||||
next_token: Option<Token>,
|
||||
}
|
||||
|
||||
impl<'x> Tokenizer<'x> {
|
||||
pub fn new(expr: &'x str) -> Self {
|
||||
Self {
|
||||
expr,
|
||||
iter: expr.chars().peekable(),
|
||||
buf: String::new(),
|
||||
depth: 0,
|
||||
next_token: None,
|
||||
comparator_depth: u32::MAX,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'x> Iterator for Tokenizer<'x> {
|
||||
type Item = Token;
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
if let Some(token) = self.next_token.take() {
|
||||
return Some(token);
|
||||
}
|
||||
|
||||
while let Some(ch) = self.iter.next() {
|
||||
match ch {
|
||||
'A'..='Z' | 'a'..='z' | '0'..='9' | '_' => {
|
||||
self.buf.push(ch);
|
||||
}
|
||||
_ => {
|
||||
let mut depth = self.depth;
|
||||
let prev_token = if !self.buf.is_empty() {
|
||||
Token::from(std::mem::take(&mut self.buf)).into()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let token = match ch {
|
||||
'&' | '|' => {
|
||||
if matches!(self.iter.next(), Some(c) if c == ch) {
|
||||
Token::Logical(if ch == '&' { Logical::And } else { Logical::Or })
|
||||
} else {
|
||||
eprintln!("Warning: Single {ch} in meta expression {}", self.expr);
|
||||
return None;
|
||||
}
|
||||
}
|
||||
'!' => Token::Logical(Logical::Not),
|
||||
'=' => match self.iter.next() {
|
||||
Some('=') => Token::Comparator(Comparator::Eq),
|
||||
Some('>') => Token::Comparator(Comparator::Ge),
|
||||
Some('<') => Token::Comparator(Comparator::Le),
|
||||
_ => {
|
||||
eprintln!("Warning: Single = in meta expression {}", self.expr);
|
||||
Token::Comparator(Comparator::Eq)
|
||||
}
|
||||
},
|
||||
'>' => match self.iter.peek() {
|
||||
Some('=') => {
|
||||
self.iter.next();
|
||||
Token::Comparator(Comparator::Ge)
|
||||
}
|
||||
_ => Token::Comparator(Comparator::Gt),
|
||||
},
|
||||
'<' => match self.iter.peek() {
|
||||
Some('=') => {
|
||||
self.iter.next();
|
||||
Token::Comparator(Comparator::Le)
|
||||
}
|
||||
_ => Token::Comparator(Comparator::Lt),
|
||||
},
|
||||
'(' => {
|
||||
self.depth += 1;
|
||||
Token::OpenParen
|
||||
}
|
||||
')' => {
|
||||
if self.depth == 0 {
|
||||
eprintln!(
|
||||
"Warning: Unmatched close parenthesis in meta expression {}",
|
||||
self.expr
|
||||
);
|
||||
return None;
|
||||
}
|
||||
self.depth -= 1;
|
||||
depth = self.depth;
|
||||
|
||||
Token::CloseParen
|
||||
}
|
||||
'+' => Token::Operation(Operation::Add),
|
||||
'*' => Token::Operation(Operation::Multiply),
|
||||
'/' => Token::Operation(Operation::Divide),
|
||||
'-' => Token::Operation(Operation::Subtract),
|
||||
' ' => {
|
||||
if let Some(prev_token) = prev_token {
|
||||
return Some(prev_token);
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
eprintln!(
|
||||
"Warning: Invalid character {ch} in meta expression {}",
|
||||
self.expr
|
||||
);
|
||||
return None;
|
||||
}
|
||||
};
|
||||
|
||||
if matches!(token, Token::Comparator(_)) {
|
||||
self.comparator_depth = depth;
|
||||
}
|
||||
|
||||
return Some(if let Some(prev_token) = prev_token {
|
||||
self.next_token = Some(token);
|
||||
prev_token
|
||||
} else {
|
||||
token
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if self.depth > 0 {
|
||||
eprintln!(
|
||||
"Warning: Unmatched open parenthesis in meta expression {}",
|
||||
self.expr
|
||||
);
|
||||
None
|
||||
} else if !self.buf.is_empty() {
|
||||
Some(Token::from(std::mem::take(&mut self.buf)))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<String> for Token {
|
||||
fn from(value: String) -> Self {
|
||||
if let Ok(value) = value.parse() {
|
||||
Token::Number(value)
|
||||
} else {
|
||||
Token::Tag(value)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,172 +0,0 @@
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
pub fn replace_tags(
|
||||
pattern: &str,
|
||||
tag_start: char,
|
||||
tag_end: char,
|
||||
patterns: &HashMap<String, String>,
|
||||
) -> String {
|
||||
//print!("replacing {} ", pattern);
|
||||
let mut result = String::with_capacity(pattern.len());
|
||||
let mut chars = fix_broken_regex(pattern.trim()).chars();
|
||||
let mut tag_pre = String::new();
|
||||
let mut tag_inter = String::new();
|
||||
let mut tag_post = String::new();
|
||||
let mut is_adjacent = false;
|
||||
|
||||
'outer: while let Some(ch) = chars.next() {
|
||||
if ch == tag_start {
|
||||
let mut buf = String::new();
|
||||
for ch in chars.by_ref() {
|
||||
if ch == tag_end {
|
||||
break;
|
||||
} else if ch.is_ascii_alphanumeric() || ch.is_ascii_whitespace() {
|
||||
buf.push(ch);
|
||||
} else {
|
||||
result.push(tag_start);
|
||||
result.push_str(&buf);
|
||||
result.push(ch);
|
||||
continue 'outer;
|
||||
}
|
||||
}
|
||||
if let Some(pattern) = patterns.get(&buf) {
|
||||
if buf.starts_with("pre ") {
|
||||
tag_pre = pattern.to_string();
|
||||
} else if buf.starts_with("inter ") {
|
||||
tag_inter = pattern.to_string();
|
||||
} else if buf.starts_with("post ") {
|
||||
tag_post = pattern.to_string();
|
||||
} else {
|
||||
if !tag_pre.is_empty() {
|
||||
result.push_str(&tag_pre);
|
||||
}
|
||||
if !tag_inter.is_empty() && is_adjacent {
|
||||
result.push_str(&tag_inter);
|
||||
}
|
||||
result.push_str(pattern);
|
||||
if !tag_post.is_empty() {
|
||||
result.push_str(&tag_post);
|
||||
}
|
||||
is_adjacent = true;
|
||||
}
|
||||
} else {
|
||||
eprintln!("Warning: Unknown tag {}", buf);
|
||||
}
|
||||
} else {
|
||||
result.push(ch);
|
||||
is_adjacent = false;
|
||||
}
|
||||
}
|
||||
|
||||
//println!("to {}", result);
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
pub fn fix_broken_regex(value: &str) -> &str {
|
||||
match value {
|
||||
r"/[\042\223\224\262\263\271]{2}\S{0,16}[\042\223\224\262\263\271]{2}/" => {
|
||||
//r#"[\"\u{93}\u{94}\u{B2}\u{B3}\u{B9}]{2}\S{0,16}[\"\u{93}\u{94}\u{B2}\u{B3}\u{B9}]{2}"#
|
||||
r"/[\x22\x93\x94\xB2\xB3\xB9]{2}\S{0,16}[\x22\x93\x94\xB2\xB3\xB9]{2}/"
|
||||
}
|
||||
r"/\b_{0,3}d[_\W]?[i1!|l\xEC-\xEF][_\W]?d[_\W]?r[_\W][e3\xE8-\xEB[_\W]?xx?_{0,3}\b/i" => {
|
||||
r"/\b_{0,3}d[_\W]?[i1!|l\xEC-\xEF][_\W]?d[_\W]?r[_\W][e3\xE8-\xEB][_\W]?xx?_{0,3}\b/i"
|
||||
}
|
||||
r#"/<!--(?:\s{1,10}[-\w'"]{1,40}){100}/im"# => r#"/<!--(?:\s{1,10}[-\w'"]{1,40}){5}/im"#,
|
||||
r"/\015/" => r"/\x0D/",
|
||||
r"/[({[<][. ]*(?-i:\xbc\xba[. ]*\xc0\xce[. ]*)?(?-i:\xb1\xa4(?:[. ]*|[\x00-\x7f]{0,3})\xb0\xed|\xc1\xa4[. ]*\xba\xb8|\xc8\xab[. ]*\xba\xb8)[. ]*[)}\]>]/" => {
|
||||
r"/[\(\{\[\<][. ]*(?-i:\xbc\xba[. ]*\xc0\xce[. ]*)?(?-i:\xb1\xa4(?:[. ]*|[\x00-\x7f]{0,3})\xb0\xed|\xc1\xa4[. ]*\xba\xb8|\xc8\xab[. ]*\xba\xb8)[. ]*[\)\}\]\>]/"
|
||||
}
|
||||
_ => value,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn import_regex(value: &str) -> (String, String, HashSet<String>) {
|
||||
// Obtain separator
|
||||
let mut iter = value.chars().peekable();
|
||||
let separator = match iter.next() {
|
||||
Some('/') => Some('/'),
|
||||
Some('m') => iter.next().map(|ch| if ch == '{' { '}' } else { ch }),
|
||||
_ => None,
|
||||
}
|
||||
.unwrap_or(char::from(0));
|
||||
let mut regex = String::with_capacity(value.len());
|
||||
let mut flags = String::new();
|
||||
|
||||
let mut variables = HashSet::new();
|
||||
let mut variable_buf = String::new();
|
||||
let mut in_variable = false;
|
||||
|
||||
// Obtain regex
|
||||
let mut found_separator = false;
|
||||
while let Some(mut ch) = iter.next() {
|
||||
if ch == '%' && matches!(iter.peek(), Some('{')) {
|
||||
ch = '$';
|
||||
in_variable = true;
|
||||
} else if in_variable {
|
||||
match ch {
|
||||
'{' => {}
|
||||
'}' => {
|
||||
if !variable_buf.is_empty() {
|
||||
variables.insert(variable_buf.clone());
|
||||
variable_buf.clear();
|
||||
}
|
||||
in_variable = false;
|
||||
}
|
||||
_ => {
|
||||
variable_buf.push(ch);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ch == separator {
|
||||
if !found_separator {
|
||||
found_separator = true;
|
||||
} else {
|
||||
regex.push(ch);
|
||||
regex.push_str(&flags);
|
||||
flags.clear();
|
||||
}
|
||||
} else if !found_separator {
|
||||
regex.push(ch);
|
||||
} else {
|
||||
flags.push(ch);
|
||||
}
|
||||
}
|
||||
|
||||
(regex, flags, variables)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod test {
|
||||
use std::collections::HashSet;
|
||||
|
||||
#[test]
|
||||
fn import_regex() {
|
||||
for (expr, result, vars) in [
|
||||
(
|
||||
r"m{<img\b[^>]{0,100}\ssrc=.?https?://[^>]{6,80}(?:\?[^>]{8}|[^a-z](?![a-f]{3}|20\d\d[01]\d[0-3]\d)[0-9a-f]{8})}i",
|
||||
r"(?i)<img\b[^>]{0,100}\ssrc=.?https?://[^>]{6,80}(?:\?[^>]{8}|[^a-z](?![a-f]{3}|20\d\d[01]\d[0-3]\d)[0-9a-f]{8})",
|
||||
vec![],
|
||||
),
|
||||
(r"/\bhoodia\b/i", r"(?i)\bhoodia\b", vec![]),
|
||||
(r"/\bCurrent Price:/", r"\bCurrent Price:", vec![]),
|
||||
(
|
||||
r"m|^https?://storage\.cloud\.google\.com/.{4,128}\#%{GB_TO_ADDR}|i",
|
||||
r"(?i)^https?://storage\.cloud\.google\.com/.{4,128}\#${GB_TO_ADDR}",
|
||||
vec!["GB_TO_ADDR"],
|
||||
),
|
||||
] {
|
||||
let (mut regex, flags, regex_vars) = super::import_regex(expr);
|
||||
if !flags.is_empty() {
|
||||
regex = format!("(?{flags}){regex}");
|
||||
}
|
||||
|
||||
assert_eq!(regex, result);
|
||||
assert_eq!(
|
||||
HashSet::from_iter(vars.iter().map(|s| s.to_string())),
|
||||
regex_vars
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user