From ce456c02eba4a44dcaee2ea8a447a17ae5119356 Mon Sep 17 00:00:00 2001 From: mdecimus Date: Sun, 24 Dec 2023 11:25:50 +0100 Subject: [PATCH] Lookup store refactoring --- crates/directory/src/backend/sql/config.rs | 7 +- crates/smtp/src/config/condition.rs | 2 +- crates/smtp/src/config/scripts.rs | 14 +-- crates/smtp/src/core/mod.rs | 72 ++++----------- crates/smtp/src/scripts/event_loop.rs | 46 +++++----- crates/smtp/src/scripts/plugins/lookup.rs | 63 ++++--------- crates/store/src/backend/memory/lookup.rs | 52 +---------- crates/store/src/backend/memory/main.rs | 2 +- crates/store/src/config.rs | 30 ++---- crates/store/src/dispatch/lookup.rs | 86 ++++++++++++++---- crates/store/src/lib.rs | 23 +++-- resources/config.zip | Bin 171238 -> 171227 bytes resources/config/smtp/spamfilter.toml | 46 ++++++---- resources/config/store/mysql.toml | 2 +- .../resources/smtp/config/rules-dynvalue.toml | 15 ++- tests/resources/smtp/config/rules-eval.toml | 6 +- tests/src/directory/mod.rs | 23 ++--- tests/src/directory/sql.rs | 6 +- tests/src/imap/mod.rs | 11 +-- tests/src/jmap/mod.rs | 11 +-- tests/src/smtp/config.rs | 21 +++-- tests/src/smtp/inbound/antispam.rs | 52 ++++++----- tests/src/smtp/inbound/scripts.rs | 6 +- tests/src/smtp/mod.rs | 1 - tests/src/store/lookup.rs | 12 +++ 25 files changed, 275 insertions(+), 334 deletions(-) diff --git a/crates/directory/src/backend/sql/config.rs b/crates/directory/src/backend/sql/config.rs index 709c593c..0c351a6b 100644 --- a/crates/directory/src/backend/sql/config.rs +++ b/crates/directory/src/backend/sql/config.rs @@ -72,9 +72,10 @@ impl SqlDirectory { ("expand", &mut mappings.query_expand), ("domains", &mut mappings.query_domains), ] { - if let Some(query_) = stores.lookups.get(&format!("{}/{}", store_id, query_id)) { - *query = query_.query.to_string(); - } + *query = config + .value(("store", store_id, "query", query_id)) + .unwrap_or_default() + .to_string(); } Ok(SqlDirectory { diff --git a/crates/smtp/src/config/condition.rs b/crates/smtp/src/config/condition.rs index 3d61e0b9..9708d984 100644 --- a/crates/smtp/src/config/condition.rs +++ b/crates/smtp/src/config/condition.rs @@ -217,7 +217,7 @@ impl ConfigCondition for Config { MatchType::Lookup => { if let Some(lookup) = ctx.directory.lookups.get(value_str) { ConditionMatch::Lookup(lookup.clone().into()) - } else if let Some(lookup) = ctx.stores.lookups.get(value_str) { + } else if let Some(lookup) = ctx.stores.lookup_stores.get(value_str) { ConditionMatch::Lookup(lookup.clone().into()) } else { return Err(format!( diff --git a/crates/smtp/src/config/scripts.rs b/crates/smtp/src/config/scripts.rs index 92eec865..1ddbf1a2 100644 --- a/crates/smtp/src/config/scripts.rs +++ b/crates/smtp/src/config/scripts.rs @@ -121,7 +121,7 @@ impl ConfigSieve for Config { .with_valid_notification_uri("mailto") .with_valid_ext_lists( ctx.stores - .lookups + .lookup_stores .keys() .chain(ctx.directory.lookups.keys()) .map(|k| k.to_string()), @@ -193,18 +193,6 @@ impl ConfigSieve for Config { Ok(SieveCore { runtime, scripts: ctx.scripts.clone(), - lookup: ctx - .stores - .lookups - .iter() - .map(|(k, v)| (k.to_string(), v.clone().into())) - .chain( - ctx.directory - .lookups - .iter() - .map(|(k, v)| (k.to_string(), v.clone().into())), - ) - .collect(), lookup_stores: ctx.stores.lookup_stores.clone(), directories: ctx.directory.directories.clone(), default_directory: self diff --git a/crates/smtp/src/core/mod.rs b/crates/smtp/src/core/mod.rs index a17d1dcb..8fff4c64 100644 --- a/crates/smtp/src/core/mod.rs +++ b/crates/smtp/src/core/mod.rs @@ -22,7 +22,6 @@ */ use std::{ - cmp::Ordering, hash::Hash, net::IpAddr, sync::{atomic::AtomicU32, Arc}, @@ -41,7 +40,7 @@ use smtp_proto::{ }, IntoString, }; -use store::{LookupStore, Row, Value}; +use store::{LookupKey, LookupStore, LookupValue, Value}; use tokio::{ io::{AsyncRead, AsyncWrite}, sync::mpsc, @@ -65,6 +64,7 @@ use crate::{ }, queue::{self, DomainPart, QueueId, QuotaLimiter}, reporting, + scripts::plugins::lookup::VariableExists, }; use self::throttle::{Limiter, ThrottleKey, ThrottleKeyHasherBuilder}; @@ -119,7 +119,6 @@ pub struct SieveCore { pub sign: Vec>, pub directories: AHashMap>, pub lookup_stores: AHashMap, - pub lookup: AHashMap, pub default_lookup_store: Option, pub default_directory: Option>, } @@ -161,7 +160,7 @@ pub struct TlsConnectors { #[derive(Clone)] pub enum Lookup { - Store(Arc), + Store(LookupStore), Directory(directory::Lookup), } @@ -283,68 +282,35 @@ impl SessionData { } impl Lookup { - pub async fn contains(&self, item: impl Into>) -> Option { + pub async fn contains(&self, item: &str) -> Option { match self { - Lookup::Store(lookup) => lookup + Lookup::Store(LookupStore::Query(lookup)) => lookup .store .query::(&lookup.query, vec![item.into()]) .await .ok(), + Lookup::Store(store) => store + .key_get::(LookupKey::Key(item.to_string().into_bytes())) + .await + .ok() + .map(|v| !matches!(v, LookupValue::None)), Lookup::Directory(lookup) => match lookup { - directory::Lookup::DomainExists(directory) => directory - .is_local_domain(item.into().to_str().as_ref()) - .await - .ok(), - directory::Lookup::EmailExists(directory) => { - directory.rcpt(item.into().to_str().as_ref()).await.ok() + directory::Lookup::DomainExists(directory) => { + directory.is_local_domain(item).await.ok() } + directory::Lookup::EmailExists(directory) => directory.rcpt(item).await.ok(), }, } } - - pub async fn lookup(&self, items: Vec>) -> Option { - match self { - Lookup::Store(lookup) => lookup - .store - .query::>(&lookup.query, items) - .await - .ok() - .map(|row| { - let mut row = row.map(|row| row.values).unwrap_or_default(); - match row.len().cmp(&1) { - Ordering::Equal if !matches!(row.first(), Some(Value::Null)) => { - row.pop().map(into_sieve_value).unwrap() - } - Ordering::Less => Variable::default(), - _ => Variable::Array( - row.into_iter() - .map(into_sieve_value) - .collect::>() - .into(), - ), - } - }), - Lookup::Directory(_) => None, - } - } - - pub async fn query(&self, items: Vec>) -> Option>> { - match self { - Lookup::Store(lookup) => lookup - .store - .query::>(&lookup.query, items) - .await - .ok() - .map(|row| row.map(|row| row.values).unwrap_or_default()), - Lookup::Directory(_) => None, - } - } } impl PartialEq for Lookup { fn eq(&self, other: &Self) -> bool { match (self, other) { - (Lookup::Store(a), Lookup::Store(b)) => a.query == b.query, + (Lookup::Store(LookupStore::Query(a)), Lookup::Store(LookupStore::Query(b))) => { + a.query == b.query + } + (Lookup::Store(LookupStore::Store(_)), Lookup::Store(LookupStore::Store(_))) => true, (Lookup::Directory(a), Lookup::Directory(b)) => matches!( (a, b), ( @@ -389,8 +355,8 @@ pub fn to_store_value(value: &Variable) -> Value<'static> { } } -impl From> for Lookup { - fn from(lookup: Arc) -> Self { +impl From for Lookup { + fn from(lookup: LookupStore) -> Self { Lookup::Store(lookup) } } diff --git a/crates/smtp/src/scripts/event_loop.rs b/crates/smtp/src/scripts/event_loop.rs index 8e543742..e2db65ab 100644 --- a/crates/smtp/src/scripts/event_loop.rs +++ b/crates/smtp/src/scripts/event_loop.rs @@ -32,15 +32,18 @@ use smtp_proto::{ MAIL_BY_TRACE, MAIL_RET_FULL, MAIL_RET_HDRS, RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_NEVER, RCPT_NOTIFY_SUCCESS, }; -use store::backend::memory::MemoryStore; +use store::{backend::memory::MemoryStore, LookupKey, LookupStore, LookupValue}; use tokio::runtime::Handle; use crate::{ - core::{Lookup, SMTP}, + core::SMTP, queue::{DomainPart, InstantFromTimestamp, Message}, }; -use super::{plugins::PluginContext, ScriptModification, ScriptParameters, ScriptResult}; +use super::{ + plugins::{lookup::VariableExists, PluginContext}, + ScriptModification, ScriptParameters, ScriptResult, +}; impl SMTP { pub fn run_script_blocking( @@ -92,14 +95,18 @@ impl SMTP { } => { input = false.into(); 'outer: for list in lists { - if let Some(list) = self.sieve.lookup.get(&list) { + if let Some(store) = self.sieve.lookup_stores.get(&list) { for value in &values { - let result = if !matches!(match_as, MatchAs::Lowercase) { - handle.block_on(list.contains(value)) - } else { - handle.block_on(list.contains(&value.to_lowercase())) - }; - if let Some(true) = result { + if let Ok(LookupValue::Value { .. }) = handle.block_on( + store.key_get::(LookupKey::Key( + if !matches!(match_as, MatchAs::Lowercase) { + value.clone() + } else { + value.to_lowercase() + } + .into_bytes(), + )), + ) { input = true.into(); break 'outer; } @@ -165,18 +172,13 @@ impl SMTP { } } Recipient::List(list) => { - if let Some(list) = self.sieve.lookup.get(&list) { - if let Lookup::Store(list) = list { - if let store::LookupStore::Memory(list) = &list.store { - if let MemoryStore::List(list) = list.as_ref() { - for rcpt in &list.set { - handle.block_on( - message.add_recipient( - rcpt, - &self.queue.config, - ), - ); - } + if let Some(list) = self.sieve.lookup_stores.get(&list) { + if let LookupStore::Memory(list) = list { + if let MemoryStore::List(list) = list.as_ref() { + for rcpt in &list.set { + handle.block_on( + message.add_recipient(rcpt, &self.queue.config), + ); } } } diff --git a/crates/smtp/src/scripts/plugins/lookup.rs b/crates/smtp/src/scripts/plugins/lookup.rs index b96879b3..afd832e6 100644 --- a/crates/smtp/src/scripts/plugins/lookup.rs +++ b/crates/smtp/src/scripts/plugins/lookup.rs @@ -29,11 +29,11 @@ use std::{ use mail_auth::flate2; use sieve::{runtime::Variable, FunctionMap}; -use store::{Deserialize, LookupKey, LookupValue}; +use store::{Deserialize, LookupKey, LookupValue, Value}; use crate::{ config::scripts::{RemoteList, SieveContext}, - core::to_store_value, + core::into_sieve_value, USER_AGENT, }; @@ -61,32 +61,6 @@ pub fn register_local_domain(plugin_id: u32, fnc_map: &mut FunctionMap) -> Variable { let store = match &ctx.arguments[0] { - Variable::String(v) if v.contains('/') => { - if let Some(lookup) = ctx.core.sieve.lookup.get(v.as_ref()) { - return match &ctx.arguments[1] { - Variable::Array(items) => { - for item in items.iter() { - if !item.is_empty() - && ctx - .handle - .block_on(lookup.contains(to_store_value(item))) - .unwrap_or(false) - { - return true.into(); - } - } - false - } - v if !v.is_empty() => ctx - .handle - .block_on(lookup.contains(to_store_value(v))) - .unwrap_or(false), - _ => false, - } - .into(); - } - None - } Variable::String(v) if !v.is_empty() => ctx.core.sieve.lookup_stores.get(v.as_ref()), _ => ctx.core.sieve.default_lookup_store.as_ref(), }; @@ -133,23 +107,6 @@ pub fn exec(ctx: PluginContext<'_>) -> Variable { pub fn exec_get(ctx: PluginContext<'_>) -> Variable { let store = match &ctx.arguments[0] { - Variable::String(v) if v.contains('/') => { - if let Some(lookup) = ctx.core.sieve.lookup.get(v.as_ref()) { - let items = match &ctx.arguments[1] { - Variable::Array(l) => l.iter().map(to_store_value).collect(), - v if !v.is_empty() => vec![to_store_value(v)], - _ => vec![], - }; - return if !items.is_empty() { - ctx.handle - .block_on(lookup.lookup(items)) - .unwrap_or_default() - } else { - Variable::default() - }; - } - None - } Variable::String(v) if !v.is_empty() => ctx.core.sieve.lookup_stores.get(v.as_ref()), _ => ctx.core.sieve.default_lookup_store.as_ref(), }; @@ -467,10 +424,10 @@ pub fn exec_local_domain(ctx: PluginContext<'_>) -> Variable { } #[derive(Debug, PartialEq, Eq)] -pub(super) struct VariableWrapper(Variable); +pub struct VariableWrapper(Variable); #[derive(Debug, PartialEq, Eq)] -pub(super) struct VariableExists; +pub struct VariableExists; impl Deserialize for VariableWrapper { fn deserialize(bytes: &[u8]) -> store::Result { @@ -493,3 +450,15 @@ impl VariableWrapper { self.0 } } + +impl From> for VariableExists { + fn from(_: Value<'static>) -> Self { + VariableExists + } +} + +impl From> for VariableWrapper { + fn from(value: Value<'static>) -> Self { + VariableWrapper(into_sieve_value(value)) + } +} diff --git a/crates/store/src/backend/memory/lookup.rs b/crates/store/src/backend/memory/lookup.rs index 41a4dc08..4538fd21 100644 --- a/crates/store/src/backend/memory/lookup.rs +++ b/crates/store/src/backend/memory/lookup.rs @@ -21,57 +21,9 @@ * for more details. */ -use crate::{IntoRows, QueryResult, QueryType, Row, Value}; +use crate::{IntoRows, Row}; -use super::{LookupList, MatchType, MemoryStore}; - -impl MemoryStore { - pub(crate) fn query( - &self, - _: &str, - params: Vec>, - ) -> crate::Result { - let exists = match T::query_type() { - QueryType::Exists => true, - QueryType::QueryOne => false, - QueryType::QueryAll | QueryType::Execute => { - return Err(crate::Error::InternalError( - "Unsupported query type".to_string(), - )) - } - }; - - let needle = params.first().map(|v| v.to_str()).unwrap_or_default(); - - match self { - MemoryStore::List(list) => { - let found = list.contains(needle.as_ref()); - if exists { - Ok(T::from_exists(found)) - } else { - Ok(T::from_query_one(Some(Row { - values: vec![Value::Bool(found)], - }))) - } - } - MemoryStore::Map(map) => { - if let Some(value) = map.get(needle.as_ref()) { - if exists { - Ok(T::from_exists(true)) - } else { - Ok(T::from_query_one(Some(Row { - values: vec![value.clone()], - }))) - } - } else if exists { - Ok(T::from_exists(false)) - } else { - Ok(T::from_query_one(None::)) - } - } - } - } -} +use super::{LookupList, MatchType}; impl IntoRows for Option { fn into_row(self) -> Option { diff --git a/crates/store/src/backend/memory/main.rs b/crates/store/src/backend/memory/main.rs index 9274331f..ec56b2d3 100644 --- a/crates/store/src/backend/memory/main.rs +++ b/crates/store/src/backend/memory/main.rs @@ -54,7 +54,7 @@ impl MemoryStore { pub async fn open(config: &Config, prefix: impl AsKey) -> crate::Result { let prefix = prefix.as_key(); - let lookup_type = config.property_require::((&prefix, "type"))?; + let lookup_type = config.property_require::((&prefix, "format"))?; let format = LookupFormat { lookup_type, comment: config.value((&prefix, "comment")).map(|s| s.to_string()), diff --git a/crates/store/src/config.rs b/crates/store/src/config.rs index 355e4106..7d66a3e4 100644 --- a/crates/store/src/config.rs +++ b/crates/store/src/config.rs @@ -24,12 +24,12 @@ use std::sync::Arc; use async_trait::async_trait; -use utils::config::{cron::SimpleCron, utils::AsKey, Config}; +use utils::config::{cron::SimpleCron, Config}; use crate::{ backend::{fs::FsStore, memory::MemoryStore}, write::purge::{PurgeSchedule, PurgeStore}, - Lookup, LookupStore, Store, Stores, + LookupStore, QueryStore, Store, Stores, }; #[cfg(feature = "s3")] @@ -178,21 +178,9 @@ impl ConfigStore for Config { continue; } "memory" => { - let prefix = prefix.as_key(); - for lookup_id in self.sub_keys((&prefix, "lookup")) { - config.lookups.insert( - format!("{store_id}/{lookup_id}"), - Arc::new(Lookup { - store: MemoryStore::open( - self, - (prefix.as_str(), "lookup", lookup_id), - ) - .await? - .into(), - query: String::new(), - }), - ); - } + config + .lookup_stores + .insert(store_id, MemoryStore::open(self, prefix).await?.into()); continue; } @@ -202,15 +190,15 @@ impl ConfigStore for Config { } }; - // Add queries + // Add queries as lookup stores let lookup_store: LookupStore = lookup_store.into(); for lookup_id in self.sub_keys(("store", id, "query")) { - config.lookups.insert( + config.lookup_stores.insert( format!("{store_id}/{lookup_id}"), - Arc::new(Lookup { + LookupStore::Query(Arc::new(QueryStore { store: lookup_store.clone(), query: self.property_require(("store", id, "query", lookup_id))?, - }), + })), ); } config.lookup_stores.insert(store_id, lookup_store.clone()); diff --git a/crates/store/src/dispatch/lookup.rs b/crates/store/src/dispatch/lookup.rs index 580f9f43..a1893778 100644 --- a/crates/store/src/dispatch/lookup.rs +++ b/crates/store/src/dispatch/lookup.rs @@ -21,6 +21,7 @@ * for more details. */ +use crate::{backend::memory::MemoryStore, Row}; #[allow(unused_imports)] use crate::{ write::{ @@ -39,21 +40,14 @@ impl LookupStore { params: Vec>, ) -> crate::Result { let result = match self { - LookupStore::Store(store) => match store { - #[cfg(feature = "sqlite")] - Store::SQLite(store) => store.query(query, params).await, - #[cfg(feature = "postgres")] - Store::PostgreSQL(store) => store.query(query, params).await, - #[cfg(feature = "mysql")] - Store::MySQL(store) => store.query(query, params).await, - _ => Err(crate::Error::InternalError( - "Store does not support queries".into(), - )), - }, - LookupStore::Memory(store) => store.query(query, params), - #[cfg(feature = "redis")] - LookupStore::Redis(_) => Err(crate::Error::InternalError( - "Redis does not support queries".into(), + #[cfg(feature = "sqlite")] + LookupStore::Store(Store::SQLite(store)) => store.query(query, params).await, + #[cfg(feature = "postgres")] + LookupStore::Store(Store::PostgreSQL(store)) => store.query(query, params).await, + #[cfg(feature = "mysql")] + LookupStore::Store(Store::MySQL(store)) => store.query(query, params).await, + _ => Err(crate::Error::InternalError( + "Store does not support queries".into(), )), }; @@ -89,11 +83,21 @@ impl LookupStore { } #[cfg(feature = "redis")] LookupStore::Redis(store) => store.key_set(key, value).await, - LookupStore::Memory(_) => unimplemented!(), + LookupStore::Query(lookup) => lookup + .store + .query::( + &lookup.query, + vec![String::from_utf8(key).unwrap_or_default().into()], + ) + .await + .map(|_| ()), + LookupStore::Memory(_) => Err(crate::Error::InternalError( + "This store does not support key_set".into(), + )), } } - pub async fn key_get( + pub async fn key_get> + std::fmt::Debug + 'static>( &self, key: LookupKey, ) -> crate::Result> { @@ -120,7 +124,38 @@ impl LookupStore { }, #[cfg(feature = "redis")] LookupStore::Redis(store) => store.key_get(key).await, - LookupStore::Memory(_) => unimplemented!(), + LookupStore::Memory(store) => { + let key = String::from(key); + match store.as_ref() { + MemoryStore::List(list) => Ok(if list.contains(&key) { + LookupValue::Value { + value: T::from(Value::Bool(true)), + expires: 0, + } + } else { + LookupValue::None + }), + MemoryStore::Map(map) => Ok(map + .get(&key) + .map(|value| LookupValue::Value { + value: T::from(value.to_owned()), + expires: 0, + }) + .unwrap_or(LookupValue::None)), + } + } + LookupStore::Query(lookup) => lookup + .store + .query::>(&lookup.query, vec![String::from(key).into()]) + .await + .map(|row| { + row.and_then(|row| row.values.into_iter().next()) + .map(|value| LookupValue::Value { + value: T::from(value), + expires: 0, + }) + .unwrap_or(LookupValue::None) + }), } } @@ -169,7 +204,7 @@ impl LookupStore { } #[cfg(feature = "redis")] LookupStore::Redis(_) => {} - LookupStore::Memory(_) => {} + LookupStore::Memory(_) | LookupStore::Query(_) => {} } Ok(()) @@ -190,3 +225,16 @@ impl Deserialize for LookupValue { }) } } + +impl From> for String { + fn from(value: Value<'static>) -> Self { + match value { + Value::Text(string) => string.into_owned(), + Value::Blob(bytes) => String::from_utf8_lossy(bytes.as_ref()).into_owned(), + Value::Bool(boolean) => boolean.to_string(), + Value::Null => String::new(), + Value::Integer(num) => num.to_string(), + Value::Float(num) => num.to_string(), + } + } +} diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index f780b6fb..06f2aca0 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -194,13 +194,6 @@ pub struct Stores { pub blob_stores: AHashMap, pub fts_stores: AHashMap, pub lookup_stores: AHashMap, - pub lookups: AHashMap>, -} - -#[derive(Clone)] -pub struct Lookup { - pub store: LookupStore, - pub query: String, } #[derive(Clone)] @@ -235,11 +228,17 @@ pub enum FtsStore { #[derive(Clone)] pub enum LookupStore { Store(Store), + Query(Arc), Memory(Arc), #[cfg(feature = "redis")] Redis(Arc), } +pub struct QueryStore { + pub store: LookupStore, + pub query: String, +} + #[cfg(feature = "sqlite")] impl From for Store { fn from(store: SqliteStore) -> Self { @@ -364,6 +363,16 @@ impl<'x> Value<'x> { } } +impl From for String { + fn from(value: LookupKey) -> Self { + let key = match value { + LookupKey::Key(key) | LookupKey::Counter(key) => key, + }; + String::from_utf8(key) + .unwrap_or_else(|err| String::from_utf8_lossy(&err.into_bytes()).into_owned()) + } +} + #[derive(Clone, Debug)] pub struct Row { pub values: Vec>, diff --git a/resources/config.zip b/resources/config.zip index 2b1276b08b6541164e8b612b8dbec941f8fb15ab..b6b723a8001fed1a389b66ec88e55fb0e20d2c90 100644 GIT binary patch delta 2455 zcmZXWdpy*67sqG5zcOwulS^aVmJ-$_dD!HB*$IhImMIe{N=#TasVB@}Sh?2VOYX5d zkuEm5yb;Oo^{D6rBKMNw3_|0)9YoQ=a1iOe&>BY=X}rk{?7cqzgIP)gEgXe zGGwG;zRz@bSP2DGfWw&*aX5LL0;Dm4QltnDv*XA4+N0}im>3SfVF8cBZIH&{aGlU4 zU{Fu8ZDd6=(dKDf|KcZa2}SWmec|FDH4??#q{=QMbxb;Fs&;y;+HpMNas?qP--pP5 z$F~^%IM(t!QHxr9WBOdj8DF-OMRAADvJQhjP$0^4^}jDP_$;3JvqKkt$Cb#jXM`68 zznhkKwBpC^Jpv~AH#vG9b{5<$A^K3NT2jcNnCjkd?zV6CuZY#v_1tlbQ}1jRcE6^( zQpJN`qJ3DHEn#gY-Nf!??W*VSI-;NJx0;4&+`H3|qw4Omr?4k(YP(}ESF4EM)V4?C z9`Og;4>!ELPP=>Ec=>j?YQ$YFa>IELVs_WFsH!y~X<0c@xn*V`KXlwa*trO^*Mh+# zCT?a29voY7d!5i}F{ZA5=X^DDZs%#k65i~J=zDg6PEK9L_;ptQEYWCblFoT*@_0+- zKK)XS=)Q%KhczKdY6@YOeclLDLYpe*|>+M-X6c? zCVs*0&pO_=6?2vGAQ97RZ3@i=!vEHYnha;&pXn{!6p`B;_x??=WYvuH^88FiZi`*l z={)*C$hZ@QN+`X}V)q*rQ7sK5dQ|D=Vx3Fb$_CzdYq)6CuU=c*%)nPizOEZqAlHthG&G=}WB%L7iag|h>;He}HhQXGYZucDB^6KCKO zQl;n*4r8k~7Xm>c0o_2wGhjeU$TrjcmGC4n1(rZ`#VC)Hov-aUIwFITrK$EbKh5~)x44z1yK`2(8T3@R zed$F`Xm@=n{nzJK-#&agdTxt#%SLwQ2`$E9;cKMso&aWG%&p=^lE<4v&(xlPlCw^c zqrwwG^Vtr*=2A+W=8GO0<4rNGFGg)Ewp|*Mq%!O%R|u+SC@Lj-#KxKubyxG?a)ZmS zhOHg8w>$cf(=}dKCbN7rU$PGs5SsSli){Q=jt8Z`%uyFR&K;YrXQ&jFms?IOKXSIE zR?3(SQM~2)w7(564k=8`(RzC;qRgHdnJa8FmQzjgk`&s2Q)y**WNz7DB*YL2+!Llva z9~a!#ZPv~E&iO3)gI|e!KwxDk`NQ0zQAFM-&L-)Mc=9ef!)>^ZO&&RX?a9l{0ikBu zOxeu_e!Gie0}h<(_xwaXYuF{H7NOT4sea}MYQOPlv-MfJSl{DmRl4{=|%5+cp zK6Z^;?!FxGuB%ra9}O-i?PuN`uT2I()^Cicf3L6RP^pffK@; z{_!@3_Do>7`VxjLCNRbu!#XB7jIV?w7VyLx3Xp>GF~Ay2`C|3NfQ>M074g;~ zH%vlHAYT($vVj}6OK&~8!v?OHxYatu@wz0;*=HpN^M$f0gks`p>lBb53%n@WaC8bv!PZ|Ritt4W>OcwZOhp|4QvpOB z5Mf6u(xsXS@^qM?P{Ydz4cZ8h>~}yGmaHOMiU1L=plz+S6{PuK3nXFe6(pG53n0+} zd;{lJt>aDtdA<`6gPt6OZXOh%`yA9#irbo{1UucpCa9l=1gTVkATJJ%GuL2rY z7X;u?5FkU+Rj?f{2ZOI9Na%Y9heKaJxMJ3uU;srzfbyza6}o2wCG>b1*Hc(zYjo{H7@HJG-0c6yaQ4Tl;7Qt;WbL(Wqbz~m37w{McS=uDwIhR3e-x#H_x+5}e znc~^>#}Qh$;ZE8$GVs@YT=8klx?ta_U~Z4J@QY`m)mEWqr?gkZ>pmOPOVq>lt;Rzh z+46G7=PR39Q^TvSuf$Ep#*{6TNK7nE?|K~<{if#D!q4)*8h!IJ;5v!BhzxWWC29mQ z$NY2&Mony5{kCI=CX-^}*13U#$=0QoAdgo$(HrTXxC7tk!#N%FuV4PC7JBwfwP}`Z z{ENckQT>~|6;)^MD$|VnmTs+~NA38$UVX7>q(J5Jlu!95r_N`txniqxZzc0|Mk*Zw zR|wM`zK`@CG>z0A3R}jpIdg~0O#I55=H=`fwmF7=iJrM}LQ8J?`BY4wdir>nSG|3e z`hR}>_FgpV)%CWg(>Fp5x>`-DcClOknK|pt9Ic7JQ@_`LP5WW=bJBqd`c$nJMI^1Q zJ3d6LuhC$<5&i06Ik6u+cx8@seF)L{}4{Jfpe@NnHnr_KfywW z$hGu|TGgzl{oUcRM;@9xj}u}gxmLTq9*eZ#6-R}S;`df;KgyA^sChivtLuO!yqO5e zI;oT>r+G5-h|3FRe1rl*X_B>YHkqo5*{QvKyD{Yy*>Kj@EMN5_m$@}^k7S?3evX#f zq*CvGiAhHFm747iUIE>C`=4m;HL#xFVQyl4fkWy37-{Kk`0?(AK(o0C>5peOr+8SMe83i5aFuq|KBF#@zsp+NKx^{P5s?blbTIP%Z)IUZQkBq?LCk6 zkZR-GgAc9^9#nH^<8%mp=-W4YHIeW#>Cwf*CDAE7DWQGxa_U!`M(@dJ2AkX6L<~3H zD^?5)(Imyolf;?PhGKng^on=nnN_2ijh8-WFbsFlLJSqo+d37ZbU*b0sZ#VSV}rN* zAsw16wTXL)DNWWpTN!UMROoP;#O)=^(R}(0f<@okby-QL7J=S_FRc0_o)~Hc>4*51 z#i;DeFU{{;d6LDn9}}r#^=c4;=3JI*@t&W!<{rX@`hlpO}VeOo^rlq(un&)^yF^0fYY(Alf$2sBHCR` zW8M+h=!KjKyI6E+x;SgLH}qQri`2F1)MXTx_A+$t!69+F^M>pn`YC!a?BgxY;*zIH zXZX9;`Y?$q(bDYBsr$OMqk3+22d?-hE1OG}&krvRdbGV7jhKD9GP4-7X5MF+t~e*8 zckb0fy=fADPgk3583DJr8 zy$Z>zi*0LJmYej{0$#cg-FdbslRKQ5r{DmU=Oexb+_``-no_7Pibt{TZJ$~eqV`|=>&pEL$6dXx%e;`aECCXnV1USowXw}~P zyjutEN)hca8rstK%g!$VL;3sI1Jk%ve@olBU9DFVy0fD+8S1gt^s|G@8;fHN2Z0*hxB zIE+ZZ>seSVB`A{(B7wVzz*LrvO?b1606572Sty@_#cbRpV9w?MH!vb60Ad6n1#7Z_ zI^1*_sDRkb0^0R5Z~^N?K2U}omw_WlRpSE{cwY@j!`{o7gHJBz(6~+DfaYRl%h~ZK zD9*#2`waw!?*>2u#^qu8GY$ESDjdths;6cwFx%u~%lHS(&j(Zlg-8Ji1>-gXQ)U5H zOGi5apxa?-#S5`8T?hHh78p^8Wn6X?nE%BG!7c*8b^+qh9mU2rdkL7ED7Jz^Z$40g z3QTNzsIS1%?+YXa%el?Oma~g2pqH4y4V%w_A`pdSz~&<0jvu4|I7`9Gp;n9utQiIr zV@hp=UB#Hd{Gc)f6odpXKuoaUCjgax!UVMpFoyR^;pB0D|BV*mP#IPseSX6q3P;ajyR6Cv637VDXM;e1#KS^1un{IEVAl!mzZAvR A-T(jq diff --git a/resources/config/smtp/spamfilter.toml b/resources/config/smtp/spamfilter.toml index e4960cf3..e43326f6 100644 --- a/resources/config/smtp/spamfilter.toml +++ b/resources/config/smtp/spamfilter.toml @@ -2,58 +2,64 @@ # SMTP Spam & Phishing filter configuration ############################################# -[store."spam"] +[store."spam/free-domains"] type = "memory" - -[store."spam".lookup."free-domains"] -type = "glob" +format = "glob" comment = '#' values = ["https://get.stalw.art/resources/config/spamfilter/maps/domains_free.list", "file+fallback://%{BASE_PATH}%/etc/spamfilter/maps/domains_free.list"] -[store."spam".lookup."disposable-domains"] -type = "glob" +[store."spam/disposable-domains"] +type = "memory" +format = "glob" comment = '#' values = ["https://get.stalw.art/resources/config/spamfilter/maps/domains_disposable.list", "file+fallback://%{BASE_PATH}%/etc/spamfilter/maps/domains_disposable.list"] -[store."spam".lookup."redirectors"] -type = "glob" +[store."spam/redirectors"] +type = "memory" +format = "glob" comment = '#' values = ["https://get.stalw.art/resources/config/spamfilter/maps/url_redirectors.list", "file+fallback://%{BASE_PATH}%/etc/spamfilter/maps/url_redirectors.list"] -[store."spam".lookup."domains-allow"] -type = "glob" +[store."spam/domains-allow"] +type = "memory" +format = "glob" comment = '#' values = ["https://get.stalw.art/resources/config/spamfilter/maps/allow_domains.list", "file+fallback://%{BASE_PATH}%/etc/spamfilter/maps/allow_domains.list"] -[store."spam".lookup."dmarc-allow"] -type = "glob" +[store."spam/dmarc-allow"] +type = "memory" +format = "glob" comment = '#' values = ["https://get.stalw.art/resources/config/spamfilter/maps/allow_dmarc.list", "file+fallback://%{BASE_PATH}%/etc/spamfilter/maps/allow_dmarc.list"] -[store."spam".lookup."spf-dkim-allow"] -type = "glob" +[store."spam/spf-dkim-allow"] +type = "memory" +format = "glob" comment = '#' values = ["https://get.stalw.art/resources/config/spamfilter/maps/allow_spf_dkim.list", "file+fallback://%{BASE_PATH}%/etc/spamfilter/maps/allow_spf_dkim.list"] -[store."spam".lookup."mime-types"] -type = "map" +[store."spam/mime-types"] +type = "memory" +format = "map" comment = '#' values = ["https://get.stalw.art/resources/config/spamfilter/maps/mime_types.map", "file+fallback://%{BASE_PATH}%/etc/spamfilter/maps/mime_types.map"] -[store."spam".lookup."trap-address"] -type = "glob" +[store."spam/trap-address"] +type = "memory" +format = "glob" comment = '#' values = "file://%{BASE_PATH}%/etc/spamfilter/maps/spam_trap.list" -[store."spam".lookup."scores"] -type = "map" +[store."spam/scores"] +type = "memory" +format = "map" values = "file://%{BASE_PATH}%/etc/spamfilter/maps/scores.map" [sieve.trusted.scripts] diff --git a/resources/config/store/mysql.toml b/resources/config/store/mysql.toml index bd95dd9c..356f4e18 100644 --- a/resources/config/store/mysql.toml +++ b/resources/config/store/mysql.toml @@ -14,7 +14,7 @@ disable = true [store."mysql".timeout] wait = "15s" -#[store."postgresql".pool] +#[store."mysql".pool] #max-connections = 10 #min-connections = 5 diff --git a/tests/resources/smtp/config/rules-dynvalue.toml b/tests/resources/smtp/config/rules-dynvalue.toml index 7e750fbe..1f83e6c6 100644 --- a/tests/resources/smtp/config/rules-dynvalue.toml +++ b/tests/resources/smtp/config/rules-dynvalue.toml @@ -61,22 +61,19 @@ test = [ ] expect = false -[store."list_mx"] +[store."list_mx/domains"] type = "memory" -[store."list_mx".lookup."domains"] -type = "list" +format = "list" values = ["mx"] -[store."list_foo"] +[store."list_foo/domains"] type = "memory" -[store."list_foo".lookup."domains"] -type = "list" +format = "list" values = ["foo"] -[store."list_123"] +[store."list_123/domains"] type = "memory" -[store."list_123".lookup."domains"] -type = "list" +format = "list" values = ["123"] [maybe-eval."dyn_mx"] diff --git a/tests/resources/smtp/config/rules-eval.toml b/tests/resources/smtp/config/rules-eval.toml index 3133c327..1a658753 100644 --- a/tests/resources/smtp/config/rules-eval.toml +++ b/tests/resources/smtp/config/rules-eval.toml @@ -164,9 +164,7 @@ nested-none-of-false = { none-of = [ ]} ]} -[store."list"] +[store."list/domains"] type = "memory" - -[store."list".lookup."domains"] -type = "list" +format = "list" values = ["mydomain1.org", "foo.net", "otherdomain.net"] diff --git a/tests/src/directory/mod.rs b/tests/src/directory/mod.rs index af6a19ff..36a50114 100644 --- a/tests/src/directory/mod.rs +++ b/tests/src/directory/mod.rs @@ -523,24 +523,25 @@ impl core::fmt::Debug for Item { #[ignore] async fn lookup_local() { const LOOKUP_CONFIG: &str = r#" - [store."local"] + [store."local/regex"] type = "memory" - - [store."local".lookup."regex"] - type = "regex" + format = "regex" values = ["^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$", "^(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$"] - [store."local".lookup."glob"] - type = "glob" + [store."local/glob"] + type = "memory" + format = "glob" values = ["*@example.org", "test@*", "localhost", "*+*@*.domain.net"] - [store."local".lookup."list"] - type = "list" + [store."local/list"] + type = "memory" + format = "list" values = ["abc", "xyz", "123"] - [store."local".lookup."suffix"] - type = "glob" + [store."local/suffix"] + type = "memory" + format = "glob" comment = "//" values = ["https://publicsuffix.org/list/public_suffix_list.dat", "fallback+file://%PATH%/public_suffix_list.dat.gz"] "#; @@ -570,7 +571,7 @@ async fn lookup_local() { .parse_stores() .await .unwrap() - .lookups; + .lookup_stores; for (lookup, item, expect) in [ ("glob", "user@example.org", true), diff --git a/tests/src/directory/sql.rs b/tests/src/directory/sql.rs index 3166851f..e3dd5990 100644 --- a/tests/src/directory/sql.rs +++ b/tests/src/directory/sql.rs @@ -47,9 +47,9 @@ async fn sql_directory() { let mut config = DirectoryTest::new(directory_id.into()).await; let lookups = config .stores - .lookups - .into_iter() - .map(|(k, v)| (k, Lookup::from(v))) + .lookup_stores + .iter() + .map(|(k, v)| (k.clone(), Lookup::from(v.clone()))) .collect::>(); println!("Testing SQL directory {:?}", directory_id); diff --git a/tests/src/imap/mod.rs b/tests/src/imap/mod.rs index d0b0c368..c403a291 100644 --- a/tests/src/imap/mod.rs +++ b/tests/src/imap/mod.rs @@ -233,15 +233,14 @@ email = "address" quota = "quota" type = "type" -[store."local"] +[store."local/domains"] type = "memory" - -[store."local".lookup."domains"] -type = "list" +format = "list" values = ["example.com"] -[store."local".lookup."remote-domains"] -type = "list" +[store."local/remote-domains"] +type = "memory" +format = "list" values = ["remote.org", "foobar.com", "test.com", "other_domain.com"] [oauth] diff --git a/tests/src/jmap/mod.rs b/tests/src/jmap/mod.rs index 9ecb89bd..8abb8929 100644 --- a/tests/src/jmap/mod.rs +++ b/tests/src/jmap/mod.rs @@ -239,15 +239,14 @@ email = "address" quota = "quota" type = "type" -[store."local"] +[store."local/domains"] type = "memory" - -[store."local".lookup."domains"] -type = "list" +format = "list" values = ["example.com"] -[store."local".lookup."remote-domains"] -type = "list" +[store."local/remote-domains"] +type = "memory" +format = "list" values = ["remote.org", "foobar.com", "test.com", "other_domain.com"] [oauth] diff --git a/tests/src/smtp/config.rs b/tests/src/smtp/config.rs index 9e2d1c00..0fd2f1d3 100644 --- a/tests/src/smtp/config.rs +++ b/tests/src/smtp/config.rs @@ -33,6 +33,7 @@ use std::{ use store::{ backend::memory::{LookupList, MemoryStore}, config::ConfigStore, + LookupStore, }; use tokio::net::TcpSocket; @@ -80,13 +81,13 @@ fn parse_conditions() { ..Default::default() }]; let mut context = ConfigContext::new(&servers); - let list = Arc::new(store::Lookup { + let list = LookupStore::Query(Arc::new(store::QueryStore { store: MemoryStore::List(LookupList::default()).into(), query: "abc".into(), - }); + })); context .stores - .lookups + .lookup_stores .insert("test-list".to_string(), list.clone()); let mut conditions = config.parse_conditions(&context).unwrap(); @@ -648,6 +649,12 @@ async fn eval_dynvalue() { "failed for test {test_name:?}" ); } + let wrapped_stores = context + .stores + .lookup_stores + .iter() + .map(|(k, v)| (k.clone(), Arc::new(v.clone()))) + .collect::>(); for test_name in config.sub_keys("maybe-eval") { //println!("============= Testing {:?} ==================", key); @@ -670,11 +677,7 @@ async fn eval_dynvalue() { ) .unwrap() .unwrap() - .map_if_block( - &context.stores.lookups, - ("maybe-eval", test_name, "test"), - "test", - ) + .map_if_block(&wrapped_stores, ("maybe-eval", test_name, "test"), "test") .unwrap(); let expected = config .value_require(("maybe-eval", test_name, "expect")) @@ -685,6 +688,8 @@ async fn eval_dynvalue() { .await .into_value(&envelope) .unwrap() + .as_ref() + .clone() .into(); assert!(lookup.contains(expected).await.unwrap()); diff --git a/tests/src/smtp/inbound/antispam.rs b/tests/src/smtp/inbound/antispam.rs index 6613f71e..84362f79 100644 --- a/tests/src/smtp/inbound/antispam.rs +++ b/tests/src/smtp/inbound/antispam.rs @@ -54,48 +54,50 @@ path = "%PATH%/test_antispam.db" #type = "redis" #url = "redis://127.0.0.1" -[store."default"] +[store."default/domains"] type = "memory" - -[store."default".lookup."domains"] -type = "list" +format = "list" values = ["local-domain.org"] -[store."spam"] +[store."spam/free-domains"] type = "memory" - -[store."spam".lookup."free-domains"] -type = "glob" +format = "glob" comment = '#' values = ["gmail.com", "googlemail.com", "yahoomail.com", "*.freemail.org"] -[store."spam".lookup."disposable-domains"] -type = "glob" +[store."spam/disposable-domains"] +type = "memory" +format = "glob" comment = '#' values = ["guerrillamail.com", "*.disposable.org"] -[store."spam".lookup."redirectors"] -type = "glob" +[store."spam/redirectors"] +type = "memory" +format = "glob" comment = '#' values = ["bit.ly", "redirect.io", "redirect.me", "redirect.org", "redirect.com", "redirect.net", "t.ly", "tinyurl.com"] -[store."spam".lookup."dmarc-allow"] -type = "glob" +[store."spam/dmarc-allow"] +type = "memory" +format = "glob" comment = '#' values = ["dmarc-allow.org"] -[store."spam".lookup."spf-dkim-allow"] -type = "glob" +[store."spam/spf-dkim-allow"] +type = "memory" +format = "glob" comment = '#' values = ["spf-dkim-allow.org"] -[store."spam".lookup."domains-allow"] -type = "glob" +[store."spam/domains-allow"] +type = "memory" +format = "glob" values = [] -[store."spam".lookup."mime-types"] -type = "map" +[store."spam/mime-types"] +type = "memory" +format = "map" comment = '#' values = ["html text/html|BAD", "pdf application/pdf|NZ", @@ -104,13 +106,15 @@ values = ["html text/html|BAD", "js BAD|NZ", "hta BAD|NZ"] -[store."spam".lookup."trap-address"] -type = "glob" +[store."spam/trap-address"] +type = "memory" +format = "glob" comment = '#' values = ["spamtrap@*"] -[store."spam".lookup."scores"] -type = "map" +[store."spam/scores"] +type = "memory" +format = "map" values = "file://%CFG_PATH%/maps/scores.map" [resolver] diff --git a/tests/src/smtp/inbound/scripts.rs b/tests/src/smtp/inbound/scripts.rs index 2d27cf30..5bb99c2a 100644 --- a/tests/src/smtp/inbound/scripts.rs +++ b/tests/src/smtp/inbound/scripts.rs @@ -49,11 +49,9 @@ max-connections = 10 min-connections = 0 idle-timeout = "5m" -[store."local"] +[store."local/invalid-ehlos"] type = "memory" - -[store."local".lookup."invalid-ehlos"] -type = "list" +format = "list" values = ["spammer.org", "spammer.net"] [session.data.pipe."test"] diff --git a/tests/src/smtp/mod.rs b/tests/src/smtp/mod.rs index 0121c343..e170053d 100644 --- a/tests/src/smtp/mod.rs +++ b/tests/src/smtp/mod.rs @@ -433,7 +433,6 @@ impl TestConfig for SieveCore { SieveCore { runtime: Runtime::new_with_context(SieveContext::default()), scripts: AHashMap::new(), - lookup: AHashMap::new(), from_addr: "MAILER-DAEMON@example.org".to_string(), from_name: "Mailer Daemon".to_string(), return_path: "".to_string(), diff --git a/tests/src/store/lookup.rs b/tests/src/store/lookup.rs index 53c9036e..b37f6b29 100644 --- a/tests/src/store/lookup.rs +++ b/tests/src/store/lookup.rs @@ -37,6 +37,18 @@ pub async fn lookup_tests() { println!("Testing lookup store {}...", store_id); if let LookupStore::Store(store) = &store { store.destroy().await; + } else { + // Reset redis counter + store + .key_set( + "abc".as_bytes().to_vec(), + LookupValue::Value { + value: "0".as_bytes().to_vec(), + expires: 0, + }, + ) + .await + .unwrap(); } // Test key