diff --git a/Cargo.lock b/Cargo.lock index ddf6e225..0d0aaf18 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1822,12 +1822,10 @@ dependencies = [ "futures", "jmap_proto", "ldap3", - "lru-cache", "mail-builder", "mail-parser", "mail-send", "md5", - "parking_lot", "password-hash", "pbkdf2", "proc_macros", diff --git a/crates/common/src/manager/boot.rs b/crates/common/src/manager/boot.rs index 06dce60e..491b2a98 100644 --- a/crates/common/src/manager/boot.rs +++ b/crates/common/src/manager/boot.rs @@ -326,7 +326,7 @@ impl BootManager { } // Parse in-memory stores - stores.parse_in_memory(&mut config).await; + stores.parse_in_memory(&mut config, false).await; // Parse settings let core = Core::parse(&mut config, stores, manager).await; diff --git a/crates/common/src/manager/reload.rs b/crates/common/src/manager/reload.rs index 38c6c15c..6b4765f4 100644 --- a/crates/common/src/manager/reload.rs +++ b/crates/common/src/manager/reload.rs @@ -53,7 +53,7 @@ impl Server { pub async fn reload_lookups(&self) -> trc::Result { let mut config = self.core.storage.config.build_config("lookup").await?; let mut stores = Stores::default(); - stores.parse_static_stores(&mut config); + stores.parse_static_stores(&mut config, true); let mut core = self.core.as_ref().clone(); for (id, store) in stores.in_memory_stores { @@ -79,7 +79,7 @@ impl Server { purge_schedules: Default::default(), }; stores.parse_stores(&mut config).await; - stores.parse_in_memory(&mut config).await; + stores.parse_in_memory(&mut config, true).await; // Parse tracers let tracers = Telemetry::parse(&mut config, &stores); diff --git a/crates/directory/Cargo.toml b/crates/directory/Cargo.toml index 87b69f93..31700538 100644 --- a/crates/directory/Cargo.toml +++ b/crates/directory/Cargo.toml @@ -21,9 +21,7 @@ rustls-pki-types = { version = "1" } ldap3 = { version = "0.11.1", default-features = false, features = ["tls-rustls"] } deadpool = { version = "0.10", features = ["managed", "rt_tokio_1"] } async-trait = "0.1.68" -parking_lot = "0.12" ahash = { version = "0.8" } -lru-cache = "0.1.2" pwhash = "1" password-hash = "0.5.0" argon2 = "0.5.0" diff --git a/crates/directory/src/core/cache.rs b/crates/directory/src/core/cache.rs index 229229e0..aef1c859 100644 --- a/crates/directory/src/core/cache.rs +++ b/crates/directory/src/core/cache.rs @@ -4,27 +4,18 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{ - borrow::Borrow, - hash::Hash, - time::{Duration, Instant}, -}; +use std::time::Duration; -use parking_lot::Mutex; -use utils::config::{utils::AsKey, Config}; +use utils::{ + cache::CacheWithTtl, + config::{utils::AsKey, Config}, +}; use crate::backend::RcptType; pub struct CachedDirectory { - cached_domains: Mutex>, - cached_rcpts: Mutex>, -} - -#[allow(clippy::type_complexity)] -#[derive(Debug)] -pub struct LookupCache { - cache_pos: lru_cache::LruCache, - cache_neg: lru_cache::LruCache, + cached_domains: CacheWithTtl, + cached_rcpts: CacheWithTtl, ttl_pos: Duration, ttl_neg: Duration, } @@ -32,97 +23,45 @@ pub struct LookupCache { impl CachedDirectory { pub fn try_from_config(config: &mut Config, prefix: impl AsKey) -> Option { let prefix = prefix.as_key(); - let cached_entries = config.property((&prefix, "cache.entries"))?; - let cache_ttl_positive = config - .property((&prefix, "cache.ttl.positive")) - .unwrap_or(Duration::from_secs(86400)); - let cache_ttl_negative = config - .property((&prefix, "cache.ttl.negative")) - .unwrap_or_else(|| Duration::from_secs(3600)); + let cached_size = config + .property_or_default::>((&prefix, "cache.size"), "1048576") + .unwrap_or_default()?; Some(CachedDirectory { - cached_domains: Mutex::new(LookupCache::new( - cached_entries, - cache_ttl_positive, - cache_ttl_negative, - )), - cached_rcpts: Mutex::new(LookupCache::new( - cached_entries, - cache_ttl_positive, - cache_ttl_negative, - )), + cached_domains: CacheWithTtl::new(50, cached_size), + cached_rcpts: CacheWithTtl::new(100, cached_size), + ttl_pos: config + .property((&prefix, "cache.ttl.positive")) + .unwrap_or(Duration::from_secs(86400)), + ttl_neg: config + .property((&prefix, "cache.ttl.negative")) + .unwrap_or_else(|| Duration::from_secs(3600)), }) } pub fn get_rcpt(&self, address: &str) -> Option { - self.cached_rcpts.lock().get(address).map(Into::into) + self.cached_rcpts.get(address).map(Into::into) } pub fn set_rcpt(&self, address: &str, exists: &RcptType) { - match exists { - RcptType::Mailbox => self.cached_rcpts.lock().insert_pos(address.to_string()), - RcptType::Invalid => self.cached_rcpts.lock().insert_neg(address.to_string()), - RcptType::List(_) => {} - } + let (exists, ttl) = match exists { + RcptType::Mailbox => (true, self.ttl_pos), + RcptType::Invalid => (false, self.ttl_neg), + RcptType::List(_) => return, + }; + + self.cached_rcpts.insert(address.to_string(), exists, ttl); } pub fn get_domain(&self, domain: &str) -> Option { - self.cached_domains.lock().get(domain) + self.cached_domains.get(domain) } pub fn set_domain(&self, domain: &str, exists: bool) { - if exists { - self.cached_domains.lock().insert_pos(domain.to_string()); - } else { - self.cached_domains.lock().insert_neg(domain.to_string()); - } - } -} - -impl LookupCache { - pub fn new(capacity: usize, ttl_pos: Duration, ttl_neg: Duration) -> Self { - Self { - cache_pos: lru_cache::LruCache::with_hasher(capacity, ahash::RandomState::new()), - cache_neg: lru_cache::LruCache::with_hasher(capacity, ahash::RandomState::new()), - ttl_pos, - ttl_neg, - } - } - - pub fn get(&mut self, name: &Q) -> Option - where - T: Borrow, - Q: Hash + Eq + ?Sized, - { - // Check positive cache - if let Some(valid_until) = self.cache_pos.get_mut(name) { - if *valid_until >= Instant::now() { - return Some(true); - } else { - self.cache_pos.remove(name); - } - } - - // Check negative cache - let valid_until = self.cache_neg.get_mut(name)?; - if *valid_until >= Instant::now() { - Some(false) - } else { - self.cache_pos.remove(name); - None - } - } - - pub fn insert_pos(&mut self, item: T) { - self.cache_pos.insert(item, Instant::now() + self.ttl_pos); - } - - pub fn insert_neg(&mut self, item: T) { - self.cache_neg.insert(item, Instant::now() + self.ttl_neg); - } - - pub fn clear(&mut self) { - self.cache_pos.clear(); - self.cache_neg.clear(); + self.cached_domains.insert( + domain.to_string(), + exists, + if exists { self.ttl_pos } else { self.ttl_neg }, + ); } } diff --git a/crates/store/src/backend/http/config.rs b/crates/store/src/backend/http/config.rs index 7931748e..51498c5d 100644 --- a/crates/store/src/backend/http/config.rs +++ b/crates/store/src/backend/http/config.rs @@ -19,7 +19,7 @@ use crate::{InMemoryStore, Stores}; use super::{HttpStore, HttpStoreConfig, HttpStoreFormat}; impl Stores { - pub fn parse_http_stores(&mut self, config: &mut Config) { + pub fn parse_http_stores(&mut self, config: &mut Config, is_reload: bool) { // Parse remote lists for id in config .sub_keys("http-lookup", ".url") @@ -104,12 +104,13 @@ impl Stores { entry.insert(InMemoryStore::Http(store.into())); } - Entry::Occupied(e) => { + Entry::Occupied(e) if !is_reload => { config.new_build_error( ("http-lookup", e.key().as_str()), - "An im-memory store with this id already exists", + "An in-memory store with this id already exists", ); } + _ => {} } } } diff --git a/crates/store/src/backend/memory/mod.rs b/crates/store/src/backend/memory/mod.rs index 14edc875..efdfef74 100644 --- a/crates/store/src/backend/memory/mod.rs +++ b/crates/store/src/backend/memory/mod.rs @@ -14,7 +14,7 @@ use crate::{InMemoryStore, Stores, Value}; pub type StaticMemoryStore = GlobMap>; impl Stores { - pub fn parse_static_stores(&mut self, config: &mut Config) { + pub fn parse_static_stores(&mut self, config: &mut Config, is_reload: bool) { let mut lookups = AHashMap::new(); let mut errors = Vec::new(); @@ -86,12 +86,13 @@ impl Stores { Entry::Vacant(entry) => { entry.insert(InMemoryStore::Static(store.into())); } - Entry::Occupied(e) => { + Entry::Occupied(e) if !is_reload => { config.new_build_error( ("lookup", e.key().as_str()), - "An im-memory store with this id already exists", + "An in-memory store with this id already exists", ); } + _ => {} } } } diff --git a/crates/store/src/config.rs b/crates/store/src/config.rs index 931dfc85..35839b43 100644 --- a/crates/store/src/config.rs +++ b/crates/store/src/config.rs @@ -47,9 +47,9 @@ enum CompositeStore { } impl Stores { - pub async fn parse_all(config: &mut Config) -> Self { + pub async fn parse_all(config: &mut Config, is_reload: bool) -> Self { let mut stores = Self::parse(config).await; - stores.parse_in_memory(config).await; + stores.parse_in_memory(config, is_reload).await; stores } @@ -311,12 +311,12 @@ impl Stores { } } - pub async fn parse_in_memory(&mut self, config: &mut Config) { + pub async fn parse_in_memory(&mut self, config: &mut Config, is_reload: bool) { // Parse memory stores - self.parse_static_stores(config); + self.parse_static_stores(config, is_reload); // Parse http stores - self.parse_http_stores(config); + self.parse_http_stores(config, is_reload); // Parse purge schedules if let Some(store) = config diff --git a/crates/utils/src/cache.rs b/crates/utils/src/cache.rs index f7ead68e..60167613 100644 --- a/crates/utils/src/cache.rs +++ b/crates/utils/src/cache.rs @@ -301,6 +301,12 @@ impl CacheItemWeight for IpAddr { } } +impl CacheItemWeight for bool { + fn weight(&self) -> u64 { + std::mem::size_of::() as u64 + } +} + impl TtlEntry { pub fn new(value: T, expires: Duration) -> Self { Self { diff --git a/tests/src/directory/mod.rs b/tests/src/directory/mod.rs index b253c058..dfb1b894 100644 --- a/tests/src/directory/mod.rs +++ b/tests/src/directory/mod.rs @@ -352,7 +352,7 @@ impl DirectoryTest { config_file.replace("type = \"memory\"", "type = \"memory\"\ndisable = true") } let mut config = utils::config::Config::new(&config_file).unwrap(); - let stores = Stores::parse_all(&mut config).await; + let stores = Stores::parse_all(&mut config, false).await; let directories = Directories::parse( &mut config, &stores, diff --git a/tests/src/imap/mod.rs b/tests/src/imap/mod.rs index 96c20f95..25c1b670 100644 --- a/tests/src/imap/mod.rs +++ b/tests/src/imap/mod.rs @@ -325,7 +325,7 @@ async fn init_imap_tests(store_id: &str, delete_if_exists: bool) -> IMAPTest { servers.bind_and_drop_priv(&mut config); // Build stores - let stores = Stores::parse_all(&mut config).await; + let stores = Stores::parse_all(&mut config, false).await; // Parse core let tracers = Telemetry::parse(&mut config, &stores); diff --git a/tests/src/jmap/mod.rs b/tests/src/jmap/mod.rs index 026e3984..1abed89d 100644 --- a/tests/src/jmap/mod.rs +++ b/tests/src/jmap/mod.rs @@ -575,7 +575,7 @@ async fn init_jmap_tests(store_id: &str, delete_if_exists: bool) -> JMAPTest { servers.bind_and_drop_priv(&mut config); // Build stores - let stores = Stores::parse_all(&mut config).await; + let stores = Stores::parse_all(&mut config, false).await; // Parse core let config_manager = ConfigManager { diff --git a/tests/src/smtp/inbound/antispam.rs b/tests/src/smtp/inbound/antispam.rs index 60f77a8b..468e9cfe 100644 --- a/tests/src/smtp/inbound/antispam.rs +++ b/tests/src/smtp/inbound/antispam.rs @@ -154,7 +154,7 @@ async fn antispam() { // Parse config let mut config = Config::new(&config).unwrap(); config.resolve_all_macros().await; - let stores = Stores::parse_all(&mut config).await; + let stores = Stores::parse_all(&mut config, false).await; let mut core = Core::parse(&mut config, stores, Default::default()) .await .enable_enterprise(); diff --git a/tests/src/smtp/inbound/auth.rs b/tests/src/smtp/inbound/auth.rs index fee57498..21d37ec5 100644 --- a/tests/src/smtp/inbound/auth.rs +++ b/tests/src/smtp/inbound/auth.rs @@ -75,7 +75,7 @@ async fn auth() { let tmp_dir = TempDir::new("smtp_auth_test", true); let mut config = Config::new(tmp_dir.update_config(CONFIG)).unwrap(); - let stores = Stores::parse_all(&mut config).await; + let stores = Stores::parse_all(&mut config, false).await; let core = Core::parse(&mut config, stores, Default::default()).await; config.assert_no_errors(); diff --git a/tests/src/smtp/inbound/data.rs b/tests/src/smtp/inbound/data.rs index 7b7552a7..5b0c8f38 100644 --- a/tests/src/smtp/inbound/data.rs +++ b/tests/src/smtp/inbound/data.rs @@ -109,7 +109,7 @@ async fn data() { // Create temp dir for queue let tmp_dir = TempDir::new("smtp_data_test", true); let mut config = Config::new(tmp_dir.update_config(CONFIG)).unwrap(); - let stores = Stores::parse_all(&mut config).await; + let stores = Stores::parse_all(&mut config, false).await; let core = Core::parse(&mut config, stores, Default::default()).await; config.assert_no_errors(); diff --git a/tests/src/smtp/inbound/dmarc.rs b/tests/src/smtp/inbound/dmarc.rs index afacd416..2192470b 100644 --- a/tests/src/smtp/inbound/dmarc.rs +++ b/tests/src/smtp/inbound/dmarc.rs @@ -96,7 +96,7 @@ async fn dmarc() { let tmp_dir = TempDir::new("smtp_dmarc_test", true); let mut config = Config::new(tmp_dir.update_config(CONFIG.to_string() + SIGNATURES)).unwrap(); - let stores = Stores::parse_all(&mut config).await; + let stores = Stores::parse_all(&mut config, false).await; let core = Core::parse(&mut config, stores, Default::default()).await; let test = TestSMTP::from_core(core); diff --git a/tests/src/smtp/inbound/mail.rs b/tests/src/smtp/inbound/mail.rs index 56bfa21c..9ff1f72d 100644 --- a/tests/src/smtp/inbound/mail.rs +++ b/tests/src/smtp/inbound/mail.rs @@ -74,7 +74,7 @@ async fn mail() { let tmp_dir = TempDir::new("smtp_mail_test", true); let mut config = Config::new(tmp_dir.update_config(CONFIG)).unwrap(); - let stores = Stores::parse_all(&mut config).await; + let stores = Stores::parse_all(&mut config, false).await; let core = Core::parse(&mut config, stores, Default::default()).await; let server = TestSMTP::from_core(core).server; diff --git a/tests/src/smtp/inbound/milter.rs b/tests/src/smtp/inbound/milter.rs index 5dd711bb..c708a2c0 100644 --- a/tests/src/smtp/inbound/milter.rs +++ b/tests/src/smtp/inbound/milter.rs @@ -103,7 +103,7 @@ async fn milter_session() { // Configure tests let tmp_dir = TempDir::new("smtp_milter_test", true); let mut config = Config::new(tmp_dir.update_config(CONFIG_MILTER)).unwrap(); - let stores = Stores::parse_all(&mut config).await; + let stores = Stores::parse_all(&mut config, false).await; let core = Core::parse(&mut config, stores, Default::default()).await; let _rx = spawn_mock_milter_server(); tokio::time::sleep(Duration::from_millis(100)).await; @@ -236,7 +236,7 @@ async fn mta_hook_session() { // Configure tests let tmp_dir = TempDir::new("smtp_mta_hook_test", true); let mut config = Config::new(tmp_dir.update_config(CONFIG_JMILTER)).unwrap(); - let stores = Stores::parse_all(&mut config).await; + let stores = Stores::parse_all(&mut config, false).await; let core = Core::parse(&mut config, stores, Default::default()).await; let _rx = spawn_mock_mta_hook_server(); tokio::time::sleep(Duration::from_millis(100)).await; diff --git a/tests/src/smtp/inbound/rcpt.rs b/tests/src/smtp/inbound/rcpt.rs index 764478e7..3a5da304 100644 --- a/tests/src/smtp/inbound/rcpt.rs +++ b/tests/src/smtp/inbound/rcpt.rs @@ -89,7 +89,7 @@ async fn rcpt() { let tmp_dir = TempDir::new("smtp_rcpt_test", true); let mut config = Config::new(tmp_dir.update_config(CONFIG)).unwrap(); - let stores = Stores::parse_all(&mut config).await; + let stores = Stores::parse_all(&mut config, false).await; let core = Core::parse(&mut config, stores, Default::default()).await; // RCPT without MAIL FROM diff --git a/tests/src/smtp/inbound/scripts.rs b/tests/src/smtp/inbound/scripts.rs index 72edc4cd..c2dfcbbe 100644 --- a/tests/src/smtp/inbound/scripts.rs +++ b/tests/src/smtp/inbound/scripts.rs @@ -133,7 +133,7 @@ async fn sieve_scripts() { let tmp_dir = TempDir::new("smtp_sieve_test", true); let mut config = Config::new(tmp_dir.update_config(config)).unwrap(); config.resolve_all_macros().await; - let stores = Stores::parse_all(&mut config).await; + let stores = Stores::parse_all(&mut config, false).await; let core = Core::parse(&mut config, stores, Default::default()).await; config.assert_no_errors(); diff --git a/tests/src/smtp/inbound/sign.rs b/tests/src/smtp/inbound/sign.rs index 291de254..02fa40a8 100644 --- a/tests/src/smtp/inbound/sign.rs +++ b/tests/src/smtp/inbound/sign.rs @@ -128,7 +128,7 @@ async fn sign_and_seal() { let tmp_dir = TempDir::new("smtp_sign_test", true); let mut config = Config::new(tmp_dir.update_config(CONFIG.to_string() + SIGNATURES)).unwrap(); - let stores = Stores::parse_all(&mut config).await; + let stores = Stores::parse_all(&mut config, false).await; let core = Core::parse(&mut config, stores, Default::default()).await; let test = TestSMTP::from_core(core); diff --git a/tests/src/smtp/inbound/throttle.rs b/tests/src/smtp/inbound/throttle.rs index 54dbf706..28cfdc9e 100644 --- a/tests/src/smtp/inbound/throttle.rs +++ b/tests/src/smtp/inbound/throttle.rs @@ -49,7 +49,7 @@ async fn throttle_inbound() { let tmp_dir = TempDir::new("smtp_inbound_throttle", true); let mut config = Config::new(tmp_dir.update_config(CONFIG)).unwrap(); - let stores = Stores::parse_all(&mut config).await; + let stores = Stores::parse_all(&mut config, false).await; let core = Core::parse(&mut config, stores, Default::default()).await; // Test connection concurrency limit diff --git a/tests/src/smtp/inbound/vrfy.rs b/tests/src/smtp/inbound/vrfy.rs index 516b899f..88952ec4 100644 --- a/tests/src/smtp/inbound/vrfy.rs +++ b/tests/src/smtp/inbound/vrfy.rs @@ -73,7 +73,7 @@ async fn vrfy_expn() { let tmp_dir = TempDir::new("smtp_vrfy_test", true); let mut config = Config::new(tmp_dir.update_config(CONFIG)).unwrap(); - let stores = Stores::parse_all(&mut config).await; + let stores = Stores::parse_all(&mut config, false).await; let core = Core::parse(&mut config, stores, Default::default()).await; config.assert_no_errors(); diff --git a/tests/src/smtp/lookup/sql.rs b/tests/src/smtp/lookup/sql.rs index f72bb829..83119c8b 100644 --- a/tests/src/smtp/lookup/sql.rs +++ b/tests/src/smtp/lookup/sql.rs @@ -22,7 +22,8 @@ use utils::config::Config; use crate::{ directory::DirectoryStore, smtp::{ - session::{TestSession, VerifyResponse}, DnsCache, TempDir, TestSMTP + session::{TestSession, VerifyResponse}, + DnsCache, TempDir, TestSMTP, }, }; use smtp::{core::Session, queue::RecipientDomain}; @@ -102,7 +103,7 @@ async fn lookup_sql() { // Parse settings let temp_dir = TempDir::new("smtp_lookup_tests", true); let mut config = Config::new(temp_dir.update_config(CONFIG)).unwrap(); - let stores = Stores::parse_all(&mut config).await; + let stores = Stores::parse_all(&mut config, false).await; let core = Core::parse(&mut config, stores, Default::default()).await; diff --git a/tests/src/smtp/mod.rs b/tests/src/smtp/mod.rs index 7b6e9ac3..06946725 100644 --- a/tests/src/smtp/mod.rs +++ b/tests/src/smtp/mod.rs @@ -201,7 +201,7 @@ impl TestSMTP { let mut config = Config::new(temp_dir.update_config(add_test_certs(CONFIG) + config.as_ref())).unwrap(); config.resolve_all_macros().await; - let stores = Stores::parse_all(&mut config).await; + let stores = Stores::parse_all(&mut config, false).await; let core = Core::parse(&mut config, stores, Default::default()).await; let data = Data::parse(&mut config); diff --git a/tests/src/store/blob.rs b/tests/src/store/blob.rs index 5ebc5a47..0163a492 100644 --- a/tests/src/store/blob.rs +++ b/tests/src/store/blob.rs @@ -18,7 +18,7 @@ pub async fn blob_tests() { let temp_dir = TempDir::new("blob_tests", true); let mut config = Config::new(CONFIG.replace("{TMP}", temp_dir.path.as_path().to_str().unwrap())).unwrap(); - let stores = Stores::parse_all(&mut config).await; + let stores = Stores::parse_all(&mut config, false).await; for (store_id, blob_store) in &stores.blob_stores { println!("Testing blob store {}...", store_id); diff --git a/tests/src/store/lookup.rs b/tests/src/store/lookup.rs index 71c15392..8a4ef6c0 100644 --- a/tests/src/store/lookup.rs +++ b/tests/src/store/lookup.rs @@ -21,7 +21,7 @@ pub async fn lookup_tests() { Config::new(CONFIG.replace("{TMP}", temp_dir.path.as_path().to_str().unwrap())) .unwrap() .assert_no_errors(); - let stores = Stores::parse_all(&mut config).await; + let stores = Stores::parse_all(&mut config, false).await; let rate = Rate { requests: 1, period: Duration::from_secs(1), diff --git a/tests/src/store/mod.rs b/tests/src/store/mod.rs index 76c3bb2b..cc789182 100644 --- a/tests/src/store/mod.rs +++ b/tests/src/store/mod.rs @@ -81,7 +81,7 @@ pub async fn store_tests() { let mut config = Config::new(CONFIG.replace("{TMP}", &temp_dir.path.to_string_lossy())) .unwrap() .assert_no_errors(); - let stores = Stores::parse_all(&mut config).await; + let stores = Stores::parse_all(&mut config, false).await; let store_id = std::env::var("STORE") .expect("Missing store type. Try running `STORE= cargo test`");