diff --git a/.gitignore b/.gitignore index 017b8293..b6a70171 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ *_failed stalwart.toml run.sh +_ignore diff --git a/Cargo.lock b/Cargo.lock index ebcd2228..1b9cc078 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -25,7 +25,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "433cfd6710c9986c576a25ca913c39d66a6474107b406f34f91d4a8923395241" dependencies = [ "cfg-if", - "cipher", + "cipher 0.4.4", "cpufeatures", ] @@ -37,7 +37,7 @@ checksum = "209b47e8954a928e1d72e86eca7000ebb6655fe1436d33eefc2201cad027e237" dependencies = [ "aead", "aes", - "cipher", + "cipher 0.4.4", "ctr", "ghash", "subtle", @@ -51,7 +51,7 @@ checksum = "ae0784134ba9375416d469ec31e7c5f9fa94405049cf08c5ce5b4698be673e0d" dependencies = [ "aead", "aes", - "cipher", + "cipher 0.4.4", "ctr", "polyval", "subtle", @@ -106,6 +106,17 @@ version = "1.0.71" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" +[[package]] +name = "argon2" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95c2fcf79ad1932ac6269a738109997a83c227c09b75842ae564dc8ede6a861c" +dependencies = [ + "base64ct", + "blake2", + "password-hash 0.5.0", +] + [[package]] name = "arrayref" version = "0.3.7" @@ -304,9 +315,9 @@ checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "bb8" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1627eccf3aa91405435ba240be23513eeca466b5dc33866422672264de061582" +checksum = "98b4b0f25f18bcdc3ac72bdb486ed0acf7e185221fd4dc985bc15db5800b0ba2" dependencies = [ "async-trait", "futures-channel", @@ -388,6 +399,15 @@ dependencies = [ "crunchy", ] +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest 0.10.7", +] + [[package]] name = "blake3" version = "1.3.3" @@ -399,7 +419,16 @@ dependencies = [ "cc", "cfg-if", "constant_time_eq 0.2.5", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "block-buffer" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4" +dependencies = [ + "generic-array", ] [[package]] @@ -411,6 +440,17 @@ dependencies = [ "generic-array", ] +[[package]] +name = "blowfish" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32fa6a061124e37baba002e496d203e23ba3d7b73750be82dbfbc92913048a5b" +dependencies = [ + "byteorder", + "cipher 0.2.5", + "opaque-debug", +] + [[package]] name = "bumpalo" version = "3.12.2" @@ -505,6 +545,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "cipher" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f8e7987cbd042a63249497f41aed09f8e65add917ea6566effbc56578d6801" +dependencies = [ + "generic-array", +] + [[package]] name = "cipher" version = "0.4.4" @@ -700,10 +749,20 @@ dependencies = [ ] [[package]] -name = "csv" -version = "1.2.1" +name = "crypto-mac" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b015497079b9a9d69c02ad25de6c0a6edef051ea6360a327d0bd05802ef64ad" +checksum = "bff07008ec701e8028e2ceb8f83f0e4274ee62bd2dbdc4fefff2e9a91824081a" +dependencies = [ + "generic-array", + "subtle", +] + +[[package]] +name = "csv" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626ae34994d3d8d668f4269922248239db4ae42d538b14c398b74a52208e8086" dependencies = [ "csv-core", "itoa", @@ -726,7 +785,7 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" dependencies = [ - "cipher", + "cipher 0.4.4", ] [[package]] @@ -784,13 +843,22 @@ dependencies = [ "rusticata-macros", ] +[[package]] +name = "digest" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066" +dependencies = [ + "generic-array", +] + [[package]] name = "digest" version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", + "block-buffer 0.10.4", "const-oid", "crypto-common", "subtle", @@ -801,16 +869,29 @@ name = "directory" version = "0.1.0" dependencies = [ "ahash 0.8.3", + "argon2", "async-trait", "bb8", "ldap3", + "lru-cache", + "mail-builder", + "mail-parser", "mail-send", + "md5", + "parking_lot", + "password-hash 0.5.0", + "pbkdf2 0.12.1", + "pwhash", "rustls 0.21.1", + "scrypt", + "sha1", + "sha2 0.10.6", "smtp-proto", "sqlx", "tokio", "tokio-rustls 0.24.0", "tracing", + "utils", ] [[package]] @@ -837,7 +918,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0997c976637b606099b9985693efa3581e84e41f5c11ba5255f88711058ad428" dependencies = [ "der 0.7.6", - "digest", + "digest 0.10.7", "elliptic-curve", "rfc6979", "signature", @@ -858,7 +939,7 @@ dependencies = [ "once_cell", "openssl", "serde", - "sha2", + "sha2 0.10.6", "thiserror", ] @@ -879,7 +960,7 @@ checksum = "968405c8fdc9b3bf4df0a6638858cc0b52462836ab6b1c87377785dd09cf1c0b" dependencies = [ "base16ct", "crypto-bigint", - "digest", + "digest 0.10.7", "ff", "generic-array", "group", @@ -1387,7 +1468,17 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "791a029f6b9fc27657f6f188ec6e5e43f6911f6f878e0dc5501396e09809d437" dependencies = [ - "hmac", + "hmac 0.12.1", +] + +[[package]] +name = "hmac" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1441c6b1e930e2817404b5046f1f989899143a12bf92de603b69f4e0aee1e15" +dependencies = [ + "crypto-mac", + "digest 0.9.0", ] [[package]] @@ -1396,7 +1487,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] @@ -1711,7 +1802,7 @@ dependencies = [ "reqwest", "serde", "serde_json", - "sha2", + "sha2 0.10.6", "sieve-rs", "smtp", "smtp-proto", @@ -2060,13 +2151,24 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "md-5" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5a279bb9607f9f53c22d496eade00d138d1bdcccd07d74650387cf94942a15" +dependencies = [ + "block-buffer 0.9.0", + "digest 0.9.0", + "opaque-debug", +] + [[package]] name = "md-5" version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] @@ -2431,7 +2533,7 @@ dependencies = [ "ecdsa", "elliptic-curve", "primeorder", - "sha2", + "sha2 0.10.6", ] [[package]] @@ -2468,6 +2570,17 @@ dependencies = [ "subtle", ] +[[package]] +name = "password-hash" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" +dependencies = [ + "base64ct", + "rand_core", + "subtle", +] + [[package]] name = "paste" version = "1.0.12" @@ -2480,10 +2593,22 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" dependencies = [ - "digest", - "hmac", - "password-hash", - "sha2", + "digest 0.10.7", + "hmac 0.12.1", + "password-hash 0.4.2", + "sha2 0.10.6", +] + +[[package]] +name = "pbkdf2" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0ca0b5a68607598bf3bad68f32227a8164f6254833f84eafaac409cd6746c31" +dependencies = [ + "digest 0.10.7", + "hmac 0.12.1", + "password-hash 0.5.0", + "sha2 0.10.6", ] [[package]] @@ -2758,6 +2883,21 @@ dependencies = [ "prost", ] +[[package]] +name = "pwhash" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419a3ad8fa9f9d445e69d9b185a24878ae6e6f55c96e4512f4a0e28cd3bc5c56" +dependencies = [ + "blowfish", + "byteorder", + "hmac 0.10.1", + "md-5 0.9.1", + "rand", + "sha-1", + "sha2 0.9.9", +] + [[package]] name = "quick-error" version = "1.2.3" @@ -2959,7 +3099,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" dependencies = [ - "hmac", + "hmac 0.12.1", "subtle", ] @@ -3006,7 +3146,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55a77d189da1fee555ad95b7e50e7457d91c0e089ec68ca69ad2989413bbdab4" dependencies = [ "byteorder", - "digest", + "digest 0.10.7", "num-bigint-dig", "num-integer", "num-iter", @@ -3139,6 +3279,15 @@ version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" +[[package]] +name = "salsa20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" +dependencies = [ + "cipher 0.4.4", +] + [[package]] name = "schannel" version = "0.1.21" @@ -3163,6 +3312,18 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "scrypt" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" +dependencies = [ + "password-hash 0.5.0", + "pbkdf2 0.12.1", + "salsa20", + "sha2 0.10.6", +] + [[package]] name = "sct" version = "0.7.0" @@ -3278,6 +3439,19 @@ dependencies = [ "syn 2.0.16", ] +[[package]] +name = "sha-1" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99cd6713db3cf16b6c84e06321e049a9b9f699826e16096d23bbcc44d15d51a6" +dependencies = [ + "block-buffer 0.9.0", + "cfg-if", + "cpufeatures", + "digest 0.9.0", + "opaque-debug", +] + [[package]] name = "sha1" version = "0.10.5" @@ -3286,7 +3460,20 @@ checksum = "f04293dc80c3993519f2d7f6f511707ee7094fe0c6d3406feb330cdb3540eba3" dependencies = [ "cfg-if", "cpufeatures", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" +dependencies = [ + "block-buffer 0.9.0", + "cfg-if", + "cpufeatures", + "digest 0.9.0", + "opaque-debug", ] [[package]] @@ -3297,7 +3484,7 @@ checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" dependencies = [ "cfg-if", "cpufeatures", - "digest", + "digest 0.10.7", ] [[package]] @@ -3344,7 +3531,7 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e1788eed21689f9cf370582dfc467ef36ed9c707f073528ddafa8d83e3b8500" dependencies = [ - "digest", + "digest 0.10.7", "rand_core", ] @@ -3376,6 +3563,7 @@ dependencies = [ "ahash 0.8.3", "blake3", "dashmap", + "directory", "form_urlencoded", "http-body-util", "hyper 1.0.0-rc.3", @@ -3396,7 +3584,7 @@ dependencies = [ "serde", "serde_json", "sha1", - "sha2", + "sha2 0.10.6", "sieve-rs", "smtp-proto", "sqlx", @@ -3515,7 +3703,7 @@ dependencies = [ "rustls-pemfile", "serde", "serde_json", - "sha2", + "sha2 0.10.6", "smallvec", "sqlformat", "thiserror", @@ -3554,7 +3742,7 @@ dependencies = [ "quote", "serde", "serde_json", - "sha2", + "sha2 0.10.6", "sqlx-core", "sqlx-mysql", "sqlx-postgres", @@ -3577,7 +3765,7 @@ dependencies = [ "byteorder", "bytes", "crc", - "digest", + "digest 0.10.7", "dotenvy", "either", "futures-channel", @@ -3587,10 +3775,10 @@ dependencies = [ "generic-array", "hex", "hkdf", - "hmac", + "hmac 0.12.1", "itoa", "log", - "md-5", + "md-5 0.10.5", "memchr", "once_cell", "percent-encoding", @@ -3598,7 +3786,7 @@ dependencies = [ "rsa", "serde", "sha1", - "sha2", + "sha2 0.10.6", "smallvec", "sqlx-core", "stringprep", @@ -3626,18 +3814,18 @@ dependencies = [ "futures-util", "hex", "hkdf", - "hmac", + "hmac 0.12.1", "home", "itoa", "log", - "md-5", + "md-5 0.10.5", "memchr", "once_cell", "rand", "serde", "serde_json", "sha1", - "sha2", + "sha2 0.10.6", "smallvec", "sqlx-core", "stringprep", @@ -3725,9 +3913,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" [[package]] name = "subtle" -version = "2.5.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" +checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" @@ -3802,12 +3990,12 @@ dependencies = [ "chrono", "csv", "dashmap", + "directory", "ece", "flate2", "futures", "http-body-util", "hyper 1.0.0-rc.3", - "jmap", "jmap-client", "jmap_proto", "mail-auth", @@ -4846,8 +5034,8 @@ dependencies = [ "crc32fast", "crossbeam-utils", "flate2", - "hmac", - "pbkdf2", + "hmac 0.12.1", + "pbkdf2 0.11.0", "sha1", "time 0.3.21", "zstd", diff --git a/crates/directory/Cargo.toml b/crates/directory/Cargo.toml index 96c55123..27a809f1 100644 --- a/crates/directory/Cargo.toml +++ b/crates/directory/Cargo.toml @@ -16,6 +16,7 @@ rustls = "0.21.0" sqlx = { version = "0.7.0-alpha.3", features = [ "runtime-tokio-rustls", "postgres", "mysql", "sqlite" ] } ldap3 = { version = "0.11.1", default-features = false, features = ["tls-rustls"] } bb8 = "0.8.1" +parking_lot = "0.12" async-trait = "0.1.68" ahash = { version = "0.8" } tracing = "0.1" diff --git a/crates/directory/src/cache/config.rs b/crates/directory/src/cache/config.rs new file mode 100644 index 00000000..d2d4cba3 --- /dev/null +++ b/crates/directory/src/cache/config.rs @@ -0,0 +1,41 @@ +use std::{sync::Arc, time::Duration}; + +use parking_lot::lock_api::Mutex; +use utils::config::Config; + +use crate::Directory; + +use super::{lru::LookupCache, CachedDirectory}; + +impl CachedDirectory { + pub fn try_from_config( + config: &Config, + prefix: &str, + inner: T, + ) -> utils::config::Result> { + if let Some(cached_entries) = config.property((prefix, "cache.entries"))? { + let cache_ttl_positive = config + .property((prefix, "cache.ttl.positive"))? + .unwrap_or(Duration::from_secs(86400)); + let cache_ttl_negative = config + .property((prefix, "cache.ttl.positive"))? + .unwrap_or_else(|| Duration::from_secs(3600)); + + Ok(Arc::new(CachedDirectory { + inner, + cached_domains: Mutex::new(LookupCache::new( + cached_entries, + cache_ttl_positive, + cache_ttl_negative, + )), + cached_rcpts: Mutex::new(LookupCache::new( + cached_entries, + cache_ttl_positive, + cache_ttl_negative, + )), + })) + } else { + Ok(Arc::new(inner)) + } + } +} diff --git a/crates/directory/src/cache/lookup.rs b/crates/directory/src/cache/lookup.rs new file mode 100644 index 00000000..2e4a7c92 --- /dev/null +++ b/crates/directory/src/cache/lookup.rs @@ -0,0 +1,77 @@ +use mail_send::Credentials; + +use crate::{Directory, Principal}; + +use super::CachedDirectory; + +#[async_trait::async_trait] +impl Directory for CachedDirectory { + async fn authenticate( + &self, + credentials: &Credentials, + ) -> crate::Result> { + self.inner.authenticate(credentials).await + } + + async fn principal_by_name(&self, name: &str) -> crate::Result> { + self.inner.principal_by_name(name).await + } + + async fn principal_by_id(&self, id: u32) -> crate::Result> { + self.inner.principal_by_id(id).await + } + + async fn member_of(&self, _principal: &Principal) -> crate::Result> { + self.inner.member_of(_principal).await + } + + async fn emails_by_id(&self, id: u32) -> crate::Result> { + self.inner.emails_by_id(id).await + } + + async fn ids_by_email(&self, address: &str) -> crate::Result> { + self.inner.ids_by_email(address).await + } + + async fn rcpt(&self, address: &str) -> crate::Result { + if let Some(result) = { + let result = self.cached_rcpts.lock().get(address); + result + } { + Ok(result) + } else if self.inner.rcpt(address).await? { + self.cached_rcpts.lock().insert_pos(address.to_string()); + Ok(true) + } else { + self.cached_rcpts.lock().insert_neg(address.to_string()); + Ok(false) + } + } + + async fn vrfy(&self, address: &str) -> crate::Result> { + self.inner.vrfy(address).await + } + + async fn expn(&self, address: &str) -> crate::Result> { + self.inner.expn(address).await + } + + async fn query(&self, query: &str, params: &[&str]) -> crate::Result { + self.inner.query(query, params).await + } + + async fn is_local_domain(&self, domain: &str) -> crate::Result { + if let Some(result) = { + let result = self.cached_domains.lock().get(domain); + result + } { + Ok(result) + } else if self.inner.is_local_domain(domain).await? { + self.cached_domains.lock().insert_pos(domain.to_string()); + Ok(true) + } else { + self.cached_domains.lock().insert_neg(domain.to_string()); + Ok(false) + } + } +} diff --git a/crates/directory/src/cache/lru.rs b/crates/directory/src/cache/lru.rs new file mode 100644 index 00000000..5f85d303 --- /dev/null +++ b/crates/directory/src/cache/lru.rs @@ -0,0 +1,85 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of the Stalwart SMTP Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{ + borrow::Borrow, + hash::Hash, + time::{Duration, Instant}, +}; + +#[allow(clippy::type_complexity)] +#[derive(Debug)] +pub struct LookupCache { + cache_pos: lru_cache::LruCache, + cache_neg: lru_cache::LruCache, + ttl_pos: Duration, + ttl_neg: Duration, +} + +impl LookupCache { + pub fn new(capacity: usize, ttl_pos: Duration, ttl_neg: Duration) -> Self { + Self { + cache_pos: lru_cache::LruCache::with_hasher(capacity, ahash::RandomState::new()), + cache_neg: lru_cache::LruCache::with_hasher(capacity, ahash::RandomState::new()), + ttl_pos, + ttl_neg, + } + } + + pub fn get(&mut self, name: &Q) -> Option + where + T: Borrow, + Q: Hash + Eq, + { + // Check positive cache + if let Some(valid_until) = self.cache_pos.get_mut(name) { + if *valid_until >= Instant::now() { + return Some(true); + } else { + self.cache_pos.remove(name); + } + } + + // Check negative cache + let valid_until = self.cache_neg.get_mut(name)?; + if *valid_until >= Instant::now() { + Some(false) + } else { + self.cache_pos.remove(name); + None + } + } + + pub fn insert_pos(&mut self, item: T) { + self.cache_pos.insert(item, Instant::now() + self.ttl_pos); + } + + pub fn insert_neg(&mut self, item: T) { + self.cache_neg.insert(item, Instant::now() + self.ttl_neg); + } + + pub fn clear(&mut self) { + self.cache_pos.clear(); + self.cache_neg.clear(); + } +} diff --git a/crates/directory/src/cache/mod.rs b/crates/directory/src/cache/mod.rs new file mode 100644 index 00000000..cfe9c3ff --- /dev/null +++ b/crates/directory/src/cache/mod.rs @@ -0,0 +1,15 @@ +use parking_lot::Mutex; + +use crate::Directory; + +use self::lru::LookupCache; + +pub mod config; +pub mod lookup; +pub mod lru; + +pub struct CachedDirectory { + inner: T, + cached_domains: Mutex>, + cached_rcpts: Mutex>, +} diff --git a/crates/directory/src/config.rs b/crates/directory/src/config.rs index af609791..58129aad 100644 --- a/crates/directory/src/config.rs +++ b/crates/directory/src/config.rs @@ -5,7 +5,7 @@ use std::{ sync::Arc, time::Duration, }; -use utils::config::Config; +use utils::config::{utils::AsKey, Config}; use ahash::{AHashMap, AHashSet}; @@ -16,6 +16,7 @@ use crate::{ pub trait ConfigDirectory { fn parse_directory(&self) -> utils::config::Result; + fn parse_lookup_list(&self, key: impl AsKey) -> utils::config::Result>; } impl ConfigDirectory for Config { @@ -25,14 +26,17 @@ impl ConfigDirectory for Config { lookups: AHashMap::new(), }; for id in self.sub_keys("directory") { + // Parse domains list + let domains = self.parse_lookup_list(("directory", id, "lookup.domains"))?; + // Parse directory let protocol = self.value_require(("directory", id, "protocol"))?; let directory = match protocol { - "ldap" => LdapDirectory::from_config(self, ("directory", id))?, - "sql" => SqlDirectory::from_config(self, ("directory", id))?, - "imap" => ImapDirectory::from_config(self, ("directory", id))?, - "smtp" => SmtpDirectory::from_config(self, ("directory", id), false)?, - "lmtp" => SmtpDirectory::from_config(self, ("directory", id), true)?, + "ldap" => LdapDirectory::from_config(self, ("directory", id), domains)?, + "sql" => SqlDirectory::from_config(self, ("directory", id), domains)?, + "imap" => ImapDirectory::from_config(self, ("directory", id), domains)?, + "smtp" => SmtpDirectory::from_config(self, ("directory", id), domains, false)?, + "lmtp" => SmtpDirectory::from_config(self, ("directory", id), domains, true)?, "memory" => MemoryDirectory::from_config(self, ("directory", id))?, unknown => { return Err(format!("Unknown directory type: {unknown:?}")); @@ -50,30 +54,9 @@ impl ConfigDirectory for Config { .to_string(), } } else { - let mut list = AHashSet::new(); - for (_, value) in self.values(("directory", id, "lookup", lookup_id)) { - if let Some(path) = value.strip_prefix("file://") { - for line in BufReader::new(File::open(path).map_err(|err| { - format!( - "Failed to read file {path:?} for list {id}/{lookup_id}: {err}" - ) - })?) - .lines() - { - let line_ = line.map_err(|err| { - format!("Failed to read file {path:?} for list {id}/{lookup_id}: {err}") - })?; - let line = line_.trim(); - if !line.is_empty() { - list.insert(line.to_string()); - } - } - } else { - list.insert(value.to_string()); - } + Lookup::List { + list: self.parse_lookup_list(("directory", id, "lookup", lookup_id))?, } - - Lookup::List { list } }; config .lookups @@ -85,6 +68,36 @@ impl ConfigDirectory for Config { Ok(config) } + + fn parse_lookup_list(&self, key: impl AsKey) -> utils::config::Result> { + let mut list = AHashSet::new(); + for (_, value) in self.values(key.clone()) { + if let Some(path) = value.strip_prefix("file://") { + for line in BufReader::new(File::open(path).map_err(|err| { + format!( + "Failed to read file {path:?} for list {}: {err}", + key.as_key() + ) + })?) + .lines() + { + let line_ = line.map_err(|err| { + format!( + "Failed to read file {path:?} for list {}: {err}", + key.as_key() + ) + })?; + let line = line_.trim(); + if !line.is_empty() { + list.insert(line.to_string()); + } + } + } else { + list.insert(value.to_string()); + } + } + Ok(list) + } } pub(crate) fn build_pool( diff --git a/crates/directory/src/imap/config.rs b/crates/directory/src/imap/config.rs index 1ee78545..563e9b48 100644 --- a/crates/directory/src/imap/config.rs +++ b/crates/directory/src/imap/config.rs @@ -1,9 +1,10 @@ use std::sync::Arc; +use ahash::AHashSet; use mail_send::smtp::tls::build_tls_connector; use utils::config::{utils::AsKey, Config}; -use crate::{config::build_pool, imap::ImapConnectionManager, Directory}; +use crate::{cache::CachedDirectory, config::build_pool, imap::ImapConnectionManager, Directory}; use super::ImapDirectory; @@ -11,6 +12,7 @@ impl ImapDirectory { pub fn from_config( config: &Config, prefix: impl AsKey, + domains: AHashSet, ) -> utils::config::Result> { let prefix = prefix.as_key(); let address = config.value_require((&prefix, "address"))?; @@ -29,8 +31,13 @@ impl ImapDirectory { mechanisms: 0.into(), }; - Ok(Arc::new(ImapDirectory { - pool: build_pool(config, &prefix, manager)?, - })) + CachedDirectory::try_from_config( + config, + &prefix, + ImapDirectory { + pool: build_pool(config, &prefix, manager)?, + domains, + }, + ) } } diff --git a/crates/directory/src/imap/lookup.rs b/crates/directory/src/imap/lookup.rs index 43df2d87..f3ae3493 100644 --- a/crates/directory/src/imap/lookup.rs +++ b/crates/directory/src/imap/lookup.rs @@ -86,4 +86,8 @@ impl Directory for ImapDirectory { async fn query(&self, _query: &str, _params: &[&str]) -> crate::Result { Err(DirectoryError::unsupported("imap", "query")) } + + async fn is_local_domain(&self, domain: &str) -> crate::Result { + Ok(self.domains.contains(domain)) + } } diff --git a/crates/directory/src/imap/mod.rs b/crates/directory/src/imap/mod.rs index c3e63853..6593e6ee 100644 --- a/crates/directory/src/imap/mod.rs +++ b/crates/directory/src/imap/mod.rs @@ -6,12 +6,14 @@ pub mod tls; use std::{fmt::Display, sync::atomic::AtomicU64, time::Duration}; +use ahash::AHashSet; use bb8::Pool; use tokio::io::{AsyncRead, AsyncWrite}; use tokio_rustls::TlsConnector; pub struct ImapDirectory { pool: Pool, + domains: AHashSet, } pub struct ImapConnectionManager { diff --git a/crates/directory/src/ldap/config.rs b/crates/directory/src/ldap/config.rs index f0cd3dcb..db07e0da 100644 --- a/crates/directory/src/ldap/config.rs +++ b/crates/directory/src/ldap/config.rs @@ -1,9 +1,10 @@ use std::sync::Arc; +use ahash::AHashSet; use ldap3::LdapConnSettings; use utils::config::{utils::AsKey, Config}; -use crate::{config::build_pool, Directory}; +use crate::{cache::CachedDirectory, config::build_pool, Directory}; use super::{Bind, LdapConnectionManager, LdapDirectory, LdapFilter, LdapMappings}; @@ -11,6 +12,7 @@ impl LdapDirectory { pub fn from_config( config: &Config, prefix: impl AsKey, + domains: AHashSet, ) -> utils::config::Result> { let prefix = prefix.as_key(); let bind_dn = if let Some(dn) = config.value((&prefix, "bind.dn")) { @@ -42,6 +44,7 @@ impl LdapDirectory { filter_id: LdapFilter::from_config(config, (&prefix, "filter.id"))?, filter_verify: LdapFilter::from_config(config, (&prefix, "filter.verify"))?, filter_expand: LdapFilter::from_config(config, (&prefix, "filter.expand"))?, + filter_domains: LdapFilter::from_config(config, (&prefix, "filter.domains"))?, obj_user: config .value_require((&prefix, "object-classes.user"))? .to_string(), @@ -98,10 +101,15 @@ impl LdapDirectory { .attrs_email .extend(mappings.attr_email_address.iter().cloned()); - Ok(Arc::new(LdapDirectory { - mappings, - pool: build_pool(config, &prefix, manager)?, - })) + CachedDirectory::try_from_config( + config, + &prefix, + LdapDirectory { + mappings, + pool: build_pool(config, &prefix, manager)?, + domains, + }, + ) } } diff --git a/crates/directory/src/ldap/lookup.rs b/crates/directory/src/ldap/lookup.rs index 906b2dc5..aa7f4085 100644 --- a/crates/directory/src/ldap/lookup.rs +++ b/crates/directory/src/ldap/lookup.rs @@ -142,19 +142,20 @@ impl Directory for LdapDirectory { } async fn rcpt(&self, address: &str) -> crate::Result { - let (rs, _res) = self - .pool + self.pool .get() .await? - .search( + .streaming_search( &self.mappings.base_dn, Scope::Subtree, &self.mappings.filter_email.build(address), &self.mappings.attr_email_address, ) .await? - .success()?; - Ok(!rs.is_empty()) + .next() + .await + .map(|entry| entry.is_some()) + .map_err(|e| e.into()) } async fn vrfy(&self, address: &str) -> crate::Result> { @@ -250,6 +251,26 @@ impl Directory for LdapDirectory { .await? .is_some()) } + + async fn is_local_domain(&self, domain: &str) -> crate::Result { + if self.domains.contains(domain) { + return Ok(true); + } + self.pool + .get() + .await? + .streaming_search( + &self.mappings.base_dn, + Scope::Subtree, + &self.mappings.filter_domains.build(domain), + Vec::::new(), + ) + .await? + .next() + .await + .map(|entry| entry.is_some()) + .map_err(|e| e.into()) + } } impl LdapDirectory { diff --git a/crates/directory/src/ldap/mod.rs b/crates/directory/src/ldap/mod.rs index 8f65c365..a5b891b2 100644 --- a/crates/directory/src/ldap/mod.rs +++ b/crates/directory/src/ldap/mod.rs @@ -1,3 +1,4 @@ +use ahash::AHashSet; use bb8::Pool; use ldap3::{ldap_escape, LdapConnSettings}; @@ -8,6 +9,7 @@ pub mod pool; pub struct LdapDirectory { pool: Pool, mappings: LdapMappings, + domains: AHashSet, } #[derive(Debug, Default)] @@ -19,6 +21,7 @@ pub struct LdapMappings { filter_id: LdapFilter, filter_verify: LdapFilter, filter_expand: LdapFilter, + filter_domains: LdapFilter, obj_user: String, obj_group: String, attr_name: Vec, diff --git a/crates/directory/src/lib.rs b/crates/directory/src/lib.rs index a551f56f..763249f9 100644 --- a/crates/directory/src/lib.rs +++ b/crates/directory/src/lib.rs @@ -6,6 +6,7 @@ use imap::ImapError; use ldap3::LdapError; use mail_send::Credentials; +pub mod cache; pub mod config; pub mod imap; pub mod ldap; @@ -53,6 +54,7 @@ pub trait Directory: Sync + Send { async fn member_of(&self, principal: &Principal) -> Result>; async fn emails_by_id(&self, id: u32) -> Result>; async fn ids_by_email(&self, email: &str) -> Result>; + async fn is_local_domain(&self, domain: &str) -> crate::Result; async fn rcpt(&self, address: &str) -> crate::Result; async fn vrfy(&self, address: &str) -> Result>; async fn expn(&self, address: &str) -> Result>; @@ -119,7 +121,7 @@ impl Debug for Lookup { } } -#[derive(Default, Clone)] +#[derive(Default, Clone, Debug)] pub struct DirectoryConfig { pub directories: AHashMap>, pub lookups: AHashMap>, diff --git a/crates/directory/src/memory/lookup.rs b/crates/directory/src/memory/lookup.rs index e56fc739..65b16572 100644 --- a/crates/directory/src/memory/lookup.rs +++ b/crates/directory/src/memory/lookup.rs @@ -112,4 +112,12 @@ impl Directory for MemoryDirectory { async fn query(&self, _query: &str, _params: &[&str]) -> crate::Result { Err(DirectoryError::unsupported("memory", "query")) } + + async fn is_local_domain(&self, domain: &str) -> crate::Result { + let domain = format!("@{domain}"); + Ok(self + .emails_to_ids + .keys() + .any(|email| email.ends_with(&domain))) + } } diff --git a/crates/directory/src/smtp/config.rs b/crates/directory/src/smtp/config.rs index b5a95e01..88262800 100644 --- a/crates/directory/src/smtp/config.rs +++ b/crates/directory/src/smtp/config.rs @@ -1,9 +1,10 @@ use std::sync::Arc; +use ahash::AHashSet; use mail_send::{smtp::tls::build_tls_connector, SmtpClientBuilder}; use utils::config::{utils::AsKey, Config}; -use crate::{config::build_pool, smtp::SmtpConnectionManager, Directory}; +use crate::{cache::CachedDirectory, config::build_pool, smtp::SmtpConnectionManager, Directory}; use super::SmtpDirectory; @@ -11,6 +12,7 @@ impl SmtpDirectory { pub fn from_config( config: &Config, prefix: impl AsKey, + domains: AHashSet, is_lmtp: bool, ) -> utils::config::Result> { let prefix = prefix.as_key(); @@ -39,8 +41,13 @@ impl SmtpDirectory { max_auth_errors: config.property_or_static((&prefix, "limits.auth-errors"), "3")?, }; - Ok(Arc::new(SmtpDirectory { - pool: build_pool(config, &prefix, manager)?, - })) + CachedDirectory::try_from_config( + config, + &prefix, + SmtpDirectory { + pool: build_pool(config, &prefix, manager)?, + domains, + }, + ) } } diff --git a/crates/directory/src/smtp/lookup.rs b/crates/directory/src/smtp/lookup.rs index 69737a50..6004bd14 100644 --- a/crates/directory/src/smtp/lookup.rs +++ b/crates/directory/src/smtp/lookup.rs @@ -81,6 +81,10 @@ impl Directory for SmtpDirectory { async fn query(&self, _query: &str, _params: &[&str]) -> crate::Result { Err(DirectoryError::unsupported("smtp", "query")) } + + async fn is_local_domain(&self, domain: &str) -> crate::Result { + Ok(self.domains.contains(domain)) + } } impl SmtpClient { diff --git a/crates/directory/src/smtp/mod.rs b/crates/directory/src/smtp/mod.rs index e1976217..397006d8 100644 --- a/crates/directory/src/smtp/mod.rs +++ b/crates/directory/src/smtp/mod.rs @@ -2,6 +2,7 @@ pub mod config; pub mod lookup; pub mod pool; +use ahash::AHashSet; use bb8::Pool; use mail_send::SmtpClientBuilder; use smtp_proto::EhloResponse; @@ -10,6 +11,7 @@ use tokio_rustls::client::TlsStream; pub struct SmtpDirectory { pool: Pool, + domains: AHashSet, } pub struct SmtpConnectionManager { diff --git a/crates/directory/src/sql/config.rs b/crates/directory/src/sql/config.rs index 75aabbbc..3abd4f6a 100644 --- a/crates/directory/src/sql/config.rs +++ b/crates/directory/src/sql/config.rs @@ -1,9 +1,10 @@ use std::sync::Arc; +use ahash::AHashSet; use sqlx::any::{install_default_drivers, AnyPoolOptions}; use utils::config::{utils::AsKey, Config}; -use crate::Directory; +use crate::{cache::CachedDirectory, Directory}; use super::{SqlDirectory, SqlMappings}; @@ -11,6 +12,7 @@ impl SqlDirectory { pub fn from_config( config: &Config, prefix: impl AsKey, + domains: AHashSet, ) -> utils::config::Result> { let prefix = prefix.as_key(); let address = config.value_require((&prefix, "address"))?; @@ -63,6 +65,10 @@ impl SqlDirectory { .value((&prefix, "query.expand")) .unwrap_or_default() .to_string(), + query_domains: config + .value((&prefix, "query.domains")) + .unwrap_or_default() + .to_string(), column_name: config .value((&prefix, "columns.name")) .unwrap_or_default() @@ -89,6 +95,14 @@ impl SqlDirectory { .to_string(), }; - Ok(Arc::new(SqlDirectory { pool, mappings })) + CachedDirectory::try_from_config( + config, + &prefix, + SqlDirectory { + pool, + mappings, + domains, + }, + ) } } diff --git a/crates/directory/src/sql/lookup.rs b/crates/directory/src/sql/lookup.rs index 9191d5da..b4e86bf4 100644 --- a/crates/directory/src/sql/lookup.rs +++ b/crates/directory/src/sql/lookup.rs @@ -120,6 +120,19 @@ impl Directory for SqlDirectory { .map(|r| r.is_some()) .map_err(Into::into) } + + async fn is_local_domain(&self, domain: &str) -> crate::Result { + if self.domains.contains(domain) { + return Ok(true); + } + + sqlx::query(&self.mappings.query_domains) + .bind(domain) + .fetch_optional(&self.pool) + .await + .map(|id| id.is_some()) + .map_err(Into::into) + } } impl SqlMappings { diff --git a/crates/directory/src/sql/mod.rs b/crates/directory/src/sql/mod.rs index 6bb74e93..79b59f98 100644 --- a/crates/directory/src/sql/mod.rs +++ b/crates/directory/src/sql/mod.rs @@ -1,3 +1,4 @@ +use ahash::AHashSet; use sqlx::{Any, Pool}; pub mod config; @@ -6,6 +7,7 @@ pub mod lookup; pub struct SqlDirectory { pool: Pool, mappings: SqlMappings, + domains: AHashSet, } #[derive(Debug)] @@ -16,6 +18,7 @@ pub(crate) struct SqlMappings { query_members: String, query_recipients: String, query_emails: String, + query_domains: String, query_verify: String, query_expand: String, column_name: String, diff --git a/crates/smtp/src/core/mod.rs b/crates/smtp/src/core/mod.rs index c5134448..4469f513 100644 --- a/crates/smtp/src/core/mod.rs +++ b/crates/smtp/src/core/mod.rs @@ -235,7 +235,6 @@ pub struct SessionParameters { pub rcpt_errors_wait: Duration, pub rcpt_max: usize, pub rcpt_dsn: bool, - pub rcpt_lookup_domain: Option>, pub rcpt_directory: Option>, pub can_expn: bool, pub can_vrfy: bool, @@ -464,7 +463,6 @@ impl Session { rcpt_errors_wait: Default::default(), rcpt_max: Default::default(), rcpt_dsn: Default::default(), - rcpt_lookup_domain: Default::default(), rcpt_directory: Default::default(), max_message_size: Default::default(), iprev: crate::config::VerifyStrategy::Disable, diff --git a/crates/smtp/src/core/params.rs b/crates/smtp/src/core/params.rs index 47a0d4c5..e001970a 100644 --- a/crates/smtp/src/core/params.rs +++ b/crates/smtp/src/core/params.rs @@ -53,6 +53,15 @@ impl Session { let ec = &self.core.session.config.extensions; self.params.can_expn = *ec.expn.eval(self).await; self.params.can_vrfy = *ec.vrfy.eval(self).await; + self.params.rcpt_directory = self + .core + .session + .config + .rcpt + .directory + .eval(self) + .await + .clone(); } pub async fn eval_post_auth_params(&mut self) { @@ -60,6 +69,15 @@ impl Session { let ec = &self.core.session.config.extensions; self.params.can_expn = *ec.expn.eval(self).await; self.params.can_vrfy = *ec.vrfy.eval(self).await; + self.params.rcpt_directory = self + .core + .session + .config + .rcpt + .directory + .eval(self) + .await + .clone(); } pub async fn eval_rcpt_params(&mut self) { @@ -69,7 +87,6 @@ impl Session { self.params.rcpt_errors_max = *rc.errors_max.eval(self).await; self.params.rcpt_errors_wait = *rc.errors_wait.eval(self).await; self.params.rcpt_max = *rc.max_recipients.eval(self).await; - self.params.rcpt_lookup_domain = rc.lookup_domains.eval(self).await.clone(); self.params.rcpt_directory = rc.directory.eval(self).await.clone(); self.params.rcpt_dsn = *self.core.session.config.extensions.dsn.eval(self).await; diff --git a/crates/smtp/src/inbound/ehlo.rs b/crates/smtp/src/inbound/ehlo.rs index d040386a..468395cb 100644 --- a/crates/smtp/src/inbound/ehlo.rs +++ b/crates/smtp/src/inbound/ehlo.rs @@ -329,7 +329,7 @@ impl Session { } } -trait ToDnsbl { +pub trait ToDnsbl { fn to_dnsbl(&self, host: &str) -> String; } @@ -360,29 +360,3 @@ impl ToDnsbl for IpAddr { } } } - -#[cfg(test)] -mod test { - use std::net::IpAddr; - - use crate::inbound::ehlo::ToDnsbl; - - #[test] - fn ip_to_dnsbl() { - assert_eq!( - "2001:DB8:abc:123::42" - .parse::() - .unwrap() - .to_dnsbl("zen.spamhaus.org"), - "2.4.0.0.0.0.0.0.0.0.0.0.0.0.0.0.3.2.1.0.c.b.a.0.8.b.d.0.1.0.0.2.zen.spamhaus.org" - ); - - assert_eq!( - "1.2.3.4" - .parse::() - .unwrap() - .to_dnsbl("zen.spamhaus.org"), - "4.3.2.1.zen.spamhaus.org" - ); - } -} diff --git a/crates/smtp/src/inbound/rcpt.rs b/crates/smtp/src/inbound/rcpt.rs index f9a42c1b..cd5af8b4 100644 --- a/crates/smtp/src/inbound/rcpt.rs +++ b/crates/smtp/src/inbound/rcpt.rs @@ -71,13 +71,10 @@ impl Session { }; // Verify address - let cache = "true"; - if let (Some(domain_lookup), Some(address_lookup)) = - (&self.params.rcpt_lookup_domain, &self.params.auth_directory) - { - if let Some(is_local_domain) = domain_lookup.contains(&rcpt.domain).await { + if let Some(directory) = &self.params.rcpt_directory { + if let Ok(is_local_domain) = directory.is_local_domain(&rcpt.domain).await { if is_local_domain { - if let Ok(is_local_address) = address_lookup.rcpt(&rcpt.address_lcase).await { + if let Ok(is_local_address) = directory.rcpt(&rcpt.address_lcase).await { if !is_local_address { tracing::debug!(parent: &self.span, context = "rcpt", diff --git a/crates/smtp/src/outbound/mta_sts/parse.rs b/crates/smtp/src/outbound/mta_sts/parse.rs index f762a7fa..9c49e0c0 100644 --- a/crates/smtp/src/outbound/mta_sts/parse.rs +++ b/crates/smtp/src/outbound/mta_sts/parse.rs @@ -86,53 +86,3 @@ impl Policy { } } -#[cfg(test)] -mod tests { - use crate::outbound::mta_sts::{Mode, MxPattern, Policy}; - - #[test] - fn parse_policy() { - for (policy, expected_policy) in [ - ( - r"version: STSv1 -mode: enforce -mx: mail.example.com -mx: *.example.net -mx: backupmx.example.com -max_age: 604800", - Policy { - id: "abc".to_string(), - mode: Mode::Enforce, - mx: vec![ - MxPattern::Equals("mail.example.com".to_string()), - MxPattern::StartsWith("example.net".to_string()), - MxPattern::Equals("backupmx.example.com".to_string()), - ], - max_age: 604800, - }, - ), - ( - r"version: STSv1 -mode: testing -mx: gmail-smtp-in.l.google.com -mx: *.gmail-smtp-in.l.google.com -max_age: 86400 -", - Policy { - id: "abc".to_string(), - mode: Mode::Testing, - mx: vec![ - MxPattern::Equals("gmail-smtp-in.l.google.com".to_string()), - MxPattern::StartsWith("gmail-smtp-in.l.google.com".to_string()), - ], - max_age: 86400, - }, - ), - ] { - assert_eq!( - Policy::parse(policy, expected_policy.id.to_string()).unwrap(), - expected_policy - ); - } - } -} diff --git a/crates/smtp/src/reporting/mod.rs b/crates/smtp/src/reporting/mod.rs index f34e5d0e..c774ef1e 100644 --- a/crates/smtp/src/reporting/mod.rs +++ b/crates/smtp/src/reporting/mod.rs @@ -334,50 +334,3 @@ impl ReportValue { } } } - -#[cfg(test)] -mod tests { - use mail_parser::DateTime; - - use crate::config::AggregateFrequency; - - #[test] - fn aggregate_to_timestamp() { - for (freq, date, expected) in [ - ( - AggregateFrequency::Hourly, - "2023-01-24T09:10:40Z", - "2023-01-24T09:00:00Z", - ), - ( - AggregateFrequency::Daily, - "2023-01-24T09:10:40Z", - "2023-01-24T00:00:00Z", - ), - ( - AggregateFrequency::Weekly, - "2023-01-24T09:10:40Z", - "2023-01-22T00:00:00Z", - ), - ( - AggregateFrequency::Weekly, - "2023-01-28T23:59:59Z", - "2023-01-22T00:00:00Z", - ), - ( - AggregateFrequency::Weekly, - "2023-01-22T23:59:59Z", - "2023-01-22T00:00:00Z", - ), - ] { - assert_eq!( - DateTime::from_timestamp( - freq.to_timestamp_(DateTime::parse_rfc3339(date).unwrap()) as i64 - ) - .to_rfc3339(), - expected, - "failed for {freq:?} {date} {expected}" - ); - } - } -} diff --git a/resources/config/config.toml b/resources/config/config.toml index 83debf0e..f5c7e540 100644 --- a/resources/config/config.toml +++ b/resources/config/config.toml @@ -98,7 +98,7 @@ mt-priority = [ { if = "authenticated-as", ne = "", then = "mixer"}, [session.auth] mechanisms = [ { if = "listener", ne = "smtp", then = ["plain", "login"]}, { else = [] } ] -lookup = [ { if = "listener", ne = "smtp", then = "remote/imap" }, +directory = [ { if = "listener", ne = "smtp", then = "remote/imap" }, { else = false } ] require = [ { if = "listener", ne = "smtp", then = true}, { else = false } ] @@ -115,14 +115,13 @@ wait = "5s" relay = [ { if = "authenticated-as", ne = "", then = true }, { else = false } ] max-recipients = 25 - -[session.rcpt.lookup] +directory = [ { if = "authenticated-as", ne = "", then = "remote/lmtp" }, + { else = false } ] domains = "list/domains" -addresses = "remote/lmtp" -vrfy = [ { if = "authenticated-as", ne = "", then = "remote/lmtp" }, - { else = false } ] -expn = [ { if = "authenticated-as", ne = "", then = "remote/lmtp" }, - { else = false } ] + +[session.rcpt.cache] +entries = 1000 +ttl = {positive = 10, negative = 5} [session.rcpt.errors] total = 5 @@ -394,7 +393,7 @@ from-addr = "no-reply@__DOMAIN__" return-path = "" #hostname = "__HOST__" sign = ["rsa"] -use-database = "sql" +use-directory = "sql" [sieve.limits] redirects = 3 @@ -455,8 +454,8 @@ data = ''' } ''' -[management.auth] -lookup = "list/admin" +[management] +directory = "local" [list] domains = ["__DOMAIN__"] diff --git a/tests/Cargo.toml b/tests/Cargo.toml index ca980858..2ca329b8 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -11,7 +11,8 @@ foundationdb = ["store/foundation"] [dev-dependencies] store = { path = "../crates/store", features = ["test_mode"] } -jmap = { path = "../crates/jmap", features = ["test_mode"] } +directory = { path = "../crates/directory" } +#jmap = { path = "../crates/jmap", features = ["test_mode"] } jmap_proto = { path = "../crates/jmap-proto" } smtp = { path = "../crates/smtp", features = ["test_mode", "local_delivery"] } smtp-proto = { git = "https://github.com/stalwartlabs/smtp-proto" } diff --git a/tests/resources/smtp/config/lists.toml b/tests/resources/smtp/config/lists.toml index 24b562d1..9ad09c52 100644 --- a/tests/resources/smtp/config/lists.toml +++ b/tests/resources/smtp/config/lists.toml @@ -14,10 +14,6 @@ lookup = true username = "hello" secret = "world" -[remote."lmtp".cache] -entries = 1000 -ttl = {positive = 10, negative = 5} - [remote."lmtp".tls] implicit = true allow-invalid-certs = true diff --git a/tests/src/directory/ldap.rs b/tests/src/directory/ldap.rs index 257d33e1..92a541da 100644 --- a/tests/src/directory/ldap.rs +++ b/tests/src/directory/ldap.rs @@ -8,12 +8,12 @@ use crate::directory::parse_config; #[tokio::test] async fn ldap_directory() { // Enable logging - tracing::subscriber::set_global_default( + /*tracing::subscriber::set_global_default( tracing_subscriber::FmtSubscriber::builder() .with_max_level(tracing::Level::DEBUG) .finish(), ) - .unwrap(); + .unwrap();*/ // Obtain directory handle let handle = parse_config().directories.remove("ldap").unwrap(); @@ -152,6 +152,10 @@ async fn ldap_directory() { vec![2, 3, 4], ); + // Domain validation + assert!(handle.is_local_domain("example.org").await.unwrap()); + assert!(!handle.is_local_domain("other.org").await.unwrap()); + // RCPT TO assert!(handle.rcpt("jane@example.org").await.unwrap()); assert!(handle.rcpt("info@example.org").await.unwrap()); diff --git a/tests/src/directory/mod.rs b/tests/src/directory/mod.rs index 8d1e769c..4df5b814 100644 --- a/tests/src/directory/mod.rs +++ b/tests/src/directory/mod.rs @@ -27,6 +27,7 @@ recipients = "SELECT id FROM emails WHERE address = ?" emails = "SELECT address FROM emails WHERE id = ? AND type != 'list' ORDER BY type DESC, address ASC" verify = "SELECT address FROM emails WHERE address LIKE '%' || ? || '%' AND type = 'primary' ORDER BY address LIMIT 5" expand = "SELECT p.address FROM emails AS p JOIN emails AS l ON p.id = l.id WHERE p.type = 'primary' AND l.address = ? AND l.type = 'list' ORDER BY p.address LIMIT 50" +domains = "SELECT 1 FROM emails WHERE address LIKE '%@' || ? LIMIT 1" [directory."sql".columns] name = "name" @@ -56,6 +57,7 @@ email = "(&(|(objectClass=posixAccount)(objectClass=posixGroup))(|(mail=?)(given id = "(|(&(objectClass=posixAccount)(uidNumber=?))(&(objectClass=posixGroup)(gidNumber=?)))" verify = "(&(|(objectClass=posixAccount)(objectClass=posixGroup))(|(mail=*?*)(givenName=*?*)))" expand = "(&(|(objectClass=posixAccount)(objectClass=posixGroup))(sn=?))" +domains = "(&(|(objectClass=posixAccount)(objectClass=posixGroup))(|(mail=*@?)(givenName=*@?)(sn=*@?)))" [directory."ldap".object-classes] user = "posixAccount" @@ -102,6 +104,10 @@ max-connections = 5 implicit = true allow-invalid-certs = true +[directory."smtp".cache] +entries = 500 +ttl = {positive = '10s', negative = '5s'} + [directory."local"] protocol = "memory" diff --git a/tests/src/directory/smtp.rs b/tests/src/directory/smtp.rs index 4866e13f..d3de3976 100644 --- a/tests/src/directory/smtp.rs +++ b/tests/src/directory/smtp.rs @@ -127,10 +127,47 @@ async fn smtp_directory() { // Shutdown shutdown.send(false).ok(); + + // Verify that caching works + TcpStream::connect("127.0.0.1:9199").await.unwrap_err(); + assert_eq!( + handle.type_name(), + "directory::cache::CachedDirectory" + ); + + let mut requests = Vec::new(); + for n in 0..100 { + let (item, expected) = &tests[n % tests.len()]; + if matches!(item, Item::IsAccount(_)) { + let item = item.append(n); + let item_clone = item.clone(); + let handle = handle.clone(); + requests.push(( + tokio::spawn(async move { + let result: LookupResult = match &item { + Item::IsAccount(v) => handle.rcpt(v).await.unwrap().into(), + _ => unreachable!(), + }; + + result + }), + item_clone, + expected.append(n), + )); + } + } + assert!(!requests.is_empty()); + for (result, item, expected_result) in requests { + assert_eq!( + result.await.unwrap(), + expected_result, + "Failed for {item:?}" + ); + } } pub fn spawn_mock_lmtp_server(max_concurrency: u64) -> watch::Sender { - let (tx, mut rx) = watch::channel(true); + let (tx, rx) = watch::channel(true); tokio::spawn(async move { let listener = TcpListener::bind("127.0.0.1:9199") @@ -140,6 +177,7 @@ pub fn spawn_mock_lmtp_server(max_concurrency: u64) -> watch::Sender { }); let acceptor = dummy_tls_acceptor(); let limited = ConcurrencyLimiter::new(max_concurrency); + let mut rx_ = rx.clone(); loop { tokio::select! { stream = listener.accept() => { @@ -147,14 +185,14 @@ pub fn spawn_mock_lmtp_server(max_concurrency: u64) -> watch::Sender { Ok((stream, _)) => { let acceptor = acceptor.clone(); let in_flight = limited.is_allowed(); - tokio::spawn(accept_smtp(stream, acceptor, in_flight)); + tokio::spawn(accept_smtp(stream, rx.clone(), acceptor, in_flight)); } Err(err) => { panic!("Something went wrong: {err}" ); } } }, - _ = rx.changed() => { + _ = rx_.changed() => { break; } }; @@ -164,7 +202,12 @@ pub fn spawn_mock_lmtp_server(max_concurrency: u64) -> watch::Sender { tx } -async fn accept_smtp(stream: TcpStream, acceptor: Arc, in_flight: Option) { +async fn accept_smtp( + stream: TcpStream, + mut rx: watch::Receiver, + acceptor: Arc, + in_flight: Option, +) { let mut stream = acceptor.accept(stream).await.unwrap(); stream .write_all(b"220 [127.0.0.1] Clueless host service ready\r\n") @@ -178,13 +221,23 @@ async fn accept_smtp(stream: TcpStream, acceptor: Arc, in_flight: O let mut buf_u8 = vec![0u8; 1024]; loop { - let br = if let Ok(br) = stream.read(&mut buf_u8).await { - br - } else { - break; + let br = tokio::select! { + br = stream.read(&mut buf_u8) => { + match br { + Ok(br) => { + br + } + Err(_) => { + break; + } + } + }, + _ = rx.changed() => { + break; + } }; + let buf = std::str::from_utf8(&buf_u8[0..br]).unwrap(); - //print!("-> {}", buf); let response = if buf.starts_with("LHLO") { "250-mx.foobar.org\r\n250 AUTH PLAIN\r\n".to_string() } else if buf.starts_with("MAIL FROM") { diff --git a/tests/src/directory/sql.rs b/tests/src/directory/sql.rs index 1e0fedff..bd5489c6 100644 --- a/tests/src/directory/sql.rs +++ b/tests/src/directory/sql.rs @@ -178,6 +178,10 @@ async fn sql_directory() { vec![2, 3, 4] ); + // Domain validation + assert!(handle.is_local_domain("example.org").await.unwrap()); + assert!(!handle.is_local_domain("other.org").await.unwrap()); + // RCPT TO assert!(handle.rcpt("jane@example.org").await.unwrap()); assert!(handle.rcpt("info@example.org").await.unwrap()); diff --git a/tests/src/jmap/mod.rs b/tests/src/jmap/mod.rs index a129816e..206df97c 100644 --- a/tests/src/jmap/mod.rs +++ b/tests/src/jmap/mod.rs @@ -162,20 +162,59 @@ throttle = "500ms" throttle = "500ms" attempts.interval = "500ms" -[jmap.auth.database] +[directory."sql"] type = "sql" address = "sqlite::memory:" -[jmap.auth.database.query] -uid-by-login = "SELECT ROWID - 1 FROM users WHERE login = ?" -login-by-uid = "SELECT login FROM users WHERE ROWID - 1 = ?" -secret-by-uid = "SELECT secret FROM users WHERE ROWID - 1 = ?" -name-by-uid = "SELECT name FROM users WHERE ROWID - 1 = ?" -gids-by-uid = "SELECT gid FROM groups WHERE uid = ?" -uids-by-address = "SELECT uid FROM emails WHERE email = ?" -addresses-by-uid = "SELECT email FROM emails WHERE uid = ?" -vrfy = "SELECT email FROM emails WHERE email LIKE '%' || ? || '%' AND is_list = false LIMIT 5" -expn = "SELECT u.login FROM users u INNER JOIN emails e ON u.rowid -1 = e.uid WHERE e.email = ? AND e.is_list = true LIMIT 5" +[directory."sql".query] +login = "SELECT id, secret, description, quota FROM accounts WHERE name = ? AND active = true AND type = 'individual'" +name = "SELECT id, type, description, quota FROM accounts WHERE name = ?" +id = "SELECT name, type, description, quota FROM accounts WHERE id = ?" +members = "SELECT gid FROM group_members WHERE uid = ?" +recipients = "SELECT id FROM emails WHERE address = ?" +emails = "SELECT address FROM emails WHERE id = ? AND type != 'list' ORDER BY type DESC" +verify = "SELECT address FROM emails WHERE address LIKE '%' || ? || '%' AND type != 'list' LIMIT 5" +expand = "SELECT p.address FROM emails AS p JOIN emails AS l ON p.id = l.id WHERE p.type = 'primary' AND l.address = ? AND l.type = 'list' LIMIT 50" + +[directory."sql".columns] +name = "name" +description = "description" +secret = "secret" +id = "id" +email = "address" +quota = "quota" +type = "type" + +[directory."ldap"] +type = "ldap" +address = "ldap://localhost:3893" +base-dn = "dc=example,dc=com" + +[directory."ldap".bind] +dn = "cn=serviceuser,ou=svcaccts,dc=example,dc=com" +secret = "mysecret" + +[directory."ldap".filter] +login = "(&(objectClass=posixAccount)(accountStatus=active)(cn=?))" +name = "(&(!(objectClass=posixAccount)(objectClass=posixGroup))(cn=?))" +email = "(&(!(objectClass=posixAccount)(objectClass=posixGroup))(!(mail=?)(mailAliases=?)(mailLists=?)))" +id = "(|(&(objectClass=posixAccount)(uidNumber=?))(&(objectClass=posixGroup)(gidNumber=?)))" +verify = "(&(!(objectClass=posixAccount)(objectClass=posixGroup))(!(mail=*?*)(mailAliases=*?*)))" +expand = "(&(!(objectClass=posixAccount)(objectClass=posixGroup))(mailLists=?))" + +[directory."ldap".object-classes] +user = "posixAccount" +group = "posixGroup" + +[directory."ldap".attributes] +name = "cn" +description = "description" +secret = "userPassword" +groups = "memberOf" +id = ["uidNumber", "gidNumber"] +email = "mail" +email-alias = "mailAliases" +quota = "diskQuota" [oauth] key = "parerga_und_paralipomena" diff --git a/tests/src/lib.rs b/tests/src/lib.rs index c461ebf5..f9c7c1b9 100644 --- a/tests/src/lib.rs +++ b/tests/src/lib.rs @@ -24,11 +24,13 @@ use std::path::PathBuf; #[cfg(test)] -pub mod jmap; +pub mod directory; +//#[cfg(test)] +//pub mod jmap; #[cfg(test)] pub mod smtp; -#[cfg(test)] -pub mod store; +//#[cfg(test)] +//pub mod store; pub fn add_test_certs(config: &str) -> String { let mut cert_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); diff --git a/tests/src/smtp/config.rs b/tests/src/smtp/config.rs index 90ab779b..eda2fd2c 100644 --- a/tests/src/smtp/config.rs +++ b/tests/src/smtp/config.rs @@ -1,11 +1,374 @@ -use std::{fs, path::PathBuf}; +use std::{fs, path::PathBuf, sync::Arc, time::Duration}; use tokio::net::TcpSocket; -use utils::config::{Config, Listener, Server, ServerProtocol}; +use utils::config::{Config, Listener, Rate, Server, ServerProtocol}; + +use ahash::{AHashMap, AHashSet}; +use directory::Lookup; + +use smtp::config::{ + condition::ConfigCondition, if_block::ConfigIf, throttle::ConfigThrottle, Condition, + ConditionMatch, Conditions, ConfigContext, EnvelopeKey, IfBlock, IfThen, IpAddrMask, + StringMatch, Throttle, THROTTLE_AUTH_AS, THROTTLE_REMOTE_IP, THROTTLE_SENDER_DOMAIN, +}; use super::add_test_certs; +#[test] +fn parse_conditions() { + let mut file = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + file.push("resources"); + file.push("smtp"); + file.push("config"); + file.push("rules.toml"); + + let config = Config::parse(&fs::read_to_string(file).unwrap()).unwrap(); + let servers = vec![Server { + id: "smtp".to_string(), + internal_id: 123, + ..Default::default() + }]; + let mut context = ConfigContext::new(&servers); + let list = Arc::new(Lookup::List { + list: AHashSet::new(), + }); + context + .directory + .lookups + .insert("test-list".to_string(), list.clone()); + + let mut conditions = config.parse_conditions(&context).unwrap(); + let expected_rules = AHashMap::from_iter([ + ( + "simple".to_string(), + Conditions { + conditions: vec![Condition::Match { + key: EnvelopeKey::Listener, + value: ConditionMatch::UInt(123), + not: false, + }], + }, + ), + ( + "is-authenticated".to_string(), + Conditions { + conditions: vec![Condition::Match { + key: EnvelopeKey::AuthenticatedAs, + value: ConditionMatch::String(StringMatch::Equal("".to_string())), + not: true, + }], + }, + ), + ( + "expanded".to_string(), + Conditions { + conditions: vec![ + Condition::Match { + key: EnvelopeKey::SenderDomain, + value: ConditionMatch::String(StringMatch::StartsWith( + "example".to_string(), + )), + not: false, + }, + Condition::JumpIfFalse { positions: 1 }, + Condition::Match { + key: EnvelopeKey::Sender, + value: ConditionMatch::Lookup(list), + not: false, + }, + ], + }, + ), + ( + "my-nested-rule".to_string(), + Conditions { + conditions: vec![ + Condition::Match { + key: EnvelopeKey::RecipientDomain, + value: ConditionMatch::String(StringMatch::Equal( + "example.org".to_string(), + )), + not: false, + }, + Condition::JumpIfTrue { positions: 9 }, + Condition::Match { + key: EnvelopeKey::RemoteIp, + value: ConditionMatch::IpAddrMask(IpAddrMask::V4 { + addr: "192.168.0.0".parse().unwrap(), + mask: u32::MAX << (32 - 24), + }), + not: false, + }, + Condition::JumpIfTrue { positions: 7 }, + Condition::Match { + key: EnvelopeKey::Recipient, + value: ConditionMatch::String(StringMatch::StartsWith( + "no-reply@".to_string(), + )), + not: false, + }, + Condition::JumpIfFalse { positions: 5 }, + Condition::Match { + key: EnvelopeKey::Sender, + value: ConditionMatch::String(StringMatch::EndsWith( + "@domain.org".to_string(), + )), + not: false, + }, + Condition::JumpIfFalse { positions: 3 }, + Condition::Match { + key: EnvelopeKey::Priority, + value: ConditionMatch::Int(1), + not: true, + }, + Condition::JumpIfTrue { positions: 1 }, + Condition::Match { + key: EnvelopeKey::Priority, + value: ConditionMatch::Int(-2), + not: false, + }, + ], + }, + ), + ]); + + for (key, rule) in expected_rules { + assert_eq!(Some(rule), conditions.remove(&key), "failed for {key}"); + } +} + +#[test] +fn parse_if_blocks() { + let mut file = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + file.push("resources"); + file.push("smtp"); + file.push("config"); + file.push("if-blocks.toml"); + + let config = Config::parse(&fs::read_to_string(file).unwrap()).unwrap(); + + // Create context and add some conditions + let context = ConfigContext::new(&[]); + let available_keys = vec![ + EnvelopeKey::Recipient, + EnvelopeKey::RecipientDomain, + EnvelopeKey::Sender, + EnvelopeKey::SenderDomain, + EnvelopeKey::AuthenticatedAs, + EnvelopeKey::Listener, + EnvelopeKey::RemoteIp, + EnvelopeKey::LocalIp, + EnvelopeKey::Priority, + ]; + + assert_eq!( + config + .parse_if_block::>("durations", &context, &available_keys) + .unwrap() + .unwrap(), + IfBlock { + if_then: vec![ + IfThen { + conditions: Conditions { + conditions: vec![Condition::Match { + key: EnvelopeKey::Sender, + value: ConditionMatch::String(StringMatch::Equal("jdoe".to_string())), + not: false + }] + }, + then: Duration::from_secs(5 * 86400).into() + }, + IfThen { + conditions: Conditions { + conditions: vec![ + Condition::Match { + key: EnvelopeKey::Priority, + value: ConditionMatch::Int(-1), + not: false + }, + Condition::JumpIfTrue { positions: 1 }, + Condition::Match { + key: EnvelopeKey::Recipient, + value: ConditionMatch::String(StringMatch::StartsWith( + "jane".to_string() + )), + not: false + } + ] + }, + then: Duration::from_secs(3600).into() + } + ], + default: None + } + ); + + assert_eq!( + config + .parse_if_block::>("string-list", &context, &available_keys) + .unwrap() + .unwrap(), + IfBlock { + if_then: vec![ + IfThen { + conditions: Conditions { + conditions: vec![Condition::Match { + key: EnvelopeKey::Sender, + value: ConditionMatch::String(StringMatch::Equal("jdoe".to_string())), + not: false + }] + }, + then: vec!["From".to_string(), "To".to_string(), "Date".to_string()] + }, + IfThen { + conditions: Conditions { + conditions: vec![ + Condition::Match { + key: EnvelopeKey::Priority, + value: ConditionMatch::Int(-1), + not: false + }, + Condition::JumpIfTrue { positions: 1 }, + Condition::Match { + key: EnvelopeKey::Recipient, + value: ConditionMatch::String(StringMatch::StartsWith( + "jane".to_string() + )), + not: false + } + ] + }, + then: vec!["Other-ID".to_string()] + } + ], + default: vec![] + } + ); + + assert_eq!( + config + .parse_if_block::>("string-list-bis", &context, &available_keys) + .unwrap() + .unwrap(), + IfBlock { + if_then: vec![ + IfThen { + conditions: Conditions { + conditions: vec![Condition::Match { + key: EnvelopeKey::Sender, + value: ConditionMatch::String(StringMatch::Equal("jdoe".to_string())), + not: false + }] + }, + then: vec!["From".to_string(), "To".to_string(), "Date".to_string()] + }, + IfThen { + conditions: Conditions { + conditions: vec![ + Condition::Match { + key: EnvelopeKey::Priority, + value: ConditionMatch::Int(-1), + not: false + }, + Condition::JumpIfTrue { positions: 1 }, + Condition::Match { + key: EnvelopeKey::Recipient, + value: ConditionMatch::String(StringMatch::StartsWith( + "jane".to_string() + )), + not: false + } + ] + }, + then: vec![] + } + ], + default: vec!["ID-Bis".to_string()] + } + ); + + assert_eq!( + config + .parse_if_block::("single-value", &context, &available_keys) + .unwrap() + .unwrap(), + IfBlock { + if_then: vec![], + default: "hello world".to_string() + } + ); + + for bad_rule in [ + "bad-multi-value", + "bad-if-without-then", + "bad-if-without-else", + "bad-multiple-else", + ] { + if let Ok(value) = config.parse_if_block::(bad_rule, &context, &available_keys) { + panic!("Condition {bad_rule:?} had unexpected result {value:?}"); + } + } +} + +#[test] +fn parse_throttle() { + let mut file = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + file.push("resources"); + file.push("smtp"); + file.push("config"); + file.push("throttle.toml"); + + let available_keys = vec![ + EnvelopeKey::Recipient, + EnvelopeKey::RecipientDomain, + EnvelopeKey::Sender, + EnvelopeKey::SenderDomain, + EnvelopeKey::AuthenticatedAs, + EnvelopeKey::Listener, + EnvelopeKey::RemoteIp, + EnvelopeKey::LocalIp, + EnvelopeKey::Priority, + ]; + + let config = Config::parse(&fs::read_to_string(file).unwrap()).unwrap(); + let context = ConfigContext::new(&[]); + let throttle = config + .parse_throttle("throttle", &context, &available_keys, u16::MAX) + .unwrap(); + + assert_eq!( + throttle, + vec![ + Throttle { + conditions: Conditions { + conditions: vec![Condition::Match { + key: EnvelopeKey::RemoteIp, + value: ConditionMatch::IpAddrMask(IpAddrMask::V4 { + addr: "127.0.0.1".parse().unwrap(), + mask: u32::MAX + }), + not: false + }] + }, + keys: THROTTLE_REMOTE_IP | THROTTLE_AUTH_AS, + concurrency: 100.into(), + rate: Rate { + requests: 50, + period: Duration::from_secs(30) + } + .into() + }, + Throttle { + conditions: Conditions { conditions: vec![] }, + keys: THROTTLE_SENDER_DOMAIN, + concurrency: 10000.into(), + rate: None + } + ] + ); +} + #[test] fn parse_servers() { let mut file = PathBuf::from(env!("CARGO_MANIFEST_DIR")); diff --git a/tests/src/smtp/inbound/auth.rs b/tests/src/smtp/inbound/auth.rs index 2afc13e7..04aed0a2 100644 --- a/tests/src/smtp/inbound/auth.rs +++ b/tests/src/smtp/inbound/auth.rs @@ -21,10 +21,9 @@ * for more details. */ -use std::sync::Arc; - -use ahash::AHashSet; +use directory::config::ConfigDirectory; use smtp_proto::{AUTH_LOGIN, AUTH_PLAIN}; +use utils::config::Config; use crate::smtp::{ session::{TestSession, VerifyResponse}, @@ -33,30 +32,44 @@ use crate::smtp::{ use smtp::{ config::ConfigContext, core::{Session, State, SMTP}, - lookup::Lookup, }; +const DIRECTORY: &str = r#" +[directory."local"] +protocol = "memory" + +[[directory."local".users]] +name = "john" +description = "John Doe" +secret = "secret" +email = ["john@example.org", "jdoe@example.org", "john.doe@example.org"] +email-list = ["info@example.org"] +member-of = ["sales"] + +[[directory."local".users]] +name = "jane" +description = "Jane Doe" +secret = "p4ssw0rd" +email = "jane@example.org" +email-list = ["info@example.org"] +member-of = ["sales", "support"] +"#; + #[tokio::test] async fn auth() { let mut core = SMTP::test(); let mut ctx = ConfigContext::new(&[]); - ctx.lookup.insert( - "plain".to_string(), - Arc::new(Lookup::List(AHashSet::from_iter([ - "john:secret".to_string(), - "jane:p4ssw0rd".to_string(), - ]))), - ); + ctx.directory = Config::parse(DIRECTORY).unwrap().parse_directory().unwrap(); let mut config = &mut core.session.config.auth; config.require = r"[{if = 'remote-ip', eq = '10.0.0.1', then = true}, {else = false}]" .parse_if(&ctx); - config.lookup = r"[{if = 'remote-ip', eq = '10.0.0.1', then = 'plain'}, + config.directory = r"[{if = 'remote-ip', eq = '10.0.0.1', then = 'local'}, {else = false}]" .parse_if::>(&ctx) - .map_if_block(&ctx.lookup, "", "") + .map_if_block(&ctx.directory.directories, "", "") .unwrap(); config.errors_max = r"[{if = 'remote-ip', eq = '10.0.0.1', then = 2}, {else = 3}]" diff --git a/tests/src/smtp/inbound/data.rs b/tests/src/smtp/inbound/data.rs index 89ed1a8f..191bf858 100644 --- a/tests/src/smtp/inbound/data.rs +++ b/tests/src/smtp/inbound/data.rs @@ -21,9 +21,8 @@ * for more details. */ -use std::sync::Arc; - -use ahash::AHashSet; +use directory::config::ConfigDirectory; +use utils::config::Config; use crate::smtp::{ inbound::{TestMessage, TestQueueEvent}, @@ -33,28 +32,59 @@ use crate::smtp::{ use smtp::{ config::{ConfigContext, IfBlock}, core::{Session, SMTP}, - lookup::Lookup, }; +const DIRECTORY: &str = r#" +[directory."local"] +protocol = "memory" + +[[directory."local".users]] +name = "john" +description = "John Doe" +secret = "secret" +email = ["john@foobar.org", "jdoe@example.org", "john.doe@example.org"] + +[[directory."local".users]] +name = "jane" +description = "Jane Doe" +secret = "p4ssw0rd" +email = "jane@domain.net" + +[[directory."local".users]] +name = "bill" +description = "Bill Foobar" +secret = "p4ssw0rd" +email = "bill@foobar.org" + +[[directory."local".users]] +name = "mike" +description = "Mike Foobar" +secret = "p4ssw0rd" +email = "mike@test.com" + +[directory."local".lookup] +domains = ["foobar.org", "domain.net", "test.com"] +"#; + #[tokio::test] async fn data() { + // Enable logging + /*tracing::subscriber::set_global_default( + tracing_subscriber::FmtSubscriber::builder() + .with_max_level(tracing::Level::DEBUG) + .finish(), + ) + .unwrap();*/ let mut core = SMTP::test(); // Create temp dir for queue let mut qr = core.init_test_queue("smtp_data_test"); - + let directory = Config::parse(DIRECTORY).unwrap().parse_directory().unwrap(); let mut config = &mut core.session.config.rcpt; - config.lookup_domains = IfBlock::new(Some(Arc::new(Lookup::List(AHashSet::from_iter([ - "foobar.org".to_string(), - "domain.net".to_string(), - "test.com".to_string(), - ]))))); - config.lookup_addresses = IfBlock::new(Some(Arc::new(Lookup::List(AHashSet::from_iter([ - "bill@foobar.org".to_string(), - "john@foobar.org".to_string(), - "jane@domain.net".to_string(), - "mike@test.com".to_string(), - ]))))); + config.lookup_domains = IfBlock::new(Some( + directory.lookups.get("local/domains").unwrap().clone(), + )); + config.directory = IfBlock::new(Some(directory.directories.get("local").unwrap().clone())); let mut config = &mut core.session.config; config.data.add_auth_results = "[{if = 'remote-ip', eq = '10.0.0.3', then = true}, diff --git a/tests/src/smtp/inbound/dmarc.rs b/tests/src/smtp/inbound/dmarc.rs index 5ee0cec6..0fa12aaf 100644 --- a/tests/src/smtp/inbound/dmarc.rs +++ b/tests/src/smtp/inbound/dmarc.rs @@ -26,7 +26,7 @@ use std::{ time::{Duration, Instant}, }; -use ahash::AHashSet; +use directory::config::ConfigDirectory; use mail_auth::{ common::{parse::TxtRecordParser, verify::DomainKey}, dkim::DomainKeyReport, @@ -34,7 +34,7 @@ use mail_auth::{ report::DmarcResult, spf::Spf, }; -use utils::config::Rate; +use utils::config::{Config, Rate}; use crate::smtp::{ inbound::{sign::TextConfigContext, TestMessage, TestQueueEvent, TestReportingEvent}, @@ -44,9 +44,22 @@ use crate::smtp::{ use smtp::{ config::{AggregateFrequency, ConfigContext, IfBlock, VerifyStrategy}, core::{Session, SMTP}, - lookup::Lookup, }; +const DIRECTORY: &str = r#" +[directory."local"] +protocol = "memory" + +[[directory."local".users]] +name = "john" +description = "John Doe" +secret = "secret" +email = ["jdoe@example.com"] + +[directory."local".lookup] +domains = ["example.com"] +"#; + #[tokio::test] async fn dmarc() { let mut core = SMTP::test(); @@ -119,14 +132,12 @@ async fn dmarc() { // Create report channels let mut rr = core.init_test_report(); - + let directory = Config::parse(DIRECTORY).unwrap().parse_directory().unwrap(); let mut config = &mut core.session.config.rcpt; - config.lookup_domains = IfBlock::new(Some(Arc::new(Lookup::List(AHashSet::from_iter([ - "example.com".to_string(), - ]))))); - config.lookup_addresses = IfBlock::new(Some(Arc::new(Lookup::List(AHashSet::from_iter([ - "jdoe@example.com".to_string(), - ]))))); + config.lookup_domains = IfBlock::new(Some( + directory.lookups.get("local/domains").unwrap().clone(), + )); + config.directory = IfBlock::new(Some(directory.directories.get("local").unwrap().clone())); let mut config = &mut core.session.config; config.data.add_auth_results = IfBlock::new(true); diff --git a/tests/src/smtp/inbound/rcpt.rs b/tests/src/smtp/inbound/rcpt.rs index 4e65f126..96cbe55a 100644 --- a/tests/src/smtp/inbound/rcpt.rs +++ b/tests/src/smtp/inbound/rcpt.rs @@ -21,10 +21,11 @@ * for more details. */ -use std::{sync::Arc, time::Duration}; +use std::time::Duration; -use ahash::AHashSet; +use directory::config::ConfigDirectory; use smtp_proto::{RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_SUCCESS}; +use utils::config::Config; use crate::smtp::{ session::{TestSession, VerifyResponse}, @@ -33,25 +34,51 @@ use crate::smtp::{ use smtp::{ config::{ConfigContext, IfBlock}, core::{Session, State, SMTP}, - lookup::Lookup, }; +const DIRECTORY: &str = r#" +[directory."local"] +protocol = "memory" + +[[directory."local".users]] +name = "john" +description = "John Doe" +secret = "secret" +email = "john@foobar.org" + +[[directory."local".users]] +name = "jane" +description = "Jane Doe" +secret = "p4ssw0rd" +email = "jane@foobar.org" + +[[directory."local".users]] +name = "bill" +description = "Bill Foobar" +secret = "p4ssw0rd" +email = "bill@foobar.org" + +[[directory."local".users]] +name = "mike" +description = "Mike Foobar" +secret = "p4ssw0rd" +email = "mike@foobar.org" + +[directory."local".lookup] +domains = ["foobar.org"] +"#; + #[tokio::test] async fn rcpt() { let mut core = SMTP::test(); - let list_addresses = Lookup::List(AHashSet::from_iter([ - "jane@foobar.org".to_string(), - "bill@foobar.org".to_string(), - "mike@foobar.org".to_string(), - "john@foobar.org".to_string(), - ])); - let list_domains = Lookup::List(AHashSet::from_iter(["foobar.org".to_string()])); - - let mut config = &mut core.session.config.rcpt; let mut config_ext = &mut core.session.config.extensions; - config.lookup_domains = IfBlock::new(Some(Arc::new(list_domains))); - config.lookup_addresses = IfBlock::new(Some(Arc::new(list_addresses))); + let directory = Config::parse(DIRECTORY).unwrap().parse_directory().unwrap(); + let mut config = &mut core.session.config.rcpt; + config.lookup_domains = IfBlock::new(Some( + directory.lookups.get("local/domains").unwrap().clone(), + )); + config.directory = IfBlock::new(Some(directory.directories.get("local").unwrap().clone())); config.max_recipients = r"[{if = 'remote-ip', eq = '10.0.0.1', then = 3}, {else = 5}]" .parse_if(&ConfigContext::new(&[])); diff --git a/tests/src/smtp/inbound/scripts.rs b/tests/src/smtp/inbound/scripts.rs index 9a61696c..c7075aa6 100644 --- a/tests/src/smtp/inbound/scripts.rs +++ b/tests/src/smtp/inbound/scripts.rs @@ -28,23 +28,27 @@ use crate::smtp::{ session::{TestSession, VerifyResponse}, TestConfig, TestSMTP, }; +use directory::config::ConfigDirectory; use smtp::{ - config::{ - database::ConfigDatabase, list::ConfigList, scripts::ConfigSieve, session::ConfigSession, - ConfigContext, EnvelopeKey, IfBlock, - }, + config::{scripts::ConfigSieve, session::ConfigSession, ConfigContext, EnvelopeKey, IfBlock}, core::{Session, SMTP}, }; use utils::config::Config; const CONFIG: &str = r#" -[database."sql"] +[directory."sql"] +protocol = "sql" address = "sqlite://%PATH%/test.db?mode=rwc" + +[directory."sql".pool] max-connections = 10 min-connections = 0 idle-timeout = "5m" -[list] +[directory."local"] +protocol = "memory" + +[directory."local".lookup] invalid-ehlos = ["spammer.org", "spammer.net"] [session.data.pipe."test"] @@ -59,7 +63,7 @@ from-addr = "sieve@foobar.org" return-path = "" hostname = "mx.foobar.org" sign = ["rsa"] -use-database = "sql" +use-directory = "sql" [sieve.limits] redirects = 3 @@ -81,7 +85,7 @@ if string "${env.remote_ip}" "10.0.0.88" { ehlo = ''' require ["variables", "extlists", "reject"]; -if string :list "${env.helo_domain}" "list/invalid-ehlos" { +if string :list "${env.helo_domain}" "local/invalid-ehlos" { reject "551 5.1.1 Your domain '${env.helo_domain}' has been blacklisted."; } ''' @@ -96,7 +100,7 @@ if envelope :localpart :is "from" "spammer" { execute :query "CREATE TABLE IF NOT EXISTS blocked_senders (addr TEXT PRIMARY KEY)"; execute :query "INSERT OR IGNORE INTO blocked_senders (addr) VALUES (?)" "marketing@spam-domain.com"; -if execute :query "SELECT EXISTS(SELECT 1 FROM blocked_senders WHERE addr=? LIMIT 1)" ["${envelope.from}"] { +if execute :query "SELECT 1 FROM blocked_senders WHERE addr=? LIMIT 1" ["${envelope.from}"] { reject "Your address has been blocked."; } ''' @@ -109,7 +113,7 @@ if envelope :domain :is "to" "foobar.org" { set "triplet" "${env.remote_ip}.${envelope.from}.${envelope.to}"; - if not execute :query "SELECT EXISTS(SELECT 1 FROM greylist WHERE addr=? LIMIT 1)" ["${triplet}"] { + if not execute :query "SELECT 1 FROM greylist WHERE addr=? LIMIT 1" ["${triplet}"] { execute :query "INSERT INTO greylist (addr) VALUES (?)" ["${triplet}"]; reject "422 4.2.2 You have been greylisted '${triplet}'."; } @@ -176,8 +180,7 @@ async fn sieve_scripts() { .replace("%CFG_PATH%", pipe_path.as_path().to_str().unwrap()), ) .unwrap(); - config.parse_lists(&mut ctx).unwrap(); - config.parse_databases(&mut ctx).unwrap(); + ctx.directory = config.parse_directory().unwrap(); let pipes = config.parse_pipes(&ctx, &[EnvelopeKey::RemoteIp]).unwrap(); core.sieve = config.parse_sieve(&mut ctx).unwrap(); let config = &mut core.session.config; diff --git a/tests/src/smtp/inbound/sign.rs b/tests/src/smtp/inbound/sign.rs index 8e8828db..bdeb3a2e 100644 --- a/tests/src/smtp/inbound/sign.rs +++ b/tests/src/smtp/inbound/sign.rs @@ -21,12 +21,9 @@ * for more details. */ -use std::{ - sync::Arc, - time::{Duration, Instant}, -}; +use std::time::{Duration, Instant}; -use ahash::AHashSet; +use directory::config::ConfigDirectory; use mail_auth::{ common::{parse::TxtRecordParser, verify::DomainKey}, spf::Spf, @@ -41,7 +38,6 @@ use crate::smtp::{ use smtp::{ config::{auth::ConfigAuth, ConfigContext, IfBlock, VerifyStrategy}, core::{Session, SMTP}, - lookup::Lookup, }; const SIGNATURES: &str = " @@ -94,6 +90,20 @@ canonicalization = 'relaxed/simple' set-body-length = false "; +const DIRECTORY: &str = r#" +[directory."local"] +protocol = "memory" + +[[directory."local".users]] +name = "john" +description = "John Doe" +secret = "secret" +email = ["jdoe@example.com"] + +[directory."local".lookup] +domains = ["example.com"] +"#; + #[tokio::test] async fn sign_and_seal() { let mut core = SMTP::test(); @@ -140,13 +150,12 @@ async fn sign_and_seal() { Instant::now() + Duration::from_secs(5), ); + let directory = Config::parse(DIRECTORY).unwrap().parse_directory().unwrap(); let mut config = &mut core.session.config.rcpt; - config.lookup_domains = IfBlock::new(Some(Arc::new(Lookup::List(AHashSet::from_iter([ - "example.com".to_string(), - ]))))); - config.lookup_addresses = IfBlock::new(Some(Arc::new(Lookup::List(AHashSet::from_iter([ - "jdoe@example.com".to_string(), - ]))))); + config.lookup_domains = IfBlock::new(Some( + directory.lookups.get("local/domains").unwrap().clone(), + )); + config.directory = IfBlock::new(Some(directory.directories.get("local").unwrap().clone())); let mut config = &mut core.session.config; config.data.add_auth_results = IfBlock::new(true); diff --git a/tests/src/smtp/inbound/vrfy.rs b/tests/src/smtp/inbound/vrfy.rs index be79ff24..361f3c1c 100644 --- a/tests/src/smtp/inbound/vrfy.rs +++ b/tests/src/smtp/inbound/vrfy.rs @@ -21,51 +21,61 @@ * for more details. */ -use std::sync::Arc; - -use ahash::AHashSet; +use directory::config::ConfigDirectory; +use utils::config::Config; use crate::smtp::{ session::{TestSession, VerifyResponse}, ParseTestConfig, TestConfig, }; use smtp::{ - config::ConfigContext, + config::{ConfigContext, IfBlock}, core::{Session, SMTP}, - lookup::Lookup, }; +const DIRECTORY: &str = r#" +[directory."local"] +protocol = "memory" + +[[directory."local".users]] +name = "john" +description = "John Doe" +secret = "secret" +email = ["john@foobar.org"] +email-list = ["sales@foobar.org"] + +[[directory."local".users]] +name = "jane" +description = "Jane Doe" +secret = "p4ssw0rd" +email = "jane@foobar.org" +email-list = ["sales@foobar.org"] + +[[directory."local".users]] +name = "bill" +description = "Bill Foobar" +secret = "p4ssw0rd" +email = "bill@foobar.org" +email-list = ["sales@foobar.org"] + +"#; + #[tokio::test] async fn vrfy_expn() { let mut core = SMTP::test(); - let mut ctx = ConfigContext::new(&[]); - ctx.lookup.insert( - "vrfy".to_string(), - Arc::new(Lookup::List(AHashSet::from_iter([ - "john@foobar.org:john@foobar.org".to_string(), - "john:john@foobar.org".to_string(), - ]))), - ); - ctx.lookup.insert( - "expn".to_string(), - Arc::new(Lookup::List(AHashSet::from_iter([ - "sales:john@foobar.org,bill@foobar.org,jane@foobar.org".to_string(), - "support:mike@foobar.org".to_string(), - ]))), - ); + let ctx = ConfigContext::new(&[]); + let directory = Config::parse(DIRECTORY).unwrap().parse_directory().unwrap(); let mut config = &mut core.session.config.rcpt; + config.directory = IfBlock::new(Some(directory.directories.get("local").unwrap().clone())); - config.lookup_vrfy = r"[{if = 'remote-ip', eq = '10.0.0.1', then = 'vrfy'}, + let mut config = &mut core.session.config.extensions; + config.vrfy = r"[{if = 'remote-ip', eq = '10.0.0.1', then = true}, {else = false}]" - .parse_if::>(&ctx) - .map_if_block(&ctx.lookup, "", "") - .unwrap(); - config.lookup_expn = r"[{if = 'remote-ip', eq = '10.0.0.1', then = 'expn'}, + .parse_if(&ctx); + config.expn = r"[{if = 'remote-ip', eq = '10.0.0.1', then = true}, {else = false}]" - .parse_if::>(&ctx) - .map_if_block(&ctx.lookup, "", "") - .unwrap(); + .parse_if(&ctx); // EHLO should not avertise VRFY/EXPN to 10.0.0.2 let mut session = Session::test(core); @@ -77,7 +87,7 @@ async fn vrfy_expn() { .assert_not_contains("EXPN") .assert_not_contains("VRFY"); session.cmd("VRFY john", "252 2.5.1").await; - session.cmd("EXPN sales", "252 2.5.1").await; + session.cmd("EXPN sales@foobar.org", "252 2.5.1").await; // EHLO should advertise VRFY/EXPN for 10.0.0.1 session.data.remote_ip = "10.0.0.1".parse().unwrap(); @@ -93,14 +103,14 @@ async fn vrfy_expn() { // Successful EXPN session - .cmd("EXPN sales", "250") + .cmd("EXPN sales@foobar.org", "250") .await .assert_contains("250-john@foobar.org") - .assert_contains("250-bill@foobar.org") - .assert_contains("250 jane@foobar.org"); + .assert_contains("250-jane@foobar.org") + .assert_contains("250 bill@foobar.org"); // Non-existent VRFY - session.cmd("VRFY bill", "550 5.1.2").await; + session.cmd("VRFY robert", "550 5.1.2").await; // Non-existent EXPN session.cmd("EXPN procurement", "550 5.1.2").await; diff --git a/tests/src/smtp/lookup/imap.rs b/tests/src/smtp/lookup/imap.rs deleted file mode 100644 index f1e3d615..00000000 --- a/tests/src/smtp/lookup/imap.rs +++ /dev/null @@ -1,232 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of the Stalwart SMTP Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::sync::Arc; - -use mail_parser::decoders::base64::base64_decode; -use mail_send::Credentials; -use tokio::{ - io::{AsyncReadExt, AsyncWriteExt}, - net::{TcpListener, TcpStream}, - sync::watch, -}; -use tokio_rustls::TlsAcceptor; - -use smtp::config::{remote::ConfigHost, ConfigContext}; -use utils::{ - config::Config, - ipc::{Item, LookupResult}, - listener::limiter::{ConcurrencyLimiter, InFlight}, -}; - -use crate::smtp::lookup::{TestItem, TestLookupResult}; - -use super::dummy_tls_acceptor; - -const REMOTE: &str = " -[remote.imap] -address = 127.0.0.1 -port = 9998 -concurrency = 5 -protocol = 'imap' - -[remote.imap.limits] -errors = 3 -requests = 5 - -[remote.imap.cache] -entries = 500 -ttl = {positive = '10s', negative = '5s'} - -[remote.imap.tls] -implicit = true -allow-invalid-certs = true -"; - -#[tokio::test] -async fn lookup_imap() { - // Enable logging - /*tracing::subscriber::set_global_default( - tracing_subscriber::FmtSubscriber::builder() - .with_max_level(tracing::Level::DEBUG) - .finish(), - ) - .unwrap();*/ - - // Spawn mock LMTP server - let shutdown = spawn_mock_imap_server(5); - - // Spawn lookup client - let mut ctx = ConfigContext::new(&[]); - let config = Config::parse(REMOTE).unwrap(); - config.parse_remote_hosts(&mut ctx).unwrap(); - let lookup = ctx.hosts.remove("imap").unwrap().spawn(&config); - - // Basic lookup - let tests = vec![ - ( - Item::Authenticate(Credentials::Plain { - username: "john".to_string(), - secret: "ok".to_string(), - }), - LookupResult::True, - ), - ( - Item::Authenticate(Credentials::Plain { - username: "john".to_string(), - secret: "bad".to_string(), - }), - LookupResult::False, - ), - ]; - - for (item, expected) in &tests { - assert_eq!(&lookup.lookup(item.clone()).await.unwrap(), expected); - } - - // Concurrent requests - let mut requests = Vec::new(); - for n in 0..100 { - let (item, expected) = &tests[n % tests.len()]; - let item = item.append(n); - let item_clone = item.clone(); - let lookup = lookup.clone(); - requests.push(( - tokio::spawn(async move { lookup.lookup(item).await }), - item_clone, - expected.append(n), - )); - } - for (result, item, expected_result) in requests { - let result = result.await.unwrap(); - assert_eq!(result, Some(expected_result), "Failed for {item:?}"); - } - - // Shutdown - shutdown.send(false).ok(); - - // Verify that caching works - TcpStream::connect("127.0.0.1:9998").await.unwrap_err(); - - let mut requests = Vec::new(); - for n in 0..100 { - let (item, expected) = &tests[n % tests.len()]; - let item = item.append(n); - let item_clone = item.clone(); - let lookup = lookup.clone(); - requests.push(( - tokio::spawn(async move { lookup.lookup(item).await }), - item_clone, - expected.append(n), - )); - } - for (result, item, expected_result) in requests { - let result = result.await.unwrap(); - assert_eq!(result, Some(expected_result), "Failed for {item:?}"); - } -} - -pub fn spawn_mock_imap_server(max_concurrency: u64) -> watch::Sender { - let (tx, mut rx) = watch::channel(true); - - tokio::spawn(async move { - let listener = TcpListener::bind("127.0.0.1:9998") - .await - .unwrap_or_else(|e| { - panic!("Failed to bind mock SMTP server to 127.0.0.1:9998: {e}"); - }); - let acceptor = dummy_tls_acceptor(); - let limited = ConcurrencyLimiter::new(max_concurrency); - loop { - tokio::select! { - stream = listener.accept() => { - match stream { - Ok((stream, _)) => { - let acceptor = acceptor.clone(); - let in_flight = limited.is_allowed(); - tokio::spawn(accept_smtp(stream, acceptor, in_flight)); - } - Err(err) => { - panic!("Something went wrong: {err}" ); - } - } - }, - _ = rx.changed() => { - break; - } - }; - } - }); - - tx -} - -async fn accept_smtp(stream: TcpStream, acceptor: Arc, in_flight: Option) { - let mut stream = acceptor.accept(stream).await.unwrap(); - stream - .write_all(b"* OK Clueless host service ready\r\n") - .await - .unwrap(); - - if in_flight.is_none() { - eprintln!("WARNING: Concurrency exceeded!"); - } - - let mut buf_u8 = vec![0u8; 1024]; - - loop { - let br = if let Ok(br) = stream.read(&mut buf_u8).await { - br - } else { - break; - }; - let buf = std::str::from_utf8(&buf_u8[0..br]).unwrap(); - let (op, buf) = buf.split_once(' ').unwrap(); - - //print!("-> {}", buf); - let response = if buf.starts_with("CAPABILITY") { - format!( - "* CAPABILITY IMAP4rev2 IMAP4rev1 AUTH=PLAIN\r\n{op} OK CAPABILITY completed\r\n", - ) - } else if buf.starts_with("NOOP") { - format!("{op} OK NOOP completed\r\n") - } else if buf.starts_with("AUTHENTICATE PLAIN") { - let buf = base64_decode(buf.rsplit_once(' ').unwrap().1.as_bytes()).unwrap(); - if String::from_utf8_lossy(&buf).contains("ok") { - format!("{op} OK Great success!\r\n") - } else { - format!("{op} BAD No soup for you!\r\n") - } - } else if buf.starts_with("LOGOUT") { - format!("* BYE\r\n{op} OK LOGOUT completed\r\n") - } else { - panic!("Unknown command: {}", buf.trim()); - }; - //print!("<- {}", response); - stream.write_all(response.as_bytes()).await.unwrap(); - - if buf.contains("bye") || buf.starts_with("LOGOUT") { - return; - } - } -} diff --git a/tests/src/smtp/lookup/mod.rs b/tests/src/smtp/lookup/mod.rs index aab096f8..433fe836 100644 --- a/tests/src/smtp/lookup/mod.rs +++ b/tests/src/smtp/lookup/mod.rs @@ -21,181 +21,5 @@ * for more details. */ -use std::{io::BufReader, sync::Arc}; - -use mail_send::Credentials; -use rustls::{Certificate, PrivateKey, ServerConfig}; -use rustls_pemfile::{certs, pkcs8_private_keys}; -use tokio_rustls::TlsAcceptor; -use utils::ipc::{Item, LookupResult}; - -pub mod imap; -pub mod smtp; pub mod sql; - -const CERT: &str = "-----BEGIN CERTIFICATE----- -MIIFCTCCAvGgAwIBAgIUCgHGQYUqtelbHGVSzCVwBL3fyEUwDQYJKoZIhvcNAQEL -BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTIyMDUxNjExNDAzNFoXDTIzMDUx -NjExNDAzNFowFDESMBAGA1UEAwwJbG9jYWxob3N0MIICIjANBgkqhkiG9w0BAQEF -AAOCAg8AMIICCgKCAgEAtwS0Fzl3SjaCuKEXgZ/fdWbDoj/qDphyNCAKNevQ0+D0 -STNkWCO04aFSH0zcL8zoD9gokNos0i7OU9//ZhZQmex4V6EFdZn8bFwUWN/scUvW -HEFXVjtHldO2isZgIxH9LuwRv7KAgkISuWahqerOVDhe7SeQUV0AJGNEh3cT9PZr -gSY931BxB7n+5k8eoSk8Z1gtBzQzL62kVGpHDKfw8yX8m65owF9eLUBrNzgxmXfC -xpuHwj7hmVhS09PPKeN/RsFS8PsYO7bo0u8jEKalteumjRT7RyUEbioqfo6ZFOGj -FHPIq/uKXS9zN1fpoyNh3ur5hMznQhrqlwBM9KlM7GdBJ0pZ3ad0YjT8IL/GnGKR -85J2WZdLqaQdUZo7nV67FhqdDlNE4MdwiykTMjfmLRXGAVhAzJHKyRKNwmkI2aqe -S7aqeNgvuDBwY80Q9a2rb5py1Aw+L8yCkUBuHboToDpxSVRDNN8DrWNmmsXnxsOG -wRDODy4GICKyxlP+RFSM8xWSQ6y9ktS2OfDBm+Eqcw+3pZKhdz2wgxLkUBJ8X1eh -kJrCA/6LTuhy6m6mMjAfoSOFU7fu88jxaWPgvP7GKyH+LM/t9eucobz2ks5rtSjz -V4Dc5DCS94/OpVRHwHdaFSPbJKBN9Ev8gnNrAyx/aBPGoHBPG/QUiU7dcUNIPt0C -AwEAAaNTMFEwHQYDVR0OBBYEFI167IxBmErB11EqiPPqFLa31ZaMMB8GA1UdIwQY -MBaAFI167IxBmErB11EqiPPqFLa31ZaMMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZI -hvcNAQELBQADggIBALU00IOiH5ubEauVCmakms5ermNTZfculnhnDfWTLMeh2+a7 -G4cqADErfMhm/mmLbrw33t9s6tCAhQltvewKR40ST9uMPSyiQbYaCXd5DXnuI6Ox -JtNW+UOWIaMf8abnkdLvREOvb8dVQS1i3xq14tAjY5XgpGwCPP8m54b7N3Q7soLn -e5PDhPNTnhRIn2RLuYoZmQmMA5fcqEUDYff4epUww7PhrM1QckZligI3566NlGOf -j1G9JrivBtY0eaJtamIFnGMBT0ThDudxVja2Nv0C2Elry0p4T/o4nc4M67BJ/y1R -vjNLAgFhbxssemU3lZqSd+pykpJBwDBjFSPrZZmQcbk7H6Uz8V1xr/xuzfw6fA13 -NWZ5vLgP/DQ13sM+XFlxThKfbPMPVe/UCTvfGtNW+3XyBgPntEkR+fNEawQmzbYl -R+X1ymT9MZnEZqRMf7/UD/SYek1aUJefoew3upjMgxYVvh4F8dqJ+39F+xoFzIA2 -1dDAEMzXtjA3zKhZ2cycZbEzpJvYA3eGLuR16Suqfi4kPvfwK0mOhCxQmpayt7/X -vuEzW6dPCH8Hgbb0WvsSppGOvhdbDaZFNfFc5eNSxhyKzu3H3ACNImZRtZE+yixx -0fR8+xz9kDLf8xupV+X9heyFGHSyYU2Lveaevtr2Ij3weLRgJ6LbNALoeKXk ------END CERTIFICATE----- -"; -const PK: &str = "-----BEGIN PRIVATE KEY----- -MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQC3BLQXOXdKNoK4 -oReBn991ZsOiP+oOmHI0IAo169DT4PRJM2RYI7ThoVIfTNwvzOgP2CiQ2izSLs5T -3/9mFlCZ7HhXoQV1mfxsXBRY3+xxS9YcQVdWO0eV07aKxmAjEf0u7BG/soCCQhK5 -ZqGp6s5UOF7tJ5BRXQAkY0SHdxP09muBJj3fUHEHuf7mTx6hKTxnWC0HNDMvraRU -akcMp/DzJfybrmjAX14tQGs3ODGZd8LGm4fCPuGZWFLT088p439GwVLw+xg7tujS -7yMQpqW166aNFPtHJQRuKip+jpkU4aMUc8ir+4pdL3M3V+mjI2He6vmEzOdCGuqX -AEz0qUzsZ0EnSlndp3RiNPwgv8acYpHzknZZl0uppB1RmjudXrsWGp0OU0Tgx3CL -KRMyN+YtFcYBWEDMkcrJEo3CaQjZqp5Ltqp42C+4MHBjzRD1ratvmnLUDD4vzIKR -QG4duhOgOnFJVEM03wOtY2aaxefGw4bBEM4PLgYgIrLGU/5EVIzzFZJDrL2S1LY5 -8MGb4SpzD7elkqF3PbCDEuRQEnxfV6GQmsID/otO6HLqbqYyMB+hI4VTt+7zyPFp -Y+C8/sYrIf4sz+3165yhvPaSzmu1KPNXgNzkMJL3j86lVEfAd1oVI9skoE30S/yC -c2sDLH9oE8agcE8b9BSJTt1xQ0g+3QIDAQABAoICABq5oxqpF5RMtXYEgAw7rkPU -h8jPkHwlIrgd3Z/WGZ53APUXfhWo0ScJiZZsgNKyF0kJBZNxaI4gq5xv3zmnFIoF -j+Ur7EIqBERGheoceMhqjI9/syMycNeeHM/S/ALjA5ewfT8C7+UVhOpx5DWNxidi -O+phlp9q9zRZEo69grqIqVYooWxUsMyyCljTQOPDw8BLjfe5VagmsRJqmolslLDM -4UBSjZVZ18S/3Wgo2oVQia660244BHWCAkZQbbXuNI2+eUAbSoSdxw3WQcaSrywL -hzyezbqr2yPDIIVuiUgVUt0Ps0P57VCCN07jlYhvCEGnClysFzD+ATefoZ0wg7za -dQu2E+d166rAjnssyhzcHMn3pxgSdtXD+dQR/xfIGbPABucCupEFqKmhLdMm9+ud -lHay87qzMpIa8cITJwEQROfXqWAhNUU98pKCOx1SVXBqQC7QVqGQ5solDf0eMSVh -ngQ6Dz2WUI2ty75LteiFwlyTgnU9nyPN0NXsrMEET2BHWre7ufTQqiULtQ7+9BwH -AMxEKvrQHjMUjdfbXuzdyc5w5mPYJZfFVSQ1HMslx66h9yCpRIsBZvUGvoaP8Tpe -nQ66FTYRbiOkkdJ7k8DtrnhsJI1oOGjnvj/rvZ8D2pvrlJcIH2AyN3MOL8Jp5Oj1 -nCFt77TwpF92pgl0g9gBAoIBAQDcarmP54QboaIQ9S2gE/4gSVC5i44iDJuSRdI8 -K081RQcWiNzqQXTRc5nqJ7KzLyPiGlg+6rWsBKLos5l4t+MdhhH+KUvk/OtT/g8V -0NZBNXLIbSb8j8ix4v3/f2qKHN3Co6QOlxb3gFvobKDdoKqUNiSH1zTZ8/Y/BzkM -jqWKhTdaLz6eyzhKfOTA4LO8kJ3VF8HUM1N9/e8Gjorl+gZpJUXUQS0+AIi8W76C -OwDrVb3BPGVnApQJfWF78h4g20RwXrx/GYUW2vOMcLjXXDV5U7+nobPUoJnLxoZC -16o88y0Ivan8dBNXsc1epyPvvEqp6MJbAyyVuNeuRJcgYA0BAoIBAQDUkGRV7fLG -wCr5rNysUO+FKzVtTJnf9KEsqAqUmmVnG4oubxAJJtiB5n2+DT+CtO8Nrtz05BbR -uxfWm+lbEw6lVMj63bywtp0NdULg7/2t+oq2Svv16KrZIRJttXMkdEiFFmkVAEhX -l8Fyl6PJPfSMwbPdXEUPUAaNrXweVFffXczHc4W2G212ZzDB0z7QQSgEntbTDFB/ -2Cg5dvuojlM9zw0fuEyLwItZs7n16j/ONZLgBHyroMU9ZPxbnLrVyoZlqtob+RWm -Ju2fSIL9QqG6O4td1TqcUBGvFQYjGvKA+q5fsG26NBJ0Ac48cNK6PS4lMkN3Av2J -ccloYaMEHAXdAoIBAE8WMCy1Ok6byUXiYxOL+OPmyoM40q/e7DcovE2AkLQhZ3Cr -fPDEucCphPFiexkV8f8fysgQeU0WgMmUH54UBPbD81LJyISKR3nkr875Ftdg8SV/ -HL0EblN9ifuR4U1bHCrJgoUFq2T09oVH7NR44Ju7bZIcIseNZK6qzcp2qGkycXD3 -gLWDX1hCxeV6+qLPFQKvuomEPRH4+jnVDXuFIaW6jPqixDP6BxXmqU2bFDJcmnBq -VkwGvc1F4qORdUP+yOi05VeJdZqEx1x92aTUXg+BgEQKnjbNxUE7o1L6hQfHjUIU -o5iEoagWkQTEXf2YBwY+EPaNBgNWxnSuAbfJHwECggEBALOF95ezTVWauzD/U6ic -+o3n/kl/Zn4FJ5KFodn7xCSe18d7uXlhO34KYqx+l+MWWMefpbGWacdcUjfImf93 -SulLgCqP12sP7/iLzp4XUpL7hOeM0NvRU2nqSpwpoUNqik0Mrlc0U+TWoGTduVCf -aMjwV65e3VyfY8mIeclLxqM5n1fcM1OoOnzDjiRE+0n7nYa5eAnq3pn6v4449TZY -belH03e0ucFWLtrltesBmj3YdWGJqJlzQOInRhNBfXJOh8+ZynfRmP0o54udPDQV -cG3PGFd5XPTjkuvhv7sqaSGRlm/um92lWOhtFfdp+i+cuDpmByCef+7zEP19aKZx -3GkCggEAFTs7KNMfvIEaLH0yQUFeq2gLmtcMofmOmeoIECycN1rG7iJo07lJLIs0 -bVODH8Z0kX8llu3cjGMAH/6R2uugJSxkmFiZKrngTzKmxDPvTCKWR4RFwXH9j8IO -cPq7FtKN4SgrPy9ciAPdkcGmu3zz/sBKOaoPwvU2PdBRT+v/aoz+GCLXAvzFlKVe -9/7zdg87ilo8+AtV+71EJeR3kyBPKS9JrWYUKfiams12+uuH4/53rMFZfNCAaZ3Z -1sdXEO4o3Loc5TX4DbO9FVdBSBe6klEXx4T0QJboO6uBvTBnnRL2SQriJQQFwYT6 -XzVV5pwOxkIDBWDIqMUfwJDChBKfpw== ------END PRIVATE KEY----- -"; - -pub fn dummy_tls_acceptor() -> Arc { - // Init server config builder with safe defaults - let config = ServerConfig::builder() - .with_safe_defaults() - .with_no_client_auth(); - - // load TLS key/cert files - let cert_file = &mut BufReader::new(CERT.as_bytes()); - let key_file = &mut BufReader::new(PK.as_bytes()); - - // convert files to key/cert objects - let cert_chain = certs(cert_file) - .unwrap() - .into_iter() - .map(Certificate) - .collect(); - let mut keys: Vec = pkcs8_private_keys(key_file) - .unwrap() - .into_iter() - .map(PrivateKey) - .collect(); - - // exit if no keys could be parsed - if keys.is_empty() { - panic!("Could not locate PKCS 8 private keys."); - } - - Arc::new(TlsAcceptor::from(Arc::new( - config.with_single_cert(cert_chain, keys.remove(0)).unwrap(), - ))) -} - -pub trait TestItem { - fn append(&self, append: usize) -> Self; -} - -impl TestItem for Item { - fn append(&self, append: usize) -> Self { - match self { - Item::IsAccount(str) => Item::IsAccount(format!("{append}{str}")), - Item::Authenticate(str) => Item::Authenticate(match str { - Credentials::Plain { username, secret } => Credentials::Plain { - username: username.to_string(), - secret: format!("{append}{secret}"), - }, - Credentials::OAuthBearer { token } => Credentials::OAuthBearer { - token: format!("{append}{token}"), - }, - Credentials::XOauth2 { username, secret } => Credentials::XOauth2 { - username: username.to_string(), - secret: format!("{append}{secret}"), - }, - }), - Item::Verify(str) => Item::Verify(format!("{append}{str}")), - Item::Expand(str) => Item::Expand(format!("{append}{str}")), - } - } -} - -pub trait TestLookupResult { - fn append(&self, append: usize) -> Self; -} - -impl TestLookupResult for LookupResult { - fn append(&self, append: usize) -> Self { - match self { - LookupResult::True => LookupResult::True, - LookupResult::False => LookupResult::False, - LookupResult::Values(v) => { - let mut r = Vec::with_capacity(v.len()); - for (pos, val) in v.iter().enumerate() { - r.push(if pos == 0 { - format!("{append}{val}") - } else { - val.to_string() - }); - } - LookupResult::Values(r) - } - } - } -} +pub mod utils; diff --git a/tests/src/smtp/lookup/smtp.rs b/tests/src/smtp/lookup/smtp.rs deleted file mode 100644 index beb4ad50..00000000 --- a/tests/src/smtp/lookup/smtp.rs +++ /dev/null @@ -1,293 +0,0 @@ -/* - * Copyright (c) 2023 Stalwart Labs Ltd. - * - * This file is part of the Stalwart SMTP Server. - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of - * the License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * in the LICENSE file at the top-level directory of this distribution. - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - * You can be released from the requirements of the AGPLv3 license by - * purchasing a commercial license. Please contact licensing@stalw.art - * for more details. -*/ - -use std::sync::Arc; - -use mail_parser::decoders::base64::base64_decode; -use mail_send::Credentials; -use tokio::{ - io::{AsyncReadExt, AsyncWriteExt}, - net::{TcpListener, TcpStream}, - sync::watch, -}; -use tokio_rustls::TlsAcceptor; - -use smtp::config::{remote::ConfigHost, ConfigContext}; -use utils::{ - config::Config, - ipc::{Item, LookupResult}, - listener::limiter::{ConcurrencyLimiter, InFlight}, -}; - -use crate::smtp::lookup::{TestItem, TestLookupResult}; - -use super::dummy_tls_acceptor; - -const REMOTE: &str = " -[remote.lmtp] -address = 127.0.0.1 -port = 9999 -protocol = 'lmtp' -concurrency = 5 - -[remote.lmtp.limits] -errors = 3 -requests = 5 - -[remote.lmtp.cache] -entries = 500 -ttl = {positive = '10s', negative = '5s'} - -[remote.lmtp.tls] -implicit = true -allow-invalid-certs = true -"; - -#[tokio::test] -async fn lookup_smtp() { - // Spawn mock LMTP server - let shutdown = spawn_mock_lmtp_server(5); - - // Spawn lookup client - let mut ctx = ConfigContext::new(&[]); - let config = Config::parse(REMOTE).unwrap(); - config.parse_remote_hosts(&mut ctx).unwrap(); - let lookup = ctx.hosts.remove("lmtp").unwrap().spawn(&config); - - // Basic lookup - let tests = vec![ - ( - Item::IsAccount("john-ok@domain".to_string()), - LookupResult::True, - ), - ( - Item::IsAccount("john-bad@domain".to_string()), - LookupResult::False, - ), - ( - Item::Verify("john-ok@domain".to_string()), - LookupResult::Values(vec!["john-ok@domain".to_string()]), - ), - ( - Item::Verify("doesnot@exist.org".to_string()), - LookupResult::False, - ), - ( - Item::Expand("sales-ok,item1,item2,item3".to_string()), - LookupResult::Values(vec![ - "sales-ok".to_string(), - "item1".to_string(), - "item2".to_string(), - "item3".to_string(), - ]), - ), - (Item::Expand("other".to_string()), LookupResult::False), - ( - Item::Authenticate(Credentials::Plain { - username: "john".to_string(), - secret: "ok".to_string(), - }), - LookupResult::True, - ), - ( - Item::Authenticate(Credentials::Plain { - username: "john".to_string(), - secret: "bad".to_string(), - }), - LookupResult::False, - ), - ]; - - for (item, expected) in &tests { - assert_eq!(&lookup.lookup(item.clone()).await.unwrap(), expected); - } - - // Concurrent requests - let mut requests = Vec::new(); - for n in 0..100 { - let (item, expected) = &tests[n % tests.len()]; - let item = item.append(n); - let item_clone = item.clone(); - let lookup = lookup.clone(); - requests.push(( - tokio::spawn(async move { lookup.lookup(item).await }), - item_clone, - expected.append(n), - )); - } - for (result, item, expected_result) in requests { - let result = result.await.unwrap(); - assert_eq!(result, Some(expected_result), "Failed for {item:?}"); - } - - // Shutdown - shutdown.send(false).ok(); - - // Verify that caching works - TcpStream::connect("127.0.0.1:9999").await.unwrap_err(); - - let mut requests = Vec::new(); - for n in 0..100 { - let (item, expected) = &tests[n % tests.len()]; - if !matches!(item, Item::Verify(_) | Item::Expand(_)) { - let item = item.append(n); - let item_clone = item.clone(); - let lookup = lookup.clone(); - requests.push(( - tokio::spawn(async move { lookup.lookup(item).await }), - item_clone, - expected.append(n), - )); - } - } - for (result, item, expected_result) in requests { - let result = result.await.unwrap(); - assert_eq!(result, Some(expected_result), "Failed for {item:?}"); - } -} - -pub fn spawn_mock_lmtp_server(max_concurrency: u64) -> watch::Sender { - let (tx, mut rx) = watch::channel(true); - - tokio::spawn(async move { - let listener = TcpListener::bind("127.0.0.1:9999") - .await - .unwrap_or_else(|e| { - panic!("Failed to bind mock SMTP server to 127.0.0.1:9999: {e}"); - }); - let acceptor = dummy_tls_acceptor(); - let limited = ConcurrencyLimiter::new(max_concurrency); - loop { - tokio::select! { - stream = listener.accept() => { - match stream { - Ok((stream, _)) => { - let acceptor = acceptor.clone(); - let in_flight = limited.is_allowed(); - tokio::spawn(accept_smtp(stream, acceptor, in_flight)); - } - Err(err) => { - panic!("Something went wrong: {err}" ); - } - } - }, - _ = rx.changed() => { - break; - } - }; - } - }); - - tx -} - -async fn accept_smtp(stream: TcpStream, acceptor: Arc, in_flight: Option) { - let mut stream = acceptor.accept(stream).await.unwrap(); - stream - .write_all(b"220 [127.0.0.1] Clueless host service ready\r\n") - .await - .unwrap(); - - if in_flight.is_none() { - eprintln!("WARNING: Concurrency exceeded!"); - } - - let mut buf_u8 = vec![0u8; 1024]; - - loop { - let br = if let Ok(br) = stream.read(&mut buf_u8).await { - br - } else { - break; - }; - let buf = std::str::from_utf8(&buf_u8[0..br]).unwrap(); - //print!("-> {}", buf); - let response = if buf.starts_with("LHLO") { - "250-mx.foobar.org\r\n250 AUTH PLAIN\r\n".to_string() - } else if buf.starts_with("MAIL FROM") { - if buf.contains("<>") || buf.contains("ok@") { - "250 OK\r\n".to_string() - } else { - "552-I do not\r\n552 like that MAIL FROM.\r\n".to_string() - } - } else if buf.starts_with("RCPT TO") { - if buf.contains("ok") { - "250 OK\r\n".to_string() - } else { - "550-I refuse to\r\n550 accept that recipient.\r\n".to_string() - } - } else if buf.starts_with("VRFY") { - if buf.contains("ok") { - format!("250 {}\r\n", buf.split_once(' ').unwrap().1) - } else { - "550-I refuse to\r\n550 verify that recipient.\r\n".to_string() - } - } else if buf.starts_with("EXPN") { - if buf.contains("ok") { - let parts = buf - .split_once(' ') - .unwrap() - .1 - .split(',') - .filter_map(|s| { - if !s.is_empty() { - s.to_string().into() - } else { - None - } - }) - .collect::>(); - let mut buf = String::with_capacity(16); - for (pos, part) in parts.iter().enumerate() { - buf.push_str("250"); - buf.push(if pos == parts.len() - 1 { ' ' } else { '-' }); - buf.push_str(part); - buf.push_str("\r\n"); - } - - buf - } else { - "550-I refuse to\r\n550 accept that recipient.\r\n".to_string() - } - } else if buf.starts_with("AUTH PLAIN") { - let buf = base64_decode(buf.rsplit_once(' ').unwrap().1.as_bytes()).unwrap(); - if String::from_utf8_lossy(&buf).contains("ok") { - "235 Great success!\r\n".to_string() - } else { - "535 No soup for you\r\n".to_string() - } - } else if buf.starts_with("QUIT") { - "250 Arrivederci!\r\n".to_string() - } else if buf.starts_with("RSET") { - "250 Your wish is my command.\r\n".to_string() - } else { - panic!("Unknown command: {}", buf.trim()); - }; - //print!("<- {}", response); - stream.write_all(response.as_bytes()).await.unwrap(); - - if buf.contains("bye") || buf.starts_with("QUIT") { - return; - } - } -} diff --git a/tests/src/smtp/lookup/sql.rs b/tests/src/smtp/lookup/sql.rs index a0a1fa75..e417b5b4 100644 --- a/tests/src/smtp/lookup/sql.rs +++ b/tests/src/smtp/lookup/sql.rs @@ -23,39 +23,49 @@ use std::time::Duration; +use directory::config::ConfigDirectory; use smtp_proto::{AUTH_LOGIN, AUTH_PLAIN}; use utils::config::Config; -use crate::smtp::{ - make_temp_dir, - session::{TestSession, VerifyResponse}, - ParseTestConfig, TestConfig, +use crate::{ + directory::sql::{create_test_directory, create_test_user_with_email, link_test_address}, + smtp::{ + session::{TestSession, VerifyResponse}, + ParseTestConfig, TestConfig, + }, }; use smtp::{ - config::{database::ConfigDatabase, ConfigContext, IfBlock}, + config::{ConfigContext, IfBlock}, core::{Session, SMTP}, - lookup::SqlDatabase, }; const CONFIG: &str = r#" -[database."sql"] -address = "sqlite://%PATH%/test.db?mode=rwc" -max-connections = 10 -min-connections = 0 -idle-timeout = "5m" +[directory."sql"] +protocol = "sql" +address = "sqlite::memory:" -[database."sql".lookup] -auth = "SELECT secret FROM users WHERE email=?" -rcpt = "SELECT EXISTS(SELECT 1 FROM users WHERE email=? LIMIT 1)" -vrfy = "SELECT email FROM users WHERE email LIKE '%' || ? || '%' LIMIT 5" -expn = "SELECT member FROM mailing_lists WHERE id = ?" -domains = "SELECT EXISTS(SELECT 1 FROM domains WHERE name=? LIMIT 1)" -is_ip_allowed = "SELECT EXISTS(SELECT 1 FROM allowed_ips WHERE addr=? LIMIT 1)" +[directory."sql".pool] +max-connections = 1 + +[directory."sql".query] +login = "SELECT id, name, type, secret, description, quota FROM accounts WHERE name = ? AND active = true AND type = 'individual'" +recipients = "SELECT id FROM emails WHERE address = ?" +name = "SELECT id, name, type, description, quota FROM accounts WHERE name = ?" +emails = "SELECT address FROM emails WHERE id = ? AND type != 'list' ORDER BY type DESC, address ASC" +verify = "SELECT address FROM emails WHERE address LIKE '%' || ? || '%' AND type = 'primary' ORDER BY address LIMIT 5" +expand = "SELECT p.address FROM emails AS p JOIN emails AS l ON p.id = l.id WHERE p.type = 'primary' AND l.address = ? AND l.type = 'list' ORDER BY p.address LIMIT 50" +domains = "SELECT 1 FROM emails WHERE address LIKE '%@' || ? LIMIT 1" + +[directory."sql".columns] +name = "name" +secret = "secret" +id = "id" +email = "address" + +[directory."sql".lookup] +domains = "SELECT name FROM domains WHERE name = ? LIMIT 1" +is_ip_allowed = "SELECT addr FROM allowed_ips WHERE addr = ? LIMIT 1" -[database."sql".cache] -enable = ["rcpt", "domains"] -entries = 1000 -ttl = {positive = "1d", negative = "1h"} "#; #[tokio::test] @@ -70,70 +80,61 @@ async fn lookup_sql() { // Parse settings let mut core = SMTP::test(); - let _temp_dir = make_temp_dir("sql_lookup_test", true); let mut ctx = ConfigContext::new(&[]); - let config = - Config::parse(&CONFIG.replace("%PATH%", _temp_dir.temp_dir.as_path().to_str().unwrap())) - .unwrap(); - config.parse_databases(&mut ctx).unwrap(); + let config = Config::parse(CONFIG).unwrap(); + ctx.directory = config.parse_directory().unwrap(); + + // Obtain directory handle + let handle = ctx.directory.directories.get("sql").unwrap().as_ref(); + + // Create tables + create_test_directory(handle).await; // Create test records - if let SqlDatabase::SqlLite(db) = ctx.databases.get("sql").unwrap() { - for query in [ - "CREATE TABLE users (email TEXT PRIMARY KEY, secret TEXT NOT NULL);", - "CREATE TABLE mailing_lists (id TEXT NOT NULL, member TEXT NOT NULL, PRIMARY KEY (id, member));", - "CREATE TABLE domains (name TEXT PRIMARY KEY, description TEXT);", - "CREATE TABLE allowed_ips (addr TEXT PRIMARY KEY);", - "INSERT INTO allowed_ips (addr) VALUES ('10.0.0.50');", - "INSERT INTO domains (name, description) VALUES ('foobar.org', 'Main domain');", - "INSERT INTO domains (name, description) VALUES ('foobar.net', 'Secondary domain');", - "INSERT INTO users (email, secret) VALUES ('jane@foobar.org', 's3cr3tp4ss');", - "INSERT INTO users (email, secret) VALUES ('john@foobar.org', 'mypassword');", - "INSERT INTO users (email, secret) VALUES ('bill@foobar.org', '123456');", - "INSERT INTO mailing_lists (id, member) VALUES ('sales@foobar.org', 'jane@foobar.org');", - "INSERT INTO mailing_lists (id, member) VALUES ('sales@foobar.org', 'john@foobar.org');", - "INSERT INTO mailing_lists (id, member) VALUES ('sales@foobar.org', 'bill@foobar.org');", - "INSERT INTO mailing_lists (id, member) VALUES ('support@foobar.org', 'mike@foobar.net');", - ] { - sqlx::query(query).execute(db).await.unwrap(); - } - } else { - panic!("Unexpected database type"); + create_test_user_with_email(handle, "jane@foobar.org", "s3cr3tp4ss", "Jane").await; + create_test_user_with_email(handle, "john@foobar.org", "mypassword", "John").await; + create_test_user_with_email(handle, "bill@foobar.org", "123456", "Bill").await; + create_test_user_with_email(handle, "mike@foobar.net", "098765", "Mike").await; + link_test_address(handle, "jane@foobar.org", "sales@foobar.org", "list").await; + link_test_address(handle, "john@foobar.org", "sales@foobar.org", "list").await; + link_test_address(handle, "bill@foobar.org", "sales@foobar.org", "list").await; + link_test_address(handle, "mike@foobar.net", "support@foobar.org", "list").await; + + for query in [ + "CREATE TABLE domains (name TEXT PRIMARY KEY, description TEXT);", + "INSERT INTO domains (name, description) VALUES ('foobar.org', 'Main domain');", + "INSERT INTO domains (name, description) VALUES ('foobar.net', 'Secondary domain');", + "CREATE TABLE allowed_ips (addr TEXT PRIMARY KEY);", + "INSERT INTO allowed_ips (addr) VALUES ('10.0.0.50');", + ] { + handle.query(query, &[]).await.unwrap(); } // Enable AUTH let mut config = &mut core.session.config.auth; - config.lookup = r"'db/sql/auth'" + config.directory = r"'sql'" .parse_if::>(&ctx) - .map_if_block(&ctx.lookup, "", "") + .map_if_block(&ctx.directory.directories, "", "") .unwrap(); config.mechanisms = IfBlock::new(AUTH_PLAIN | AUTH_LOGIN); config.errors_wait = IfBlock::new(Duration::from_millis(5)); // Enable VRFY/EXPN/RCPT let mut config = &mut core.session.config.rcpt; - config.lookup_addresses = r"'db/sql/rcpt'" + config.directory = r"'sql'" .parse_if::>(&ctx) - .map_if_block(&ctx.lookup, "", "") + .map_if_block(&ctx.directory.directories, "", "") .unwrap(); - config.lookup_domains = r"'db/sql/domains'" + config.lookup_domains = r"'sql/domains'" .parse_if::>(&ctx) - .map_if_block(&ctx.lookup, "", "") - .unwrap(); - config.lookup_expn = r"'db/sql/expn'" - .parse_if::>(&ctx) - .map_if_block(&ctx.lookup, "", "") - .unwrap(); - config.lookup_vrfy = r"'db/sql/vrfy'" - .parse_if::>(&ctx) - .map_if_block(&ctx.lookup, "", "") + .map_if_block(&ctx.directory.lookups, "", "") .unwrap(); config.relay = IfBlock::new(false); config.errors_wait = IfBlock::new(Duration::from_millis(5)); // Enable REQUIRETLS based on SQL lookup core.session.config.extensions.requiretls = - r"[{if = 'remote-ip', in-list = 'db/sql/is_ip_allowed', then = true}, + r"[{if = 'remote-ip', in-list = 'sql/is_ip_allowed', then = true}, {else = false}]" .parse_if(&ctx); let mut session = Session::test(core); diff --git a/tests/src/smtp/lookup/utils.rs b/tests/src/smtp/lookup/utils.rs index 7f9751a7..ce2d1fd0 100644 --- a/tests/src/smtp/lookup/utils.rs +++ b/tests/src/smtp/lookup/utils.rs @@ -1,10 +1,22 @@ -use std::time::{Duration, Instant}; +use std::{ + net::IpAddr, + time::{Duration, Instant}, +}; use mail_auth::{IpLookupStrategy, MX}; -use smtp::{config::IfBlock, core::SMTP, outbound::NextHop}; +use ::smtp::{config::IfBlock, core::SMTP, outbound::NextHop}; +use mail_parser::DateTime; +use smtp::{ + config::AggregateFrequency, + inbound::ehlo::ToDnsbl, + outbound::{ + lookup::ToNextHop, + mta_sts::{Mode, MxPattern, Policy}, + }, +}; -use super::ToNextHop; +use crate::smtp::TestConfig; #[tokio::test] async fn lookup_ip() { @@ -100,3 +112,108 @@ fn to_remote_hosts() { }]; assert!(mx.to_remote_hosts("domain", 10).is_none()); } + +#[test] +fn ip_to_dnsbl() { + assert_eq!( + "2001:DB8:abc:123::42" + .parse::() + .unwrap() + .to_dnsbl("zen.spamhaus.org"), + "2.4.0.0.0.0.0.0.0.0.0.0.0.0.0.0.3.2.1.0.c.b.a.0.8.b.d.0.1.0.0.2.zen.spamhaus.org" + ); + + assert_eq!( + "1.2.3.4" + .parse::() + .unwrap() + .to_dnsbl("zen.spamhaus.org"), + "4.3.2.1.zen.spamhaus.org" + ); +} + +#[test] +fn parse_policy() { + for (policy, expected_policy) in [ + ( + r"version: STSv1 +mode: enforce +mx: mail.example.com +mx: *.example.net +mx: backupmx.example.com +max_age: 604800", + Policy { + id: "abc".to_string(), + mode: Mode::Enforce, + mx: vec![ + MxPattern::Equals("mail.example.com".to_string()), + MxPattern::StartsWith("example.net".to_string()), + MxPattern::Equals("backupmx.example.com".to_string()), + ], + max_age: 604800, + }, + ), + ( + r"version: STSv1 +mode: testing +mx: gmail-smtp-in.l.google.com +mx: *.gmail-smtp-in.l.google.com +max_age: 86400 +", + Policy { + id: "abc".to_string(), + mode: Mode::Testing, + mx: vec![ + MxPattern::Equals("gmail-smtp-in.l.google.com".to_string()), + MxPattern::StartsWith("gmail-smtp-in.l.google.com".to_string()), + ], + max_age: 86400, + }, + ), + ] { + assert_eq!( + Policy::parse(policy, expected_policy.id.to_string()).unwrap(), + expected_policy + ); + } +} + +#[test] +fn aggregate_to_timestamp() { + for (freq, date, expected) in [ + ( + AggregateFrequency::Hourly, + "2023-01-24T09:10:40Z", + "2023-01-24T09:00:00Z", + ), + ( + AggregateFrequency::Daily, + "2023-01-24T09:10:40Z", + "2023-01-24T00:00:00Z", + ), + ( + AggregateFrequency::Weekly, + "2023-01-24T09:10:40Z", + "2023-01-22T00:00:00Z", + ), + ( + AggregateFrequency::Weekly, + "2023-01-28T23:59:59Z", + "2023-01-22T00:00:00Z", + ), + ( + AggregateFrequency::Weekly, + "2023-01-22T23:59:59Z", + "2023-01-22T00:00:00Z", + ), + ] { + assert_eq!( + DateTime::from_timestamp( + freq.to_timestamp_(DateTime::parse_rfc3339(date).unwrap()) as i64 + ) + .to_rfc3339(), + expected, + "failed for {freq:?} {date} {expected}" + ); + } +} diff --git a/tests/src/smtp/management/queue.rs b/tests/src/smtp/management/queue.rs index 02bee7f8..15884020 100644 --- a/tests/src/smtp/management/queue.rs +++ b/tests/src/smtp/management/queue.rs @@ -26,11 +26,12 @@ use std::{ time::{Duration, Instant}, }; -use ahash::{AHashMap, AHashSet, HashMap, HashSet}; +use ahash::{AHashMap, HashMap, HashSet}; +use directory::config::ConfigDirectory; use hyper::{header::AUTHORIZATION, StatusCode}; use mail_auth::MX; use mail_parser::DateTime; -use utils::config::ServerProtocol; +use utils::config::{Config, ServerProtocol}; use crate::smtp::{ inbound::TestQueueEvent, management::send_manage_request, outbound::start_test_server, @@ -39,13 +40,23 @@ use crate::smtp::{ use smtp::{ config::IfBlock, core::{management::Message, Session, SMTP}, - lookup::Lookup, queue::{ manager::{Queue, SpawnQueue}, QueueId, Status, }, }; +const DIRECTORY: &str = r#" +[directory."local"] +protocol = "memory" + +[[directory."local".users]] +name = "admin" +description = "Superuser" +secret = "secret" + +"#; + #[tokio::test] #[serial_test::serial] async fn manage_queue() { @@ -80,6 +91,8 @@ async fn manage_queue() { ); // Start local management interface + let directory = Config::parse(DIRECTORY).unwrap().parse_directory().unwrap(); + core.queue.config.management_lookup = directory.directories.get("local").unwrap().clone(); core.session.config.rcpt.relay = IfBlock::new(true); core.session.config.rcpt.max_recipients = IfBlock::new(100); core.session.config.extensions.future_release = IfBlock::new(Some(Duration::from_secs(86400))); @@ -87,9 +100,6 @@ async fn manage_queue() { core.queue.config.retry = IfBlock::new(vec![Duration::from_secs(1000)]); core.queue.config.notify = IfBlock::new(vec![Duration::from_secs(2000)]); core.queue.config.expire = IfBlock::new(Duration::from_secs(3000)); - core.queue.config.management_lookup = Arc::new(Lookup::List(AHashSet::from_iter([ - "admin:secret".to_string(), - ]))); let local_qr = core.init_test_queue("smtp_manage_queue_local"); let core = Arc::new(core); local_qr.queue_rx.spawn(core.clone(), Queue::default()); diff --git a/tests/src/smtp/management/report.rs b/tests/src/smtp/management/report.rs index edb994a1..4d252e4f 100644 --- a/tests/src/smtp/management/report.rs +++ b/tests/src/smtp/management/report.rs @@ -23,7 +23,8 @@ use std::sync::Arc; -use ahash::{AHashMap, AHashSet, HashSet}; +use ahash::{AHashMap, HashSet}; +use directory::config::ConfigDirectory; use mail_auth::{ common::parse::TxtRecordParser, dmarc::Dmarc, @@ -34,7 +35,7 @@ use mail_auth::{ }, }; use tokio::sync::mpsc; -use utils::config::ServerProtocol; +use utils::config::{Config, ServerProtocol}; use crate::smtp::{ make_temp_dir, management::send_manage_request, outbound::start_test_server, TestConfig, @@ -42,13 +43,23 @@ use crate::smtp::{ use smtp::{ config::{AggregateFrequency, IfBlock}, core::{management::Report, SMTP}, - lookup::Lookup, reporting::{ scheduler::{Scheduler, SpawnReport}, DmarcEvent, TlsEvent, }, }; +const DIRECTORY: &str = r#" +[directory."local"] +protocol = "memory" + +[[directory."local".users]] +name = "admin" +description = "Superuser" +secret = "secret" + +"#; + #[tokio::test] #[serial_test::serial] async fn manage_reports() { @@ -67,9 +78,8 @@ async fn manage_reports() { config.hash = IfBlock::new(16); config.dmarc_aggregate.max_size = IfBlock::new(1024); config.tls.max_size = IfBlock::new(1024); - core.queue.config.management_lookup = Arc::new(Lookup::List(AHashSet::from_iter([ - "admin:secret".to_string(), - ]))); + let directory = Config::parse(DIRECTORY).unwrap().parse_directory().unwrap(); + core.queue.config.management_lookup = directory.directories.get("local").unwrap().clone(); let (report_tx, report_rx) = mpsc::channel(1024); core.report.tx = report_tx; let core = Arc::new(core); diff --git a/tests/src/smtp/mod.rs b/tests/src/smtp/mod.rs index b34750c9..29e7f5e4 100644 --- a/tests/src/smtp/mod.rs +++ b/tests/src/smtp/mod.rs @@ -23,8 +23,9 @@ use std::{path::PathBuf, sync::Arc, time::Duration}; -use ahash::{AHashMap, AHashSet}; +use ahash::AHashMap; use dashmap::DashMap; +use directory::memory::MemoryDirectory; use mail_auth::{ common::lru::{DnsCache, LruCache}, trust_dns_resolver::config::{ResolverConfig, ResolverOpts}, @@ -48,7 +49,6 @@ use smtp::{ throttle::ThrottleKeyHasherBuilder, QueueCore, ReportCore, Resolvers, SessionCore, SieveConfig, SieveCore, TlsConnectors, SMTP, }, - lookup::Lookup, outbound::dane::DnssecResolver, }; use utils::config::{utils::ParseValues, Config}; @@ -204,9 +204,11 @@ impl TestConfig for SessionConfig { deliver_by: IfBlock::new(None), mt_priority: IfBlock::new(None), dsn: IfBlock::new(true), + expn: IfBlock::new(true), + vrfy: IfBlock::new(true), }, auth: Auth { - lookup: IfBlock::new(None), + directory: IfBlock::new(None), mechanisms: IfBlock::new(AUTH_PLAIN | AUTH_LOGIN), require: IfBlock::new(false), errors_max: IfBlock::new(10), @@ -219,9 +221,7 @@ impl TestConfig for SessionConfig { script: IfBlock::new(None), relay: IfBlock::new(false), lookup_domains: IfBlock::new(None), - lookup_addresses: IfBlock::new(None), - lookup_expn: IfBlock::new(None), - lookup_vrfy: IfBlock::new(None), + directory: IfBlock::new(None), errors_max: IfBlock::new(3), errors_wait: IfBlock::new(Duration::from_secs(1)), max_recipients: IfBlock::new(3), @@ -314,7 +314,7 @@ impl TestConfig for QueueConfig { rcpt: vec![], rcpt_domain: vec![], }, - management_lookup: Arc::new(Lookup::List(AHashSet::default())), + management_lookup: Arc::new(MemoryDirectory::default()), } } }