diff --git a/Cargo.lock b/Cargo.lock index e21ad440..2dc884e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -990,12 +990,25 @@ dependencies = [ "ahash 0.8.11", "arc-swap", "base64 0.22.0", + "bincode", "chrono", + "decancer", "directory", "futures", + "hyper 1.2.0", + "idna 0.5.0", + "imagesize", + "infer", + "jmap_proto", "mail-auth", + "mail-parser", "mail-send", + "md5", "nlp", + "opentelemetry 0.22.0", + "opentelemetry-otlp 0.15.0", + "opentelemetry-semantic-conventions 0.14.0", + "opentelemetry_sdk 0.22.1", "parking_lot", "pem", "privdrop", @@ -1009,13 +1022,21 @@ dependencies = [ "rustls-pki-types", "serde", "serde_json", + "sha1", + "sha2 0.10.8", "sieve-rs", + "smtp-proto", "store", "tokio", "tokio-rustls 0.25.0", "tracing", + "tracing-appender", "tracing-journald", + "tracing-opentelemetry 0.23.0", + "tracing-subscriber", + "unicode-security", "utils", + "whatlang", "x509-parser 0.16.0", ] @@ -2847,7 +2868,7 @@ dependencies = [ "aes-gcm-siv", "async-stream", "async-trait", - "base64 0.21.7", + "base64 0.22.0", "bincode", "cbc", "chrono", @@ -3769,6 +3790,21 @@ dependencies = [ "urlencoding", ] +[[package]] +name = "opentelemetry" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d57987be3f2aeb70d385fff9b27fb74c5723cc9a52d904d4f9c807a0667bf" +dependencies = [ + "futures-core", + "futures-sink", + "js-sys", + "once_cell", + "pin-project-lite", + "thiserror", + "urlencoding", +] + [[package]] name = "opentelemetry-http" version = "0.10.0" @@ -3778,7 +3814,20 @@ dependencies = [ "async-trait", "bytes", "http 0.2.12", - "opentelemetry", + "opentelemetry 0.21.0", + "reqwest 0.11.26", +] + +[[package]] +name = "opentelemetry-http" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cbfa5308166ca861434f0b0913569579b8e587430a3d6bcd7fd671921ec145a" +dependencies = [ + "async-trait", + "bytes", + "http 0.2.12", + "opentelemetry 0.22.0", "reqwest 0.11.26", ] @@ -3791,16 +3840,37 @@ dependencies = [ "async-trait", "futures-core", "http 0.2.12", - "opentelemetry", - "opentelemetry-http", - "opentelemetry-proto", - "opentelemetry-semantic-conventions", - "opentelemetry_sdk", - "prost", + "opentelemetry 0.21.0", + "opentelemetry-http 0.10.0", + "opentelemetry-proto 0.4.0", + "opentelemetry-semantic-conventions 0.13.0", + "opentelemetry_sdk 0.21.2", + "prost 0.11.9", "reqwest 0.11.26", "thiserror", "tokio", - "tonic", + "tonic 0.9.2", +] + +[[package]] +name = "opentelemetry-otlp" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a016b8d9495c639af2145ac22387dcb88e44118e45320d9238fbf4e7889abcb" +dependencies = [ + "async-trait", + "futures-core", + "http 0.2.12", + "opentelemetry 0.22.0", + "opentelemetry-http 0.11.0", + "opentelemetry-proto 0.5.0", + "opentelemetry-semantic-conventions 0.14.0", + "opentelemetry_sdk 0.22.1", + "prost 0.12.3", + "reqwest 0.11.26", + "thiserror", + "tokio", + "tonic 0.11.0", ] [[package]] @@ -3809,10 +3879,22 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2e155ce5cc812ea3d1dffbd1539aed653de4bf4882d60e6e04dcf0901d674e1" dependencies = [ - "opentelemetry", - "opentelemetry_sdk", - "prost", - "tonic", + "opentelemetry 0.21.0", + "opentelemetry_sdk 0.21.2", + "prost 0.11.9", + "tonic 0.9.2", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a8fddc9b68f5b80dae9d6f510b88e02396f006ad48cac349411fbecc80caae4" +dependencies = [ + "opentelemetry 0.22.0", + "opentelemetry_sdk 0.22.1", + "prost 0.12.3", + "tonic 0.11.0", ] [[package]] @@ -3821,9 +3903,15 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f5774f1ef1f982ef2a447f6ee04ec383981a3ab99c8e77a1a7b30182e65bbc84" dependencies = [ - "opentelemetry", + "opentelemetry 0.21.0", ] +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9ab5bd6c42fb9349dcf28af2ba9a0667f697f9bdcca045d39f2cec5543e2910" + [[package]] name = "opentelemetry_sdk" version = "0.21.2" @@ -3837,7 +3925,29 @@ dependencies = [ "futures-util", "glob", "once_cell", - "opentelemetry", + "opentelemetry 0.21.0", + "ordered-float", + "percent-encoding", + "rand", + "thiserror", + "tokio", + "tokio-stream", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e90c7113be649e31e9a0f8b5ee24ed7a16923b322c3c5ab6367469c049d6b7e" +dependencies = [ + "async-trait", + "crossbeam-channel", + "futures-channel", + "futures-executor", + "futures-util", + "glob", + "once_cell", + "opentelemetry 0.22.0", "ordered-float", "percent-encoding", "rand", @@ -4276,7 +4386,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.11.9", +] + +[[package]] +name = "prost" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c289cda302b98a28d40c8b3b90498d6e526dd24ac2ecea73e4e491685b94a" +dependencies = [ + "bytes", + "prost-derive 0.12.3", ] [[package]] @@ -4292,6 +4412,19 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "prost-derive" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efb6c9a1dd1def8e2124d17e83a20af56f1570d6c2d2bd9e266ccb768df3840e" +dependencies = [ + "anyhow", + "itertools 0.11.0", + "proc-macro2", + "quote", + "syn 2.0.52", +] + [[package]] name = "proxy-header" version = "0.1.0" @@ -6246,7 +6379,34 @@ dependencies = [ "hyper-timeout", "percent-encoding", "pin-project", - "prost", + "prost 0.11.9", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76c4eb7a4e9ef9d4763600161f12f5070b92a578e1b634db88a6887844c91a13" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64 0.21.7", + "bytes", + "h2 0.3.24", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.28", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost 0.12.3", "tokio", "tokio-stream", "tower", @@ -6362,14 +6522,32 @@ checksum = "c67ac25c5407e7b961fafc6f7e9aa5958fd297aada2d20fa2ae1737357e55596" dependencies = [ "js-sys", "once_cell", - "opentelemetry", - "opentelemetry_sdk", + "opentelemetry 0.21.0", + "opentelemetry_sdk 0.21.2", "smallvec", "tracing", "tracing-core", "tracing-log", "tracing-subscriber", - "web-time", + "web-time 0.2.4", +] + +[[package]] +name = "tracing-opentelemetry" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9be14ba1bbe4ab79e9229f7f89fab8d120b865859f10527f31c033e599d2284" +dependencies = [ + "js-sys", + "once_cell", + "opentelemetry 0.22.0", + "opentelemetry_sdk 0.22.1", + "smallvec", + "tracing", + "tracing-core", + "tracing-log", + "tracing-subscriber", + "web-time 1.1.0", ] [[package]] @@ -6589,10 +6767,10 @@ dependencies = [ "lru-cache", "mail-auth", "mail-send", - "opentelemetry", - "opentelemetry-otlp", - "opentelemetry-semantic-conventions", - "opentelemetry_sdk", + "opentelemetry 0.21.0", + "opentelemetry-otlp 0.14.0", + "opentelemetry-semantic-conventions 0.13.0", + "opentelemetry_sdk 0.21.2", "parking_lot", "pem", "privdrop", @@ -6613,7 +6791,7 @@ dependencies = [ "tracing", "tracing-appender", "tracing-journald", - "tracing-opentelemetry", + "tracing-opentelemetry 0.22.0", "tracing-subscriber", "webpki-roots 0.26.1", "x509-parser 0.16.0", @@ -6782,6 +6960,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webpki" version = "0.22.4" diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 3265a4a2..2a43e039 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -9,9 +9,12 @@ utils = { path = "../utils" } nlp = { path = "../nlp" } store = { path = "../store" } directory = { path = "../directory" } +jmap_proto = { path = "../jmap-proto" } sieve-rs = { version = "0.4" } +mail-parser = { version = "0.9", features = ["full_encoding", "ludicrous_mode"] } mail-auth = { version = "0.3" } mail-send = { version = "0.4", default-features = false, features = ["cram-md5"] } +smtp-proto = { version = "0.1", features = ["serde_support"] } ahash = { version = "0.8.0", features = ["serde"] } parking_lot = "0.12.1" regex = "1.7.0" @@ -33,7 +36,28 @@ base64 = "0.22" x509-parser = "0.16.0" pem = "3.0" chrono = "0.4" +hyper = { version = "1.0.1", features = ["server", "http1", "http2"] } +tracing-subscriber = { version = "0.3", features = ["env-filter"] } +tracing-appender = "0.2" +tracing-opentelemetry = "0.23.0" +opentelemetry = { version = "0.22.0" } +opentelemetry_sdk = { version = "0.22.1", features = ["rt-tokio"] } +opentelemetry-otlp = { version = "0.15.0", features = ["http-proto", "reqwest-client"] } +opentelemetry-semantic-conventions = { version = "0.14.0" } +imagesize = "0.12" +sha1 = "0.10" +sha2 = "0.10.6" +md5 = "0.7.0" +whatlang = "0.16" +idna = "0.5" +decancer = "3.0.1" +unicode-security = "0.1.0" +infer = "0.15.0" +bincode = "1.3.1" [target.'cfg(unix)'.dependencies] privdrop = "0.5.3" tracing-journald = "0.3" + +[features] +test_mode = [] diff --git a/crates/common/src/addresses.rs b/crates/common/src/addresses.rs index 6ad0a186..459ba0f9 100644 --- a/crates/common/src/addresses.rs +++ b/crates/common/src/addresses.rs @@ -114,7 +114,7 @@ impl Core { } impl AddressMapping { - pub fn try_parse(config: &mut Config, key: impl AsKey) -> Self { + pub fn parse(config: &mut Config, key: impl AsKey) -> Self { let key = key.as_key(); if let Some(value) = config.value(key.as_str()) { match value { diff --git a/crates/common/src/config/imap.rs b/crates/common/src/config/imap.rs new file mode 100644 index 00000000..9d07fa23 --- /dev/null +++ b/crates/common/src/config/imap.rs @@ -0,0 +1,48 @@ +use std::time::Duration; + +use utils::config::{Config, Rate}; + +pub struct ImapConfig { + pub max_request_size: usize, + pub max_auth_failures: u32, + pub name_shared: String, + pub allow_plain_auth: bool, + + pub timeout_auth: Duration, + pub timeout_unauth: Duration, + pub timeout_idle: Duration, + + pub rate_requests: Option, + pub rate_concurrent: Option, +} + +impl ImapConfig { + pub fn parse(config: &mut Config) -> Self { + ImapConfig { + max_request_size: config + .property_or_default_("imap.request.max-size", "52428800") + .unwrap_or(52428800), + max_auth_failures: config + .property_or_default_("imap.auth.max-failures", "3") + .unwrap_or(3), + name_shared: config + .value("imap.folders.name.shared") + .unwrap_or("Shared Folders") + .to_string(), + timeout_auth: config + .property_or_default_("imap.timeout.authenticated", "30m") + .unwrap_or_else(|| Duration::from_secs(1800)), + timeout_unauth: config + .property_or_default_("imap.timeout.anonymous", "1m") + .unwrap_or_else(|| Duration::from_secs(60)), + timeout_idle: config + .property_or_default_("imap.timeout.idle", "30m") + .unwrap_or_else(|| Duration::from_secs(1800)), + rate_requests: config.property_or_default_("imap.rate-limit.requests", "2000/1m"), + rate_concurrent: config.property_("imap.rate-limit.concurrent"), + allow_plain_auth: config + .property_or_default_("imap.auth.allow-plain-text", "false") + .unwrap_or(false), + } + } +} diff --git a/crates/common/src/config/jmap/capabilities.rs b/crates/common/src/config/jmap/capabilities.rs new file mode 100644 index 00000000..d2b84d94 --- /dev/null +++ b/crates/common/src/config/jmap/capabilities.rs @@ -0,0 +1,178 @@ +use ahash::AHashSet; +use jmap_proto::{ + request::capability::{ + BlobCapabilities, Capabilities, Capability, CoreCapabilities, EmptyCapabilities, + MailCapabilities, SieveAccountCapabilities, SieveSessionCapabilities, + SubmissionCapabilities, + }, + types::type_state::DataType, +}; +use utils::{config::Config, map::vec_map::VecMap}; + +use super::settings::JmapConfig; + +#[derive(Default)] +pub struct BaseCapabilities { + pub session: VecMap, + pub account: VecMap, +} + +impl JmapConfig { + pub fn add_capabilites(&mut self, config: &mut Config) { + // Add core capabilities + self.capabilities.session.append( + Capability::Core, + Capabilities::Core(CoreCapabilities { + max_size_upload: self.upload_max_size, + max_concurrent_upload: self.upload_max_concurrent as usize, + max_size_request: self.request_max_size, + max_concurrent_requests: self.request_max_concurrent as usize, + max_calls_in_request: self.request_max_calls, + max_objects_in_get: self.get_max_objects, + max_objects_in_set: self.set_max_objects, + collation_algorithms: vec![ + "i;ascii-numeric".to_string(), + "i;ascii-casemap".to_string(), + "i;unicode-casemap".to_string(), + ], + }), + ); + + // Add email capabilities + self.capabilities.session.append( + Capability::Mail, + Capabilities::Empty(EmptyCapabilities::default()), + ); + self.capabilities.account.append( + Capability::Mail, + Capabilities::Mail(MailCapabilities { + max_mailboxes_per_email: None, + max_mailbox_depth: self.mailbox_max_depth, + max_size_mailbox_name: self.mailbox_name_max_len, + max_size_attachments_per_email: self.mail_attachments_max_size, + email_query_sort_options: [ + "receivedAt", + "size", + "from", + "to", + "subject", + "sentAt", + "hasKeyword", + "allInThreadHaveKeyword", + "someInThreadHaveKeyword", + ] + .iter() + .map(|s| s.to_string()) + .collect(), + may_create_top_level_mailbox: true, + }), + ); + + // Add submission capabilities + self.capabilities.session.append( + Capability::Submission, + Capabilities::Empty(EmptyCapabilities::default()), + ); + self.capabilities.account.append( + Capability::Submission, + Capabilities::Submission(SubmissionCapabilities { + max_delayed_send: 86400 * 30, + submission_extensions: VecMap::from_iter([ + ("FUTURERELEASE".to_string(), Vec::new()), + ("SIZE".to_string(), Vec::new()), + ("DSN".to_string(), Vec::new()), + ("DELIVERYBY".to_string(), Vec::new()), + ("MT-PRIORITY".to_string(), vec!["MIXER".to_string()]), + ("REQUIRETLS".to_string(), vec![]), + ]), + }), + ); + + // Add vacation response capabilities + self.capabilities.session.append( + Capability::VacationResponse, + Capabilities::Empty(EmptyCapabilities::default()), + ); + self.capabilities.account.append( + Capability::VacationResponse, + Capabilities::Empty(EmptyCapabilities::default()), + ); + + // Add Sieve capabilities + let mut notification_methods = Vec::new(); + + for (_, uri) in config.values("sieve.untrusted.notification-uris") { + notification_methods.push(uri.to_string()); + } + if notification_methods.is_empty() { + notification_methods.push("mailto".to_string()); + } + + let mut capabilities: AHashSet = + AHashSet::from_iter(sieve::compiler::grammar::Capability::all().iter().cloned()); + + for (_, capability) in config.values("sieve.untrusted.disabled-capabilities") { + capabilities.remove(&sieve::compiler::grammar::Capability::parse(capability)); + } + + let mut extensions = capabilities + .into_iter() + .map(|c| c.to_string()) + .collect::>(); + extensions.sort_unstable(); + + self.capabilities.session.append( + Capability::Sieve, + Capabilities::SieveSession(SieveSessionCapabilities::default()), + ); + self.capabilities.account.append( + Capability::Sieve, + Capabilities::SieveAccount(SieveAccountCapabilities { + max_script_name: self.sieve_max_script_name, + max_script_size: config + .property_("sieve.untrusted.max-script-size") + .unwrap_or(1024 * 1024), + max_scripts: self.sieve_max_scripts, + max_redirects: config + .property_("sieve.untrusted.max-redirects") + .unwrap_or(1), + extensions, + notification_methods: if !notification_methods.is_empty() { + notification_methods.into() + } else { + None + }, + ext_lists: None, + }), + ); + + // Add Blob capabilities + self.capabilities.session.append( + Capability::Blob, + Capabilities::Empty(EmptyCapabilities::default()), + ); + self.capabilities.account.append( + Capability::Blob, + Capabilities::Blob(BlobCapabilities { + max_size_blob_set: (self.request_max_size * 3 / 4) - 512, + max_data_sources: self.request_max_calls, + supported_type_names: vec![ + DataType::Email, + DataType::Thread, + DataType::SieveScript, + ], + supported_digest_algorithms: vec!["sha", "sha-256", "sha-512"], + }), + ); + + // Add Quota capabilities + self.capabilities.session.append( + Capability::Quota, + Capabilities::Empty(EmptyCapabilities::default()), + ); + self.capabilities.account.append( + Capability::Quota, + Capabilities::Empty(EmptyCapabilities::default()), + ); + } +} diff --git a/crates/common/src/config/jmap/mod.rs b/crates/common/src/config/jmap/mod.rs new file mode 100644 index 00000000..0b6492c2 --- /dev/null +++ b/crates/common/src/config/jmap/mod.rs @@ -0,0 +1,2 @@ +pub mod capabilities; +pub mod settings; diff --git a/crates/common/src/config/jmap/settings.rs b/crates/common/src/config/jmap/settings.rs new file mode 100644 index 00000000..57346cc0 --- /dev/null +++ b/crates/common/src/config/jmap/settings.rs @@ -0,0 +1,274 @@ +use std::{str::FromStr, time::Duration}; + +use mail_parser::HeaderName; +use nlp::language::Language; +use store::rand::{distributions::Alphanumeric, thread_rng, Rng}; +use utils::config::{cron::SimpleCron, utils::ParseValue, Config, Rate}; + +use super::capabilities::BaseCapabilities; + +pub struct JmapConfig { + pub default_language: Language, + pub query_max_results: usize, + pub changes_max_results: usize, + pub snippet_max_results: usize, + + pub request_max_size: usize, + pub request_max_calls: usize, + pub request_max_concurrent: u64, + + pub get_max_objects: usize, + pub set_max_objects: usize, + + pub upload_max_size: usize, + pub upload_max_concurrent: u64, + + pub upload_tmp_quota_size: usize, + pub upload_tmp_quota_amount: usize, + pub upload_tmp_ttl: u64, + + pub mailbox_max_depth: usize, + pub mailbox_name_max_len: usize, + pub mail_attachments_max_size: usize, + pub mail_parse_max_items: usize, + pub mail_max_size: usize, + + pub sieve_max_script_name: usize, + pub sieve_max_scripts: usize, + + pub session_cache_ttl: Duration, + pub rate_authenticated: Option, + pub rate_authenticate_req: Option, + pub rate_anonymous: Option, + pub rate_use_forwarded: bool, + + pub event_source_throttle: Duration, + pub push_max_total: usize, + pub push_attempt_interval: Duration, + pub push_attempts_max: u32, + pub push_retry_interval: Duration, + pub push_timeout: Duration, + pub push_verify_timeout: Duration, + pub push_throttle: Duration, + + pub web_socket_throttle: Duration, + pub web_socket_timeout: Duration, + pub web_socket_heartbeat: Duration, + + pub oauth_key: String, + pub oauth_expiry_user_code: u64, + pub oauth_expiry_auth_code: u64, + pub oauth_expiry_token: u64, + pub oauth_expiry_refresh_token: u64, + pub oauth_expiry_refresh_token_renew: u64, + pub oauth_max_auth_attempts: u32, + + pub spam_header: Option<(HeaderName<'static>, String)>, + + pub http_headers: Vec<(hyper::header::HeaderName, hyper::header::HeaderValue)>, + + pub encrypt: bool, + pub encrypt_append: bool, + + pub principal_allow_lookups: bool, + + pub capabilities: BaseCapabilities, + pub session_purge_frequency: SimpleCron, +} + +impl JmapConfig { + pub fn parse(config: &mut Config) -> Self { + let mut jmap = JmapConfig { + default_language: Language::from_iso_639( + config + .value("storage.full-text.default-language") + .unwrap_or("en"), + ) + .unwrap_or(Language::English), + query_max_results: config + .property_("jmap.protocol.query.max-results") + .unwrap_or(5000), + changes_max_results: config + .property_("jmap.protocol.changes.max-results") + .unwrap_or(5000), + snippet_max_results: config + .property_("jmap.protocol.search-snippet.max-results") + .unwrap_or(100), + request_max_size: config + .property_("jmap.protocol.request.max-size") + .unwrap_or(10000000), + request_max_calls: config + .property_("jmap.protocol.request.max-calls") + .unwrap_or(16), + request_max_concurrent: config + .property_("jmap.protocol.request.max-concurrent") + .unwrap_or(4), + get_max_objects: config + .property_("jmap.protocol.get.max-objects") + .unwrap_or(500), + set_max_objects: config + .property_("jmap.protocol.set.max-objects") + .unwrap_or(500), + upload_max_size: config + .property_("jmap.protocol.upload.max-size") + .unwrap_or(50000000), + upload_max_concurrent: config + .property_("jmap.protocol.upload.max-concurrent") + .unwrap_or(4), + upload_tmp_quota_size: config + .property_("jmap.protocol.upload.quota.size") + .unwrap_or(50000000), + upload_tmp_quota_amount: config + .property_("jmap.protocol.upload.quota.files") + .unwrap_or(1000), + upload_tmp_ttl: config + .property_or_default_::("jmap.protocol.upload.ttl", "1h") + .unwrap_or_else(|| Duration::from_secs(3600)) + .as_secs(), + mailbox_max_depth: config.property_("jmap.mailbox.max-depth").unwrap_or(10), + mailbox_name_max_len: config + .property_("jmap.mailbox.max-name-length") + .unwrap_or(255), + mail_attachments_max_size: config + .property_("jmap.email.max-attachment-size") + .unwrap_or(50000000), + mail_max_size: config.property_("jmap.email.max-size").unwrap_or(75000000), + mail_parse_max_items: config.property_("jmap.email.parse.max-items").unwrap_or(10), + sieve_max_script_name: config + .property_("sieve.untrusted.limits.name-length") + .unwrap_or(512), + sieve_max_scripts: config + .property_("sieve.untrusted.limits.max-scripts") + .unwrap_or(256), + capabilities: BaseCapabilities::default(), + session_cache_ttl: config + .property_("cache.session.ttl") + .unwrap_or(Duration::from_secs(3600)), + rate_authenticated: config.property_or_default_("jmap.rate-limit.account", "1000/1m"), + rate_authenticate_req: config + .property_or_default_("authentication.rate-limit", "10/1m"), + rate_anonymous: config.property_or_default_("jmap.rate-limit.anonymous", "100/1m"), + rate_use_forwarded: config + .property_("jmap.rate-limit.use-forwarded") + .unwrap_or(false), + oauth_key: config + .value("oauth.key") + .map(|s| s.to_string()) + .unwrap_or_else(|| { + thread_rng() + .sample_iter(Alphanumeric) + .take(64) + .map(char::from) + .collect::() + }), + oauth_expiry_user_code: config + .property_or_default_::("oauth.expiry.user-code", "30m") + .unwrap_or_else(|| Duration::from_secs(30 * 60)) + .as_secs(), + oauth_expiry_auth_code: config + .property_or_default_::("oauth.expiry.auth-code", "10m") + .unwrap_or_else(|| Duration::from_secs(10 * 60)) + .as_secs(), + oauth_expiry_token: config + .property_or_default_::("oauth.expiry.token", "1h") + .unwrap_or_else(|| Duration::from_secs(60 * 60)) + .as_secs(), + oauth_expiry_refresh_token: config + .property_or_default_::("oauth.expiry.refresh-token", "30d") + .unwrap_or_else(|| Duration::from_secs(30 * 24 * 60 * 60)) + .as_secs(), + oauth_expiry_refresh_token_renew: config + .property_or_default_::("oauth.expiry.refresh-token-renew", "4d") + .unwrap_or_else(|| Duration::from_secs(4 * 24 * 60 * 60)) + .as_secs(), + oauth_max_auth_attempts: config + .property_or_default_("oauth.auth.max-attempts", "3") + .unwrap_or(10), + event_source_throttle: config + .property_or_default_("jmap.event-source.throttle", "1s") + .unwrap_or_else(|| Duration::from_secs(1)), + web_socket_throttle: config + .property_or_default_("jmap.web-socket.throttle", "1s") + .unwrap_or_else(|| Duration::from_secs(1)), + web_socket_timeout: config + .property_or_default_("jmap.web-socket.timeout", "10m") + .unwrap_or_else(|| Duration::from_secs(10 * 60)), + web_socket_heartbeat: config + .property_or_default_("jmap.web-socket.heartbeat", "1m") + .unwrap_or_else(|| Duration::from_secs(60)), + push_max_total: config + .property_or_default_("jmap.push.max-total", "100") + .unwrap_or(100), + principal_allow_lookups: config + .property_("jmap.principal.allow-lookups") + .unwrap_or(true), + encrypt: config + .property_or_default_("storage.encryption.enable", "true") + .unwrap_or(true), + encrypt_append: config + .property_or_default_("storage.encryption.append", "false") + .unwrap_or(false), + spam_header: config.value("spam.header.is-spam").and_then(|v| { + v.split_once(':').map(|(k, v)| { + ( + mail_parser::HeaderName::parse(k.trim().to_string()).unwrap(), + v.trim().to_string(), + ) + }) + }), + http_headers: config + .values("server.http.headers") + .map(|(_, v)| { + if let Some((k, v)) = v.split_once(':') { + Ok(( + hyper::header::HeaderName::from_str(k.trim()).map_err(|err| { + format!( + "Invalid header found in property \"server.http.headers\": {}", + err + ) + })?, + hyper::header::HeaderValue::from_str(v.trim()).map_err(|err| { + format!( + "Invalid header found in property \"server.http.headers\": {}", + err + ) + })?, + )) + } else { + Err(format!( + "Invalid header found in property \"server.http.headers\": {}", + v + )) + } + }) + .collect::, String>>() + .map_err(|e| config.new_parse_error("server.http.headers", e)) + .unwrap_or_default(), + push_attempt_interval: config + .property_or_default_("jmap.push.attempts.interval", "1m") + .unwrap_or_else(|| Duration::from_secs(60)), + push_attempts_max: config + .property_or_default_("jmap.push.attempts.max", "3") + .unwrap_or(3), + push_retry_interval: config + .property_or_default_("jmap.push.retry.interval", "1s") + .unwrap_or_else(|| Duration::from_secs(1)), + push_timeout: config + .property_or_default_("jmap.push.timeout.request", "10s") + .unwrap_or_else(|| Duration::from_secs(10)), + push_verify_timeout: config + .property_or_default_("jmap.push.timeout.verify", "1m") + .unwrap_or_else(|| Duration::from_secs(60)), + push_throttle: config + .property_or_default_("jmap.push.throttle", "1s") + .unwrap_or_else(|| Duration::from_secs(1)), + session_purge_frequency: config + .property_or_default_::("jmap.session.purge.frequency", "15 * *") + .unwrap_or_else(|| SimpleCron::parse_value("15 * *", "").unwrap()), + }; + + // Add capabilities + jmap.add_capabilites(config); + jmap + } +} diff --git a/crates/common/src/config/mod.rs b/crates/common/src/config/mod.rs index cc9cc674..6a4fb36f 100644 --- a/crates/common/src/config/mod.rs +++ b/crates/common/src/config/mod.rs @@ -1,4 +1,7 @@ +pub mod imap; +pub mod jmap; pub mod scripts; pub mod server; pub mod smtp; pub mod storage; +pub mod tracers; diff --git a/crates/common/src/config/scripts.rs b/crates/common/src/config/scripts.rs index 2ca53e8d..356af515 100644 --- a/crates/common/src/config/scripts.rs +++ b/crates/common/src/config/scripts.rs @@ -1,33 +1,31 @@ -use std::{collections::HashSet, sync::Arc, time::Instant}; +use std::{ + collections::HashSet, + sync::Arc, + time::{Duration, Instant}, +}; use ahash::AHashMap; use nlp::bayes::cache::BayesTokenCache; use parking_lot::RwLock; -use sieve::{Compiler, Runtime, Sieve}; -use utils::suffixlist::PublicSuffix; +use sieve::{compiler::grammar::Capability, Compiler, Runtime, Sieve}; +use store::Stores; +use utils::config::Config; -use super::smtp::auth::DkimSigner; +use crate::scripts::{functions::register_functions, plugins::RegisterSievePlugins}; -pub struct SieveCore { +use super::smtp::parse_server_hostname; + +pub struct Scripting { pub untrusted_compiler: Compiler, pub untrusted_runtime: Runtime<()>, - pub trusted_runtime: Runtime, + pub trusted_runtime: Runtime<()>, pub from_addr: String, pub from_name: String, pub return_path: String, - pub sign: Vec>, + pub sign: Vec, pub scripts: AHashMap>, -} - -#[derive(Default)] -pub struct SieveContext { - pub psl: PublicSuffix, pub bayes_cache: BayesTokenCache, - pub remote_lists: RemoteLists, -} - -pub struct RemoteLists { - pub lists: RwLock>, + pub remote_lists: RwLock>, } pub struct RemoteList { @@ -35,10 +33,294 @@ pub struct RemoteList { pub expires: Instant, } -impl Default for RemoteLists { - fn default() -> Self { - Self { - lists: RwLock::new(AHashMap::new()), +impl Scripting { + pub async fn parse(config: &mut Config, stores: &Stores) -> Self { + // Parse untrusted compiler + let untrusted_compiler = Compiler::new() + .with_max_script_size( + config + .property_("sieve.untrusted.limits.script-size") + .unwrap_or(1024 * 1024), + ) + .with_max_string_size( + config + .property_("sieve.untrusted.limits.string-length") + .unwrap_or(4096), + ) + .with_max_variable_name_size( + config + .property_("sieve.untrusted.limits.variable-name-length") + .unwrap_or(32), + ) + .with_max_nested_blocks( + config + .property_("sieve.untrusted.limits.nested-blocks") + .unwrap_or(15), + ) + .with_max_nested_tests( + config + .property_("sieve.untrusted.limits.nested-tests") + .unwrap_or(15), + ) + .with_max_nested_foreverypart( + config + .property_("sieve.untrusted.limits.nested-foreverypart") + .unwrap_or(3), + ) + .with_max_match_variables( + config + .property_("sieve.untrusted.limits.match-variables") + .unwrap_or(30), + ) + .with_max_local_variables( + config + .property_("sieve.untrusted.limits.local-variables") + .unwrap_or(128), + ) + .with_max_header_size( + config + .property_("sieve.untrusted.limits.header-size") + .unwrap_or(1024), + ) + .with_max_includes( + config + .property_("sieve.untrusted.limits.includes") + .unwrap_or(3), + ); + + // Parse untrusted runtime + let untrusted_runtime = Runtime::new() + .with_max_nested_includes( + config + .property_("sieve.untrusted.limits.nested-includes") + .unwrap_or(3), + ) + .with_cpu_limit( + config + .property_("sieve.untrusted.limits.cpu") + .unwrap_or(5000), + ) + .with_max_variable_size( + config + .property_("sieve.untrusted.limits.variable-size") + .unwrap_or(4096), + ) + .with_max_redirects( + config + .property_("sieve.untrusted.limits.redirects") + .unwrap_or(1), + ) + .with_max_received_headers( + config + .property_("sieve.untrusted.limits.received-headers") + .unwrap_or(10), + ) + .with_max_header_size( + config + .property_("sieve.untrusted.limits.header-size") + .unwrap_or(1024), + ) + .with_max_out_messages( + config + .property_("sieve.untrusted.limits.outgoing-messages") + .unwrap_or(3), + ) + .with_default_vacation_expiry( + config + .property_::("sieve.untrusted.default-expiry.vacation") + .unwrap_or(Duration::from_secs(30 * 86400)) + .as_secs(), + ) + .with_default_duplicate_expiry( + config + .property_::("sieve.untrusted.default-expiry.duplicate") + .unwrap_or(Duration::from_secs(7 * 86400)) + .as_secs(), + ) + .without_capabilities( + config + .values("sieve.untrusted.disable-capabilities") + .map(|(_, v)| v), + ) + .with_valid_notification_uris({ + let values = config + .values("sieve.untrusted.notification-uris") + .map(|(_, v)| v.to_string()) + .collect::>(); + if !values.is_empty() { + values + } else { + vec!["mailto".to_string()] + } + }) + .with_protected_headers({ + let values = config + .values("sieve.untrusted.protected-headers") + .map(|(_, v)| v.to_string()) + .collect::>(); + if !values.is_empty() { + values + } else { + vec![ + "Original-Subject".to_string(), + "Original-From".to_string(), + "Received".to_string(), + "Auto-Submitted".to_string(), + ] + } + }) + .with_vacation_default_subject( + config + .value("sieve.untrusted.vacation.default-subject") + .unwrap_or("Automated reply") + .to_string(), + ) + .with_vacation_subject_prefix( + config + .value("sieve.untrusted.vacation.subject-prefix") + .unwrap_or("Auto: ") + .to_string(), + ) + .with_env_variable("name", "Stalwart JMAP") + .with_env_variable("version", env!("CARGO_PKG_VERSION")) + .with_env_variable("location", "MS") + .with_env_variable("phase", "during"); + + // Parse trusted compiler and runtime + let mut fnc_map = register_functions().register_plugins(); + + // Allocate compiler and runtime + let trusted_compiler = Compiler::new() + .with_max_string_size(52428800) + .with_max_variable_name_size(100) + .with_max_nested_blocks(50) + .with_max_nested_tests(50) + .with_max_nested_foreverypart(10) + .with_max_local_variables(8192) + .with_max_header_size(10240) + .with_max_includes(10) + .with_no_capability_check( + config + .property_or_default_("sieve.trusted.no-capability-check", "true") + .unwrap_or(true), + ) + .register_functions(&mut fnc_map); + + let mut trusted_runtime = Runtime::new_with_context(()) + .without_capabilities([ + Capability::FileInto, + Capability::Vacation, + Capability::VacationSeconds, + Capability::Fcc, + Capability::Mailbox, + Capability::MailboxId, + Capability::MboxMetadata, + Capability::ServerMetadata, + Capability::ImapSieve, + Capability::Duplicate, + ]) + .with_capability(Capability::Expressions) + .with_capability(Capability::While) + .with_max_variable_size( + config + .property_or_default_("sieve.trusted.limits.variable-size", "52428800") + .unwrap_or(52428800), + ) + .with_max_header_size(10240) + .with_valid_notification_uri("mailto") + .with_valid_ext_lists(stores.lookup_stores.keys().map(|k| k.to_string())) + .with_functions(&mut fnc_map); + + if let Some(value) = config.property_("sieve.trusted.limits.redirects") { + trusted_runtime.set_max_redirects(value); + } + if let Some(value) = config.property_("sieve.trusted.limits.out-messages") { + trusted_runtime.set_max_out_messages(value); + } + if let Some(value) = config.property_("sieve.trusted.limits.cpu") { + trusted_runtime.set_cpu_limit(value); + } + if let Some(value) = config.property_("sieve.trusted.limits.nested-includes") { + trusted_runtime.set_max_nested_includes(value); + } + if let Some(value) = config.property_("sieve.trusted.limits.received-headers") { + trusted_runtime.set_max_received_headers(value); + } + if let Some(value) = config.property_::("sieve.trusted.limits.duplicate-expiry") { + trusted_runtime.set_default_duplicate_expiry(value.as_secs()); + } + let hostname = if let Some(hostname) = config.value("sieve.trusted.hostname") { + hostname.to_string() + } else { + parse_server_hostname(config) + .and_then(|h| h.into_default_string()) + .unwrap_or_else(|| "localhost".to_string()) + }; + trusted_runtime.set_local_hostname(hostname.clone()); + + // Parse scripts + let mut scripts = AHashMap::new(); + for id in config + .sub_keys("sieve.trusted.scripts", ".contents") + .map(|s| s.to_string()) + .collect::>() + { + // Skip sub-scripts + if config + .property_(("sieve.trusted.scripts", id.as_str(), "snippet")) + .unwrap_or(false) + { + continue; + } + + let script = config + .value(("sieve.trusted.scripts", id.as_str(), "contents")) + .unwrap(); + + match trusted_compiler.compile(script.as_bytes()) { + Ok(compiled) => { + scripts.insert(id, compiled.into()); + } + Err(err) => config.new_build_error( + ("sieve.trusted.scripts", id.as_str(), "contents"), + format!("Failed to compile Sieve script: {err}"), + ), + } + } + + Scripting { + untrusted_compiler, + untrusted_runtime, + trusted_runtime, + from_addr: config + .value("sieve.trusted.from-addr") + .map(|a| a.to_string()) + .unwrap_or(format!("MAILER-DAEMON@{hostname}")), + from_name: config + .value("sieve.trusted.from-name") + .unwrap_or("Mailer Daemon") + .to_string(), + return_path: config + .value("sieve.trusted.return-path") + .unwrap_or_default() + .to_string(), + sign: config + .values("sieve.trusted.sign") + .map(|(_, v)| v.to_string()) + .collect(), + scripts, + bayes_cache: BayesTokenCache::new( + config + .property_or_default_("cache.bayes.capacity", "8192") + .unwrap_or(8192), + config + .property_or_default_("cache.bayes.ttl.positive", "1h") + .unwrap_or_else(|| Duration::from_secs(3600)), + config + .property_or_default_("cache.bayes.ttl.negative", "1h") + .unwrap_or_else(|| Duration::from_secs(3600)), + ), + remote_lists: Default::default(), } } } diff --git a/crates/common/src/config/server/tls.rs b/crates/common/src/config/server/tls.rs index c0b74fde..c43906dd 100644 --- a/crates/common/src/config/server/tls.rs +++ b/crates/common/src/config/server/tls.rs @@ -114,11 +114,11 @@ impl ConfigBuilder { } pub fn parse_acmes(&mut self, config: &mut Config) { - let acme_ids = config - .sub_keys("acme", ".cache") + for acme_id in config + .sub_keys("acme", ".directory") .map(|s| s.to_string()) - .collect::>(); - for acme_id in acme_ids { + .collect::>() + { let directory = config .value(("acme", acme_id.as_str(), "directory")) .unwrap_or(LETS_ENCRYPT_PRODUCTION_DIRECTORY) @@ -149,23 +149,11 @@ impl ConfigBuilder { continue; } - // Find which domains are covered by this ACME manager - let mut domains = Vec::new(); - for id in config.sub_keys("server.listener", ".protocol") { - match ( - config.value_or_else(("server.listener", id, "tls.acme"), "server.tls.acme"), - config.value_or_else(("server.listener", id, "hostname"), "server.hostname"), - ) { - (Some(listener_acme), Some(hostname)) if listener_acme == acme_id => { - let hostname = hostname.trim().to_lowercase(); - - if !domains.contains(&hostname) { - domains.push(hostname); - } - } - _ => (), - } - } + // Domains covered by this ACME manager + let domains = config + .values(("acme", acme_id.as_str(), "domains")) + .map(|(_, v)| v.to_string()) + .collect::>(); if !domains.is_empty() { match AcmeManager::new( diff --git a/crates/common/src/config/smtp/auth.rs b/crates/common/src/config/smtp/auth.rs index 6be905d8..7128ead7 100644 --- a/crates/common/src/config/smtp/auth.rs +++ b/crates/common/src/config/smtp/auth.rs @@ -1,13 +1,19 @@ -use std::sync::Arc; +use std::{sync::Arc, time::Duration}; use ahash::AHashMap; use mail_auth::{ - common::crypto::{Ed25519Key, RsaKey, Sha256}, - dkim::Done, + common::crypto::{Algorithm, Ed25519Key, HashAlgorithm, RsaKey, Sha256, SigningKey}, + dkim::{Canonicalization, Done}, +}; +use mail_parser::decoders::base64::base64_decode; +use utils::config::{ + utils::{AsKey, ParseValue}, + Config, }; -use utils::config::utils::{AsKey, ParseValue}; -use crate::expr::{self, if_block::IfBlock, Constant, ConstantValue}; +use crate::expr::{self, if_block::IfBlock, tokenizer::TokenMap, Constant, ConstantValue}; + +use super::*; pub struct MailAuthConfig { pub dkim: DkimAuthConfig, @@ -50,6 +56,12 @@ pub enum VerifyStrategy { Disable, } +#[derive(Debug, Clone)] +pub struct DkimCanonicalization { + pub headers: Canonicalization, + pub body: Canonicalization, +} + pub enum DkimSigner { RsaSha256(mail_auth::dkim::DkimSigner, Done>), Ed25519Sha256(mail_auth::dkim::DkimSigner), @@ -87,6 +99,267 @@ impl Default for MailAuthConfig { } } +impl MailAuthConfig { + pub fn parse(config: &mut Config) -> Self { + let sender_vars = TokenMap::default() + .with_smtp_variables(&[ + V_SENDER, + V_SENDER_DOMAIN, + V_PRIORITY, + V_AUTHENTICATED_AS, + V_LISTENER, + V_REMOTE_IP, + V_LOCAL_IP, + ]) + .with_constants::(); + let conn_vars = TokenMap::default() + .with_smtp_variables(&[V_LISTENER, V_REMOTE_IP, V_LOCAL_IP]) + .with_constants::(); + let mut mail_auth = Self::default(); + + for (value, key, token_map) in [ + (&mut mail_auth.dkim.verify, "auth.dkim.verify", &sender_vars), + (&mut mail_auth.dkim.sign, "auth.dkim.sign", &sender_vars), + (&mut mail_auth.arc.verify, "auth.arc.verify", &sender_vars), + (&mut mail_auth.arc.seal, "auth.arc.seal", &sender_vars), + ( + &mut mail_auth.spf.verify_ehlo, + "auth.spf.verify.ehlo", + &conn_vars, + ), + ( + &mut mail_auth.spf.verify_mail_from, + "auth.spf.verify.mail-from", + &conn_vars, + ), + ( + &mut mail_auth.dmarc.verify, + "auth.dmarc.verify", + &sender_vars, + ), + ( + &mut mail_auth.iprev.verify, + "auth.iprev.verify", + &sender_vars, + ), + ] { + if let Some(if_block) = IfBlock::try_parse(config, key, token_map) { + *value = if_block; + } + } + + // Parse signatures + for id in config + .sub_keys("signature", ".algorithm") + .map(|k| k.to_string()) + .collect::>() + { + let id = id.to_string(); + if let Some((signer, sealer)) = build_signature(config, &id) { + mail_auth.signers.insert(id.clone(), Arc::new(signer)); + mail_auth.sealers.insert(id, Arc::new(sealer)); + } + } + + mail_auth + } +} + +fn build_signature(config: &mut Config, id: &str) -> Option<(DkimSigner, ArcSealer)> { + match config.property_require_::(("signature", id, "algorithm"))? { + Algorithm::RsaSha256 => { + let pk = config + .value_require_(("signature", id, "private-key"))? + .to_string(); + let key = RsaKey::::from_rsa_pem(&pk) + .or_else(|_| RsaKey::::from_pkcs8_pem(&pk)) + .map_err(|err| { + config.new_build_error( + ("signature", id, "private-key"), + format!("Failed to build RSA key: {err}",), + ) + }) + .ok()?; + let key_clone = RsaKey::::from_rsa_pem(&pk) + .or_else(|_| RsaKey::::from_pkcs8_pem(&pk)) + .map_err(|err| { + config.new_build_error( + ("signature", id, "private-key"), + format!("Failed to build RSA key: {err}",), + ) + }) + .ok()?; + let (signer, sealer) = parse_signature(config, id, key_clone, key)?; + (DkimSigner::RsaSha256(signer), ArcSealer::RsaSha256(sealer)).into() + } + Algorithm::Ed25519Sha256 => { + let mut public_key = vec![]; + let mut private_key = vec![]; + + for (key, key_bytes) in [ + (("signature", id, "public-key"), &mut public_key), + (("signature", id, "private-key"), &mut private_key), + ] { + let mut contents = config.value_require_(key)?.as_bytes().iter().copied(); + let mut base64 = vec![]; + + 'outer: while let Some(ch) = contents.next() { + if !ch.is_ascii_whitespace() { + if ch == b'-' { + for ch in contents.by_ref() { + if ch == b'\n' { + break; + } + } + } else { + base64.push(ch); + } + + for ch in contents.by_ref() { + if ch == b'-' { + break 'outer; + } else if !ch.is_ascii_whitespace() { + base64.push(ch); + } + } + } + } + + *key_bytes = base64_decode(&base64) + .ok_or_else(|| { + config.new_build_error( + ("signature", id), + format!("Failed to base64 decode key for {}.", key.as_key(),), + ) + }) + .ok()?; + } + + let key = Ed25519Key::from_pkcs8_maybe_unchecked_der(&private_key) + .or_else(|_| Ed25519Key::from_seed_and_public_key(&private_key, &public_key)) + .map_err(|err| { + config.new_build_error( + ("signature", id), + format!("Failed to build ED25519 key for signature {id:?}: {err}"), + ) + }) + .ok()?; + let key_clone = Ed25519Key::from_pkcs8_maybe_unchecked_der(&private_key) + .or_else(|_| Ed25519Key::from_seed_and_public_key(&private_key, &public_key)) + .map_err(|err| { + config.new_build_error( + ("signature", id), + format!("Failed to build ED25519 key for signature {id:?}: {err}"), + ) + }) + .ok()?; + + let (signer, sealer) = parse_signature(config, id, key_clone, key)?; + ( + DkimSigner::Ed25519Sha256(signer), + ArcSealer::Ed25519Sha256(sealer), + ) + .into() + } + Algorithm::RsaSha1 => { + config.new_build_error( + ("signature", id), + format!("Could not build signature {id:?}: SHA1 signatures are deprecated.",), + ); + None + } + } +} + +fn parse_signature>( + config: &mut Config, + id: &str, + key_dkim: T, + key_arc: U, +) -> Option<( + mail_auth::dkim::DkimSigner, + mail_auth::arc::ArcSealer, +)> { + let domain = config + .value_require_(("signature", id, "domain"))? + .to_string(); + let selector = config + .value_require_(("signature", id, "selector"))? + .to_string(); + let mut headers = config + .values(("signature", id, "headers")) + .filter_map(|(_, v)| { + if !v.is_empty() { + v.to_string().into() + } else { + None + } + }) + .collect::>(); + if headers.is_empty() { + headers = vec![ + "From".to_string(), + "To".to_string(), + "Date".to_string(), + "Subject".to_string(), + "Message-ID".to_string(), + ]; + } + + let mut signer = mail_auth::dkim::DkimSigner::from_key(key_dkim) + .domain(&domain) + .selector(&selector) + .headers(headers.clone()); + if !headers + .iter() + .any(|h| h.eq_ignore_ascii_case("DKIM-Signature")) + { + headers.push("DKIM-Signature".to_string()); + } + let mut sealer = mail_auth::arc::ArcSealer::from_key(key_arc) + .domain(domain) + .selector(selector) + .headers(headers); + + if let Some(c) = config.property_::(("signature", id, "canonicalization")) + { + signer = signer + .body_canonicalization(c.body) + .header_canonicalization(c.headers); + sealer = sealer + .body_canonicalization(c.body) + .header_canonicalization(c.headers); + } + + if let Some(c) = config.property_::(("signature", id, "expire")) { + signer = signer.expiration(c.as_secs()); + sealer = sealer.expiration(c.as_secs()); + } + + if let Some(true) = config.property_::(("signature", id, "set-body-length")) { + signer = signer.body_length(true); + sealer = sealer.body_length(true); + } + + if let Some(true) = config.property_::(("signature", id, "report")) { + signer = signer.reporting(true); + } + + if let Some(auid) = config.property_::(("signature", id, "auid")) { + signer = signer.agent_user_identifier(auid); + } + + if let Some(atps) = config.property_::(("signature", id, "third-party")) { + signer = signer.atps(atps); + } + + if let Some(atpsh) = config.property_::(("signature", id, "third-party-algo")) { + signer = signer.atpsh(atpsh); + } + + Some((signer, sealer)) +} + impl<'x> TryFrom> for VerifyStrategy { type Error = (); @@ -128,4 +401,40 @@ impl ParseValue for VerifyStrategy { } } -impl ConstantValue for VerifyStrategy {} +impl ConstantValue for VerifyStrategy { + fn add_constants(token_map: &mut TokenMap) { + token_map + .add_constant("relaxed", VerifyStrategy::Relaxed) + .add_constant("strict", VerifyStrategy::Strict) + .add_constant("disable", VerifyStrategy::Disable) + .add_constant("disabled", VerifyStrategy::Disable) + .add_constant("never", VerifyStrategy::Disable) + .add_constant("none", VerifyStrategy::Disable); + } +} + +impl ParseValue for DkimCanonicalization { + fn parse_value(key: impl AsKey, value: &str) -> utils::config::Result { + if let Some((headers, body)) = value.split_once('/') { + Ok(DkimCanonicalization { + headers: Canonicalization::parse_value(key.clone(), headers.trim())?, + body: Canonicalization::parse_value(key, body.trim())?, + }) + } else { + let c = Canonicalization::parse_value(key, value)?; + Ok(DkimCanonicalization { + headers: c, + body: c, + }) + } + } +} + +impl Default for DkimCanonicalization { + fn default() -> Self { + Self { + headers: Canonicalization::Relaxed, + body: Canonicalization::Relaxed, + } + } +} diff --git a/crates/common/src/config/smtp/mod.rs b/crates/common/src/config/smtp/mod.rs index 862cba4a..d1c63b0d 100644 --- a/crates/common/src/config/smtp/mod.rs +++ b/crates/common/src/config/smtp/mod.rs @@ -1,17 +1,19 @@ -use utils::{config::Rate, expr::Expression}; +use utils::config::{Config, Rate}; pub mod auth; pub mod queue; pub mod report; pub mod resolver; pub mod session; +pub mod throttle; + +use crate::expr::{if_block::IfBlock, tokenizer::TokenMap, Expression, ExpressionItem, Token}; use self::{ auth::MailAuthConfig, queue::QueueConfig, report::ReportConfig, resolver::Resolvers, session::SessionConfig, }; -#[derive(Default)] pub struct SmtpConfig { pub session: SessionConfig, pub queue: QueueConfig, @@ -39,3 +41,49 @@ pub const THROTTLE_MX: u16 = 1 << 6; pub const THROTTLE_REMOTE_IP: u16 = 1 << 7; pub const THROTTLE_LOCAL_IP: u16 = 1 << 8; pub const THROTTLE_HELO_DOMAIN: u16 = 1 << 9; + +pub const V_RECIPIENT: u32 = 0; +pub const V_RECIPIENT_DOMAIN: u32 = 1; +pub const V_SENDER: u32 = 2; +pub const V_SENDER_DOMAIN: u32 = 3; +pub const V_MX: u32 = 4; +pub const V_HELO_DOMAIN: u32 = 5; +pub const V_AUTHENTICATED_AS: u32 = 6; +pub const V_LISTENER: u32 = 7; +pub const V_REMOTE_IP: u32 = 8; +pub const V_LOCAL_IP: u32 = 9; +pub const V_PRIORITY: u32 = 10; + +pub const VARIABLES_MAP: &[(&str, u32)] = &[ + ("rcpt", V_RECIPIENT), + ("rcpt_domain", V_RECIPIENT_DOMAIN), + ("sender", V_SENDER), + ("sender_domain", V_SENDER_DOMAIN), + ("mx", V_MX), + ("helo_domain", V_HELO_DOMAIN), + ("authenticated_as", V_AUTHENTICATED_AS), + ("listener", V_LISTENER), + ("remote_ip", V_REMOTE_IP), + ("local_ip", V_LOCAL_IP), + ("priority", V_PRIORITY), +]; + +impl TokenMap { + pub fn with_smtp_variables(mut self, variables: &[u32]) -> Self { + for (name, idx) in VARIABLES_MAP { + if variables.contains(idx) { + self.tokens.insert(name, Token::Variable(*idx)); + } + } + + self + } +} + +pub(crate) fn parse_server_hostname(config: &mut Config) -> Option { + IfBlock::try_parse( + config, + "server.hostname", + &TokenMap::default().with_smtp_variables(&[V_LISTENER, V_REMOTE_IP, V_LOCAL_IP]), + ) +} diff --git a/crates/common/src/config/smtp/queue.rs b/crates/common/src/config/smtp/queue.rs index 7f2fa43d..ddf5af47 100644 --- a/crates/common/src/config/smtp/queue.rs +++ b/crates/common/src/config/smtp/queue.rs @@ -5,12 +5,14 @@ use mail_auth::IpLookupStrategy; use mail_send::Credentials; use utils::config::{ utils::{AsKey, ParseValue}, - ServerProtocol, + Config, ServerProtocol, }; use crate::expr::{if_block::IfBlock, Constant, ConstantValue, Expression, Variable}; -use super::Throttle; +use self::throttle::{parse_throttle, parse_throttle_key}; + +use super::*; pub struct QueueConfig { // Schedule @@ -156,6 +158,300 @@ impl Default for QueueConfig { } } +impl QueueConfig { + pub fn parse(config: &mut Config) -> Self { + let mut queue = QueueConfig::default(); + let rcpt_vars = TokenMap::default().with_smtp_variables(&[ + V_RECIPIENT_DOMAIN, + V_SENDER, + V_SENDER_DOMAIN, + V_PRIORITY, + ]); + let sender_vars = + TokenMap::default().with_smtp_variables(&[V_SENDER, V_SENDER_DOMAIN, V_PRIORITY]); + let mx_vars = TokenMap::default().with_smtp_variables(&[ + V_RECIPIENT_DOMAIN, + V_SENDER, + V_SENDER_DOMAIN, + V_PRIORITY, + V_MX, + ]); + let host_vars = TokenMap::default().with_smtp_variables(&[ + V_RECIPIENT_DOMAIN, + V_SENDER, + V_SENDER_DOMAIN, + V_PRIORITY, + V_LOCAL_IP, + V_REMOTE_IP, + V_MX, + ]); + let ip_strategy_vars = sender_vars.clone().with_constants::(); + let dane_vars = mx_vars.clone().with_constants::(); + let mta_sts_vars = rcpt_vars.clone().with_constants::(); + + // Parse default server hostname + if let Some(hostname) = parse_server_hostname(config) { + queue.hostname = hostname.into_default("queue.outbound.hostname"); + } + + for (value, key, token_map) in [ + (&mut queue.retry, "queue.schedule.retry", &host_vars), + (&mut queue.notify, "queue.schedule.notify", &rcpt_vars), + (&mut queue.expire, "queue.schedule.expire", &rcpt_vars), + (&mut queue.hostname, "queue.outbound.hostname", &sender_vars), + (&mut queue.max_mx, "queue.outbound.limits.mx", &rcpt_vars), + ( + &mut queue.max_multihomed, + "queue.outbound.limits.multihomed", + &rcpt_vars, + ), + ( + &mut queue.ip_strategy, + "queue.outbound.ip-strategy", + &ip_strategy_vars, + ), + ( + &mut queue.source_ip.ipv4, + "queue.outbound.source-ip.v4", + &mx_vars, + ), + ( + &mut queue.source_ip.ipv6, + "queue.outbound.source-ip.v6", + &mx_vars, + ), + (&mut queue.next_hop, "queue.outbound.next-hop", &rcpt_vars), + (&mut queue.tls.dane, "queue.outbound.tls.dane", &dane_vars), + ( + &mut queue.tls.mta_sts, + "queue.outbound.tls.mta-sts", + &mta_sts_vars, + ), + ( + &mut queue.tls.start, + "queue.outbound.tls.starttls", + &dane_vars, + ), + ( + &mut queue.tls.invalid_certs, + "queue.outbound.tls.allow-invalid-certs", + &mx_vars, + ), + ( + &mut queue.timeout.connect, + "queue.outbound.timeouts.connect", + &host_vars, + ), + ( + &mut queue.timeout.greeting, + "queue.outbound.timeouts.greeting", + &host_vars, + ), + ( + &mut queue.timeout.tls, + "queue.outbound.timeouts.tls", + &host_vars, + ), + ( + &mut queue.timeout.ehlo, + "queue.outbound.timeouts.ehlo", + &host_vars, + ), + ( + &mut queue.timeout.mail, + "queue.outbound.timeouts.mail-from", + &host_vars, + ), + ( + &mut queue.timeout.rcpt, + "queue.outbound.timeouts.rcpt-to", + &host_vars, + ), + ( + &mut queue.timeout.data, + "queue.outbound.timeouts.data", + &host_vars, + ), + ( + &mut queue.timeout.mta_sts, + "queue.outbound.timeouts.mta-sts", + &host_vars, + ), + (&mut queue.dsn.name, "report.dsn.from-name", &sender_vars), + ( + &mut queue.dsn.address, + "report.dsn.from-address", + &sender_vars, + ), + (&mut queue.dsn.sign, "report.dsn.sign", &sender_vars), + ] { + if let Some(if_block) = IfBlock::try_parse(config, key, token_map) { + *value = if_block; + } + } + + // Parse queue quotas and throttles + queue.throttle = parse_queue_throttle(config); + queue.quota = parse_queue_quota(config); + queue + } +} + +fn parse_queue_throttle(config: &mut Config) -> QueueThrottle { + // Parse throttle + let mut throttle = QueueThrottle { + sender: Vec::new(), + rcpt: Vec::new(), + host: Vec::new(), + }; + + let all_throttles = parse_throttle( + config, + "queue.throttle", + &TokenMap::default().with_smtp_variables(&[ + V_RECIPIENT_DOMAIN, + V_SENDER, + V_SENDER_DOMAIN, + V_PRIORITY, + V_MX, + V_REMOTE_IP, + V_LOCAL_IP, + ]), + THROTTLE_RCPT_DOMAIN + | THROTTLE_SENDER + | THROTTLE_SENDER_DOMAIN + | THROTTLE_MX + | THROTTLE_REMOTE_IP + | THROTTLE_LOCAL_IP, + ); + for t in all_throttles { + if (t.keys & (THROTTLE_MX | THROTTLE_REMOTE_IP | THROTTLE_LOCAL_IP)) != 0 + || t.expr + .items() + .iter() + .any(|c| matches!(c, ExpressionItem::Variable(V_MX | V_REMOTE_IP | V_LOCAL_IP))) + { + throttle.host.push(t); + } else if (t.keys & (THROTTLE_RCPT_DOMAIN)) != 0 + || t.expr + .items() + .iter() + .any(|c| matches!(c, ExpressionItem::Variable(V_RECIPIENT_DOMAIN))) + { + throttle.rcpt.push(t); + } else { + throttle.sender.push(t); + } + } + + throttle +} + +fn parse_queue_quota(config: &mut Config) -> QueueQuotas { + let mut capacities = QueueQuotas { + sender: Vec::new(), + rcpt: Vec::new(), + rcpt_domain: Vec::new(), + }; + + for quota_id in config + .sub_keys("queue.quota", "") + .map(|s| s.to_string()) + .collect::>() + { + if let Some(quota) = parse_queue_quota_item(config, ("queue.quota", "a_id)) { + if (quota.keys & THROTTLE_RCPT) != 0 + || quota + .expr + .items() + .iter() + .any(|c| matches!(c, ExpressionItem::Variable(V_RECIPIENT))) + { + capacities.rcpt.push(quota); + } else if (quota.keys & THROTTLE_RCPT_DOMAIN) != 0 + || quota + .expr + .items() + .iter() + .any(|c| matches!(c, ExpressionItem::Variable(V_RECIPIENT_DOMAIN))) + { + capacities.rcpt_domain.push(quota); + } else { + capacities.sender.push(quota); + } + } + } + + capacities +} + +fn parse_queue_quota_item(config: &mut Config, prefix: impl AsKey) -> Option { + let prefix = prefix.as_key(); + let mut keys = 0; + for (key_, value) in config + .values((&prefix, "key")) + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect::>() + { + match parse_throttle_key(&value) { + Ok(key) => { + if (key + & (THROTTLE_RCPT_DOMAIN + | THROTTLE_RCPT + | THROTTLE_SENDER + | THROTTLE_SENDER_DOMAIN)) + != 0 + { + keys |= key; + } else { + let err = format!("Quota key {value:?} is not available in this context"); + config.new_build_error(key_, err); + } + } + Err(err) => { + config.new_parse_error(key_, err); + } + } + } + + let quota = QueueQuota { + expr: Expression::try_parse( + config, + (prefix.as_str(), "match"), + &TokenMap::default().with_smtp_variables(&[ + V_RECIPIENT, + V_RECIPIENT_DOMAIN, + V_SENDER, + V_SENDER_DOMAIN, + V_PRIORITY, + ]), + ) + .unwrap_or_default(), + keys, + size: config + .property_::((prefix.as_str(), "size")) + .filter(|&v| v > 0), + messages: config + .property_::((prefix.as_str(), "messages")) + .filter(|&v| v > 0), + }; + + // Validate + if quota.size.is_none() && quota.messages.is_none() { + config.new_parse_error( + prefix.as_str(), + concat!( + "Queue quota needs to define a ", + "valid 'size' and/or 'messages' property." + ) + .to_string(), + ); + None + } else { + Some(quota) + } +} + impl ParseValue for RequireOptional { fn parse_value(key: impl AsKey, value: &str) -> utils::config::Result { match value { @@ -194,7 +490,18 @@ impl From for Constant { } } -impl ConstantValue for RequireOptional {} +impl ConstantValue for RequireOptional { + fn add_constants(token_map: &mut crate::expr::tokenizer::TokenMap) { + token_map + .add_constant("optional", RequireOptional::Optional) + .add_constant("require", RequireOptional::Require) + .add_constant("required", RequireOptional::Require) + .add_constant("disable", RequireOptional::Disable) + .add_constant("disabled", RequireOptional::Disable) + .add_constant("none", RequireOptional::Disable) + .add_constant("false", RequireOptional::Disable); + } +} impl<'x> TryFrom> for IpLookupStrategy { type Error = (); @@ -225,4 +532,12 @@ impl From for Constant { } } -impl ConstantValue for IpLookupStrategy {} +impl ConstantValue for IpLookupStrategy { + fn add_constants(token_map: &mut crate::expr::tokenizer::TokenMap) { + token_map + .add_constant("ipv4_only", IpLookupStrategy::Ipv4Only) + .add_constant("ipv6_only", IpLookupStrategy::Ipv6Only) + .add_constant("ipv6_then_ipv4", IpLookupStrategy::Ipv6thenIpv4) + .add_constant("ipv4_then_ipv6", IpLookupStrategy::Ipv4thenIpv6); + } +} diff --git a/crates/common/src/config/smtp/report.rs b/crates/common/src/config/smtp/report.rs index 29e7bc93..b416a8ef 100644 --- a/crates/common/src/config/smtp/report.rs +++ b/crates/common/src/config/smtp/report.rs @@ -1,11 +1,16 @@ use std::time::Duration; use utils::{ - config::utils::{AsKey, ParseValue}, + config::{ + utils::{AsKey, ParseValue}, + Config, + }, snowflake::SnowflakeIdGenerator, }; -use crate::expr::{if_block::IfBlock, Constant, ConstantValue, Variable}; +use crate::expr::{if_block::IfBlock, tokenizer::TokenMap, Constant, ConstantValue, Variable}; + +use super::*; pub struct ReportConfig { pub submitter: IfBlock, @@ -58,6 +63,147 @@ pub enum AggregateFrequency { Never, } +impl ReportConfig { + pub fn parse(config: &mut Config) -> Self { + let sender_vars = TokenMap::default().with_smtp_variables(&[ + V_SENDER, + V_SENDER_DOMAIN, + V_PRIORITY, + V_AUTHENTICATED_AS, + V_LISTENER, + V_REMOTE_IP, + V_LOCAL_IP, + ]); + let rcpt_vars = TokenMap::default().with_smtp_variables(&[ + V_SENDER, + V_SENDER_DOMAIN, + V_PRIORITY, + V_REMOTE_IP, + V_LOCAL_IP, + V_RECIPIENT_DOMAIN, + ]); + + let default_hostname_if_block = parse_server_hostname(config); + let default_hostname = default_hostname_if_block + .as_ref() + .and_then(|i| i.default_string()) + .unwrap_or("localhost") + .to_string(); + + Self { + submitter: IfBlock::try_parse( + config, + "report.submitter", + &TokenMap::default().with_smtp_variables(&[V_RECIPIENT_DOMAIN]), + ) + .unwrap_or_else(|| { + default_hostname_if_block + .map(|i| i.into_default("report.submitter")) + .unwrap_or_else(|| IfBlock::new("localhost".to_string())) + }), + analysis: ReportAnalysis { + addresses: config + .properties_::("report.analysis.addresses") + .into_iter() + .map(|(_, m)| m) + .collect(), + forward: config.property_("report.analysis.forward").unwrap_or(true), + store: config.property_("report.analysis.store"), + report_id: config + .property_::("storage.cluster.node-id") + .map(SnowflakeIdGenerator::with_node_id) + .unwrap_or_default(), + }, + dkim: Report::parse(config, "dkim", &default_hostname, &sender_vars), + spf: Report::parse(config, "spf", &default_hostname, &sender_vars), + dmarc: Report::parse(config, "dmarc", &default_hostname, &sender_vars), + dmarc_aggregate: AggregateReport::parse( + config, + "dmarc", + &default_hostname, + &sender_vars.with_constants::(), + ), + tls: AggregateReport::parse( + config, + "tls", + &default_hostname, + &rcpt_vars.with_constants::(), + ), + } + } +} + +impl Report { + pub fn parse( + config: &mut Config, + id: &str, + default_hostname: &str, + token_map: &TokenMap, + ) -> Self { + let mut report = Self { + name: IfBlock::new(format!("{} Reporting", id.to_ascii_uppercase())), + address: IfBlock::new(format!("MAILER-DAEMON@{default_hostname}")), + subject: IfBlock::new(format!("{} Report", id.to_ascii_uppercase())), + sign: Default::default(), + send: Default::default(), + }; + for (value, key) in [ + (&mut report.name, "from-name"), + (&mut report.address, "from-address"), + (&mut report.subject, "subject"), + (&mut report.sign, "sign"), + (&mut report.send, "send"), + ] { + if let Some(if_block) = IfBlock::try_parse(config, ("report", id, key), token_map) { + *value = if_block; + } + } + + report + } +} + +impl AggregateReport { + pub fn parse( + config: &mut Config, + id: &str, + default_hostname: &str, + token_map: &TokenMap, + ) -> Self { + let rcpt_vars = TokenMap::default().with_smtp_variables(&[V_RECIPIENT_DOMAIN]); + + let mut report = Self { + name: IfBlock::new(format!("{} Aggregate Report", id.to_ascii_uppercase())), + address: IfBlock::new(format!("noreply-{id}@{default_hostname}")), + org_name: Default::default(), + contact_info: Default::default(), + send: IfBlock::new(AggregateFrequency::Never), + sign: Default::default(), + max_size: IfBlock::new(25 * 1024 * 1024), + }; + + for (value, key, token_map) in [ + (&mut report.name, "aggregate.from-name", &rcpt_vars), + (&mut report.address, "aggregate.from-address", &rcpt_vars), + (&mut report.org_name, "aggregate.org-name", &rcpt_vars), + ( + &mut report.contact_info, + "aggregate.contact-info", + &rcpt_vars, + ), + (&mut report.send, "aggregate.send", token_map), + (&mut report.sign, "aggregate.sign", &rcpt_vars), + (&mut report.max_size, "aggregate.max-size", &rcpt_vars), + ] { + if let Some(if_block) = IfBlock::try_parse(config, ("report", id, key), token_map) { + *value = if_block; + } + } + + report + } +} + impl Default for ReportConfig { fn default() -> Self { Self { @@ -144,4 +290,39 @@ impl<'x> TryFrom> for AggregateFrequency { } } -impl ConstantValue for AggregateFrequency {} +impl ConstantValue for AggregateFrequency { + fn add_constants(token_map: &mut crate::expr::tokenizer::TokenMap) { + token_map + .add_constant("never", AggregateFrequency::Never) + .add_constant("hourly", AggregateFrequency::Hourly) + .add_constant("hour", AggregateFrequency::Hourly) + .add_constant("daily", AggregateFrequency::Daily) + .add_constant("day", AggregateFrequency::Daily) + .add_constant("weekly", AggregateFrequency::Weekly) + .add_constant("week", AggregateFrequency::Weekly) + .add_constant("never", AggregateFrequency::Never) + .add_constant("disable", AggregateFrequency::Never) + .add_constant("false", AggregateFrequency::Never); + } +} + +impl ParseValue for AddressMatch { + fn parse_value(key: impl AsKey, value: &str) -> utils::config::Result { + if let Some(value) = value.strip_prefix('*').map(|v| v.trim()) { + if !value.is_empty() { + return Ok(AddressMatch::EndsWith(value.to_lowercase())); + } + } else if let Some(value) = value.strip_suffix('*').map(|v| v.trim()) { + if !value.is_empty() { + return Ok(AddressMatch::StartsWith(value.to_lowercase())); + } + } else if value.contains('@') { + return Ok(AddressMatch::Equals(value.trim().to_lowercase())); + } + Err(format!( + "Invalid address match value {:?} for key {:?}.", + value, + key.as_key() + )) + } +} diff --git a/crates/common/src/config/smtp/resolver.rs b/crates/common/src/config/smtp/resolver.rs index 50d9907e..8abdf6dd 100644 --- a/crates/common/src/config/smtp/resolver.rs +++ b/crates/common/src/config/smtp/resolver.rs @@ -1,19 +1,24 @@ -use std::sync::Arc; +use std::{ + net::{IpAddr, Ipv4Addr, SocketAddr}, + sync::Arc, +}; use mail_auth::{ common::lru::{DnsCache, LruCache}, hickory_resolver::{ - config::{ResolverConfig, ResolverOpts}, + config::{NameServerConfig, Protocol, ResolverConfig, ResolverOpts}, system_conf::read_system_conf, AsyncResolver, TokioAsyncResolver, }, Resolver, }; +use utils::{config::Config, suffixlist::PublicSuffix}; pub struct Resolvers { pub dns: Resolver, pub dnssec: DnssecResolver, pub cache: DnsRecordCache, + pub psl: PublicSuffix, } pub struct DnssecResolver { @@ -61,6 +66,154 @@ pub struct Policy { pub max_age: u64, } +impl Resolvers { + pub async fn parse(config: &mut Config) -> Self { + let (resolver_config, mut opts) = match config + .value_require_("resolver.type") + .unwrap_or("system") + { + "cloudflare" => (ResolverConfig::cloudflare(), ResolverOpts::default()), + "cloudflare-tls" => (ResolverConfig::cloudflare_tls(), ResolverOpts::default()), + "quad9" => (ResolverConfig::quad9(), ResolverOpts::default()), + "quad9-tls" => (ResolverConfig::quad9_tls(), ResolverOpts::default()), + "google" => (ResolverConfig::google(), ResolverOpts::default()), + "system" => read_system_conf() + .map_err(|err| { + config.new_build_error( + "resolver.type", + format!("Failed to read system DNS config: {err}"), + ) + }) + .unwrap_or_else(|_| (ResolverConfig::cloudflare(), ResolverOpts::default())), + "custom" => { + let mut resolver_config = ResolverConfig::new(); + for url in config + .values("resolver.custom") + .map(|(_, v)| v.to_string()) + .collect::>() + { + let (proto, host) = if let Some((proto, host)) = url + .split_once("://") + .map(|(a, b)| (a.to_string(), b.to_string())) + { + ( + match proto.as_str() { + "udp" => Protocol::Udp, + "tcp" => Protocol::Tcp, + "tls" => Protocol::Tls, + _ => { + config.new_parse_error( + "resolver.custom", + format!("Invalid custom resolver protocol {url:?}"), + ); + Protocol::Udp + } + }, + host.to_string(), + ) + } else { + (Protocol::Udp, url) + }; + let (host, port) = if let Some((host, port)) = host.split_once(':') { + ( + host.to_string(), + port.parse::() + .map_err(|err| { + config.new_parse_error( + "resolver.custom", + format!("Invalid custom resolver port {port:?}: {err}"), + ); + }) + .unwrap_or(53), + ) + } else { + (host, 53) + }; + let host = host + .parse::() + .map_err(|err| { + config.new_parse_error( + "resolver.custom", + format!("Invalid custom resolver IP {host:?}: {err}"), + ) + }) + .unwrap_or(IpAddr::V4(Ipv4Addr::new(8, 8, 8, 8))); + resolver_config + .add_name_server(NameServerConfig::new(SocketAddr::new(host, port), proto)); + } + if !resolver_config.name_servers().is_empty() { + (resolver_config, ResolverOpts::default()) + } else { + config.new_parse_error( + "resolver.custom", + "At least one custom resolver must be specified.", + ); + (ResolverConfig::cloudflare(), ResolverOpts::default()) + } + } + other => { + let err = format!("Unknown resolver type {other:?}."); + config.new_parse_error("resolver.custom", err); + (ResolverConfig::cloudflare(), ResolverOpts::default()) + } + }; + if let Some(concurrency) = config.property_("resolver.concurrency") { + opts.num_concurrent_reqs = concurrency; + } + if let Some(timeout) = config.property_("resolver.timeout") { + opts.timeout = timeout; + } + if let Some(preserve) = config.property_("resolver.preserve-intermediates") { + opts.preserve_intermediates = preserve; + } + if let Some(try_tcp_on_error) = config.property_("resolver.try-tcp-on-error") { + opts.try_tcp_on_error = try_tcp_on_error; + } + if let Some(attempts) = config.property_("resolver.attempts") { + opts.attempts = attempts; + } + + // Prepare DNSSEC resolver options + let config_dnssec = resolver_config.clone(); + let mut opts_dnssec = opts.clone(); + opts_dnssec.validate = true; + + let mut capacities = [1024usize; 5]; + for (pos, key) in ["txt", "mx", "ipv4", "ipv6", "ptr"].into_iter().enumerate() { + if let Some(capacity) = config.property_(("cache.resolver", key)) { + capacities[pos] = capacity; + } + } + + Resolvers { + dns: Resolver::with_capacities( + resolver_config, + opts, + capacities[0], + capacities[1], + capacities[2], + capacities[3], + capacities[4], + ) + .unwrap(), + dnssec: DnssecResolver { + resolver: AsyncResolver::tokio(config_dnssec, opts_dnssec), + }, + cache: DnsRecordCache { + tlsa: LruCache::with_capacity( + config.property_("cache.resolver.tlsa.size").unwrap_or(1024), + ), + mta_sts: LruCache::with_capacity( + config + .property_("cache.resolver.mta-sts.size") + .unwrap_or(1024), + ), + }, + psl: PublicSuffix::parse(config, "resolver.public-suffix").await, + } + } +} + impl Default for Resolvers { fn default() -> Self { let (config, opts) = match read_system_conf() { @@ -82,6 +235,7 @@ impl Default for Resolvers { tlsa: LruCache::with_capacity(1024), mta_sts: LruCache::with_capacity(1024), }, + psl: PublicSuffix::default(), } } } diff --git a/crates/common/src/config/smtp/session.rs b/crates/common/src/config/smtp/session.rs index 0052be13..f69c8165 100644 --- a/crates/common/src/config/smtp/session.rs +++ b/crates/common/src/config/smtp/session.rs @@ -1,8 +1,19 @@ -use std::{net::SocketAddr, time::Duration}; +use std::{ + net::{SocketAddr, ToSocketAddrs}, + time::Duration, +}; -use crate::expr::if_block::IfBlock; +use smtp_proto::*; +use utils::config::{ + utils::{AsKey, ParseValue}, + Config, +}; -use super::Throttle; +use crate::expr::{if_block::IfBlock, tokenizer::TokenMap, Constant, ConstantValue, Variable}; + +use self::throttle::parse_throttle; + +use super::*; pub struct SessionConfig { pub timeout: IfBlock, @@ -19,6 +30,7 @@ pub struct SessionConfig { pub extensions: Extensions, } +#[derive(Default)] pub struct SessionThrottle { pub connect: Vec, pub mail_from: Vec, @@ -27,6 +39,7 @@ pub struct SessionThrottle { pub struct Connect { pub script: IfBlock, + pub greeting: IfBlock, } pub struct Ehlo { @@ -76,7 +89,7 @@ pub struct Rcpt { // Limits pub max_recipients: IfBlock, - // Catch-all and subadressing + // Catch-all and sub-adressing pub catch_all: AddressMapping, pub subaddressing: AddressMapping, } @@ -108,6 +121,7 @@ pub struct Data { pub add_date: IfBlock, } +// Ceci n'est pas une pipe pub struct Pipe { pub command: IfBlock, pub arguments: IfBlock, @@ -137,6 +151,197 @@ pub enum MilterVersion { V6, } +impl SessionConfig { + pub fn parse(config: &mut Config) -> Self { + let has_conn_vars = + TokenMap::default().with_smtp_variables(&[V_LISTENER, V_REMOTE_IP, V_LOCAL_IP]); + let has_ehlo_hars = TokenMap::default().with_smtp_variables(&[ + V_LISTENER, + V_REMOTE_IP, + V_LOCAL_IP, + V_HELO_DOMAIN, + ]); + let has_sender_vars = TokenMap::default().with_smtp_variables(&[ + V_LISTENER, + V_REMOTE_IP, + V_LOCAL_IP, + V_SENDER, + V_SENDER_DOMAIN, + V_AUTHENTICATED_AS, + ]); + let has_rcpt_vars = TokenMap::default().with_smtp_variables(&[ + V_SENDER, + V_SENDER_DOMAIN, + V_RECIPIENT, + V_RECIPIENT_DOMAIN, + V_AUTHENTICATED_AS, + V_LISTENER, + V_REMOTE_IP, + V_LOCAL_IP, + V_HELO_DOMAIN, + ]); + + let mut session = SessionConfig::default(); + session.rcpt.catch_all = AddressMapping::parse(config, "session.rcpt.catch-all"); + session.rcpt.subaddressing = AddressMapping::parse(config, "session.rcpt.sub-addressing"); + session.data.milters = config + .sub_keys("session.data.milter", "") + .map(|s| s.to_string()) + .collect::>() + .into_iter() + .filter_map(|id| parse_milter(config, &id, &has_rcpt_vars)) + .collect(); + session.data.pipe_commands = config + .sub_keys("session.data.pipe", "") + .map(|s| s.to_string()) + .collect::>() + .into_iter() + .filter_map(|id| parse_pipe(config, &id, &has_rcpt_vars)) + .collect(); + session.throttle = SessionThrottle::parse(config); + + session + } +} + +impl SessionThrottle { + pub fn parse(config: &mut Config) -> Self { + let mut throttle = SessionThrottle::default(); + let all_throttles = parse_throttle( + config, + "session.throttle", + &TokenMap::default().with_smtp_variables(&[ + V_SENDER, + V_SENDER_DOMAIN, + V_RECIPIENT, + V_RECIPIENT_DOMAIN, + V_AUTHENTICATED_AS, + V_LISTENER, + V_REMOTE_IP, + V_LOCAL_IP, + V_PRIORITY, + V_HELO_DOMAIN, + ]), + THROTTLE_LISTENER + | THROTTLE_REMOTE_IP + | THROTTLE_LOCAL_IP + | THROTTLE_AUTH_AS + | THROTTLE_HELO_DOMAIN + | THROTTLE_RCPT + | THROTTLE_RCPT_DOMAIN + | THROTTLE_SENDER + | THROTTLE_SENDER_DOMAIN, + ); + for t in all_throttles { + if (t.keys & (THROTTLE_RCPT | THROTTLE_RCPT_DOMAIN)) != 0 + || t.expr.items().iter().any(|c| { + matches!( + c, + ExpressionItem::Variable(V_RECIPIENT | V_RECIPIENT_DOMAIN) + ) + }) + { + throttle.rcpt_to.push(t); + } else if (t.keys + & (THROTTLE_SENDER + | THROTTLE_SENDER_DOMAIN + | THROTTLE_HELO_DOMAIN + | THROTTLE_AUTH_AS)) + != 0 + || t.expr.items().iter().any(|c| { + matches!( + c, + ExpressionItem::Variable( + V_SENDER | V_SENDER_DOMAIN | V_HELO_DOMAIN | V_AUTHENTICATED_AS + ) + ) + }) + { + throttle.mail_from.push(t); + } else { + throttle.connect.push(t); + } + } + + throttle + } +} + +fn parse_pipe(config: &mut Config, id: &str, token_map: &TokenMap) -> Option { + Some(Pipe { + command: IfBlock::try_parse(config, ("session.data.pipe", id, "command"), token_map)?, + arguments: IfBlock::try_parse(config, ("session.data.pipe", id, "arguments"), token_map)?, + timeout: IfBlock::try_parse(config, ("session.data.pipe", id, "timeout"), token_map) + .unwrap_or_else(|| IfBlock::new(Duration::from_secs(30))), + }) +} + +fn parse_milter(config: &mut Config, id: &str, token_map: &TokenMap) -> Option { + let hostname = config + .value_require_(("session.data.milter", id, "hostname"))? + .to_string(); + let port = config.property_require_(("session.data.milter", id, "port"))?; + Some(Milter { + enable: IfBlock::try_parse(config, ("session.data.milter", id, "enable"), token_map) + .unwrap_or_default(), + addrs: format!("{}:{}", hostname, port) + .to_socket_addrs() + .map_err(|err| { + config.new_build_error( + ("session.data.milter", id, "hostname"), + format!("Unable to resolve milter hostname {hostname}: {err}"), + ) + }) + .ok()? + .collect(), + hostname, + port, + timeout_connect: config + .property_or_default_(("session.data.milter", id, "timeout.connect"), "30s") + .unwrap_or_else(|| Duration::from_secs(30)), + timeout_command: config + .property_or_default_(("session.data.milter", id, "timeout.command"), "30s") + .unwrap_or_else(|| Duration::from_secs(30)), + timeout_data: config + .property_or_default_(("session.data.milter", id, "timeout.data"), "60s") + .unwrap_or_else(|| Duration::from_secs(60)), + tls: config + .property_or_default_(("session.data.milter", id, "tls"), "false") + .unwrap_or_default(), + tls_allow_invalid_certs: config + .property_or_default_(("session.data.milter", id, "allow-invalid-certs"), "false") + .unwrap_or_default(), + tempfail_on_error: config + .property_or_default_( + ("session.data.milter", id, "options.tempfail-on-error"), + "true", + ) + .unwrap_or(true), + max_frame_len: config + .property_or_default_( + ("session.data.milter", id, "options.max-response-size"), + "52428800", + ) + .unwrap_or(52428800), + protocol_version: match config + .property_or_default::(("session.data.milter", id, "options.version"), "6") + .unwrap_or(6) + { + 6 => MilterVersion::V6, + 2 => MilterVersion::V2, + v => { + config.new_parse_error( + ("session.data.milter", id, "options.version"), + format!("Unsupported milter protocol version {v}"), + ); + MilterVersion::V6 + } + }, + flags_actions: config.property_(("session.data.milter", id, "options.flags.actions")), + flags_protocol: config.property_(("session.data.milter", id, "options.flags.protocol")), + }) +} + impl Default for SessionConfig { fn default() -> Self { Self { @@ -150,6 +355,7 @@ impl Default for SessionConfig { }, connect: Connect { script: Default::default(), + greeting: IfBlock::new("Stalwart ESMTP at your service".to_string()), }, ehlo: Ehlo { script: Default::default(), @@ -209,3 +415,109 @@ impl Default for SessionConfig { } } } + +#[derive(Default)] +pub struct Mechanism(u64); + +impl ParseValue for Mechanism { + fn parse_value(key: impl AsKey, value: &str) -> utils::config::Result { + Ok(Mechanism(match value.to_ascii_uppercase().as_str() { + "LOGIN" => AUTH_LOGIN, + "PLAIN" => AUTH_PLAIN, + "XOAUTH2" => AUTH_XOAUTH2, + "OAUTHBEARER" => AUTH_OAUTHBEARER, + /*"SCRAM-SHA-256-PLUS" => AUTH_SCRAM_SHA_256_PLUS, + "SCRAM-SHA-256" => AUTH_SCRAM_SHA_256, + "SCRAM-SHA-1-PLUS" => AUTH_SCRAM_SHA_1_PLUS, + "SCRAM-SHA-1" => AUTH_SCRAM_SHA_1, + "XOAUTH" => AUTH_XOAUTH, + "9798-M-DSA-SHA1" => AUTH_9798_M_DSA_SHA1, + "9798-M-ECDSA-SHA1" => AUTH_9798_M_ECDSA_SHA1, + "9798-M-RSA-SHA1-ENC" => AUTH_9798_M_RSA_SHA1_ENC, + "9798-U-DSA-SHA1" => AUTH_9798_U_DSA_SHA1, + "9798-U-ECDSA-SHA1" => AUTH_9798_U_ECDSA_SHA1, + "9798-U-RSA-SHA1-ENC" => AUTH_9798_U_RSA_SHA1_ENC, + "EAP-AES128" => AUTH_EAP_AES128, + "EAP-AES128-PLUS" => AUTH_EAP_AES128_PLUS, + "ECDH-X25519-CHALLENGE" => AUTH_ECDH_X25519_CHALLENGE, + "ECDSA-NIST256P-CHALLENGE" => AUTH_ECDSA_NIST256P_CHALLENGE, + "EXTERNAL" => AUTH_EXTERNAL, + "GS2-KRB5" => AUTH_GS2_KRB5, + "GS2-KRB5-PLUS" => AUTH_GS2_KRB5_PLUS, + "GSS-SPNEGO" => AUTH_GSS_SPNEGO, + "GSSAPI" => AUTH_GSSAPI, + "KERBEROS_V4" => AUTH_KERBEROS_V4, + "KERBEROS_V5" => AUTH_KERBEROS_V5, + "NMAS-SAMBA-AUTH" => AUTH_NMAS_SAMBA_AUTH, + "NMAS_AUTHEN" => AUTH_NMAS_AUTHEN, + "NMAS_LOGIN" => AUTH_NMAS_LOGIN, + "NTLM" => AUTH_NTLM, + "OAUTH10A" => AUTH_OAUTH10A, + "OPENID20" => AUTH_OPENID20, + "OTP" => AUTH_OTP, + "SAML20" => AUTH_SAML20, + "SECURID" => AUTH_SECURID, + "SKEY" => AUTH_SKEY, + "SPNEGO" => AUTH_SPNEGO, + "SPNEGO-PLUS" => AUTH_SPNEGO_PLUS, + "SXOVER-PLUS" => AUTH_SXOVER_PLUS, + "CRAM-MD5" => AUTH_CRAM_MD5, + "DIGEST-MD5" => AUTH_DIGEST_MD5, + "ANONYMOUS" => AUTH_ANONYMOUS,*/ + _ => { + return Err(format!( + "Unsupported mechanism {:?} for property {:?}.", + value, + key.as_key() + )) + } + })) + } +} + +impl<'x> TryFrom> for Mechanism { + type Error = (); + + fn try_from(value: Variable<'x>) -> Result { + match value { + Variable::Integer(value) => Ok(Mechanism(value as u64)), + Variable::Array(items) => { + let mut mechanism = 0; + + for item in items { + match item { + Variable::Integer(value) => mechanism |= value as u64, + _ => return Err(()), + } + } + + Ok(Mechanism(mechanism)) + } + _ => Err(()), + } + } +} + +impl From for Constant { + fn from(value: Mechanism) -> Self { + Constant::Integer(value.0 as i64) + } +} + +impl ConstantValue for Mechanism { + fn add_constants(token_map: &mut crate::expr::tokenizer::TokenMap) { + todo!() + } +} + +impl From for u64 { + fn from(value: Mechanism) -> Self { + value.0 + } +} + +impl From for Mechanism { + fn from(value: u64) -> Self { + Mechanism(value) + } +} diff --git a/crates/common/src/config/smtp/throttle.rs b/crates/common/src/config/smtp/throttle.rs new file mode 100644 index 00000000..f00553dc --- /dev/null +++ b/crates/common/src/config/smtp/throttle.rs @@ -0,0 +1,127 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use utils::config::{utils::AsKey, Config, Rate}; + +use crate::expr::{tokenizer::TokenMap, Expression}; + +use super::*; + +pub(crate) fn parse_throttle( + config: &mut Config, + prefix: impl AsKey, + token_map: &TokenMap, + available_throttle_keys: u16, +) -> Vec { + let prefix_ = prefix.as_key(); + let mut throttles = Vec::new(); + for throttle_id in config + .sub_keys(prefix, "") + .map(|s| s.to_string()) + .collect::>() + { + let throttle_id = throttle_id.as_str(); + if let Some(throttle) = parse_throttle_item( + config, + (&prefix_, throttle_id), + token_map, + available_throttle_keys, + ) { + throttles.push(throttle); + } + } + + throttles +} + +fn parse_throttle_item( + config: &mut Config, + prefix: impl AsKey, + token_map: &TokenMap, + available_throttle_keys: u16, +) -> Option { + let prefix = prefix.as_key(); + let mut keys = 0; + for (key_, value) in config + .values((&prefix, "key")) + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect::>() + { + match parse_throttle_key(&value) { + Ok(key) => { + if (key & available_throttle_keys) != 0 { + keys |= key; + } else { + let err = format!("Throttle key {value:?} is not available in this context"); + config.new_build_error(key_, err); + } + } + Err(err) => { + config.new_parse_error(key_, err); + } + } + } + + let throttle = Throttle { + expr: Expression::try_parse(config, (prefix.as_str(), "match"), token_map) + .unwrap_or_default(), + keys, + concurrency: config + .property_::((prefix.as_str(), "concurrency")) + .filter(|&v| v > 0), + rate: config + .property_::((prefix.as_str(), "rate")) + .filter(|v| v.requests > 0), + }; + + // Validate + if throttle.rate.is_none() && throttle.concurrency.is_none() { + config.new_parse_error( + prefix.as_str(), + concat!( + "Throttle needs to define a ", + "valid 'rate' and/or 'concurrency' property." + ) + .to_string(), + ); + None + } else { + Some(throttle) + } +} + +pub(crate) fn parse_throttle_key(value: &str) -> utils::config::Result { + match value { + "rcpt" => Ok(THROTTLE_RCPT), + "rcpt_domain" => Ok(THROTTLE_RCPT_DOMAIN), + "sender" => Ok(THROTTLE_SENDER), + "sender_domain" => Ok(THROTTLE_SENDER_DOMAIN), + "authenticated_as" => Ok(THROTTLE_AUTH_AS), + "listener" => Ok(THROTTLE_LISTENER), + "mx" => Ok(THROTTLE_MX), + "remote_ip" => Ok(THROTTLE_REMOTE_IP), + "local_ip" => Ok(THROTTLE_LOCAL_IP), + "helo_domain" => Ok(THROTTLE_HELO_DOMAIN), + _ => Err(format!("Invalid throttle key {value:?}")), + } +} diff --git a/crates/common/src/config/tracers.rs b/crates/common/src/config/tracers.rs new file mode 100644 index 00000000..9880a899 --- /dev/null +++ b/crates/common/src/config/tracers.rs @@ -0,0 +1,168 @@ +use std::{collections::HashMap, str::FromStr}; + +use opentelemetry_otlp::{HttpExporterBuilder, TonicExporterBuilder, WithExportConfig}; +use tracing::Level; +use tracing_appender::rolling::RollingFileAppender; +use utils::config::Config; + +pub enum Tracer { + Stdout { + level: Level, + ansi: bool, + }, + Log { + level: Level, + appender: RollingFileAppender, + ansi: bool, + }, + Journal { + level: Level, + }, + Otel { + level: Level, + tracer: OtelTracer, + }, +} + +pub enum OtelTracer { + Gprc(TonicExporterBuilder), + Http(HttpExporterBuilder), +} + +pub struct Tracers { + pub tracers: Vec, +} + +impl Tracers { + pub fn parse(config: &mut Config) -> Self { + let mut tracers = Vec::new(); + + for tracer_id in config + .sub_keys("tracer", ".type") + .map(|s| s.to_string()) + .collect::>() + { + let id = tracer_id.as_str(); + let level = Level::from_str(config.value(("tracer", id, "level")).unwrap_or("info")) + .map_err(|err| { + config.new_parse_error( + ("tracer", id, "level"), + format!("Invalid log level: {err}"), + ) + }) + .unwrap_or(Level::INFO); + match config + .value(("tracer", id, "type")) + .unwrap_or_default() + .to_string() + .as_str() + { + "log" => { + if let Some(path) = config + .value_require_(("tracer", id, "path")) + .map(|s| s.to_string()) + { + let prefix = config.value(("tracer", id, "prefix")).unwrap_or("stalwart"); + let appender = + match config.value(("tracer", id, "rotate")).unwrap_or("daily") { + "daily" => tracing_appender::rolling::daily(path, prefix), + "hourly" => tracing_appender::rolling::hourly(path, prefix), + "minutely" => tracing_appender::rolling::minutely(path, prefix), + "never" => tracing_appender::rolling::never(path, prefix), + rotate => { + let appender = tracing_appender::rolling::daily(path, prefix); + let err = format!("Invalid rotate value: {rotate}"); + config.new_parse_error(("tracer", id, "rotate"), err); + appender + } + }; + tracers.push(Tracer::Log { + level, + appender, + ansi: config + .property_or_default_(("tracer", id, "ansi"), "true") + .unwrap_or(true), + }); + } + } + "stdout" => { + tracers.push(Tracer::Stdout { + level, + ansi: config + .property_or_default_(("tracer", id, "ansi"), "true") + .unwrap_or(true), + }); + } + "otel" | "open-telemetry" => { + match config + .value_require_(("tracer", id, "transport")) + .unwrap_or_default() + { + "gprc" => { + let mut exporter = opentelemetry_otlp::new_exporter().tonic(); + if let Some(endpoint) = config.value(("tracer", id, "endpoint")) { + exporter = exporter.with_endpoint(endpoint); + } + tracers.push(Tracer::Otel { + level, + tracer: OtelTracer::Gprc(exporter), + }); + } + "http" => { + if let Some(endpoint) = config + .value_require_(("tracer", id, "endpoint")) + .map(|s| s.to_string()) + { + let mut headers = HashMap::new(); + let mut err = None; + for (_, value) in config.values(("tracer", id, "headers")) { + if let Some((key, value)) = value.split_once(':') { + headers.insert( + key.trim().to_string(), + value.trim().to_string(), + ); + } else { + err = format!("Invalid open-telemetry header {value:?}") + .into(); + break; + } + } + if let Some(err) = err { + config.new_parse_error(("tracer", id, "headers"), err); + } + + let mut exporter = opentelemetry_otlp::new_exporter() + .http() + .with_endpoint(endpoint); + if !headers.is_empty() { + exporter = exporter.with_headers(headers); + } + + tracers.push(Tracer::Otel { + level, + tracer: OtelTracer::Http(exporter), + }); + } + } + "" => {} + transport => { + let err = format!("Invalid transport: {transport}"); + config.new_parse_error(("tracer", id, "transport"), err); + } + } + } + "journal" => { + tracers.push(Tracer::Journal { level }); + } + unknown => { + config.new_parse_error( + ("tracer", id, "type"), + format!("Unknown tracer type: {unknown}"), + ); + } + } + } + + Tracers { tracers } + } +} diff --git a/crates/common/src/expr/if_block.rs b/crates/common/src/expr/if_block.rs index 933ba30a..ea98a401 100644 --- a/crates/common/src/expr/if_block.rs +++ b/crates/common/src/expr/if_block.rs @@ -28,6 +28,7 @@ use crate::expr::{Constant, Expression}; use super::{ parser::ExpressionParser, tokenizer::{TokenMap, Tokenizer}, + ExpressionItem, }; #[derive(Debug, Clone, Default)] @@ -60,8 +61,12 @@ impl IfBlock { } impl Expression { - pub fn try_parse(config: &mut Config, key: &str, token_map: &TokenMap) -> Option { - if let Some(expr) = config.value_or_warn(key) { + pub fn try_parse( + config: &mut Config, + key: impl AsKey, + token_map: &TokenMap, + ) -> Option { + if let Some(expr) = config.value(key.as_key()) { match ExpressionParser::new(Tokenizer::new(expr, token_map)).parse() { Ok(expr) => Some(expr), Err(err) => { @@ -194,4 +199,32 @@ impl IfBlock { Some(if_block) } } + + pub fn into_default(self, key: impl Into) -> IfBlock { + IfBlock { + key: key.into(), + if_then: Default::default(), + default: self.default, + } + } + + pub fn default_string(&self) -> Option<&str> { + for expr_item in &self.default.items { + if let ExpressionItem::Constant(Constant::String(value)) = expr_item { + return Some(value.as_str()); + } + } + + None + } + + pub fn into_default_string(self) -> Option { + for expr_item in self.default.items { + if let ExpressionItem::Constant(Constant::String(value)) = expr_item { + return Some(value); + } + } + + None + } } diff --git a/crates/common/src/expr/mod.rs b/crates/common/src/expr/mod.rs index ca0e6393..a135bc1e 100644 --- a/crates/common/src/expr/mod.rs +++ b/crates/common/src/expr/mod.rs @@ -26,6 +26,8 @@ use std::{borrow::Cow, time::Duration}; use regex::Regex; use utils::config::utils::ParseValue; +use self::tokenizer::TokenMap; + pub mod eval; pub mod functions; pub mod if_block; @@ -289,9 +291,12 @@ impl Eq for Token {} pub trait ConstantValue: ParseValue + for<'x> TryFrom> + Into + Sized { + fn add_constants(token_map: &mut TokenMap); } -impl ConstantValue for Duration {} +impl ConstantValue for Duration { + fn add_constants(_: &mut TokenMap) {} +} impl<'x> TryFrom> for Duration { type Error = (); diff --git a/crates/common/src/expr/tokenizer.rs b/crates/common/src/expr/tokenizer.rs index 48f735ff..e62b27e7 100644 --- a/crates/common/src/expr/tokenizer.rs +++ b/crates/common/src/expr/tokenizer.rs @@ -29,7 +29,7 @@ use utils::config::utils::ParseValue; use super::{ functions::{ASYNC_FUNCTIONS, FUNCTIONS}, - BinaryOperator, Constant, Token, UnaryOperator, + BinaryOperator, Constant, ConstantValue, Token, UnaryOperator, }; pub struct Tokenizer<'x> { @@ -45,9 +45,9 @@ pub struct Tokenizer<'x> { is_eof: bool, } -#[derive(Debug, Default)] +#[derive(Debug, Default, Clone)] pub struct TokenMap { - tokens: AHashMap<&'static str, Token>, + pub tokens: AHashMap<&'static str, Token>, } impl<'x> Tokenizer<'x> { @@ -359,7 +359,7 @@ impl TokenMap { self } - pub fn with_constants(mut self, consts: I) -> Self + pub fn set_constants(mut self, consts: I) -> Self where I: IntoIterator, T: Into, @@ -370,4 +370,14 @@ impl TokenMap { self } + + pub fn with_constants(mut self) -> Self { + T::add_constants(&mut self); + self + } + + pub fn add_constant(&mut self, name: &'static str, constant: impl Into) -> &mut Self { + self.tokens.insert(name, Token::Constant(constant.into())); + self + } } diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 69c09e0a..ff3d25f3 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -2,7 +2,9 @@ use std::{net::IpAddr, sync::Arc}; use ahash::AHashMap; use config::{ - scripts::SieveCore, + imap::ImapConfig, + jmap::settings::JmapConfig, + scripts::Scripting, server::Server, smtp::{ auth::{ArcSealer, DkimSigner}, @@ -10,23 +12,46 @@ use config::{ SmtpConfig, }, storage::Storage, + tracers::{OtelTracer, Tracer, Tracers}, }; use directory::{Directory, Principal, QueryBy}; +use expr::if_block::IfBlock; use listener::{acme::AcmeManager, blocked::BlockedIps, tls::Certificate}; use mail_send::Credentials; +use opentelemetry::KeyValue; +use opentelemetry_sdk::{ + trace::{self, Sampler}, + Resource, +}; +use opentelemetry_semantic_conventions::resource::{SERVICE_NAME, SERVICE_VERSION}; use sieve::Sieve; use store::LookupStore; +use tracing::{level_filters::LevelFilter, Level}; +use tracing_appender::non_blocking::WorkerGuard; +use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter, Layer}; pub mod addresses; pub mod config; pub mod expr; pub mod listener; +pub mod scripts; + +pub static USER_AGENT: &str = concat!("StalwartMail/", env!("CARGO_PKG_VERSION"),); +pub static DAEMON_NAME: &str = concat!("Stalwart Mail Server v", env!("CARGO_PKG_VERSION"),); pub struct Core { pub storage: Storage, - pub sieve: SieveCore, + pub sieve: Scripting, + pub network: Network, pub smtp: SmtpConfig, + pub jmap: JmapConfig, + pub imap: ImapConfig, +} + +pub struct Network { pub blocked_ips: BlockedIps, + pub hostname: IfBlock, + pub url: IfBlock, } pub struct ConfigBuilder { @@ -34,6 +59,7 @@ pub struct ConfigBuilder { pub certificates: AHashMap>, pub certificates_sni: AHashMap>, pub acme_managers: AHashMap>, + pub tracers: Vec, pub core: Core, } @@ -43,6 +69,17 @@ pub enum AuthResult { Banned, } +pub trait IntoString: Sized { + fn into_string(self) -> String; +} + +impl IntoString for Vec { + fn into_string(self) -> String { + String::from_utf8(self) + .unwrap_or_else(|err| String::from_utf8_lossy(err.as_bytes()).into_owned()) + } +} + impl Core { pub fn get_directory(&self, name: &str) -> Option<&Arc> { self.storage.directories.get(name) @@ -172,3 +209,141 @@ impl Core { } } } + +#[derive(Default)] +pub struct TracerResult { + pub guards: Vec, + pub errors: Vec, +} + +impl Tracers { + pub fn enable(self) -> TracerResult { + let mut layers = Vec::new(); + let mut level = Level::TRACE; + + for tracer in &self.tracers { + let tracer_level = *match tracer { + Tracer::Stdout { level, .. } + | Tracer::Log { level, .. } + | Tracer::Journal { level } + | Tracer::Otel { level, .. } => level, + }; + + if tracer_level > level { + level = tracer_level; + } + } + + let mut result = TracerResult::default(); + match EnvFilter::builder().parse(format!( + "smtp={level},imap={level},jmap={level},store={level},utils={level},directory={level}" + )) { + Ok(layer) => { + layers.push(layer.boxed()); + } + Err(err) => { + result + .errors + .push(format!("Failed to set env filter: {err}")); + } + } + + for tracer in self.tracers { + match tracer { + Tracer::Stdout { level, ansi } => { + layers.push( + tracing_subscriber::fmt::layer() + .with_ansi(ansi) + .with_filter(LevelFilter::from_level(level)) + .boxed(), + ); + } + Tracer::Log { + level, + appender, + ansi, + } => { + let (non_blocking, guard) = tracing_appender::non_blocking(appender); + result.guards.push(guard); + layers.push( + tracing_subscriber::fmt::layer() + .with_writer(non_blocking) + .with_ansi(ansi) + .with_filter(LevelFilter::from_level(level)) + .boxed(), + ); + } + Tracer::Otel { level, tracer } => { + let tracer = match tracer { + OtelTracer::Gprc(exporter) => opentelemetry_otlp::new_pipeline() + .tracing() + .with_exporter(exporter), + OtelTracer::Http(exporter) => opentelemetry_otlp::new_pipeline() + .tracing() + .with_exporter(exporter), + } + .with_trace_config( + trace::config() + .with_resource(Resource::new(vec![ + KeyValue::new(SERVICE_NAME, "stalwart-mail".to_string()), + KeyValue::new( + SERVICE_VERSION, + env!("CARGO_PKG_VERSION").to_string(), + ), + ])) + .with_sampler(Sampler::AlwaysOn), + ) + .install_batch(opentelemetry_sdk::runtime::Tokio); + + match tracer { + Ok(tracer) => { + layers.push( + tracing_opentelemetry::layer() + .with_tracer(tracer) + .with_filter(LevelFilter::from_level(level)) + .boxed(), + ); + } + Err(err) => { + result + .errors + .push(format!("Failed to start OpenTelemetry: {err}")); + } + } + } + Tracer::Journal { level } => { + #[cfg(unix)] + { + match tracing_journald::layer() { + Ok(layer) => { + layers.push( + layer.with_filter(LevelFilter::from_level(level)).boxed(), + ); + } + Err(err) => { + result + .errors + .push(format!("Failed to start Journald: {err}")); + } + } + } + + #[cfg(not(unix))] + { + result + .errors + .push("Journald is only available on Unix systems.".to_string()); + } + } + } + } + + if let Err(err) = tracing_subscriber::registry().with(layers).try_init() { + result + .errors + .push(format!("Failed to start tracing: {err}")); + } + + result + } +} diff --git a/crates/common/src/listener/blocked.rs b/crates/common/src/listener/blocked.rs index 001faaa3..154824a7 100644 --- a/crates/common/src/listener/blocked.rs +++ b/crates/common/src/listener/blocked.rs @@ -78,7 +78,7 @@ impl BlockedIps { impl Core { pub async fn is_fail2banned(&self, ip: IpAddr, login: String) -> store::Result { - if let Some(rate) = &self.blocked_ips.limiter_rate { + if let Some(rate) = &self.network.blocked_ips.limiter_rate { let is_allowed = self .storage .lookup @@ -93,7 +93,7 @@ impl Core { .is_none(); if !is_allowed { // Add IP to blocked list - self.blocked_ips.ip_addresses.write().insert(ip); + self.network.blocked_ips.ip_addresses.write().insert(ip); // Write blocked IP to config self.storage @@ -112,13 +112,14 @@ impl Core { } pub fn has_fail2ban(&self) -> bool { - self.blocked_ips.limiter_rate.is_some() + self.network.blocked_ips.limiter_rate.is_some() } pub fn is_ip_blocked(&self, ip: &IpAddr) -> bool { - self.blocked_ips.ip_addresses.read().contains(ip) - || (self.blocked_ips.has_networks + self.network.blocked_ips.ip_addresses.read().contains(ip) + || (self.network.blocked_ips.has_networks && self + .network .blocked_ips .ip_networks .iter() diff --git a/crates/common/src/scripts/functions/array.rs b/crates/common/src/scripts/functions/array.rs new file mode 100644 index 00000000..46a6e9b7 --- /dev/null +++ b/crates/common/src/scripts/functions/array.rs @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::collections::{HashMap, HashSet}; + +use sieve::{runtime::Variable, Context}; + +pub fn fn_count<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + match &v[0] { + Variable::Array(a) => a.len(), + v => { + if !v.is_empty() { + 1 + } else { + 0 + } + } + } + .into() +} + +pub fn fn_sort<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + let is_asc = v[1].to_bool(); + let mut arr = (*v[0].to_array()).clone(); + if is_asc { + arr.sort_unstable_by(|a, b| b.cmp(a)); + } else { + arr.sort_unstable(); + } + arr.into() +} + +pub fn fn_dedup<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + let arr = v[0].to_array(); + let mut result = Vec::with_capacity(arr.len()); + + for item in arr.iter() { + if !result.contains(item) { + result.push(item.clone()); + } + } + + result.into() +} + +pub fn fn_cosine_similarity<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + let mut word_freq: HashMap = HashMap::new(); + + for (idx, var) in v.into_iter().enumerate() { + match var { + Variable::Array(l) => { + for item in l.iter() { + word_freq.entry(item.clone()).or_insert([0, 0])[idx] += 1; + } + } + _ => { + for char in var.to_string().chars() { + word_freq.entry(char.to_string().into()).or_insert([0, 0])[idx] += 1; + } + } + } + } + + let mut dot_product = 0; + let mut magnitude_a = 0; + let mut magnitude_b = 0; + + for (_word, count) in word_freq.iter() { + dot_product += count[0] * count[1]; + magnitude_a += count[0] * count[0]; + magnitude_b += count[1] * count[1]; + } + + if magnitude_a != 0 && magnitude_b != 0 { + dot_product as f64 / (magnitude_a as f64).sqrt() / (magnitude_b as f64).sqrt() + } else { + 0.0 + } + .into() +} + +pub fn fn_jaccard_similarity<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + let mut word_freq = [HashSet::new(), HashSet::new()]; + + for (idx, var) in v.into_iter().enumerate() { + match var { + Variable::Array(l) => { + for item in l.iter() { + word_freq[idx].insert(item.clone()); + } + } + _ => { + for char in var.to_string().chars() { + word_freq[idx].insert(char.to_string().into()); + } + } + } + } + + let intersection_size = word_freq[0].intersection(&word_freq[1]).count(); + let union_size = word_freq[0].union(&word_freq[1]).count(); + + if union_size != 0 { + intersection_size as f64 / union_size as f64 + } else { + 0.0 + } + .into() +} + +pub fn fn_is_intersect<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + match (&v[0], &v[1]) { + (Variable::Array(a), Variable::Array(b)) => a.iter().any(|x| b.contains(x)), + (Variable::Array(a), item) | (item, Variable::Array(a)) => a.contains(item), + _ => false, + } + .into() +} + +pub fn fn_winnow<'x>(_: &'x Context<'x, ()>, mut v: Vec) -> Variable { + match v.remove(0) { + Variable::Array(a) => a + .iter() + .filter(|i| !i.is_empty()) + .cloned() + .collect::>() + .into(), + v => v, + } +} diff --git a/crates/common/src/scripts/functions/email.rs b/crates/common/src/scripts/functions/email.rs new file mode 100644 index 00000000..d522e339 --- /dev/null +++ b/crates/common/src/scripts/functions/email.rs @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use sieve::{runtime::Variable, Context}; + +use super::ApplyString; + +pub fn fn_is_email<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + let mut last_ch = 0; + let mut in_quote = false; + let mut at_count = 0; + let mut dot_count = 0; + let mut lp_len = 0; + let mut value = 0; + + for ch in v[0].to_string().bytes() { + match ch { + b'0'..=b'9' + | b'a'..=b'z' + | b'A'..=b'Z' + | b'!' + | b'#' + | b'$' + | b'%' + | b'&' + | b'\'' + | b'*' + | b'+' + | b'-' + | b'/' + | b'=' + | b'?' + | b'^' + | b'_' + | b'`' + | b'{' + | b'|' + | b'}' + | b'~' + | 0x7f..=u8::MAX => { + value += 1; + } + b'.' if !in_quote => { + if last_ch != b'.' && last_ch != b'@' && value != 0 { + value += 1; + if at_count == 1 { + dot_count += 1; + } + } else { + return false.into(); + } + } + b'@' if !in_quote => { + at_count += 1; + lp_len = value; + value = 0; + } + b'>' | b':' | b',' | b' ' if in_quote => { + value += 1; + } + b'\"' if !in_quote || last_ch != b'\\' => { + in_quote = !in_quote; + } + b'\\' if in_quote && last_ch != b'\\' => (), + _ => { + if !in_quote { + return false.into(); + } + } + } + + last_ch = ch; + } + + (at_count == 1 && dot_count > 0 && lp_len > 0 && value > 0).into() +} + +pub fn fn_email_part<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + v[0].transform(|s| { + s.rsplit_once('@') + .map(|(u, d)| match v[1].to_string().as_ref() { + "local" => Variable::from(u.trim()), + "domain" => Variable::from(d.trim()), + _ => Variable::default(), + }) + .unwrap_or_default() + }) +} diff --git a/crates/common/src/scripts/functions/header.rs b/crates/common/src/scripts/functions/header.rs new file mode 100644 index 00000000..fd08e4af --- /dev/null +++ b/crates/common/src/scripts/functions/header.rs @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use mail_parser::{parsers::fields::thread::thread_name, HeaderName, HeaderValue, MimeHeaders}; +use sieve::{compiler::ReceivedPart, runtime::Variable, Context}; + +use super::ApplyString; + +pub fn fn_received_part<'x>(ctx: &'x Context<'x, ()>, v: Vec) -> Variable { + if let (Ok(part), Some(HeaderValue::Received(rcvd))) = ( + ReceivedPart::try_from(v[1].to_string().as_ref()), + ctx.message() + .part(ctx.part()) + .and_then(|p| { + p.headers + .iter() + .filter(|h| h.name == HeaderName::Received) + .nth((v[0].to_integer() as usize).saturating_sub(1)) + }) + .map(|h| &h.value), + ) { + part.eval(rcvd).unwrap_or_default() + } else { + Variable::default() + } +} + +pub fn fn_is_encoding_problem<'x>(ctx: &'x Context<'x, ()>, _: Vec) -> Variable { + ctx.message() + .part(ctx.part()) + .map(|p| p.is_encoding_problem) + .unwrap_or_default() + .into() +} + +pub fn fn_is_attachment<'x>(ctx: &'x Context<'x, ()>, _: Vec) -> Variable { + ctx.message().attachments.contains(&ctx.part()).into() +} + +pub fn fn_is_body<'x>(ctx: &'x Context<'x, ()>, _: Vec) -> Variable { + (ctx.message().text_body.contains(&ctx.part()) || ctx.message().html_body.contains(&ctx.part())) + .into() +} + +pub fn fn_attachment_name<'x>(ctx: &'x Context<'x, ()>, _: Vec) -> Variable { + ctx.message() + .part(ctx.part()) + .and_then(|p| p.attachment_name()) + .unwrap_or_default() + .into() +} + +pub fn fn_mime_part_len<'x>(ctx: &'x Context<'x, ()>, _: Vec) -> Variable { + ctx.message() + .part(ctx.part()) + .map(|p| p.len()) + .unwrap_or_default() + .into() +} + +pub fn fn_thread_name<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + v[0].transform(|s| thread_name(s).into()) +} + +pub fn fn_is_header_utf8_valid<'x>(ctx: &'x Context<'x, ()>, v: Vec) -> Variable { + ctx.message() + .part(ctx.part()) + .map(|p| { + let raw = ctx.message().raw_message(); + let mut is_valid = true; + if let Some(header_name) = HeaderName::parse(v[0].to_string().as_ref()) { + for header in &p.headers { + if header.name == header_name + && raw + .get(header.offset_start()..header.offset_end()) + .and_then(|raw| std::str::from_utf8(raw).ok()) + .is_none() + { + is_valid = false; + break; + } + } + } else { + is_valid = raw + .get(p.raw_header_offset()..p.raw_body_offset()) + .and_then(|raw| std::str::from_utf8(raw).ok()) + .is_some(); + } + + Variable::from(is_valid) + }) + .unwrap_or(Variable::Integer(1)) +} diff --git a/crates/common/src/scripts/functions/html.rs b/crates/common/src/scripts/functions/html.rs new file mode 100644 index 00000000..f0c8cb20 --- /dev/null +++ b/crates/common/src/scripts/functions/html.rs @@ -0,0 +1,439 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::borrow::Cow; + +use mail_parser::decoders::html::{add_html_token, html_to_text}; +use sieve::{runtime::Variable, Context}; + +pub fn fn_html_to_text<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + html_to_text(v[0].to_string().as_ref()).into() +} + +pub fn fn_html_has_tag<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + v[0].as_array() + .map(|arr| { + let token = v[1].to_string(); + arr.iter().any(|v| { + v.to_string() + .as_ref() + .strip_prefix('<') + .map_or(false, |tag| tag.starts_with(token.as_ref())) + }) + }) + .unwrap_or_default() + .into() +} + +pub fn fn_html_attr_size<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + let t = v[0].to_string(); + let mut dimension = None; + + if let Some(value) = get_attribute(t.as_ref(), v[1].to_string().as_ref()) { + let value = value.trim(); + if let Some(pct) = value.strip_suffix('%') { + if let Ok(pct) = pct.trim().parse::() { + dimension = ((v[2].to_integer() * pct as i64) / 100).into(); + } + } else if let Ok(value) = value.parse::() { + dimension = (value as i64).into(); + } + } + + dimension.map(Variable::Integer).unwrap_or_default() +} + +pub fn fn_html_attrs<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + html_attr_tokens( + v[0].to_string().as_ref(), + v[1].to_string().as_ref(), + v[2].to_string_array(), + ) + .into() +} + +pub fn fn_html_attr<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + get_attribute(v[0].to_string().as_ref(), v[1].to_string().as_ref()) + .map(Variable::from) + .unwrap_or_default() +} + +pub fn html_to_tokens(input: &str) -> Vec { + let input = input.as_bytes(); + let mut iter = input.iter().enumerate(); + let mut tags = vec![]; + + let mut is_token_start = true; + let mut is_after_space = false; + let mut is_new_line = true; + + let mut token_start = 0; + let mut token_end = 0; + + let mut text = String::from("_"); + + while let Some((pos, &ch)) = iter.next() { + match ch { + b'<' => { + if !is_token_start { + add_html_token( + &mut text, + &input[token_start..token_end + 1], + is_after_space, + ); + is_after_space = false; + is_token_start = true; + } + if text.len() > 1 { + tags.push(Variable::String(text.into())); + text = String::from("_"); + } + + let mut tag = vec![b'<']; + if matches!(input.get(pos + 1..pos + 4), Some(b"!--")) { + let mut last_ch: u8 = 0; + for (_, &ch) in iter.by_ref() { + match ch { + b'>' if tag.len() > 3 + && matches!(tag.last(), Some(b'-')) + && matches!(tag.get(tag.len() - 2), Some(b'-')) => + { + break; + } + b' ' | b'\t' | b'\r' | b'\n' => { + if last_ch != b' ' { + tag.push(b' '); + } else { + last_ch = b' '; + } + continue; + } + _ => { + tag.push(ch); + } + } + last_ch = ch; + } + } else { + let mut in_quote = false; + let mut last_ch = b' '; + for (_, &ch) in iter.by_ref() { + match ch { + b'>' if !in_quote => { + break; + } + b'"' => { + in_quote = !in_quote; + tag.push(b'"'); + } + b' ' | b'\t' | b'\r' | b'\n' if !in_quote => { + if last_ch != b' ' { + tag.push(b' '); + last_ch = b' '; + } + continue; + } + b'/' if !in_quote => { + tag.push(b'/'); + last_ch = b' '; + continue; + } + _ => { + tag.push(if in_quote { + ch + } else { + ch.to_ascii_lowercase() + }); + } + } + last_ch = ch; + } + } + tags.push(Variable::String( + String::from_utf8(tag).unwrap_or_default().into(), + )); + continue; + } + b' ' | b'\t' | b'\r' | b'\n' => { + if !is_token_start { + add_html_token( + &mut text, + &input[token_start..token_end + 1], + is_after_space && !is_new_line, + ); + is_new_line = false; + } + is_after_space = true; + is_token_start = true; + continue; + } + b'&' if !is_token_start => { + add_html_token( + &mut text, + &input[token_start..token_end + 1], + is_after_space && !is_new_line, + ); + is_new_line = false; + is_token_start = true; + is_after_space = false; + } + b';' if !is_token_start => { + add_html_token( + &mut text, + &input[token_start..pos + 1], + is_after_space && !is_new_line, + ); + is_token_start = true; + is_after_space = false; + is_new_line = false; + continue; + } + _ => (), + } + + if is_token_start { + token_start = pos; + is_token_start = false; + } + token_end = pos; + } + + if !is_token_start { + add_html_token( + &mut text, + &input[token_start..token_end + 1], + is_after_space && !is_new_line, + ); + } + if text.len() > 1 { + tags.push(Variable::String(text.into())); + } + + tags +} + +pub fn html_attr_tokens(input: &str, tag: &str, attrs: Vec>) -> Vec { + let input = input.as_bytes(); + let mut iter = input.iter().enumerate().peekable(); + let mut tags = vec![]; + + while let Some((mut pos, &ch)) = iter.next() { + if ch == b'<' { + if !matches!(input.get(pos + 1..pos + 4), Some(b"!--")) { + let mut in_quote = false; + let mut last_ch_pos: usize = 0; + + while matches!(iter.peek(), Some((_, &ch)) if ch.is_ascii_whitespace()) { + pos += 1; + iter.next(); + } + + let found_tag = tag.is_empty() + || (matches!(input.get(pos + 1..pos + tag.len() + 1), Some(t) if t.eq_ignore_ascii_case(tag.as_bytes())) + && matches!(input.get(pos + tag.len() + 1), Some(ch) if ch.is_ascii_whitespace())); + + 'outer: while let Some((pos, &ch)) = iter.next() { + match ch { + b'>' if !in_quote => { + break; + } + b'"' => { + in_quote = !in_quote; + } + b'=' if found_tag + && !in_quote + && attrs.iter().any(|attr| matches!(input.get(last_ch_pos.saturating_sub(attr.len()) + 1..last_ch_pos + 1), Some(a) if a.eq_ignore_ascii_case(attr.as_bytes()))) + && matches!(input.get(last_ch_pos + 1), Some(ch) if ch.is_ascii_whitespace() || *ch == b'=') => + { + while matches!(iter.peek(), Some((_, &ch)) if ch.is_ascii_whitespace()) + { + iter.next(); + } + let mut tag = vec![]; + + for (_, &ch) in iter.by_ref() { + match ch { + b'>' if !in_quote => { + if !tag.is_empty() { + tags.push(Variable::String( + String::from_utf8(tag).unwrap_or_default().into(), + )); + } + break 'outer; + } + b'"' => { + if in_quote { + in_quote = false; + break; + } else { + in_quote = true; + } + } + b' ' | b'\t' | b'\r' | b'\n' if !in_quote => { + break; + } + _ => { + tag.push(ch); + } + } + } + + if !tag.is_empty() { + tags.push(Variable::String( + String::from_utf8(tag).unwrap_or_default().into(), + )); + } + } + b' ' | b'\t' | b'\r' | b'\n' => {} + _ => { + last_ch_pos = pos; + } + } + } + } else { + let mut last_ch: u8 = 0; + let mut before_last_ch: u8 = 0; + + for (_, &ch) in iter.by_ref() { + if ch == b'>' && last_ch == b'-' && before_last_ch == b'-' { + break; + } + before_last_ch = last_ch; + last_ch = ch; + } + } + } + } + + tags +} + +pub fn html_img_area(arr: &[Variable]) -> u32 { + arr.iter() + .filter_map(|v| { + let t = v.to_string(); + if t.starts_with("() { + let size = if idx == 0 { 800 } else { 600 }; + dimensions[idx] = (size * pct) / 100; + } + } else if let Ok(value) = value.parse::() { + dimensions[idx] = value; + } + } + } + + Some(dimensions[0].saturating_mul(dimensions[1])) + } else { + None + } + }) + .sum::() +} + +pub fn get_attribute<'x>(tag: &'x str, attr_name: &str) -> Option<&'x str> { + let tag = tag.as_bytes(); + let attr_name = attr_name.as_bytes(); + let mut iter = tag.iter().enumerate().peekable(); + let mut in_quote = false; + let mut start_pos = usize::MAX; + let mut end_pos = usize::MAX; + + while let Some((pos, ch)) = iter.next() { + match ch { + b'=' if !in_quote => { + if start_pos != usize::MAX + && end_pos != usize::MAX + && tag + .get(start_pos..end_pos + 1) + .map_or(false, |name| name == attr_name) + { + let mut token_start = 0; + let mut token_end = 0; + + for (pos, ch) in iter.by_ref() { + match ch { + b'"' => { + if !in_quote { + token_start = pos + 1; + in_quote = true; + } else { + token_end = pos; + break; + } + } + b' ' if !in_quote => { + if token_start != 0 { + token_end = pos; + break; + } + } + _ => { + if token_start == 0 { + token_start = pos; + } + } + } + } + + return if token_start > 0 { + if token_end == 0 { + token_end = tag.len(); + } + Some(std::str::from_utf8(&tag[token_start..token_end]).unwrap_or_default()) + } else { + None + }; + } else { + start_pos = usize::MAX; + end_pos = usize::MAX; + } + } + b'"' => { + in_quote = !in_quote; + } + b' ' => { + if !in_quote && !matches!(iter.peek(), Some((_, b'='))) { + start_pos = usize::MAX; + end_pos = usize::MAX; + } + } + _ => { + if !in_quote { + if start_pos == usize::MAX { + start_pos = pos; + } + end_pos = pos; + } + } + } + } + + None +} diff --git a/crates/common/src/scripts/functions/image.rs b/crates/common/src/scripts/functions/image.rs new file mode 100644 index 00000000..ea8d4bc6 --- /dev/null +++ b/crates/common/src/scripts/functions/image.rs @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use sieve::{runtime::Variable, Context}; + +pub fn fn_img_metadata<'x>(ctx: &'x Context<'x, ()>, v: Vec) -> Variable { + ctx.message() + .part(ctx.part()) + .map(|p| p.contents()) + .and_then(|bytes| { + let arg = v[1].to_string(); + match arg.as_ref() { + "type" => imagesize::image_type(bytes).ok().map(|t| { + Variable::from(match t { + imagesize::ImageType::Aseprite => "aseprite", + imagesize::ImageType::Avif => "avif", + imagesize::ImageType::Bmp => "bmp", + imagesize::ImageType::Dds => "dds", + imagesize::ImageType::Exr => "exr", + imagesize::ImageType::Farbfeld => "farbfeld", + imagesize::ImageType::Gif => "gif", + imagesize::ImageType::Hdr => "hdr", + imagesize::ImageType::Heif => "heif", + imagesize::ImageType::Ico => "ico", + imagesize::ImageType::Jpeg => "jpeg", + imagesize::ImageType::Jxl => "jxl", + imagesize::ImageType::Ktx2 => "ktx2", + imagesize::ImageType::Png => "png", + imagesize::ImageType::Pnm => "pnm", + imagesize::ImageType::Psd => "psd", + imagesize::ImageType::Qoi => "qoi", + imagesize::ImageType::Tga => "tga", + imagesize::ImageType::Tiff => "tiff", + imagesize::ImageType::Vtf => "vtf", + imagesize::ImageType::Webp => "webp", + }) + }), + "width" => imagesize::blob_size(bytes) + .ok() + .map(|s| Variable::Integer(s.width as i64)), + "height" => imagesize::blob_size(bytes) + .ok() + .map(|s| Variable::Integer(s.height as i64)), + "area" => imagesize::blob_size(bytes) + .ok() + .map(|s| Variable::Integer(s.width.saturating_mul(s.height) as i64)), + "dimension" => imagesize::blob_size(bytes) + .ok() + .map(|s| Variable::Integer(s.width.saturating_add(s.height) as i64)), + _ => None, + } + }) + .unwrap_or_default() +} diff --git a/crates/common/src/scripts/functions/misc.rs b/crates/common/src/scripts/functions/misc.rs new file mode 100644 index 00000000..fc7690ab --- /dev/null +++ b/crates/common/src/scripts/functions/misc.rs @@ -0,0 +1,121 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::net::IpAddr; + +use mail_auth::common::resolver::ToReverseName; +use sha1::Sha1; +use sha2::{Sha256, Sha512}; +use sieve::{runtime::Variable, Context}; + +use super::ApplyString; + +pub fn fn_is_empty<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + match &v[0] { + Variable::String(s) => s.is_empty(), + Variable::Integer(_) | Variable::Float(_) => false, + Variable::Array(a) => a.is_empty(), + } + .into() +} + +pub fn fn_is_number<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + matches!(&v[0], Variable::Integer(_) | Variable::Float(_)).into() +} + +pub fn fn_is_ip_addr<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + v[0].to_string().parse::().is_ok().into() +} + +pub fn fn_is_ipv4_addr<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + v[0].to_string() + .parse::() + .map_or(false, |ip| matches!(ip, IpAddr::V4(_))) + .into() +} + +pub fn fn_is_ipv6_addr<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + v[0].to_string() + .parse::() + .map_or(false, |ip| matches!(ip, IpAddr::V6(_))) + .into() +} + +pub fn fn_ip_reverse_name<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + v[0].to_string() + .parse::() + .map(|ip| ip.to_reverse_name()) + .unwrap_or_default() + .into() +} + +pub fn fn_detect_file_type<'x>(ctx: &'x Context<'x, ()>, v: Vec) -> Variable { + ctx.message() + .part(ctx.part()) + .and_then(|p| infer::get(p.contents())) + .map(|t| { + Variable::from( + if v[0].to_string() != "ext" { + t.mime_type() + } else { + t.extension() + } + .to_string(), + ) + }) + .unwrap_or_default() +} + +pub fn fn_hash<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + use sha1::Digest; + let hash = v[1].to_string(); + + v[0].transform(|value| match hash.as_ref() { + "md5" => format!("{:x}", md5::compute(value.as_bytes())).into(), + "sha1" => { + let mut hasher = Sha1::new(); + hasher.update(value.as_bytes()); + format!("{:x}", hasher.finalize()).into() + } + "sha256" => { + let mut hasher = Sha256::new(); + hasher.update(value.as_bytes()); + format!("{:x}", hasher.finalize()).into() + } + "sha512" => { + let mut hasher = Sha512::new(); + hasher.update(value.as_bytes()); + format!("{:x}", hasher.finalize()).into() + } + _ => Variable::default(), + }) +} + +pub fn fn_is_var_names<'x>(ctx: &'x Context<'x, ()>, _: Vec) -> Variable { + Variable::Array( + ctx.global_variable_names() + .map(|v| Variable::from(v.to_uppercase())) + .collect::>() + .into(), + ) +} diff --git a/crates/common/src/scripts/functions/mod.rs b/crates/common/src/scripts/functions/mod.rs new file mode 100644 index 00000000..02d8dcad --- /dev/null +++ b/crates/common/src/scripts/functions/mod.rs @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +mod array; +mod email; +mod header; +pub mod html; +mod image; +mod misc; +pub mod text; +mod unicode; +mod url; + +use sieve::{runtime::Variable, FunctionMap}; + +use self::{ + array::*, email::*, header::*, html::*, image::*, misc::*, text::*, unicode::*, url::*, +}; + +pub fn register_functions() -> FunctionMap<()> { + FunctionMap::new() + .with_function("trim", fn_trim) + .with_function("trim_start", fn_trim_start) + .with_function("trim_end", fn_trim_end) + .with_function("len", fn_len) + .with_function("count", fn_count) + .with_function("is_empty", fn_is_empty) + .with_function("is_number", fn_is_number) + .with_function("is_ascii", fn_is_ascii) + .with_function("to_lowercase", fn_to_lowercase) + .with_function("to_uppercase", fn_to_uppercase) + .with_function("detect_language", fn_detect_language) + .with_function("is_email", fn_is_email) + .with_function("thread_name", fn_thread_name) + .with_function("html_to_text", fn_html_to_text) + .with_function("is_uppercase", fn_is_uppercase) + .with_function("is_lowercase", fn_is_lowercase) + .with_function("has_digits", fn_has_digits) + .with_function("count_spaces", fn_count_spaces) + .with_function("count_uppercase", fn_count_uppercase) + .with_function("count_lowercase", fn_count_lowercase) + .with_function("count_chars", fn_count_chars) + .with_function("dedup", fn_dedup) + .with_function("lines", fn_lines) + .with_function("is_header_utf8_valid", fn_is_header_utf8_valid) + .with_function("img_metadata", fn_img_metadata) + .with_function("is_ip_addr", fn_is_ip_addr) + .with_function("is_ipv4_addr", fn_is_ipv4_addr) + .with_function("is_ipv6_addr", fn_is_ipv6_addr) + .with_function("ip_reverse_name", fn_ip_reverse_name) + .with_function("winnow", fn_winnow) + .with_function("has_zwsp", fn_has_zwsp) + .with_function("has_obscured", fn_has_obscured) + .with_function("is_single_script", fn_is_single_script) + .with_function("puny_decode", fn_puny_decode) + .with_function("unicode_skeleton", fn_unicode_skeleton) + .with_function("cure_text", fn_cure_text) + .with_function("detect_file_type", fn_detect_file_type) + .with_function_args("sort", fn_sort, 2) + .with_function_args("email_part", fn_email_part, 2) + .with_function_args("eq_ignore_case", fn_eq_ignore_case, 2) + .with_function_args("contains", fn_contains, 2) + .with_function_args("contains_ignore_case", fn_contains_ignore_case, 2) + .with_function_args("starts_with", fn_starts_with, 2) + .with_function_args("ends_with", fn_ends_with, 2) + .with_function_args("received_part", fn_received_part, 2) + .with_function_args("cosine_similarity", fn_cosine_similarity, 2) + .with_function_args("jaccard_similarity", fn_jaccard_similarity, 2) + .with_function_args("levenshtein_distance", fn_levenshtein_distance, 2) + .with_function_args("html_has_tag", fn_html_has_tag, 2) + .with_function_args("html_attr", fn_html_attr, 2) + .with_function_args("html_attrs", fn_html_attrs, 3) + .with_function_args("html_attr_size", fn_html_attr_size, 3) + .with_function_args("uri_part", fn_uri_part, 2) + .with_function_args("substring", fn_substring, 3) + .with_function_args("split", fn_split, 2) + .with_function_args("rsplit", fn_rsplit, 2) + .with_function_args("split_once", fn_split_once, 2) + .with_function_args("rsplit_once", fn_rsplit_once, 2) + .with_function_args("strip_prefix", fn_strip_prefix, 2) + .with_function_args("strip_suffix", fn_strip_suffix, 2) + .with_function_args("is_intersect", fn_is_intersect, 2) + .with_function_args("hash", fn_hash, 2) + .with_function_no_args("is_encoding_problem", fn_is_encoding_problem) + .with_function_no_args("is_attachment", fn_is_attachment) + .with_function_no_args("is_body", fn_is_body) + .with_function_no_args("var_names", fn_is_var_names) + .with_function_no_args("attachment_name", fn_attachment_name) + .with_function_no_args("mime_part_len", fn_mime_part_len) +} + +pub trait ApplyString<'x> { + fn transform(&self, f: impl Fn(&'_ str) -> Variable) -> Variable; +} + +impl<'x> ApplyString<'x> for Variable { + fn transform(&self, f: impl Fn(&'_ str) -> Variable) -> Variable { + match self { + Variable::String(s) => f(s), + Variable::Array(list) => list + .iter() + .map(|v| match v { + Variable::String(s) => f(s), + v => f(v.to_string().as_ref()), + }) + .collect::>() + .into(), + v => f(v.to_string().as_ref()), + } + } +} diff --git a/crates/common/src/scripts/functions/text.rs b/crates/common/src/scripts/functions/text.rs new file mode 100644 index 00000000..2af43f56 --- /dev/null +++ b/crates/common/src/scripts/functions/text.rs @@ -0,0 +1,306 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use sieve::{runtime::Variable, Context}; + +use super::ApplyString; + +pub fn fn_trim<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + v[0].transform(|s| Variable::from(s.trim())) +} + +pub fn fn_trim_end<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + v[0].transform(|s| Variable::from(s.trim_end())) +} + +pub fn fn_trim_start<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + v[0].transform(|s| Variable::from(s.trim_start())) +} + +pub fn fn_len<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + match &v[0] { + Variable::String(s) => s.len(), + Variable::Array(a) => a.len(), + v => v.to_string().len(), + } + .into() +} + +pub fn fn_to_lowercase<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + v[0].transform(|s| Variable::from(s.to_lowercase())) +} + +pub fn fn_to_uppercase<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + v[0].transform(|s| Variable::from(s.to_uppercase())) +} + +pub fn fn_is_uppercase<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + v[0].transform(|s| { + s.chars() + .filter(|c| c.is_alphabetic()) + .all(|c| c.is_uppercase()) + .into() + }) +} + +pub fn fn_is_lowercase<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + v[0].transform(|s| { + s.chars() + .filter(|c| c.is_alphabetic()) + .all(|c| c.is_lowercase()) + .into() + }) +} + +pub fn fn_has_digits<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + v[0].transform(|s| s.chars().any(|c| c.is_ascii_digit()).into()) +} + +pub fn tokenize_words(v: &Variable) -> Variable { + v.to_string() + .split_whitespace() + .filter(|word| word.chars().all(|c| c.is_alphanumeric())) + .map(|word| Variable::from(word.to_string())) + .collect::>() + .into() +} + +pub fn fn_count_spaces<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + v[0].to_string() + .as_ref() + .chars() + .filter(|c| c.is_whitespace()) + .count() + .into() +} + +pub fn fn_count_uppercase<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + v[0].to_string() + .as_ref() + .chars() + .filter(|c| c.is_alphabetic() && c.is_uppercase()) + .count() + .into() +} + +pub fn fn_count_lowercase<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + v[0].to_string() + .as_ref() + .chars() + .filter(|c| c.is_alphabetic() && c.is_lowercase()) + .count() + .into() +} + +pub fn fn_count_chars<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + v[0].to_string().as_ref().chars().count().into() +} + +pub fn fn_eq_ignore_case<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + v[0].to_string() + .eq_ignore_ascii_case(v[1].to_string().as_ref()) + .into() +} + +pub fn fn_contains<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + match &v[0] { + Variable::String(s) => s.contains(v[1].to_string().as_ref()), + Variable::Array(arr) => arr.contains(&v[1]), + val => val.to_string().contains(v[1].to_string().as_ref()), + } + .into() +} + +pub fn fn_contains_ignore_case<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + let needle = v[1].to_string(); + match &v[0] { + Variable::String(s) => s.to_lowercase().contains(&needle.to_lowercase()), + Variable::Array(arr) => arr.iter().any(|v| match v { + Variable::String(s) => s.eq_ignore_ascii_case(needle.as_ref()), + _ => false, + }), + val => val.to_string().contains(needle.as_ref()), + } + .into() +} + +pub fn fn_starts_with<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + v[0].to_string() + .starts_with(v[1].to_string().as_ref()) + .into() +} + +pub fn fn_ends_with<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + v[0].to_string().ends_with(v[1].to_string().as_ref()).into() +} + +pub fn fn_lines<'x>(_: &'x Context<'x, ()>, mut v: Vec) -> Variable { + match v.remove(0) { + Variable::String(s) => s + .lines() + .map(|s| Variable::from(s.to_string())) + .collect::>() + .into(), + val => val, + } +} + +pub fn fn_substring<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + v[0].to_string() + .chars() + .skip(v[1].to_usize()) + .take(v[2].to_usize()) + .collect::() + .into() +} + +pub fn fn_strip_prefix<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + let prefix = v[1].to_string(); + v[0].transform(|s| { + s.strip_prefix(prefix.as_ref()) + .map(Variable::from) + .unwrap_or_default() + }) +} + +pub fn fn_strip_suffix<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + let suffix = v[1].to_string(); + v[0].transform(|s| { + s.strip_suffix(suffix.as_ref()) + .map(Variable::from) + .unwrap_or_default() + }) +} + +pub fn fn_split<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + v[0].to_string() + .split(v[1].to_string().as_ref()) + .map(|s| Variable::from(s.to_string())) + .collect::>() + .into() +} + +pub fn fn_rsplit<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + v[0].to_string() + .rsplit(v[1].to_string().as_ref()) + .map(|s| Variable::from(s.to_string())) + .collect::>() + .into() +} + +pub fn fn_split_once<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + v[0].to_string() + .split_once(v[1].to_string().as_ref()) + .map(|(a, b)| { + Variable::Array( + vec![Variable::from(a.to_string()), Variable::from(b.to_string())].into(), + ) + }) + .unwrap_or_default() +} + +pub fn fn_rsplit_once<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + v[0].to_string() + .rsplit_once(v[1].to_string().as_ref()) + .map(|(a, b)| { + Variable::Array( + vec![Variable::from(a.to_string()), Variable::from(b.to_string())].into(), + ) + }) + .unwrap_or_default() +} + +/** + * `levenshtein-rs` - levenshtein + * + * MIT licensed. + * + * Copyright (c) 2016 Titus Wormer + */ +pub fn fn_levenshtein_distance<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + let a = v[0].to_string(); + let b = v[1].to_string(); + + let mut result = 0; + + /* Shortcut optimizations / degenerate cases. */ + if a == b { + return result.into(); + } + + let length_a = a.chars().count(); + let length_b = b.chars().count(); + + if length_a == 0 { + return length_b.into(); + } else if length_b == 0 { + return length_a.into(); + } + + /* Initialize the vector. + * + * This is why it’s fast, normally a matrix is used, + * here we use a single vector. */ + let mut cache: Vec = (1..).take(length_a).collect(); + let mut distance_a; + let mut distance_b; + + /* Loop. */ + for (index_b, code_b) in b.chars().enumerate() { + result = index_b; + distance_a = index_b; + + for (index_a, code_a) in a.chars().enumerate() { + distance_b = if code_a == code_b { + distance_a + } else { + distance_a + 1 + }; + + distance_a = cache[index_a]; + + result = if distance_a > result { + if distance_b > result { + result + 1 + } else { + distance_b + } + } else if distance_b > distance_a { + distance_a + 1 + } else { + distance_b + }; + + cache[index_a] = result; + } + } + + result.into() +} + +pub fn fn_detect_language<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + whatlang::detect_lang(v[0].to_string().as_ref()) + .map(|l| l.code()) + .unwrap_or("unknown") + .into() +} diff --git a/crates/common/src/scripts/functions/unicode.rs b/crates/common/src/scripts/functions/unicode.rs new file mode 100644 index 00000000..41097ae3 --- /dev/null +++ b/crates/common/src/scripts/functions/unicode.rs @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use sieve::{runtime::Variable, Context}; +use unicode_security::MixedScript; + +pub fn fn_is_ascii<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + match &v[0] { + Variable::String(s) => s.chars().all(|c| c.is_ascii()), + Variable::Integer(_) | Variable::Float(_) => true, + Variable::Array(a) => a.iter().all(|v| match v { + Variable::String(s) => s.chars().all(|c| c.is_ascii()), + _ => true, + }), + } + .into() +} + +pub fn fn_has_zwsp<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + match &v[0] { + Variable::String(s) => s.chars().any(|c| c.is_zwsp()), + Variable::Array(a) => a.iter().any(|v| match v { + Variable::String(s) => s.chars().any(|c| c.is_zwsp()), + _ => true, + }), + Variable::Integer(_) | Variable::Float(_) => false, + } + .into() +} + +pub fn fn_has_obscured<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + match &v[0] { + Variable::String(s) => s.chars().any(|c| c.is_obscured()), + Variable::Array(a) => a.iter().any(|v| match v { + Variable::String(s) => s.chars().any(|c| c.is_obscured()), + _ => true, + }), + Variable::Integer(_) | Variable::Float(_) => false, + } + .into() +} + +trait CharUtils { + fn is_zwsp(&self) -> bool; + fn is_obscured(&self) -> bool; +} + +impl CharUtils for char { + fn is_zwsp(&self) -> bool { + matches!( + self, + '\u{200B}' | '\u{200C}' | '\u{200D}' | '\u{FEFF}' | '\u{00AD}' + ) + } + + fn is_obscured(&self) -> bool { + matches!( + self, + '\u{200B}'..='\u{200F}' + | '\u{2028}'..='\u{202F}' + | '\u{205F}'..='\u{206F}' + | '\u{FEFF}' + ) + } +} + +pub fn fn_cure_text<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + decancer::cure(v[0].to_string().as_ref(), decancer::Options::default()) + .map(|s| s.into_str()) + .unwrap_or_default() + .into() +} + +pub fn fn_unicode_skeleton<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + unicode_security::skeleton(v[0].to_string().as_ref()) + .collect::() + .into() +} + +pub fn fn_is_single_script<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + let text = v[0].to_string(); + if !text.is_empty() { + text.as_ref().is_single_script() + } else { + true + } + .into() +} diff --git a/crates/common/src/scripts/functions/url.rs b/crates/common/src/scripts/functions/url.rs new file mode 100644 index 00000000..bcb70d8a --- /dev/null +++ b/crates/common/src/scripts/functions/url.rs @@ -0,0 +1,75 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use hyper::Uri; +use sieve::{runtime::Variable, Context}; + +use super::ApplyString; + +pub fn fn_uri_part<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + let part = v[1].to_string(); + v[0].transform(|uri| { + uri.parse::() + .ok() + .and_then(|uri| match part.as_ref() { + "scheme" => uri.scheme_str().map(|s| Variable::from(s.to_string())), + "host" => uri.host().map(|s| Variable::from(s.to_string())), + "scheme_host" => uri + .scheme_str() + .and_then(|s| (s, uri.host()?).into()) + .map(|(s, h)| Variable::from(format!("{}://{}", s, h))), + "path" => Variable::from(uri.path().to_string()).into(), + "port" => uri.port_u16().map(|port| Variable::Integer(port as i64)), + "query" => uri.query().map(|s| Variable::from(s.to_string())), + "path_query" => uri.path_and_query().map(|s| Variable::from(s.to_string())), + "authority" => uri.authority().map(|s| Variable::from(s.to_string())), + _ => None, + }) + .unwrap_or_default() + }) +} + +pub fn fn_puny_decode<'x>(_: &'x Context<'x, ()>, v: Vec) -> Variable { + v[0].transform(|domain| { + if domain.contains("xn--") { + let mut decoded = String::with_capacity(domain.len()); + for part in domain.split('.') { + if !decoded.is_empty() { + decoded.push('.'); + } + + if let Some(puny) = part + .strip_prefix("xn--") + .and_then(idna::punycode::decode_to_string) + { + decoded.push_str(&puny); + } else { + decoded.push_str(part); + } + } + decoded.into() + } else { + domain.into() + } + }) +} diff --git a/crates/common/src/scripts/mod.rs b/crates/common/src/scripts/mod.rs new file mode 100644 index 00000000..2aec6814 --- /dev/null +++ b/crates/common/src/scripts/mod.rs @@ -0,0 +1,50 @@ +use std::sync::Arc; + +use sieve::{runtime::Variable, Envelope}; +use store::Value; + +use crate::IntoString; + +pub mod functions; +pub mod plugins; + +#[derive(Debug)] +pub enum ScriptModification { + SetEnvelope { + name: Envelope, + value: String, + }, + AddHeader { + name: Arc, + value: Arc, + }, +} + +pub fn into_sieve_value(value: Value) -> Variable { + match value { + Value::Integer(v) => Variable::Integer(v), + Value::Bool(v) => Variable::Integer(i64::from(v)), + Value::Float(v) => Variable::Float(v), + Value::Text(v) => Variable::String(v.into_owned().into()), + Value::Blob(v) => Variable::String(v.into_owned().into_string().into()), + Value::Null => Variable::default(), + } +} + +pub fn into_store_value(value: Variable) -> Value<'static> { + match value { + Variable::String(v) => Value::Text(v.to_string().into()), + Variable::Integer(v) => Value::Integer(v), + Variable::Float(v) => Value::Float(v), + v => Value::Text(v.to_string().into_owned().into()), + } +} + +pub fn to_store_value(value: &Variable) -> Value<'static> { + match value { + Variable::String(v) => Value::Text(v.to_string().into()), + Variable::Integer(v) => Value::Integer(*v), + Variable::Float(v) => Value::Float(*v), + v => Value::Text(v.to_string().into_owned().into()), + } +} diff --git a/crates/common/src/scripts/plugins/bayes.rs b/crates/common/src/scripts/plugins/bayes.rs new file mode 100644 index 00000000..9e8d5c89 --- /dev/null +++ b/crates/common/src/scripts/plugins/bayes.rs @@ -0,0 +1,362 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use nlp::{ + bayes::{ + cache::BayesTokenCache, tokenize::BayesTokenizer, BayesClassifier, BayesModel, TokenHash, + Weights, + }, + tokenizers::osb::{OsbToken, OsbTokenizer}, +}; +use sieve::{runtime::Variable, FunctionMap}; +use store::{write::key::KeySerializer, LookupStore, U64_LEN}; +use tokio::runtime::Handle; + +use super::PluginContext; + +pub fn register_train(plugin_id: u32, fnc_map: &mut FunctionMap<()>) { + fnc_map.set_external_function("bayes_train", plugin_id, 3); +} + +pub fn register_untrain(plugin_id: u32, fnc_map: &mut FunctionMap<()>) { + fnc_map.set_external_function("bayes_untrain", plugin_id, 3); +} + +pub fn register_classify(plugin_id: u32, fnc_map: &mut FunctionMap<()>) { + fnc_map.set_external_function("bayes_classify", plugin_id, 3); +} + +pub fn register_is_balanced(plugin_id: u32, fnc_map: &mut FunctionMap<()>) { + fnc_map.set_external_function("bayes_is_balanced", plugin_id, 3); +} + +pub fn exec_train(ctx: PluginContext<'_>) -> Variable { + train(ctx, true) +} + +pub fn exec_untrain(ctx: PluginContext<'_>) -> Variable { + train(ctx, false) +} + +fn train(ctx: PluginContext<'_>, is_train: bool) -> Variable { + let span: &tracing::Span = ctx.span; + let store = match &ctx.arguments[0] { + Variable::String(v) if !v.is_empty() => ctx.core.storage.lookups.get(v.as_ref()), + _ => Some(&ctx.core.storage.lookup), + }; + + let store = if let Some(store) = store { + store + } else { + tracing::warn!( + parent: span, + context = "sieve:bayes_train", + event = "failed", + reason = "Unknown store id", + lookup_store = ctx.arguments[0].to_string().as_ref(), + ); + return false.into(); + }; + let text = ctx.arguments[1].to_string(); + let is_spam = ctx.arguments[2].to_bool(); + if text.is_empty() { + return false.into(); + } + let handle = ctx.handle; + + // Train the model + let mut model = BayesModel::default(); + model.train( + OsbTokenizer::new( + BayesTokenizer::new(text.as_ref(), &ctx.core.smtp.resolvers.psl), + 5, + ), + is_spam, + ); + if model.weights.is_empty() { + return false.into(); + } + + tracing::debug!( + parent: span, + context = "sieve:bayes_train", + event = "train", + is_spam = is_spam, + num_tokens = model.weights.len(), + ); + + // Update weight and invalidate cache + let bayes_cache = &ctx.core.sieve.bayes_cache; + if is_train { + for (hash, weights) in model.weights { + if handle + .block_on( + store.counter_incr( + KeySerializer::new(U64_LEN) + .write(hash.h1) + .write(hash.h2) + .finalize(), + weights.into(), + None, + false, + ), + ) + .is_err() + { + return false.into(); + } + bayes_cache.invalidate(&hash); + } + + // Update training counts + let weights = if is_spam { + Weights { spam: 1, ham: 0 } + } else { + Weights { spam: 0, ham: 1 } + }; + if handle + .block_on( + store.counter_incr( + KeySerializer::new(U64_LEN) + .write(0u64) + .write(0u64) + .finalize(), + weights.into(), + None, + false, + ), + ) + .is_err() + { + return false.into(); + } + } else { + //TODO: Implement untrain + return false.into(); + } + + bayes_cache.invalidate(&TokenHash::default()); + + true.into() +} + +pub fn exec_classify(ctx: PluginContext<'_>) -> Variable { + let span = ctx.span; + let store = match &ctx.arguments[0] { + Variable::String(v) if !v.is_empty() => ctx.core.storage.lookups.get(v.as_ref()), + _ => Some(&ctx.core.storage.lookup), + }; + let store = if let Some(store) = store { + store + } else { + tracing::warn!( + parent: span, + context = "sieve:bayes_classify", + event = "failed", + reason = "Unknown store id", + lookup_id = ctx.arguments[0].to_string().as_ref(), + ); + return Variable::default(); + }; + let text = ctx.arguments[1].to_string(); + if text.is_empty() { + return Variable::default(); + } + + // Create classifier from defaults + let mut classifier = BayesClassifier::default(); + if let Some(params) = ctx.arguments[2].as_array() { + if let Some(Variable::Integer(value)) = params.first() { + classifier.min_token_hits = *value as u32; + } + if let Some(Variable::Integer(value)) = params.get(1) { + classifier.min_tokens = *value as u32; + } + if let Some(Variable::Float(value)) = params.get(2) { + classifier.min_prob_strength = *value; + } + if let Some(Variable::Integer(value)) = params.get(3) { + classifier.min_learns = *value as u32; + } + } + + let handle = ctx.handle; + + // Obtain training counts + let bayes_cache = &ctx.core.sieve.bayes_cache; + let (spam_learns, ham_learns) = + if let Some(weights) = bayes_cache.get_or_update(TokenHash::default(), handle, store) { + (weights.spam, weights.ham) + } else { + tracing::warn!( + parent: span, + context = "sieve:classify", + event = "failed", + reason = "Failed to obtain training counts", + ); + return Variable::default(); + }; + + // Make sure we have enough training data + if spam_learns < classifier.min_learns || ham_learns < classifier.min_learns { + tracing::debug!( + parent: span, + context = "sieve:bayes_classify", + event = "skip-classify", + reason = "Not enough training data", + spam_learns = %spam_learns, + ham_learns = %ham_learns); + return Variable::default(); + } + + // Classify the text + classifier + .classify( + OsbTokenizer::<_, TokenHash>::new( + BayesTokenizer::new(text.as_ref(), &ctx.core.smtp.resolvers.psl), + 5, + ) + .filter_map(|t| { + OsbToken { + inner: bayes_cache.get_or_update(t.inner, handle, store)?, + idx: t.idx, + } + .into() + }), + ham_learns, + spam_learns, + ) + .map(Variable::from) + .unwrap_or_default() +} + +pub fn exec_is_balanced(ctx: PluginContext<'_>) -> Variable { + let min_balance = match &ctx.arguments[2] { + Variable::Float(n) => *n, + Variable::Integer(n) => *n as f64, + _ => 0.0, + }; + + if min_balance == 0.0 { + return true.into(); + } + + let span = ctx.span; + let store = match &ctx.arguments[0] { + Variable::String(v) if !v.is_empty() => ctx.core.storage.lookups.get(v.as_ref()), + _ => Some(&ctx.core.storage.lookup), + }; + let store = if let Some(store) = store { + store + } else { + tracing::warn!( + parent: span, + context = "sieve:bayes_is_balanced", + event = "failed", + reason = "Unknown store id", + lookup_id = ctx.arguments[0].to_string().as_ref(), + ); + return Variable::default(); + }; + let learn_spam = ctx.arguments[1].to_bool(); + + // Obtain training counts + let handle = ctx.handle; + let bayes_cache = &ctx.core.sieve.bayes_cache; + let (spam_learns, ham_learns) = + if let Some(weights) = bayes_cache.get_or_update(TokenHash::default(), handle, store) { + (weights.spam as f64, weights.ham as f64) + } else { + tracing::warn!( + parent: span, + context = "sieve:bayes_is_balanced", + event = "failed", + reason = "Failed to obtain training counts", + ); + return Variable::default(); + }; + + let result = if spam_learns > 0.0 || ham_learns > 0.0 { + if learn_spam { + (spam_learns / (ham_learns + 1.0)) <= 1.0 / min_balance + } else { + (ham_learns / (spam_learns + 1.0)) <= 1.0 / min_balance + } + } else { + true + }; + + tracing::debug!( + parent: span, + context = "sieve:bayes_is_balanced", + event = "result", + is_balanced = %result, + learn_spam = %learn_spam, + min_balance = %min_balance, + spam_learns = %spam_learns, + ham_learns = %ham_learns); + + result.into() +} + +trait LookupOrInsert { + fn get_or_update( + &self, + hash: TokenHash, + handle: &Handle, + get_token: &LookupStore, + ) -> Option; +} + +impl LookupOrInsert for BayesTokenCache { + fn get_or_update( + &self, + hash: TokenHash, + handle: &Handle, + get_token: &LookupStore, + ) -> Option { + if let Some(weights) = self.get(&hash) { + weights.unwrap_or_default().into() + } else if let Ok(num) = handle.block_on( + get_token.counter_get( + KeySerializer::new(U64_LEN) + .write(hash.h1) + .write(hash.h2) + .finalize(), + ), + ) { + if num != 0 { + let weights = Weights::from(num); + self.insert_positive(hash, weights); + weights + } else { + self.insert_negative(hash); + Weights::default() + } + .into() + } else { + // Something went wrong + None + } + } +} diff --git a/crates/common/src/scripts/plugins/dns.rs b/crates/common/src/scripts/plugins/dns.rs new file mode 100644 index 00000000..c88b1b8f --- /dev/null +++ b/crates/common/src/scripts/plugins/dns.rs @@ -0,0 +1,221 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::net::IpAddr; + +use mail_auth::{Error, IpLookupStrategy}; +use sieve::{runtime::Variable, FunctionMap}; + +use super::PluginContext; + +pub fn register(plugin_id: u32, fnc_map: &mut FunctionMap<()>) { + fnc_map.set_external_function("dns_query", plugin_id, 2); +} + +pub fn register_exists(plugin_id: u32, fnc_map: &mut FunctionMap<()>) { + fnc_map.set_external_function("dns_exists", plugin_id, 2); +} + +pub fn exec(ctx: PluginContext<'_>) -> Variable { + let entry = ctx.arguments[0].to_string(); + let record_type = ctx.arguments[1].to_string(); + + if record_type.eq_ignore_ascii_case("ip") { + match ctx.handle.block_on(ctx.core.smtp.resolvers.dns.ip_lookup( + entry.as_ref(), + IpLookupStrategy::Ipv4thenIpv6, + 10, + )) { + Ok(result) => result + .iter() + .map(|ip| Variable::from(ip.to_string())) + .collect::>() + .into(), + Err(err) => err.short_error().into(), + } + } else if record_type.eq_ignore_ascii_case("mx") { + match ctx + .handle + .block_on(ctx.core.smtp.resolvers.dns.mx_lookup(entry.as_ref())) + { + Ok(result) => result + .iter() + .flat_map(|mx| { + mx.exchanges + .iter() + .map(|host| Variable::from(format!("{} {}", mx.preference, host))) + }) + .collect::>() + .into(), + Err(err) => err.short_error().into(), + } + } else if record_type.eq_ignore_ascii_case("txt") { + #[cfg(feature = "test_mode")] + { + if entry.contains("origin") { + return Variable::from("23028|US|arin|2002-01-04".to_string()); + } + } + + match ctx + .handle + .block_on(ctx.core.smtp.resolvers.dns.txt_raw_lookup(entry.as_ref())) + { + Ok(result) => Variable::from(String::from_utf8(result).unwrap_or_default()), + Err(err) => err.short_error().into(), + } + } else if record_type.eq_ignore_ascii_case("ptr") { + if let Ok(addr) = entry.parse::() { + match ctx + .handle + .block_on(ctx.core.smtp.resolvers.dns.ptr_lookup(addr)) + { + Ok(result) => result + .iter() + .map(|host| Variable::from(host.to_string())) + .collect::>() + .into(), + Err(err) => err.short_error().into(), + } + } else { + Variable::default() + } + } else if record_type.eq_ignore_ascii_case("ipv4") { + #[cfg(feature = "test_mode")] + { + if entry.contains(".168.192.") { + let parts = entry.split('.').collect::>(); + return vec![Variable::from(format!("127.0.{}.{}", parts[1], parts[0]))].into(); + } + } + + match ctx + .handle + .block_on(ctx.core.smtp.resolvers.dns.ipv4_lookup(entry.as_ref())) + { + Ok(result) => result + .iter() + .map(|ip| Variable::from(ip.to_string())) + .collect::>() + .into(), + Err(err) => err.short_error().into(), + } + } else if record_type.eq_ignore_ascii_case("ipv6") { + match ctx + .handle + .block_on(ctx.core.smtp.resolvers.dns.ipv6_lookup(entry.as_ref())) + { + Ok(result) => result + .iter() + .map(|ip| Variable::from(ip.to_string())) + .collect::>() + .into(), + Err(err) => err.short_error().into(), + } + } else { + Variable::default() + } +} + +pub fn exec_exists(ctx: PluginContext<'_>) -> Variable { + let entry = ctx.arguments[0].to_string(); + let record_type = ctx.arguments[1].to_string(); + + if record_type.eq_ignore_ascii_case("ip") { + match ctx.handle.block_on(ctx.core.smtp.resolvers.dns.ip_lookup( + entry.as_ref(), + IpLookupStrategy::Ipv4thenIpv6, + 10, + )) { + Ok(result) => i64::from(!result.is_empty()), + Err(Error::DnsRecordNotFound(_)) => 0, + Err(_) => -1, + } + } else if record_type.eq_ignore_ascii_case("mx") { + match ctx + .handle + .block_on(ctx.core.smtp.resolvers.dns.mx_lookup(entry.as_ref())) + { + Ok(result) => i64::from(result.iter().any(|mx| !mx.exchanges.is_empty())), + Err(Error::DnsRecordNotFound(_)) => 0, + Err(_) => -1, + } + } else if record_type.eq_ignore_ascii_case("ptr") { + if let Ok(addr) = entry.parse::() { + match ctx + .handle + .block_on(ctx.core.smtp.resolvers.dns.ptr_lookup(addr)) + { + Ok(result) => i64::from(!result.is_empty()), + Err(Error::DnsRecordNotFound(_)) => 0, + Err(_) => -1, + } + } else { + -1 + } + } else if record_type.eq_ignore_ascii_case("ipv4") { + #[cfg(feature = "test_mode")] + { + if entry.starts_with("2.0.168.192.") { + return 1.into(); + } + } + + match ctx + .handle + .block_on(ctx.core.smtp.resolvers.dns.ipv4_lookup(entry.as_ref())) + { + Ok(result) => i64::from(!result.is_empty()), + Err(Error::DnsRecordNotFound(_)) => 0, + Err(_) => -1, + } + } else if record_type.eq_ignore_ascii_case("ipv6") { + match ctx + .handle + .block_on(ctx.core.smtp.resolvers.dns.ipv6_lookup(entry.as_ref())) + { + Ok(result) => i64::from(!result.is_empty()), + Err(Error::DnsRecordNotFound(_)) => 0, + Err(_) => -1, + } + } else { + -1 + } + .into() +} + +trait ShortError { + fn short_error(&self) -> &'static str; +} + +impl ShortError for mail_auth::Error { + fn short_error(&self) -> &'static str { + match self { + mail_auth::Error::DnsError(_) => "temp_fail", + mail_auth::Error::DnsRecordNotFound(_) => "not_found", + mail_auth::Error::Io(_) => "io_error", + mail_auth::Error::InvalidRecordType => "invalid_record", + _ => "unknown_error", + } + } +} diff --git a/crates/common/src/scripts/plugins/exec.rs b/crates/common/src/scripts/plugins/exec.rs new file mode 100644 index 00000000..400b7b14 --- /dev/null +++ b/crates/common/src/scripts/plugins/exec.rs @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::process::Command; + +use sieve::{runtime::Variable, FunctionMap}; + +use super::PluginContext; + +pub fn register(plugin_id: u32, fnc_map: &mut FunctionMap<()>) { + fnc_map.set_external_function("exec", plugin_id, 2); +} + +pub fn exec(ctx: PluginContext<'_>) -> Variable { + let span = ctx.span; + let mut arguments = ctx.arguments.into_iter(); + match Command::new( + arguments + .next() + .map(|a| a.to_string().into_owned()) + .unwrap_or_default(), + ) + .args( + arguments + .next() + .map(|a| a.into_string_array()) + .unwrap_or_default(), + ) + .output() + { + Ok(result) => result.status.success().into(), + Err(err) => { + tracing::warn!( + parent: span, + context = "sieve", + event = "execute-failed", + reason = %err, + ); + false.into() + } + } +} diff --git a/crates/common/src/scripts/plugins/headers.rs b/crates/common/src/scripts/plugins/headers.rs new file mode 100644 index 00000000..e6325d76 --- /dev/null +++ b/crates/common/src/scripts/plugins/headers.rs @@ -0,0 +1,47 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use sieve::{runtime::Variable, FunctionMap}; + +use crate::scripts::ScriptModification; + +use super::PluginContext; + +pub fn register(plugin_id: u32, fnc_map: &mut FunctionMap<()>) { + fnc_map.set_external_function("add_header", plugin_id, 2); +} + +pub fn exec(ctx: PluginContext<'_>) -> Variable { + if let (Variable::String(name), Variable::String(value)) = + (&ctx.arguments[0], &ctx.arguments[1]) + { + ctx.modifications.push(ScriptModification::AddHeader { + name: name.clone(), + value: value.clone(), + }); + true + } else { + false + } + .into() +} diff --git a/crates/common/src/scripts/plugins/http.rs b/crates/common/src/scripts/plugins/http.rs new file mode 100644 index 00000000..421a1dad --- /dev/null +++ b/crates/common/src/scripts/plugins/http.rs @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::time::Duration; + +use reqwest::redirect::Policy; +use sieve::{runtime::Variable, FunctionMap}; + +use super::PluginContext; + +pub fn register_header(plugin_id: u32, fnc_map: &mut FunctionMap<()>) { + fnc_map.set_external_function("http_header", plugin_id, 4); +} + +pub fn exec_header(ctx: PluginContext<'_>) -> Variable { + let url = ctx.arguments[0].to_string(); + let header = ctx.arguments[1].to_string(); + let agent = ctx.arguments[2].to_string(); + let timeout = ctx.arguments[3].to_string().parse::().unwrap_or(5000); + + #[cfg(feature = "test_mode")] + if url.contains("redirect.") { + return Variable::from(url.split_once("/?").unwrap().1.to_string()); + } + + if let Ok(client) = reqwest::Client::builder() + .user_agent(agent.as_ref()) + .timeout(Duration::from_millis(timeout)) + .redirect(Policy::none()) + .danger_accept_invalid_certs(true) + .build() + { + let _enter = ctx.handle.enter(); + ctx.handle + .block_on(client.get(url.as_ref()).send()) + .ok() + .and_then(|response| { + response + .headers() + .get(header.as_ref()) + .and_then(|h| h.to_str().ok()) + .map(|h| Variable::from(h.to_string())) + }) + .unwrap_or_default() + } else { + false.into() + } +} diff --git a/crates/common/src/scripts/plugins/lookup.rs b/crates/common/src/scripts/plugins/lookup.rs new file mode 100644 index 00000000..4e38b830 --- /dev/null +++ b/crates/common/src/scripts/plugins/lookup.rs @@ -0,0 +1,433 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::{ + collections::HashSet, + io::{BufRead, BufReader}, + time::{Duration, Instant}, +}; + +use mail_auth::flate2; +use sieve::{runtime::Variable, FunctionMap}; +use store::{Deserialize, Value}; + +use crate::{config::scripts::RemoteList, scripts::into_sieve_value, USER_AGENT}; + +use super::PluginContext; + +pub fn register(plugin_id: u32, fnc_map: &mut FunctionMap<()>) { + fnc_map.set_external_function("key_exists", plugin_id, 2); +} + +pub fn register_get(plugin_id: u32, fnc_map: &mut FunctionMap<()>) { + fnc_map.set_external_function("key_get", plugin_id, 2); +} + +pub fn register_set(plugin_id: u32, fnc_map: &mut FunctionMap<()>) { + fnc_map.set_external_function("key_set", plugin_id, 4); +} + +pub fn register_remote(plugin_id: u32, fnc_map: &mut FunctionMap<()>) { + fnc_map.set_external_function("key_exists_http", plugin_id, 3); +} + +pub fn register_local_domain(plugin_id: u32, fnc_map: &mut FunctionMap<()>) { + fnc_map.set_external_function("is_local_domain", plugin_id, 2); +} + +pub fn exec(ctx: PluginContext<'_>) -> Variable { + let store = match &ctx.arguments[0] { + Variable::String(v) if !v.is_empty() => ctx.core.storage.lookups.get(v.as_ref()), + _ => Some(&ctx.core.storage.lookup), + }; + + if let Some(store) = store { + match &ctx.arguments[1] { + Variable::Array(items) => { + for item in items.iter() { + if !item.is_empty() + && ctx + .handle + .block_on(store.key_exists(item.to_string().into_owned().into_bytes())) + .unwrap_or(false) + { + return true.into(); + } + } + false + } + v if !v.is_empty() => ctx + .handle + .block_on(store.key_exists(v.to_string().into_owned().into_bytes())) + .unwrap_or(false), + _ => false, + } + } else { + tracing::warn!( + parent: ctx.span, + context = "sieve:lookup", + event = "failed", + reason = "Unknown lookup id", + lookup_id = ctx.arguments[0].to_string().as_ref(), + ); + false + } + .into() +} + +pub fn exec_get(ctx: PluginContext<'_>) -> Variable { + let store = match &ctx.arguments[0] { + Variable::String(v) if !v.is_empty() => ctx.core.storage.lookups.get(v.as_ref()), + _ => Some(&ctx.core.storage.lookup), + }; + + if let Some(store) = store { + ctx.handle + .block_on( + store.key_get::( + ctx.arguments[1].to_string().into_owned().into_bytes(), + ), + ) + .unwrap_or_default() + .map(|v| v.into_inner()) + .unwrap_or_default() + } else { + tracing::warn!( + parent: ctx.span, + context = "sieve:key_get", + event = "failed", + reason = "Unknown store or lookup id", + lookup_id = ctx.arguments[0].to_string().as_ref(), + ); + Variable::default() + } +} + +pub fn exec_set(ctx: PluginContext<'_>) -> Variable { + let store = match &ctx.arguments[0] { + Variable::String(v) if !v.is_empty() => ctx.core.storage.lookups.get(v.as_ref()), + _ => Some(&ctx.core.storage.lookup), + }; + + if let Some(store) = store { + let expires = match &ctx.arguments[3] { + Variable::Integer(v) => Some(*v as u64), + Variable::Float(v) => Some(*v as u64), + _ => None, + }; + + ctx.handle + .block_on(store.key_set( + ctx.arguments[1].to_string().into_owned().into_bytes(), + if !ctx.arguments[2].is_empty() { + bincode::serialize(&ctx.arguments[2]).unwrap_or_default() + } else { + vec![] + }, + expires, + )) + .is_ok() + .into() + } else { + tracing::warn!( + parent: ctx.span, + context = "sieve:key_set", + event = "failed", + reason = "Unknown store id", + store_id = ctx.arguments[0].to_string().as_ref(), + ); + Variable::default() + } +} + +pub fn exec_remote(ctx: PluginContext<'_>) -> Variable { + let resource = ctx.arguments[0].to_string(); + let item = ctx.arguments[1].to_string(); + + #[cfg(feature = "test_mode")] + { + if (resource.contains("open") && item.contains("open")) + || (resource.contains("tank") && item.contains("tank")) + { + return true.into(); + } + } + + if resource.is_empty() || item.is_empty() { + return false.into(); + } + + const TIMEOUT: Duration = Duration::from_secs(45); + const RETRY: Duration = Duration::from_secs(3600); + const MAX_ENTRY_SIZE: usize = 256; + const MAX_ENTRIES: usize = 100000; + + match ctx.core.sieve.remote_lists.read().get(resource.as_ref()) { + Some(remote_list) if remote_list.expires < Instant::now() => { + return remote_list.entries.contains(item.as_ref()).into() + } + _ => {} + } + + enum Format { + List, + Csv { + column: u32, + separator: char, + skip_first: bool, + }, + } + + // Obtain parameters + let mut format = Format::List; + let mut expires = Duration::from_secs(12 * 3600); + + if let Some(arr) = ctx.arguments[2].as_array() { + // Obtain expiration + match arr.first() { + Some(Variable::Integer(v)) if *v > 0 => { + expires = Duration::from_secs(*v as u64); + } + Some(Variable::Float(v)) if *v > 0.0 => { + expires = Duration::from_secs(*v as u64); + } + _ => (), + } + + // Obtain list type + if matches!(arr.get(1), Some(Variable::String(list_type)) if list_type.eq_ignore_ascii_case("csv")) + { + format = Format::Csv { + column: arr.get(2).map(|v| v.to_integer()).unwrap_or_default() as u32, + separator: arr + .get(3) + .and_then(|v| v.to_string().chars().next()) + .unwrap_or(','), + skip_first: arr.get(4).map_or(false, |v| v.to_bool()), + }; + } + } + + // Lock remote list for writing + let mut _lock = ctx.core.sieve.remote_lists.write(); + let list = _lock + .entry(resource.to_string()) + .or_insert_with(|| RemoteList { + entries: HashSet::new(), + expires: Instant::now(), + }); + + // Make sure that the list is still expired + if list.expires > Instant::now() { + return list.entries.contains(item.as_ref()).into(); + } + + let _enter = ctx.handle.enter(); + match ctx + .handle + .block_on( + reqwest::Client::builder() + .timeout(TIMEOUT) + .user_agent(USER_AGENT) + .build() + .unwrap_or_default() + .get(resource.as_ref()) + .send(), + ) + .and_then(|r| { + if r.status().is_success() { + ctx.handle.block_on(r.bytes()).map(Ok) + } else { + Ok(Err(r)) + } + }) { + Ok(Ok(bytes)) => { + let reader: Box = if resource.ends_with(".gz") { + Box::new(flate2::read::GzDecoder::new(&bytes[..])) + } else { + Box::new(&bytes[..]) + }; + + for (pos, line) in BufReader::new(reader).lines().enumerate() { + match line { + Ok(line_) => { + // Clear list once the first entry has been successfully fetched, decompressed and UTF8-decoded + if pos == 0 { + list.entries.clear(); + } + + match &format { + Format::List => { + let line = line_.trim(); + if !line.is_empty() { + list.entries.insert(line.to_string()); + } + } + Format::Csv { + column, + separator, + skip_first, + } if pos > 0 || !*skip_first => { + let mut in_quote = false; + let mut col_num = 0; + let mut entry = String::new(); + + for ch in line_.chars() { + if ch != '"' { + if ch == *separator && !in_quote { + if col_num == *column { + break; + } else { + col_num += 1; + } + } else if col_num == *column { + entry.push(ch); + if entry.len() > MAX_ENTRY_SIZE { + break; + } + } + } else { + in_quote = !in_quote; + } + } + + if !entry.is_empty() { + list.entries.insert(entry); + } + } + _ => (), + } + } + Err(err) => { + tracing::warn!( + parent: ctx.span, + context = "sieve:key_exists_http", + event = "failed", + resource = resource.as_ref(), + reason = %err, + ); + break; + } + } + + if list.entries.len() == MAX_ENTRIES { + break; + } + } + + tracing::debug!( + parent: ctx.span, + context = "sieve:key_exists_http", + event = "fetch", + resource = resource.as_ref(), + num_entries = list.entries.len(), + ); + + // Update expiration + list.expires = Instant::now() + expires; + return list.entries.contains(item.as_ref()).into(); + } + Ok(Err(response)) => { + tracing::warn!( + parent: ctx.span, + context = "sieve:key_exists_http", + event = "failed", + resource = resource.as_ref(), + status = %response.status(), + ); + } + Err(err) => { + tracing::warn!( + parent: ctx.span, + context = "sieve:key_exists_http", + event = "failed", + resource = resource.as_ref(), + reason = %err, + ); + } + } + + // Something went wrong, try again in one hour + list.expires = Instant::now() + RETRY; + false.into() +} + +pub fn exec_local_domain(ctx: PluginContext<'_>) -> Variable { + let domain = ctx.arguments[0].to_string(); + + if !domain.is_empty() { + let directory = match &ctx.arguments[0] { + Variable::String(v) if !v.is_empty() => ctx.core.storage.directories.get(v.as_ref()), + _ => Some(&ctx.core.storage.directory), + }; + + if let Some(directory) = directory { + return ctx + .handle + .block_on(directory.is_local_domain(domain.as_ref())) + .unwrap_or_default() + .into(); + } else { + tracing::warn!( + parent: ctx.span, + context = "sieve:is_local_domain", + event = "failed", + reason = "Unknown directory", + lookup_id = ctx.arguments[0].to_string().as_ref(), + ); + } + } + + Variable::default() +} + +#[derive(Debug, PartialEq, Eq)] +pub struct VariableWrapper(Variable); + +impl Deserialize for VariableWrapper { + fn deserialize(bytes: &[u8]) -> store::Result { + Ok(VariableWrapper( + bincode::deserialize::(bytes).unwrap_or_else(|_| { + Variable::String(String::from_utf8_lossy(bytes).into_owned().into()) + }), + )) + } +} + +impl From for VariableWrapper { + fn from(value: i64) -> Self { + VariableWrapper(value.into()) + } +} + +impl VariableWrapper { + pub fn into_inner(self) -> Variable { + self.0 + } +} + +impl From> for VariableWrapper { + fn from(value: Value<'static>) -> Self { + VariableWrapper(into_sieve_value(value)) + } +} diff --git a/crates/common/src/scripts/plugins/mod.rs b/crates/common/src/scripts/plugins/mod.rs new file mode 100644 index 00000000..321a7f4a --- /dev/null +++ b/crates/common/src/scripts/plugins/mod.rs @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +pub mod bayes; +pub mod dns; +pub mod exec; +pub mod headers; +pub mod http; +pub mod lookup; +pub mod pyzor; +pub mod query; +pub mod text; + +use mail_parser::Message; +use sieve::{runtime::Variable, FunctionMap, Input}; +use tokio::runtime::Handle; + +use crate::Core; + +use super::ScriptModification; + +type RegisterPluginFnc = fn(u32, &mut FunctionMap<()>) -> (); +type ExecPluginFnc = fn(PluginContext<'_>) -> Variable; + +pub struct PluginContext<'x> { + pub span: &'x tracing::Span, + pub handle: &'x Handle, + pub core: &'x Core, + pub message: &'x Message<'x>, + pub modifications: &'x mut Vec, + pub arguments: Vec, +} + +const PLUGINS_EXEC: [ExecPluginFnc; 18] = [ + query::exec, + exec::exec, + lookup::exec, + lookup::exec_get, + lookup::exec_set, + lookup::exec_remote, + lookup::exec_local_domain, + dns::exec, + dns::exec_exists, + http::exec_header, + bayes::exec_train, + bayes::exec_untrain, + bayes::exec_classify, + bayes::exec_is_balanced, + pyzor::exec, + headers::exec, + text::exec_tokenize, + text::exec_domain_part, +]; +const PLUGINS_REGISTER: [RegisterPluginFnc; 18] = [ + query::register, + exec::register, + lookup::register, + lookup::register_get, + lookup::register_set, + lookup::register_remote, + lookup::register_local_domain, + dns::register, + dns::register_exists, + http::register_header, + bayes::register_train, + bayes::register_untrain, + bayes::register_classify, + bayes::register_is_balanced, + pyzor::register, + headers::register, + text::register_tokenize, + text::register_domain_part, +]; + +pub trait RegisterSievePlugins { + fn register_plugins(self) -> Self; +} + +impl RegisterSievePlugins for FunctionMap<()> { + fn register_plugins(mut self) -> Self { + #[cfg(feature = "test_mode")] + { + self.set_external_function("print", PLUGINS_EXEC.len() as u32, 1) + } + + for (i, fnc) in PLUGINS_REGISTER.iter().enumerate() { + fnc(i as u32, &mut self); + } + self + } +} + +impl Core { + pub fn run_plugin_blocking(&self, id: u32, ctx: PluginContext<'_>) -> Input { + #[cfg(feature = "test_mode")] + if id == PLUGINS_EXEC.len() as u32 { + return test_print(ctx); + } + + PLUGINS_EXEC + .get(id as usize) + .map(|fnc| fnc(ctx)) + .unwrap_or_default() + .into() + } +} + +#[cfg(feature = "test_mode")] +pub fn test_print(ctx: PluginContext<'_>) -> Input { + println!("{}", ctx.arguments[0].to_string()); + Input::True +} diff --git a/crates/common/src/scripts/plugins/pyzor.rs b/crates/common/src/scripts/plugins/pyzor.rs new file mode 100644 index 00000000..d7af8c9a --- /dev/null +++ b/crates/common/src/scripts/plugins/pyzor.rs @@ -0,0 +1,834 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use sieve::{runtime::Variable, FunctionMap}; + +use super::PluginContext; + +use std::{ + borrow::Cow, + io::Write, + time::{Duration, SystemTime}, +}; + +use mail_parser::{decoders::html::add_html_token, Message, PartType}; +use nlp::tokenizers::types::{TokenType, TypesTokenizer}; +use sha1::{Digest, Sha1}; +use tokio::net::UdpSocket; +use utils::suffixlist::PublicSuffix; + +const MIN_LINE_LENGTH: usize = 8; +const ATOMIC_NUM_LINES: usize = 4; +const DIGEST_SPEC: &[(usize, usize)] = &[(20, 3), (60, 3)]; + +#[derive(Default, Debug, PartialEq, Eq)] +struct PyzorResponse { + code: u32, + count: u64, + wl_count: u64, +} + +pub fn register(plugin_id: u32, fnc_map: &mut FunctionMap<()>) { + fnc_map.set_external_function("pyzor_check", plugin_id, 2); +} + +pub fn exec(ctx: PluginContext<'_>) -> Variable { + // Make sure there is at least one text part + if !ctx + .message + .parts + .iter() + .any(|p| matches!(p.body, PartType::Text(_) | PartType::Html(_))) + { + return Variable::default(); + } + + // Hash message + let request = ctx + .message + .pyzor_check_message(&ctx.core.smtp.resolvers.psl); + + #[cfg(feature = "test_mode")] + { + if request.contains("b5b476f0b5ba6e1c038361d3ded5818dd39c90a2") { + return PyzorResponse { + code: 200, + count: 1000, + wl_count: 0, + } + .into(); + } else if request.contains("d67d4b8bfc3860449e3418bb6017e2612f3e2a99") { + return PyzorResponse { + code: 200, + count: 60, + wl_count: 10, + } + .into(); + } else if request.contains("81763547012b75e57a20d18ce0b93014208cdfdb") { + return PyzorResponse { + code: 200, + count: 50, + wl_count: 20, + } + .into(); + } + } + + let span = ctx.span; + let address = ctx.arguments[0].to_string(); + let timeout = Duration::from_secs(std::cmp::max( + std::cmp::min(ctx.arguments[1].to_integer() as u64, 60), + 5, + )); + // Send message to address + match ctx + .handle + .block_on(pyzor_send_message(address.as_ref(), timeout, &request)) + { + Ok(response) => response.into(), + Err(err) => { + tracing::debug!( + parent: span, + context = "sieve:pyzor_check", + event = "failed", + reason = %err, + ); + Variable::default() + } + } +} + +impl From for Variable { + fn from(response: PyzorResponse) -> Self { + vec![ + Variable::from(response.code), + Variable::from(response.count), + Variable::from(response.wl_count), + ] + .into() + } +} + +async fn pyzor_send_message( + addr: &str, + timeout: Duration, + message: &str, +) -> std::io::Result { + let socket = UdpSocket::bind("0.0.0.0:0").await?; + tokio::time::timeout(timeout, socket.send_to(message.as_bytes(), addr)).await??; + + let mut buffer = vec![0u8; 1024]; + let (size, _) = tokio::time::timeout(timeout, socket.recv_from(&mut buffer)).await??; + + let raw_response = std::str::from_utf8(&buffer[..size]) + .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidData, err))?; + let mut response = PyzorResponse { + code: u32::MAX, + count: u64::MAX, + wl_count: u64::MAX, + }; + + for line in raw_response.lines() { + if let Some((k, v)) = line.split_once(':') { + if k.eq_ignore_ascii_case("code") { + response.code = v.trim().parse().map_err(|_| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("Invalid line: {raw_response}"), + ) + })?; + } else if k.eq_ignore_ascii_case("count") { + response.count = v.trim().parse().map_err(|_| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("Invalid line: {raw_response}"), + ) + })?; + } else if k.eq_ignore_ascii_case("wl-count") { + response.wl_count = v.trim().parse().map_err(|_| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("Invalid line: {raw_response}"), + ) + })?; + } + } + } + + if response.code != u32::MAX && response.count != u64::MAX && response.wl_count != u64::MAX { + Ok(response) + } else { + Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("Invalid response: {raw_response}"), + )) + } +} + +trait PyzorDigest { + fn pyzor_digest(&self, writer: W, psl: &PublicSuffix) -> W; +} + +pub trait PyzorCheck { + fn pyzor_check_message(&self, psl: &PublicSuffix) -> String; +} + +impl<'x, W: Write> PyzorDigest for Message<'x> { + fn pyzor_digest(&self, writer: W, psl: &PublicSuffix) -> W { + let parts = self + .parts + .iter() + .filter_map(|part| match &part.body { + PartType::Text(text) => Some(text.as_ref().into()), + PartType::Html(html) => Some(html_to_text(html.as_ref()).into()), + _ => None, + }) + .collect::>>(); + + pyzor_digest(writer, parts.iter().flat_map(|text| text.lines()), psl) + } +} + +impl<'x> PyzorCheck for Message<'x> { + fn pyzor_check_message(&self, psl: &PublicSuffix) -> String { + let time = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .map_or(0, |d| d.as_secs()); + + pyzor_create_message( + self, + psl, + time, + (time & 0xFFFF) as u16 ^ ((time >> 16) & 0xFFFF) as u16, + ) + } +} + +fn pyzor_create_message( + message: &Message<'_>, + psl: &PublicSuffix, + time: u64, + thread: u16, +) -> String { + // Hash message + let hash = message.pyzor_digest(Sha1::new(), psl).finalize(); + // Hash key + let mut hash_key = Sha1::new(); + hash_key.update("anonymous:".as_bytes()); + let hash_key = hash_key.finalize(); + + // Hash message + let message = format!( + "Op: check\nOp-Digest: {hash:x}\nThread: {thread}\nPV: 2.1\nUser: anonymous\nTime: {time}" + ); + let mut msg_hash = Sha1::new(); + msg_hash.update(message.as_bytes()); + let msg_hash = msg_hash.finalize(); + + // Sign + let mut sig = Sha1::new(); + sig.update(msg_hash); + sig.update(&format!(":{time}:{hash_key:x}")); + let sig = sig.finalize(); + + format!("{message}\nSig: {sig:x}\n") +} + +fn pyzor_digest<'x, I, W>(mut writer: W, lines: I, psl: &PublicSuffix) -> W +where + I: Iterator, + W: Write, +{ + let mut result = Vec::with_capacity(16); + + for line in lines { + let mut clean_line = String::with_capacity(line.len()); + let mut token_start = usize::MAX; + let mut token_end = usize::MAX; + + let add_line = |line: &mut String, span: &str| { + if !span.contains(char::from(0)) { + if span.len() < 10 { + line.push_str(span); + } + } else { + let span = span.replace(char::from(0), ""); + if span.len() < 10 { + line.push_str(&span); + } + } + }; + + for token in TypesTokenizer::new(line, psl) { + match token.word { + TokenType::Alphabetic(_) + | TokenType::Alphanumeric(_) + | TokenType::Integer(_) + | TokenType::Float(_) + | TokenType::Other(_) + | TokenType::Punctuation(_) => { + if token_start == usize::MAX { + token_start = token.from; + } + token_end = token.to; + } + TokenType::Space + | TokenType::Url(_) + | TokenType::UrlNoScheme(_) + | TokenType::UrlNoHost(_) + | TokenType::Email(_) => { + if token_start != usize::MAX { + add_line(&mut clean_line, &line[token_start..token_end]); + token_start = usize::MAX; + token_end = usize::MAX; + } + } + } + } + + if token_start != usize::MAX { + add_line(&mut clean_line, &line[token_start..token_end]); + } + + if clean_line.len() >= MIN_LINE_LENGTH { + result.push(clean_line); + } + } + + if result.len() > ATOMIC_NUM_LINES { + for (offset, length) in DIGEST_SPEC { + for i in 0..*length { + if let Some(line) = result.get((*offset * result.len() / 100) + i) { + let _ = writer.write_all(line.as_bytes()); + } + } + } + } else { + for line in result { + let _ = writer.write_all(line.as_bytes()); + } + } + + writer +} + +fn html_to_text(input: &str) -> String { + let mut result = String::with_capacity(input.len()); + let input = input.as_bytes(); + + let mut in_tag = false; + let mut in_comment = false; + let mut in_style = false; + let mut in_script = false; + + let mut is_token_start = true; + let mut is_after_space = false; + let mut is_tag_close = false; + + let mut token_start = 0; + let mut token_end = 0; + + let mut tag_token_pos = 0; + let mut comment_pos = 0; + + for (pos, ch) in input.iter().enumerate() { + if !in_comment { + match ch { + b'<' => { + if !(in_tag || in_style || in_script || is_token_start) { + add_html_token( + &mut result, + &input[token_start..token_end + 1], + is_after_space, + ); + is_after_space = false; + } + + tag_token_pos = 0; + in_tag = true; + is_token_start = true; + is_tag_close = false; + continue; + } + b'>' if in_tag => { + if tag_token_pos == 1 { + if let Some(tag) = input.get(token_start..token_end + 1) { + if tag.eq_ignore_ascii_case(b"style") { + in_style = !is_tag_close; + } else if tag.eq_ignore_ascii_case(b"script") { + in_script = !is_tag_close; + } + } + } + + in_tag = false; + is_token_start = true; + is_after_space = !result.is_empty(); + + continue; + } + b'/' if in_tag => { + if tag_token_pos == 0 { + is_tag_close = true; + } + continue; + } + b'!' if in_tag && tag_token_pos == 0 => { + if let Some(b"--") = input.get(pos + 1..pos + 3) { + in_comment = true; + continue; + } + } + b' ' | b'\t' | b'\r' | b'\n' => { + if !(in_tag || in_style || in_script) { + if !is_token_start { + add_html_token( + &mut result, + &input[token_start..token_end + 1], + is_after_space, + ); + } + is_after_space = true; + } + + is_token_start = true; + continue; + } + b'&' if !(in_tag || is_token_start || in_style || in_script) => { + add_html_token( + &mut result, + &input[token_start..token_end + 1], + is_after_space, + ); + is_token_start = true; + is_after_space = false; + } + b';' if !(in_tag || is_token_start || in_style || in_script) => { + add_html_token(&mut result, &input[token_start..pos + 1], is_after_space); + is_token_start = true; + is_after_space = false; + continue; + } + _ => (), + } + if is_token_start { + token_start = pos; + is_token_start = false; + if in_tag { + tag_token_pos += 1; + } + } + token_end = pos; + } else { + match ch { + b'-' => comment_pos += 1, + b'>' if comment_pos == 2 => { + comment_pos = 0; + in_comment = false; + in_tag = false; + is_token_start = true; + } + _ => comment_pos = 0, + } + } + } + + if !(in_tag || is_token_start || in_style || in_script) { + add_html_token( + &mut result, + &input[token_start..token_end + 1], + is_after_space, + ); + } + + result.shrink_to_fit(); + result +} + +#[cfg(test)] +mod test { + use std::time::Duration; + + use mail_parser::MessageParser; + use sha1::Digest; + use sha1::Sha1; + use utils::suffixlist::PublicSuffix; + + use super::pyzor_create_message; + use super::pyzor_send_message; + use super::{html_to_text, pyzor_digest, PyzorDigest}; + + use super::PyzorResponse; + + #[ignore] + #[tokio::test] + async fn send_message() { + assert_eq!( + pyzor_send_message( + "public.pyzor.org:24441", + Duration::from_secs(10), + concat!( + "Op: check\n", + "Op-Digest: b2c27325a034c581df0c9ef37e4a0d63208a3e7e\n", + "Thread: 49005\n", + "PV: 2.1\n", + "User: anonymous\n", + "Time: 1697468672\n", + "Sig: 9cf4571b85d3887fdd0d4f444fd0c164e0290722\n" + ), + ) + .await + .unwrap(), + PyzorResponse { + code: 200, + count: 0, + wl_count: 0 + } + ); + } + + #[test] + fn message_pyzor() { + let mut psl = PublicSuffix::default(); + psl.suffixes.insert("com".to_string()); + let message = pyzor_create_message( + &MessageParser::new().parse(HTML_TEXT_STYLE_SCRIPT).unwrap(), + &psl, + 1697468672, + 49005, + ); + + assert_eq!( + message, + concat!( + "Op: check\n", + "Op-Digest: b2c27325a034c581df0c9ef37e4a0d63208a3e7e\n", + "Thread: 49005\n", + "PV: 2.1\n", + "User: anonymous\n", + "Time: 1697468672\n", + "Sig: 9cf4571b85d3887fdd0d4f444fd0c164e0290722\n" + ) + ); + } + + #[test] + fn digest_pyzor() { + let mut psl = PublicSuffix::default(); + psl.suffixes.insert("com".to_string()); + + // HTML stripping + assert_eq!(html_to_text(HTML_RAW), HTML_RAW_STRIPED); + + // Token stripping + for strip_me in [ + "t@abc.com", + "t1@abc.com", + "t+a@abc.com", + "t.a@abc.com", + "0A2D3f%a#S", + "3sddkf9jdkd9", + "@@#@@@@@@@@@", + "http://spammer.com/special-offers?buy=now", + ] { + assert_eq!( + String::from_utf8(pyzor_digest( + Vec::new(), + format!("Test {strip_me} Test2").lines(), + &psl + )) + .unwrap(), + "TestTest2" + ); + } + + // Test short lines + assert_eq!( + String::from_utf8(pyzor_digest( + Vec::new(), + concat!("This line is included\n", "not this\n", "This also").lines(), + &psl + )) + .unwrap(), + "ThislineisincludedThisalso" + ); + + // Test atomic + assert_eq!( + String::from_utf8(pyzor_digest( + Vec::new(), + "All this message\nShould be included\nIn the digest".lines(), + &psl + )) + .unwrap(), + "AllthismessageShouldbeincludedInthedigest" + ); + + // Test spec + let mut text = String::new(); + for i in 0..100 { + text += &format!("Line{i} test test test\n"); + } + let mut expected = String::new(); + for i in [20, 21, 22, 60, 61, 62] { + expected += &format!("Line{i}testtesttest"); + } + assert_eq!( + String::from_utf8(pyzor_digest(Vec::new(), text.lines(), &psl)).unwrap(), + expected + ); + + // Test email parsing + for (input, expected) in [ + ( + HTML_TEXT, + concat!( + "Emailspam,alsoknownasjunkemailorbulkemail,isasubset", + "ofspaminvolvingnearlyidenticalmessagessenttonumerous", + "byemail.Clickingonlinksinspamemailmaysendusersto", + "byemail.Clickingonlinksinspamemailmaysendusersto", + "phishingwebsitesorsitesthatarehostingmalware.", + "Emailspam.Emailspam,alsoknownasjunkemailorbulkemail,", + "isasubsetofspaminvolvingnearlyidenticalmessage", + "ssenttonumerousbyemail.Clickingonlinksinspamemailmaysenduse", + "rstophishingwebsitesorsitesthatarehostingmalware." + ), + ), + (HTML_TEXT_STYLE_SCRIPT, "Thisisatest.Thisisatest."), + (TEXT_ATTACHMENT, "Thisisatestmailing"), + (TEXT_ATTACHMENT_W_NULL, "Thisisatestmailing"), + (TEXT_ATTACHMENT_W_MULTIPLE_NULLS, "Thisisatestmailing"), + (TEXT_ATTACHMENT_W_SUBJECT_NULL, "Thisisatestmailing"), + (TEXT_ATTACHMENT_W_CONTENTTYPE_NULL, "Thisisatestmailing"), + ] { + assert_eq!( + String::from_utf8( + MessageParser::new() + .parse(input) + .unwrap() + .pyzor_digest(Vec::new(), &psl) + ) + .unwrap(), + expected, + "failed for {input}" + ) + } + + // Test SHA hash + assert_eq!( + format!( + "{:x}", + MessageParser::new() + .parse(HTML_TEXT_STYLE_SCRIPT) + .unwrap() + .pyzor_digest(Sha1::new(), &psl) + .finalize() + ), + "b2c27325a034c581df0c9ef37e4a0d63208a3e7e", + ) + } + + const HTML_TEXT: &str = r#"MIME-Version: 1.0 +Sender: chirila@gapps.spamexperts.com +Received: by 10.216.157.70 with HTTP; Thu, 16 Jan 2014 00:43:31 -0800 (PST) +Date: Thu, 16 Jan 2014 10:43:31 +0200 +Delivered-To: chirila@gapps.spamexperts.com +X-Google-Sender-Auth: ybCmONS9U9D6ZUfjx-9_tY-hF2Q +Message-ID: +Subject: Test +From: Alexandru Chirila +To: Alexandru Chirila +Content-Type: multipart/alternative; boundary=001a11c25ff293069304f0126bfd + +--001a11c25ff293069304f0126bfd +Content-Type: text/plain; charset=ISO-8859-1 + +Email spam. + +Email spam, also known as junk email or unsolicited bulk email, is a subset +of electronic spam involving nearly identical messages sent to numerous +recipients by email. Clicking on links in spam email may send users to +phishing web sites or sites that are hosting malware. + +--001a11c25ff293069304f0126bfd +Content-Type: text/html; charset=ISO-8859-1 +Content-Transfer-Encoding: quoted-printable + +
Email spam.

Email spam, also= + known as junk email or unsolicited bulk email, is a subset of electronic s= +pam involving nearly identical messages sent to numerous recipients by emai= +l. Clicking on links in spam email may send users to phishing web sites or = +sites that are hosting malware.
+
+ +--001a11c25ff293069304f0126bfd-- +"#; + + const HTML_TEXT_STYLE_SCRIPT: &str = r#"MIME-Version: 1.0 +Sender: chirila@gapps.spamexperts.com +Received: by 10.216.157.70 with HTTP; Thu, 16 Jan 2014 00:43:31 -0800 (PST) +Date: Thu, 16 Jan 2014 10:43:31 +0200 +Delivered-To: chirila@gapps.spamexperts.com +X-Google-Sender-Auth: ybCmONS9U9D6ZUfjx-9_tY-hF2Q +Message-ID: +Subject: Test +From: Alexandru Chirila +To: Alexandru Chirila +Content-Type: multipart/alternative; boundary=001a11c25ff293069304f0126bfd + +--001a11c25ff293069304f0126bfd +Content-Type: text/plain; charset=ISO-8859-1 + +This is a test. + +--001a11c25ff293069304f0126bfd +Content-Type: text/html; charset=ISO-8859-1 +Content-Transfer-Encoding: quoted-printable + +
+ + +
This is a test.
+
+ +--001a11c25ff293069304f0126bfd-- +"#; + + const TEXT_ATTACHMENT: &str = r#"MIME-Version: 1.0 +Received: by 10.76.127.40 with HTTP; Fri, 17 Jan 2014 02:21:43 -0800 (PST) +Date: Fri, 17 Jan 2014 12:21:43 +0200 +Delivered-To: chirila.s.alexandru@gmail.com +Message-ID: +Subject: Test +From: Alexandru Chirila +To: Alexandru Chirila +Content-Type: multipart/mixed; boundary=f46d040a62c49bb1c804f027e8cc + +--f46d040a62c49bb1c804f027e8cc +Content-Type: multipart/alternative; boundary=f46d040a62c49bb1c404f027e8ca + +--f46d040a62c49bb1c404f027e8ca +Content-Type: text/plain; charset=ISO-8859-1 + +This is a test mailing + +--f46d040a62c49bb1c404f027e8ca-- +--f46d040a62c49bb1c804f027e8cc +Content-Type: image/png; name="tar.png" +Content-Disposition: attachment; filename="tar.png" +Content-Transfer-Encoding: base64 +X-Attachment-Id: f_hqjas5ad0 + +iVBORw0KGgoAAAANSUhEUgAAAskAAADlCAAAAACErzVVAAAACXBIWXMAAAsTAAALEwEAmpwYAAAD +QmCC +--f46d040a62c49bb1c804f027e8cc--"#; + + const TEXT_ATTACHMENT_W_NULL: &str = "MIME-Version: 1.0 +Received: by 10.76.127.40 with HTTP; Fri, 17 Jan 2014 02:21:43 -0800 (PST) +Date: Fri, 17 Jan 2014 12:21:43 +0200 +Delivered-To: chirila.s.alexandru@gmail.com +Message-ID: +Subject: Test +From: Alexandru Chirila +To: Alexandru Chirila +Content-Type: multipart/mixed; boundary=f46d040a62c49bb1c804f027e8cc + +--f46d040a62c49bb1c804f027e8cc +Content-Type: multipart/alternative; boundary=f46d040a62c49bb1c404f027e8ca + +--f46d040a62c49bb1c404f027e8ca +Content-Type: text/plain; charset=ISO-8859-1 + +This is a test ma\0iling +--f46d040a62c49bb1c804f027e8cc--"; + + const TEXT_ATTACHMENT_W_MULTIPLE_NULLS: &str = "MIME-Version: 1.0 +Received: by 10.76.127.40 with HTTP; Fri, 17 Jan 2014 02:21:43 -0800 (PST) +Date: Fri, 17 Jan 2014 12:21:43 +0200 +Delivered-To: chirila.s.alexandru@gmail.com +Message-ID: +Subject: Test +From: Alexandru Chirila +To: Alexandru Chirila +Content-Type: multipart/mixed; boundary=f46d040a62c49bb1c804f027e8cc + +--f46d040a62c49bb1c804f027e8cc +Content-Type: multipart/alternative; boundary=f46d040a62c49bb1c404f027e8ca + +--f46d040a62c49bb1c404f027e8ca +Content-Type: text/plain; charset=ISO-8859-1 + +This is a test ma\0\0\0iling +--f46d040a62c49bb1c804f027e8cc--"; + + const TEXT_ATTACHMENT_W_SUBJECT_NULL: &str = "MIME-Version: 1.0 +Received: by 10.76.127.40 with HTTP; Fri, 17 Jan 2014 02:21:43 -0800 (PST) +Date: Fri, 17 Jan 2014 12:21:43 +0200 +Delivered-To: chirila.s.alexandru@gmail.com +Message-ID: +Subject: Te\0\0\0st +From: Alexandru Chirila +To: Alexandru Chirila +Content-Type: multipart/mixed; boundary=f46d040a62c49bb1c804f027e8cc + +--f46d040a62c49bb1c804f027e8cc +Content-Type: multipart/alternative; boundary=f46d040a62c49bb1c404f027e8ca + +--f46d040a62c49bb1c404f027e8ca +Content-Type: text/plain; charset=ISO-8859-1 + +This is a test mailing +--f46d040a62c49bb1c804f027e8cc--"; + + const TEXT_ATTACHMENT_W_CONTENTTYPE_NULL: &str = "MIME-Version: 1.0 +Received: by 10.76.127.40 with HTTP; Fri, 17 Jan 2014 02:21:43 -0800 (PST) +Date: Fri, 17 Jan 2014 12:21:43 +0200 +Delivered-To: chirila.s.alexandru@gmail.com +Message-ID: +Subject: Test +From: Alexandru Chirila +To: Alexandru Chirila +Content-Type: multipart/mixed; boundary=f46d040a62c49bb1c804f027e8cc + +--f46d040a62c49bb1c804f027e8cc +Content-Type: multipart/alternative; boundary=f46d040a62c49bb1c404f027e8ca + +--f46d040a62c49bb1c404f027e8ca +Content-Type: text/plain; charset=\"iso-8859-1\0\0\0\" + +This is a test mailing +--f46d040a62c49bb1c804f027e8cc--"; + + const HTML_RAW: &str = r#"Email spam +

Email spam, also known as junk email +or unsolicited bulk email (UBE), is a subset of +electronic spam +involving nearly identical messages sent to numerous recipients by +email. Clicking on +links in spam email may send users to phishing +web sites or sites that are hosting malware."#; + + const HTML_RAW_STRIPED : &str = concat!("Email spam Email spam , also known as junk email or unsolicited bulk email ( UBE )," , + " is a subset of electronic spam involving nearly identical messages sent to numerous recipients by email" , + " . Clicking on links in spam email may send users to phishing web sites or sites that are hosting malware ."); +} diff --git a/crates/common/src/scripts/plugins/query.rs b/crates/common/src/scripts/plugins/query.rs new file mode 100644 index 00000000..71cc9478 --- /dev/null +++ b/crates/common/src/scripts/plugins/query.rs @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level store of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::cmp::Ordering; + +use crate::scripts::{into_sieve_value, to_store_value}; +use sieve::{runtime::Variable, FunctionMap}; +use store::{Rows, Value}; + +use super::PluginContext; + +pub fn register(plugin_id: u32, fnc_map: &mut FunctionMap<()>) { + fnc_map.set_external_function("query", plugin_id, 3); +} + +pub fn exec(ctx: PluginContext<'_>) -> Variable { + let span = ctx.span; + + // Obtain store name + let store = match &ctx.arguments[0] { + Variable::String(v) if !v.is_empty() => ctx.core.storage.lookups.get(v.as_ref()), + _ => Some(&ctx.core.storage.lookup), + }; + + let store = if let Some(store) = store { + store + } else { + tracing::warn!( + parent: span, + context = "sieve:query", + event = "failed", + reason = "Unknown store", + store = ctx.arguments[0].to_string().as_ref(), + ); + return false.into(); + }; + + // Obtain query string + let query = ctx.arguments[1].to_string(); + if query.is_empty() { + tracing::warn!( + parent: span, + context = "sieve:query", + event = "invalid", + reason = "Empty query string", + ); + return false.into(); + } + + // Obtain arguments + let arguments = match &ctx.arguments[2] { + Variable::Array(l) => l.iter().map(to_store_value).collect(), + v => vec![to_store_value(v)], + }; + + // Run query + if query + .as_bytes() + .get(..6) + .map_or(false, |q| q.eq_ignore_ascii_case(b"SELECT")) + { + if let Ok(mut rows) = ctx.handle.block_on(store.query::(&query, arguments)) { + match rows.rows.len().cmp(&1) { + Ordering::Equal => { + let mut row = rows.rows.pop().unwrap().values; + match row.len().cmp(&1) { + Ordering::Equal if !matches!(row.first(), Some(Value::Null)) => { + row.pop().map(into_sieve_value).unwrap() + } + Ordering::Less => Variable::default(), + _ => Variable::Array( + row.into_iter() + .map(into_sieve_value) + .collect::>() + .into(), + ), + } + } + Ordering::Less => Variable::default(), + Ordering::Greater => rows + .rows + .into_iter() + .map(|r| { + Variable::Array( + r.values + .into_iter() + .map(into_sieve_value) + .collect::>() + .into(), + ) + }) + .collect::>() + .into(), + } + } else { + false.into() + } + } else { + ctx.handle + .block_on(store.query::(&query, arguments)) + .is_ok() + .into() + } +} diff --git a/crates/common/src/scripts/plugins/text.rs b/crates/common/src/scripts/plugins/text.rs new file mode 100644 index 00000000..363c6f0f --- /dev/null +++ b/crates/common/src/scripts/plugins/text.rs @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level store of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use nlp::tokenizers::types::{TokenType, TypesTokenizer}; +use sieve::{runtime::Variable, FunctionMap}; + +use crate::scripts::functions::{html::html_to_tokens, text::tokenize_words, ApplyString}; + +use super::PluginContext; + +#[derive(PartialEq, Eq, Clone, Copy)] +enum MatchPart { + Sld, + Tld, + Host, +} + +pub fn register_tokenize(plugin_id: u32, fnc_map: &mut FunctionMap<()>) { + fnc_map.set_external_function("tokenize", plugin_id, 2); +} + +pub fn register_domain_part(plugin_id: u32, fnc_map: &mut FunctionMap<()>) { + fnc_map.set_external_function("domain_part", plugin_id, 2); +} + +pub fn exec_tokenize(ctx: PluginContext<'_>) -> Variable { + let mut v = ctx.arguments; + let (urls, urls_without_scheme, emails) = match v[1].to_string().as_ref() { + "html" => return html_to_tokens(v[0].to_string().as_ref()).into(), + "words" => return tokenize_words(&v[0]), + "uri" | "url" => (true, true, true), + "uri_strict" | "url_strict" => (true, false, false), + "email" => (false, false, true), + _ => return Variable::default(), + }; + + match v.remove(0) { + v @ (Variable::String(_) | Variable::Array(_)) => { + TypesTokenizer::new(v.to_string().as_ref(), &ctx.core.smtp.resolvers.psl) + .tokenize_numbers(false) + .tokenize_urls(urls) + .tokenize_urls_without_scheme(urls_without_scheme) + .tokenize_emails(emails) + .filter_map(|t| match t.word { + TokenType::Url(text) if urls => Variable::from(text.to_string()).into(), + TokenType::UrlNoScheme(text) if urls_without_scheme => { + Variable::from(format!("https://{text}")).into() + } + TokenType::Email(text) if emails => Variable::from(text.to_string()).into(), + _ => None, + }) + .collect::>() + .into() + } + v => v, + } +} + +pub fn exec_domain_part(ctx: PluginContext<'_>) -> Variable { + let v = ctx.arguments; + let match_part = match v[1].to_string().as_ref() { + "sld" => MatchPart::Sld, + "tld" => MatchPart::Tld, + "host" => MatchPart::Host, + _ => return Variable::default(), + }; + + v[0].transform(|domain| { + let d = domain.trim().to_lowercase(); + let mut seen_dot = false; + for (pos, ch) in d.as_bytes().iter().enumerate().rev() { + if *ch == b'.' { + if seen_dot { + let maybe_domain = + std::str::from_utf8(&d.as_bytes()[pos + 1..]).unwrap_or_default(); + if !ctx.core.smtp.resolvers.psl.contains(maybe_domain) { + return if match_part == MatchPart::Sld { + maybe_domain + } else { + std::str::from_utf8(&d.as_bytes()[..pos]).unwrap_or_default() + } + .to_string() + .into(); + } + } else if match_part == MatchPart::Tld { + return std::str::from_utf8(&d.as_bytes()[pos + 1..]) + .unwrap_or_default() + .to_string() + .into(); + } else { + seen_dot = true; + } + } + } + + if seen_dot { + if match_part == MatchPart::Sld { + d.into() + } else { + Variable::default() + } + } else if match_part == MatchPart::Host { + d.into() + } else { + Variable::default() + } + }) +} diff --git a/crates/jmap-proto/src/request/capability.rs b/crates/jmap-proto/src/request/capability.rs index 67887170..a767c931 100644 --- a/crates/jmap-proto/src/request/capability.rs +++ b/crates/jmap-proto/src/request/capability.rs @@ -21,9 +21,12 @@ * for more details. */ +use utils::map::vec_map::VecMap; + use crate::{ error::request::RequestError, parser::{json::Parser, Error, JsonObjectParser}, + types::type_state::DataType, }; #[derive(Debug, Clone, Copy, serde::Serialize, Hash, PartialEq, Eq)] @@ -50,6 +53,119 @@ pub enum Capability { Quota = 1 << 9, } +#[derive(Debug, Clone, serde::Serialize)] +#[serde(untagged)] +#[allow(dead_code)] +pub enum Capabilities { + Core(CoreCapabilities), + Mail(MailCapabilities), + Submission(SubmissionCapabilities), + WebSocket(WebSocketCapabilities), + SieveAccount(SieveAccountCapabilities), + SieveSession(SieveSessionCapabilities), + Blob(BlobCapabilities), + Empty(EmptyCapabilities), +} + +#[derive(Debug, Clone, serde::Serialize)] +pub struct CoreCapabilities { + #[serde(rename(serialize = "maxSizeUpload"))] + pub max_size_upload: usize, + #[serde(rename(serialize = "maxConcurrentUpload"))] + pub max_concurrent_upload: usize, + #[serde(rename(serialize = "maxSizeRequest"))] + pub max_size_request: usize, + #[serde(rename(serialize = "maxConcurrentRequests"))] + pub max_concurrent_requests: usize, + #[serde(rename(serialize = "maxCallsInRequest"))] + pub max_calls_in_request: usize, + #[serde(rename(serialize = "maxObjectsInGet"))] + pub max_objects_in_get: usize, + #[serde(rename(serialize = "maxObjectsInSet"))] + pub max_objects_in_set: usize, + #[serde(rename(serialize = "collationAlgorithms"))] + pub collation_algorithms: Vec, +} + +#[derive(Debug, Clone, serde::Serialize)] +pub struct WebSocketCapabilities { + #[serde(rename(serialize = "url"))] + pub url: String, + #[serde(rename(serialize = "supportsPush"))] + pub supports_push: bool, +} + +#[derive(Debug, Clone, serde::Serialize)] +pub struct SieveSessionCapabilities { + #[serde(rename(serialize = "implementation"))] + pub implementation: &'static str, +} + +#[derive(Debug, Clone, serde::Serialize)] +pub struct SieveAccountCapabilities { + #[serde(rename(serialize = "maxSizeScriptName"))] + pub max_script_name: usize, + #[serde(rename(serialize = "maxSizeScript"))] + pub max_script_size: usize, + #[serde(rename(serialize = "maxNumberScripts"))] + pub max_scripts: usize, + #[serde(rename(serialize = "maxNumberRedirects"))] + pub max_redirects: usize, + #[serde(rename(serialize = "sieveExtensions"))] + pub extensions: Vec, + #[serde(rename(serialize = "notificationMethods"))] + pub notification_methods: Option>, + #[serde(rename(serialize = "externalLists"))] + pub ext_lists: Option>, +} + +#[derive(Debug, Clone, serde::Serialize)] +pub struct MailCapabilities { + #[serde(rename(serialize = "maxMailboxesPerEmail"))] + pub max_mailboxes_per_email: Option, + #[serde(rename(serialize = "maxMailboxDepth"))] + pub max_mailbox_depth: usize, + #[serde(rename(serialize = "maxSizeMailboxName"))] + pub max_size_mailbox_name: usize, + #[serde(rename(serialize = "maxSizeAttachmentsPerEmail"))] + pub max_size_attachments_per_email: usize, + #[serde(rename(serialize = "emailQuerySortOptions"))] + pub email_query_sort_options: Vec, + #[serde(rename(serialize = "mayCreateTopLevelMailbox"))] + pub may_create_top_level_mailbox: bool, +} + +#[derive(Debug, Clone, serde::Serialize)] +pub struct SubmissionCapabilities { + #[serde(rename(serialize = "maxDelayedSend"))] + pub max_delayed_send: usize, + #[serde(rename(serialize = "submissionExtensions"))] + pub submission_extensions: VecMap>, +} + +#[derive(Debug, Clone, serde::Serialize)] +pub struct BlobCapabilities { + #[serde(rename(serialize = "maxSizeBlobSet"))] + pub max_size_blob_set: usize, + #[serde(rename(serialize = "maxDataSources"))] + pub max_data_sources: usize, + #[serde(rename(serialize = "supportedTypeNames"))] + pub supported_type_names: Vec, + #[serde(rename(serialize = "supportedDigestAlgorithms"))] + pub supported_digest_algorithms: Vec<&'static str>, +} + +#[derive(Debug, Clone, Default, serde::Serialize)] +pub struct EmptyCapabilities {} + +impl Default for SieveSessionCapabilities { + fn default() -> Self { + Self { + implementation: concat!("Stalwart JMAP v", env!("CARGO_PKG_VERSION"),), + } + } +} + impl JsonObjectParser for Capability { fn parse(parser: &mut Parser<'_>) -> crate::parser::Result where diff --git a/crates/jmap/Cargo.toml b/crates/jmap/Cargo.toml index 261bf004..0113dfce 100644 --- a/crates/jmap/Cargo.toml +++ b/crates/jmap/Cargo.toml @@ -31,7 +31,7 @@ form-data = { version = "0.5.0", features = ["sync"], default-features = false } mime = "0.3.17" futures-util = "0.3.28" async-stream = "0.3.5" -base64 = "0.21" +base64 = "0.22" p256 = { version = "0.13", features = ["ecdh"] } hkdf = "0.12.3" sha1 = "0.10" diff --git a/crates/utils/src/config/utils.rs b/crates/utils/src/config/utils.rs index 5af863a4..e279262a 100644 --- a/crates/utils/src/config/utils.rs +++ b/crates/utils/src/config/utils.rs @@ -894,6 +894,16 @@ impl AsKey for String { } } +impl AsKey for &String { + fn as_key(&self) -> String { + self.to_string() + } + + fn as_prefix(&self) -> String { + format!("{self}.") + } +} + impl AsKey for (&str, &str) { fn as_key(&self) -> String { format!("{}.{}", self.0, self.1) @@ -904,6 +914,16 @@ impl AsKey for (&str, &str) { } } +impl AsKey for (&str, &String) { + fn as_key(&self) -> String { + format!("{}.{}", self.0, self.1) + } + + fn as_prefix(&self) -> String { + format!("{}.{}.", self.0, self.1) + } +} + impl AsKey for (&String, &str) { fn as_key(&self) -> String { format!("{}.{}", self.0, self.1) diff --git a/crates/utils/src/suffixlist.rs b/crates/utils/src/suffixlist.rs index 59ebd6c7..55b41fc4 100644 --- a/crates/utils/src/suffixlist.rs +++ b/crates/utils/src/suffixlist.rs @@ -21,7 +21,12 @@ * for more details. */ +use std::io::Read; + use ahash::AHashSet; +use mail_auth::flate2::read::GzDecoder; + +use crate::config::Config; #[derive(Debug, Clone, Default)] pub struct PublicSuffix { @@ -57,3 +62,108 @@ impl From<&str> for PublicSuffix { ps } } + +impl PublicSuffix { + pub async fn parse(config: &mut Config, key: &str) -> PublicSuffix { + let values = config + .values(key) + .map(|(_, s)| s.to_string()) + .collect::>(); + let has_values = !values.is_empty(); + for (idx, value) in values.into_iter().enumerate() { + let bytes = if value.starts_with("https://") || value.starts_with("http://") { + let result = match reqwest::get(&value).await { + Ok(r) => { + if r.status().is_success() { + r.bytes().await + } else { + config.new_build_error( + format!("{value}.{idx}"), + format!( + "Failed to fetch public suffixes from {value:?}: Status {status}", + value = value, + status = r.status() + ), + ); + continue; + } + } + Err(err) => Err(err), + }; + + match result { + Ok(bytes) => bytes.to_vec(), + Err(err) => { + config.new_build_error( + format!("{value}.{idx}"), + format!("Failed to fetch public suffixes from {value:?}: {err}",), + ); + continue; + } + } + } else if let Some(filename) = value.strip_prefix("file://") { + match std::fs::read(filename) { + Ok(bytes) => bytes, + Err(err) => { + config.new_build_error( + format!("{value}.{idx}"), + format!("Failed to read public suffixes from {value:?}: {err}",), + ); + continue; + } + } + } else { + config.new_parse_error(key, format!("Invalid public suffix file {value:?}")); + continue; + }; + let bytes = if value.ends_with(".gz") { + match GzDecoder::new(&bytes[..]) + .bytes() + .collect::, _>>() + { + Ok(bytes) => bytes, + Err(err) => { + config.new_build_error( + format!("{value}.{idx}"), + format!( + "Failed to decompress public suffixes from {value:?}: {err}", + value = value, + err = err + ), + ); + continue; + } + } + } else { + bytes + }; + + match String::from_utf8(bytes) { + Ok(list) => { + return PublicSuffix::from(list.as_str()); + } + Err(err) => { + config.new_build_error( + format!("{value}.{idx}"), + format!( + "Failed to parse public suffixes from {value:?}: {err}", + value = value, + err = err + ), + ); + } + } + } + + config.new_build_error( + key, + if has_values { + "Failed to parse public suffixes from any source." + } else { + "No public suffixes list was specified." + }, + ); + + PublicSuffix::default() + } +}