diff --git a/Cargo.lock b/Cargo.lock index f56e144f..29da8ef2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1097,6 +1097,9 @@ dependencies = [ "psl", "pwhash", "quick_cache", + "rasn", + "rasn-cms", + "rasn-pkix", "rcgen 0.12.1", "regex", "registry", @@ -1107,6 +1110,7 @@ dependencies = [ "rustls 0.23.36", "rustls-pemfile 2.2.0", "rustls-pki-types", + "sequoia-openpgp", "serde", "serde_json", "sha1", @@ -7418,6 +7422,7 @@ dependencies = [ "flate2", "foundationdb", "futures", + "gethostname", "lru-cache", "lz4_flex 0.12.0", "memchr", diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 1f8907f6..bb8babbd 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -80,6 +80,10 @@ hickory-proto = "0.24" arcstr = "1.2.0" nohash-hasher = "0.2.0" quick_cache = "0.6.9" +rasn = "0.10" +rasn-cms = "0.10" +rasn-pkix = "0.10" +sequoia-openpgp = { version = "2.0", default-features = false, features = ["crypto-rust", "allow-experimental-crypto", "allow-variable-time-crypto"] } [target.'cfg(unix)'.dependencies] privdrop = "0.5.3" diff --git a/crates/common/src/auth/mod.rs b/crates/common/src/auth/mod.rs index 0ec52c4f..9d5adc47 100644 --- a/crates/common/src/auth/mod.rs +++ b/crates/common/src/auth/mod.rs @@ -79,10 +79,21 @@ pub struct AccountCache { pub quota_disk: u64, pub quota_objects: Option>, pub description: Option>, + pub encryption_key: Option, pub locale: Locale, - pub is_user: bool, + pub flags: u64, } +pub type EncryptionKeys = Box<[Box<[u8]>]>; + +pub const ACCOUNT_IS_USER: u64 = 1; +pub const ACCOUNT_FLAG_ENCRYPT_TRAIN_SPAM_FILTER: u64 = 1 << 1; +pub const ACCOUNT_FLAG_ENCRYPT_METHOD_SMIME: u64 = 1 << 2; +pub const ACCOUNT_FLAG_ENCRYPT_METHOD_PGP: u64 = 1 << 3; +pub const ACCOUNT_FLAG_ENCRYPT_ALGO_AES256: u64 = 1 << 4; +pub const ACCOUNT_FLAG_ENCRYPT_ALGO_AES128: u64 = 1 << 5; +pub const ACCOUNT_FLAG_ENCRYPT_APPEND: u64 = 1 << 6; + #[derive(Debug, Clone)] pub struct RoleCache { pub id_roles: TinyVec<[u32; 3]>, diff --git a/crates/common/src/cache/invalidate.rs b/crates/common/src/cache/invalidate.rs index 0ce9fea3..4a547aed 100644 --- a/crates/common/src/cache/invalidate.rs +++ b/crates/common/src/cache/invalidate.rs @@ -44,6 +44,7 @@ impl CacheInvalidationBuilder { let groups_changed = current.member_group_ids != new.member_group_ids; let aliases_changed = current.aliases != new.aliases; let credentials_changed = current.credentials != new.credentials; + let encryption_changed = current.encryption_at_rest != new.encryption_at_rest; if was_renamed || aliases_changed @@ -51,6 +52,7 @@ impl CacheInvalidationBuilder { || groups_changed || quota_changed || details_changed + || encryption_changed { self.invalidate(CacheInvalidation::Account(id)); } @@ -253,11 +255,30 @@ impl Server { let changes = changes.into_iter().collect::>(); self.invalidate_local_caches(&changes).await; - self.cluster_broadcast(BroadcastEvent::CacheInvalidation(changes)) + self.cluster_broadcast(BroadcastEvent::CacheInvalidate(changes)) .await; Ok(()) } + pub fn invalidate_all_local_caches(&self) { + self.inner.cache.access_tokens.clear(); + self.inner.cache.domains.clear(); + self.inner.cache.domain_names.clear(); + self.inner.cache.domain_names_negative.clear(); + self.inner.cache.emails.clear(); + self.inner.cache.emails_negative.clear(); + self.inner.cache.tenants.clear(); + self.inner.cache.files.clear(); + self.inner.cache.contacts.clear(); + self.inner.cache.events.clear(); + self.inner.cache.scheduling.clear(); + self.inner.cache.dkim_signers.clear(); + self.inner.cache.accounts.clear(); + self.inner.cache.roles.clear(); + self.inner.cache.lists.clear(); + self.inner.data.logos.lock().clear(); + } + pub async fn invalidate_local_caches(&self, changes: &[CacheInvalidation]) { let cache = &self.inner.cache; diff --git a/crates/common/src/cache/principals.rs b/crates/common/src/cache/principals.rs index 7be59382..c7260715 100644 --- a/crates/common/src/cache/principals.rs +++ b/crates/common/src/cache/principals.rs @@ -7,6 +7,9 @@ use crate::{ Server, auth::{ + ACCOUNT_FLAG_ENCRYPT_ALGO_AES128, ACCOUNT_FLAG_ENCRYPT_ALGO_AES256, + ACCOUNT_FLAG_ENCRYPT_APPEND, ACCOUNT_FLAG_ENCRYPT_METHOD_PGP, + ACCOUNT_FLAG_ENCRYPT_METHOD_SMIME, ACCOUNT_FLAG_ENCRYPT_TRAIN_SPAM_FILTER, ACCOUNT_IS_USER, AccountCache, AccountInfo, AccountTenantIds, DOMAIN_FLAG_RELAY, DOMAIN_FLAG_SUB_ADDRESSING, DomainCache, EmailAddress, EmailAddressRef, EmailCache, MailingListCache, PermissionsGroup, RoleCache, TenantCache, permissions::BuildPermissions, @@ -14,7 +17,10 @@ use crate::{ config::smtp::auth::DkimSigner, expr::if_block::BootstrapExprExt, network::{masked::MaskedAddress, mta::AddressResolver}, - storage::{ObjectQuota, TenantQuota}, + storage::{ + ObjectQuota, TenantQuota, + encryption::{EncryptionMethod, parse_public_key}, + }, }; use ahash::AHashSet; use arcstr::ArcStr; @@ -23,8 +29,8 @@ use registry::{ enums::{Locale, StorageQuota, TenantStorageQuota}, prelude::{ObjectType, Property}, structs::{ - Account, DkimSignature, Domain, MailingList, MaskedEmail, Permissions, Role, - SubAddressing, Tenant, + Account, DkimSignature, Domain, EncryptionAtRest, MailingList, MaskedEmail, + Permissions, PublicKey, Role, SubAddressing, Tenant, }, }, types::{EnumImpl, id::ObjectId}, @@ -274,6 +280,51 @@ impl Server { } } + let mut flags = ACCOUNT_IS_USER; + let encryption_settings = match account.encryption_at_rest { + EncryptionAtRest::Disabled => None, + EncryptionAtRest::Aes256(settings) => { + flags |= ACCOUNT_FLAG_ENCRYPT_ALGO_AES256; + settings.into() + } + EncryptionAtRest::Aes128(settings) => { + flags |= ACCOUNT_FLAG_ENCRYPT_ALGO_AES128; + settings.into() + } + }; + let encryption_key = if let Some(settings) = encryption_settings { + if settings.allow_spam_training { + flags |= ACCOUNT_FLAG_ENCRYPT_TRAIN_SPAM_FILTER; + } + if settings.encrypt_on_append { + flags |= ACCOUNT_FLAG_ENCRYPT_APPEND; + } + if let Some(public_key) = self + .registry() + .object::(settings.public_key) + .await + .caused_by(trc::location!())? + { + parse_public_key(&public_key) + .unwrap_or_default() + .map(|params| { + match params.method { + EncryptionMethod::PGP => { + flags |= ACCOUNT_FLAG_ENCRYPT_METHOD_PGP + } + EncryptionMethod::SMIME => { + flags |= ACCOUNT_FLAG_ENCRYPT_METHOD_SMIME + } + } + params.certs + }) + } else { + None + } + } else { + None + }; + AccountCache { id: account_id, name: name.into_boxed_str(), @@ -297,7 +348,8 @@ impl Server { quota_objects: quota_objects.map(Box::new), description: account.description.map(Into::into), locale: account.locale, - is_user: true, + encryption_key, + flags, } } Account::Group(account) => { @@ -348,8 +400,9 @@ impl Server { quota_disk, quota_objects: quota_objects.map(Box::new), description: account.description.map(Into::into), + encryption_key: None, locale: account.locale, - is_user: false, + flags: 0, } } }); @@ -445,6 +498,10 @@ impl Server { pub async fn account_info(&self, id: u32) -> trc::Result { let account = self.account(id).await?; + self.build_account_info(account).await + } + + pub async fn build_account_info(&self, account: Arc) -> trc::Result { let mut addresses = Vec::with_capacity(account.id_member_of.len() + account.addresses.len()); for address in account.addresses.iter() { @@ -477,7 +534,7 @@ impl Server { } Ok(AccountInfo { - account_id: id, + account_id: account.id, account, addresses, }) @@ -617,7 +674,7 @@ impl Server { if let Some(signature) = self.registry().object::(id.into()).await? { - match DkimSigner::new(domain.names[0].to_string(), signature) { + match DkimSigner::new(domain.names[0].to_string(), signature).await { Ok(signer) => signatures.push(signer), Err(err) => { trc::error!(err.ctx(trc::Key::Id, id).caused_by(trc::location!())); @@ -672,7 +729,7 @@ impl AccountInfo { #[inline(always)] pub fn is_user_account(&self) -> bool { - self.account.is_user + self.account.flags & ACCOUNT_IS_USER != 0 } #[inline(always)] @@ -684,6 +741,11 @@ impl AccountInfo { pub fn object_quotas(&self) -> Option<&ObjectQuota> { self.account.quota_objects.as_deref() } + + #[inline(always)] + pub fn account(&self) -> &AccountCache { + &self.account + } } impl AccountCache { @@ -709,7 +771,7 @@ impl AccountCache { #[inline(always)] pub fn is_user_account(&self) -> bool { - self.is_user + self.flags & ACCOUNT_IS_USER != 0 } #[inline(always)] diff --git a/crates/common/src/cache/reload.rs b/crates/common/src/cache/reload.rs index 1d54e036..c313812e 100644 --- a/crates/common/src/cache/reload.rs +++ b/crates/common/src/cache/reload.rs @@ -12,23 +12,26 @@ use crate::{ telemetry::Telemetry, }, ipc::{QueueEvent, RegistryChange}, + network::security::BlockedIps, }; use ahash::AHashMap; use directory::Directories; -use registry::schema::{prelude::ObjectType, structs::BlockedIp}; +use registry::{ + schema::{prelude::ObjectType, structs::BlockedIp}, + types::error::{Error, Warning}, +}; use std::sync::Arc; -use store::{InMemoryStore, LookupStores, registry::bootstrap::Bootstrap, write::now}; +use store::{LookupStores, registry::bootstrap::Bootstrap, write::now}; pub struct ReloadResult { - pub bootstrap: Bootstrap, - pub new_core: Option, - pub tracers: Option, + pub errors: Vec, + pub warnings: Vec, + pub replaced_core: bool, } impl Server { pub async fn reload_registry(&self, change: RegistryChange) -> trc::Result { - let todo = "check the different events triggering this, spam filter reload, etc. make sure all are used"; - let mut bootstrap = Bootstrap::init(self.registry().clone()).await; + let mut bootstrap = Bootstrap::new(self.registry().clone()); let object = match change { RegistryChange::Insert(id) => { if matches!(id.object(), ObjectType::BlockedIp) { @@ -42,11 +45,7 @@ impl Server { ips.blocked_ip_networks.push(ip.address); } } - return Ok(ReloadResult { - bootstrap, - new_core: None, - tracers: None, - }); + return Ok(bootstrap.into()); } else { id.object() } @@ -55,56 +54,36 @@ impl Server { RegistryChange::Reload(object) => object, }; - let mut result = ReloadResult { - bootstrap, - new_core: None, - tracers: None, - }; - match object { ObjectType::Certificate => { let mut certificates = AHashMap::new(); - parse_certificates( - &mut result.bootstrap, - &mut certificates, - &mut Default::default(), - ) - .await; + parse_certificates(&mut bootstrap, &mut certificates, &mut Default::default()) + .await; self.inner .data .tls_certificates .store(Arc::new(certificates)); } - ObjectType::MemoryLookupKey | ObjectType::MemoryLookupKeyValue => { - let mut lookup = LookupStores { - stores: self.inner.data.lookup_stores.load().as_ref().clone(), - }; - lookup - .stores - .retain(|_, store| !matches!(store, InMemoryStore::Static(_))); - lookup.parse_static(&mut result.bootstrap).await; + ObjectType::MemoryLookupKey + | ObjectType::MemoryLookupKeyValue + | ObjectType::HttpLookup + | ObjectType::StoreLookup => { + let lookup = LookupStores::build(&mut bootstrap).await; + + if bootstrap.errors.is_empty() { + self.inner.data.lookup_stores.store(Arc::new(lookup.stores)); + } } - ObjectType::HttpLookup => { - let mut lookup = LookupStores { - stores: self.inner.data.lookup_stores.load().as_ref().clone(), - }; - lookup - .stores - .retain(|_, store| !matches!(store, InMemoryStore::Http(_))); - lookup.parse_http(&mut result.bootstrap).await; - } - ObjectType::StoreLookup => { - let mut lookup = LookupStores { - stores: self.inner.data.lookup_stores.load().as_ref().clone(), - }; - lookup.stores.retain(|_, store| { - matches!(store, InMemoryStore::Static(_) | InMemoryStore::Http(_)) - }); - lookup.parse_stores(&mut result.bootstrap).await; + + ObjectType::BlockedIp => { + let blocked_ips = BlockedIps::parse(&mut bootstrap).await; + if bootstrap.errors.is_empty() { + *self.inner.data.blocked_ips.write() = blocked_ips; + } } _ => { // Load stores - let directory = Directories::build(&mut result.bootstrap).await; + let directory = Directories::build(&mut bootstrap).await; let storage = &self.core.storage; let storage = Storage { registry: storage.registry.clone(), @@ -120,38 +99,76 @@ impl Server { }; // Parse tracers - let tracers = Telemetry::parse(&mut result.bootstrap, &storage).await; + let tracers = Telemetry::parse(&mut bootstrap, &storage).await; - if result.bootstrap.errors.is_empty() { - let core = Box::pin(Core::parse(&mut result.bootstrap, storage)).await; + if bootstrap.errors.is_empty() { + let core = Box::pin(Core::parse(&mut bootstrap, storage)).await; - if result.bootstrap.errors.is_empty() { - let mut servers = Listeners::parse(&mut result.bootstrap).await; + if bootstrap.errors.is_empty() { + let mut servers = Listeners::parse(&mut bootstrap).await; servers - .parse_tcp_acceptors(&mut result.bootstrap, self.inner.clone()) + .parse_tcp_acceptors(&mut bootstrap, self.inner.clone()) .await; - if result.bootstrap.errors.is_empty() { - result.new_core = Some(core); - result.tracers = Some(tracers); + if bootstrap.errors.is_empty() { + // Update core + self.inner.shared_core.store(core.into()); + + // Update tracers + + // SPDX-SnippetBegin + // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + // SPDX-License-Identifier: LicenseRef-SEL + #[cfg(feature = "enterprise")] + tracers.update(self.inner.shared_core.load().is_enterprise_edition()); + // SPDX-SnippetEnd + #[cfg(not(feature = "enterprise"))] + tracers.update(false); + + // Reload queue settings + self.inner + .ipc + .queue_tx + .send(QueueEvent::ReloadSettings) + .await + .ok(); + + return Ok(ReloadResult { + errors: bootstrap.errors, + warnings: bootstrap.warnings, + replaced_core: true, + }); } } } } } - Ok(result) - } - - pub async fn reload_core(&self, new_core: Core) { - self.inner.shared_core.store(new_core.into()); - - // Reload queue settings - self.inner - .ipc - .queue_tx - .send(QueueEvent::ReloadSettings) - .await - .ok(); + Ok(bootstrap.into()) + } +} + +impl ReloadResult { + pub fn has_errors(&self) -> bool { + !self.errors.is_empty() + } + + pub fn log(&self) { + for error in &self.errors { + error.log(); + } + for warning in &self.warnings { + warning.log(); + } + } +} + +impl From for ReloadResult { + fn from(bootstrap: Bootstrap) -> Self { + Self { + errors: bootstrap.errors, + warnings: bootstrap.warnings, + replaced_core: false, + } } } diff --git a/crates/common/src/config/groupware.rs b/crates/common/src/config/groupware.rs index 91e58819..bc0aec13 100644 --- a/crates/common/src/config/groupware.rs +++ b/crates/common/src/config/groupware.rs @@ -6,7 +6,7 @@ use registry::schema::structs::{ AddressBook, Calendar, CalendarAlarm, CalendarScheduling, DataRetention, FileStorage, Sharing, - WebDav, + SystemSettings, WebDav, }; use std::str::FromStr; use store::registry::bootstrap::Bootstrap; @@ -90,6 +90,7 @@ impl GroupwareConfig { let file = bp.setting_infallible::().await; let share = bp.setting_infallible::().await; let dr = bp.setting_infallible::().await; + let system = bp.setting_infallible::().await; GroupwareConfig { max_request_size: dav.request_max_size as usize, @@ -134,7 +135,7 @@ impl GroupwareConfig { { Some(url.to_string()) } else { - Some(format!("https://{}/calendar/rsvp", bp.hostname())) + Some(format!("https://{}/calendar/rsvp", system.default_hostname)) } } else { None diff --git a/crates/common/src/config/inner.rs b/crates/common/src/config/inner.rs index 524b2256..f06d0f6c 100644 --- a/crates/common/src/config/inner.rs +++ b/crates/common/src/config/inner.rs @@ -24,10 +24,7 @@ use arc_swap::ArcSwap; use mail_auth::{MX, Parameters, Txt}; use mail_send::smtp::tls::build_tls_connector; use parking_lot::RwLock; -use registry::schema::{ - prelude::{Object, ObjectType}, - structs, -}; +use registry::schema::{prelude::ObjectType, structs}; use std::{ net::{IpAddr, Ipv4Addr, Ipv6Addr}, sync::Arc, diff --git a/crates/common/src/config/mailstore/email.rs b/crates/common/src/config/mailstore/email.rs index 548d86cf..bbdb80f1 100644 --- a/crates/common/src/config/mailstore/email.rs +++ b/crates/common/src/config/mailstore/email.rs @@ -15,7 +15,7 @@ use registry::{ prelude::ObjectType, structs::{ AddressBook, Authentication, Calendar, DataRetention, Domain, Email, Jmap, Search, - SieveUserInterpreter, + SieveUserInterpreter, SystemSettings, }, }, types::EnumImpl, @@ -85,11 +85,12 @@ impl EmailConfig { let jmap = bp.setting_infallible::().await; let calendar = bp.setting_infallible::().await; let address_book = bp.setting_infallible::().await; + let system = bp.setting_infallible::().await; let auth = bp.setting_infallible::().await; // Obtain default domain name let default_domain_name = if let Some(default_domain) = - bp.get_infallible::(auth.default_domain_id).await + bp.get_infallible::(system.default_domain_id).await { default_domain.name } else { @@ -97,7 +98,7 @@ impl EmailConfig { ObjectType::Authentication.singleton(), format!( "Default domain with ID {} not found", - auth.default_domain_id + system.default_domain_id ), ); "localhost.local".to_string() @@ -274,7 +275,7 @@ impl EmailConfig { data_purge_frequency: dr.data_cleanup_schedule.into(), blob_purge_frequency: dr.blob_cleanup_schedule.into(), compression: email.compression_algorithm, - default_domain_id: auth.default_domain_id.id() as u32, + default_domain_id: system.default_domain_id.id() as u32, default_domain_name, } } diff --git a/crates/common/src/config/mailstore/scripts.rs b/crates/common/src/config/mailstore/scripts.rs index f93301ec..d305db3c 100644 --- a/crates/common/src/config/mailstore/scripts.rs +++ b/crates/common/src/config/mailstore/scripts.rs @@ -18,6 +18,7 @@ use registry::{ prelude::ObjectType, structs::{ SieveSystemInterpreter, SieveSystemScript, SieveUserInterpreter, SieveUserScript, + SystemSettings, }, }, types::EnumImpl, @@ -91,6 +92,7 @@ impl Scripting { // Allocate compiler and runtime let trusted = bp.setting_infallible::().await; + let system = bp.setting_infallible::().await; let trusted_compiler = Compiler::new() .with_max_string_size(52428800) .with_max_variable_name_size(100) @@ -128,7 +130,7 @@ impl Scripting { .with_max_nested_includes(trusted.max_nested_includes as usize) .with_max_received_headers(trusted.max_received_headers as usize) .with_default_duplicate_expiry(trusted.duplicate_expiry.into_inner().as_secs()); - trusted_runtime.set_local_hostname(bp.node.hostname.clone()); + trusted_runtime.set_local_hostname(system.default_hostname.clone()); // Parse trusted scripts let mut trusted_scripts = AHashMap::new(); diff --git a/crates/common/src/config/network.rs b/crates/common/src/config/network.rs index c3004544..3d7c4e96 100644 --- a/crates/common/src/config/network.rs +++ b/crates/common/src/config/network.rs @@ -12,9 +12,8 @@ use crate::{ use ahash::AHashMap; use registry::{ schema::{ - enums::NodeShardType, prelude::ObjectType, - structs::{self, Asn, HttpForm, NodeRole, NodeShard, Rate, TaskManager}, + structs::{self, Asn, HttpForm, Rate, SystemSettings, TaskManager}, }, types::EnumImpl, }; @@ -150,9 +149,11 @@ impl ContactForm { impl Network { pub async fn parse(bp: &mut Bootstrap) -> Self { + let system = bp.setting_infallible::().await; + let mut network = Network { node_id: bp.node_id(), - server_name: bp.hostname().to_string(), + server_name: system.default_hostname, security: Security::parse(bp).await, contact_form: ContactForm::parse(bp).await, asn_geo_lookup: AsnGeoLookupConfig::parse(bp).await.unwrap_or_default(), diff --git a/crates/common/src/config/server/listener.rs b/crates/common/src/config/server/listener.rs index e644bb6e..21a3d3af 100644 --- a/crates/common/src/config/server/listener.rs +++ b/crates/common/src/config/server/listener.rs @@ -14,7 +14,7 @@ use crate::{ }; use registry::schema::{ enums::{NetworkListenerProtocol, TlsCipherSuite, TlsVersion}, - structs::NetworkListener, + structs::{NetworkListener, SystemSettings}, }; use rustls::{ ALL_VERSIONS, ServerConfig, SupportedCipherSuite, @@ -36,6 +36,8 @@ impl Listeners { // Parse servers let node_id = bp.node_id(); + let system = bp.setting_infallible::().await; + for listener in bp.list_infallible::().await { if listener.object.enable_for_nodes.is_empty() || listener @@ -44,13 +46,18 @@ impl Listeners { .values() .any(|n| n.contains(node_id)) { - servers.parse_server(bp, listener); + servers.parse_server(bp, listener, &system); } } servers } - fn parse_server(&mut self, bp: &mut Bootstrap, listener: RegistryObject) { + fn parse_server( + &mut self, + bp: &mut Bootstrap, + listener: RegistryObject, + system: &SystemSettings, + ) { let id = listener.id; let revision = listener.revision; let listener = listener.object; @@ -124,8 +131,9 @@ impl Listeners { } let span_id_gen = self.span_id_gen.clone(); + self.servers.push(Listener { - max_connections: listener.max_connections.unwrap_or(bp.node.max_connections), + max_connections: listener.max_connections.unwrap_or(system.max_connections), id: listener.name.clone(), registry_id: id, protocol, @@ -133,7 +141,7 @@ impl Listeners { proxy_networks: if !listener.override_proxy_trusted_networks.is_empty() { listener.override_proxy_trusted_networks.as_slice().to_vec() } else { - bp.node.proxy_trusted_networks.as_slice().to_vec() + system.proxy_trusted_networks.as_slice().to_vec() }, span_id_gen, }); diff --git a/crates/common/src/config/smtp/auth.rs b/crates/common/src/config/smtp/auth.rs index 556711a1..cf765b4d 100644 --- a/crates/common/src/config/smtp/auth.rs +++ b/crates/common/src/config/smtp/auth.rs @@ -124,7 +124,7 @@ impl MailAuthConfig { } impl DkimSigner { - pub fn new(domain: String, signature: DkimSignature) -> trc::Result { + pub async fn new(domain: String, signature: DkimSignature) -> trc::Result { let mut errors = vec![]; if !signature.validate(&mut errors) { return Err(trc::DkimEvent::BuildError @@ -139,7 +139,8 @@ impl DkimSigner { match signature { DkimSignature::Dkim1Ed25519Sha256(signature) => { - let private_key = simple_pem_parse(&signature.private_key).ok_or_else(|| { + let private_key = signature.private_key.pem().await?; + let private_key = simple_pem_parse(&private_key).ok_or_else(|| { trc::DkimEvent::BuildError .reason("Failed to parse ED25519 private key PEM") .details("Invalid PEM format") @@ -156,24 +157,8 @@ impl DkimSigner { ))) } DkimSignature::Dkim1RsaSha256(signature) => { - let key = PrivatePkcs1KeyDer::from_pem_slice(signature.private_key.as_bytes()) - .map(PrivateKeyDer::Pkcs1) - .or_else(|_| { - PrivatePkcs8KeyDer::from_pem_slice(signature.private_key.as_bytes()) - .map(PrivateKeyDer::Pkcs8) - }) - .map_err(|err| { - trc::DkimEvent::BuildError - .reason(err) - .details("Failed to build RSA key") - }) - .and_then(|key| { - RsaKey::::from_key_der(key).map_err(|err| { - trc::DkimEvent::BuildError - .reason(err) - .details("Failed to build RSA key") - }) - })?; + let private_key = signature.private_key.pem().await?; + let key = rsa_key_parse(private_key.as_bytes())?; Ok(DkimSigner::RsaSha256(build_dkim1_signer( domain, signature, key, @@ -183,7 +168,25 @@ impl DkimSigner { } } -impl ArcSealer { +pub fn rsa_key_parse(private_key: &[u8]) -> trc::Result> { + PrivatePkcs1KeyDer::from_pem_slice(private_key) + .map(PrivateKeyDer::Pkcs1) + .or_else(|_| PrivatePkcs8KeyDer::from_pem_slice(private_key).map(PrivateKeyDer::Pkcs8)) + .map_err(|err| { + trc::DkimEvent::BuildError + .reason(err) + .details("Failed to build RSA key") + }) + .and_then(|key| { + RsaKey::::from_key_der(key).map_err(|err| { + trc::DkimEvent::BuildError + .reason(err) + .details("Failed to build RSA key") + }) + }) +} + +/*impl ArcSealer { pub fn new(selector: String, domain: String, signature: DkimSignature) -> trc::Result { let mut errors = vec![]; if !signature.validate(&mut errors) { @@ -241,7 +244,7 @@ impl ArcSealer { } } } -} +}*/ pub fn simple_pem_parse(contents: &str) -> Option> { let mut contents = contents.as_bytes().iter().copied(); @@ -327,7 +330,7 @@ fn build_dkim1_signer( signer } -fn build_dkim1_sealer>( +/*fn build_dkim1_sealer>( domain: String, selector: String, mut signature: Dkim1Signature, @@ -378,6 +381,8 @@ fn build_dkim1_sealer>( sealer } +*/ + impl<'x> TryFrom> for VerifyStrategy { type Error = (); diff --git a/crates/common/src/enterprise/config.rs b/crates/common/src/enterprise/config.rs index 0d950172..e28d354a 100644 --- a/crates/common/src/enterprise/config.rs +++ b/crates/common/src/enterprise/config.rs @@ -19,7 +19,7 @@ use registry::schema::{ prelude::{ObjectType, Property}, structs::{ self, AiModel, Alert, CalendarAlarm, CalendarScheduling, DataRetention, SecretKeyOptional, - SecretKeyValue, SpamLlm, + SecretKeyValue, SpamLlm, SystemSettings, }, }; use std::sync::Arc; @@ -32,7 +32,10 @@ use utils::template::Template; impl Enterprise { pub async fn parse(bp: &mut Bootstrap) -> Option { - let server_hostname = bp.hostname().to_string(); + let server_hostname = bp + .setting_infallible::() + .await + .default_hostname; let mut update_license = None; let mut enterprise = bp.setting_infallible::().await; diff --git a/crates/common/src/ipc.rs b/crates/common/src/ipc.rs index f06075a5..cc129f79 100644 --- a/crates/common/src/ipc.rs +++ b/crates/common/src/ipc.rs @@ -74,7 +74,9 @@ pub enum BroadcastEvent { PushNotification(PushNotification), PushServerUpdate(u32), RegistryChange(RegistryChange), - CacheInvalidation(Vec), + CacheInvalidate(Vec), + CacheInvalidateAll, + MtaQueueStatus { is_running: bool }, } #[derive(Debug)] diff --git a/crates/common/src/manager/boot.rs b/crates/common/src/manager/boot.rs index 3d6a8afb..bc2108c2 100644 --- a/crates/common/src/manager/boot.rs +++ b/crates/common/src/manager/boot.rs @@ -235,7 +235,7 @@ impl BootManager { let registry = RegistryStore::init(PathBuf::from(config_path.unwrap())) .await .failed("⚠️ Startup failed"); - let mut bootstrap = Bootstrap::init(registry).await; + let mut bootstrap = Bootstrap::new(registry); // Start listeners let mut servers = Listeners::parse(&mut bootstrap).await; diff --git a/crates/common/src/network/security.rs b/crates/common/src/network/security.rs index e36e6916..086f3820 100644 --- a/crates/common/src/network/security.rs +++ b/crates/common/src/network/security.rs @@ -121,7 +121,6 @@ impl Security { } let security = bp.setting_infallible::().await; - let local = bp.setting_infallible::().await; let auth = bp.setting_infallible::().await; Security { fallback_admin: local.fallback_admin_user.as_ref().and_then(|user| { diff --git a/crates/common/src/storage/encryption.rs b/crates/common/src/storage/encryption.rs new file mode 100644 index 00000000..9e40c8e2 --- /dev/null +++ b/crates/common/src/storage/encryption.rs @@ -0,0 +1,164 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use mail_parser::decoders::base64::base64_decode; +use registry::schema::structs::PublicKey; +use sequoia_openpgp::{Cert, parse::Parse, policy::StandardPolicy, types::KeyFlags}; +use std::borrow::Cow; + +use crate::auth::EncryptionKeys; + +const P: StandardPolicy<'static> = StandardPolicy::new(); + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EncryptionMethod { + PGP, + SMIME, +} + +pub struct EncryptionParams { + pub certs: EncryptionKeys, + pub method: EncryptionMethod, +} + +#[allow(clippy::type_complexity)] +pub fn parse_public_key(pk: &PublicKey) -> Result, Cow<'static, str>> { + let bytes_ = pk.key.as_bytes(); + let mut bytes = bytes_.iter().enumerate(); + let mut buf = vec![]; + let mut method = None; + let mut certs: Vec> = vec![]; + + loop { + // Find start of PEM block + let mut start_pos = 0; + for (pos, &ch) in bytes.by_ref() { + if ch.is_ascii_whitespace() { + continue; + } else if ch == b'-' { + start_pos = pos; + break; + } else { + return Ok(None); + } + } + + // Find block type + for (_, &ch) in bytes.by_ref() { + match ch { + b'-' => (), + b'\n' => break, + _ => { + if ch.is_ascii() { + buf.push(ch.to_ascii_uppercase()); + } else { + return Ok(None); + } + } + } + } + if buf.is_empty() { + break; + } + + // Find type + let tag = std::str::from_utf8(&buf).unwrap(); + if tag.contains("CERTIFICATE") { + if method.is_some_and(|m| m == EncryptionMethod::PGP) { + return Err("Cannot mix OpenPGP and S/MIME certificates".into()); + } else { + method = Some(EncryptionMethod::SMIME); + } + } else if tag.contains("PGP") { + if method.is_some_and(|m| m == EncryptionMethod::SMIME) { + return Err("Cannot mix OpenPGP and S/MIME certificates".into()); + } else { + method = Some(EncryptionMethod::PGP); + } + } else { + // Ignore block + let mut found_end = false; + for (_, &ch) in bytes.by_ref() { + if ch == b'-' { + found_end = true; + } else if ch == b'\n' && found_end { + break; + } + } + buf.clear(); + continue; + } + + // Collect base64 + buf.clear(); + let mut found_end = false; + let mut end_pos = 0; + for (pos, &ch) in bytes.by_ref() { + match ch { + b'-' => { + found_end = true; + } + b'\n' => { + if found_end { + end_pos = pos; + break; + } + } + _ => { + if !ch.is_ascii_whitespace() { + buf.push(ch); + } + } + } + } + + // Decode base64 + let cert = base64_decode(&buf) + .ok_or_else(|| Cow::from("Failed to decode base64 certificate."))? + .into_boxed_slice(); + match method.unwrap() { + EncryptionMethod::PGP => match Cert::from_bytes(bytes_) { + Ok(cert) => { + if !has_pgp_keys(cert) { + return Err("Could not find any suitable keys in OpenPGP public key".into()); + } + certs.push( + bytes_ + .get(start_pos..end_pos + 1) + .unwrap_or_default() + .into(), + ); + } + Err(err) => { + return Err(format!("Failed to decode OpenPGP public key: {err}").into()); + } + }, + EncryptionMethod::SMIME => { + if let Err(err) = rasn::der::decode::(&cert) { + return Err(format!("Failed to decode X509 certificate: {err}").into()); + } + certs.push(cert); + } + } + buf.clear(); + } + + Ok(method.map(|method| EncryptionParams { + method, + certs: certs.into_boxed_slice(), + })) +} + +fn has_pgp_keys(cert: Cert) -> bool { + cert.keys() + .with_policy(&P, None) + .supported() + .alive() + .revoked(false) + .key_flags(KeyFlags::empty().set_transport_encryption()) + .next() + .is_some() +} diff --git a/crates/common/src/storage/mod.rs b/crates/common/src/storage/mod.rs index a0752c9b..9810ad73 100644 --- a/crates/common/src/storage/mod.rs +++ b/crates/common/src/storage/mod.rs @@ -14,15 +14,13 @@ use registry::{ types::EnumImpl, }; use std::sync::Arc; -use store::{ - BlobStore, InMemoryStore, RegistryStore, SearchStore, Store, registry::RegistryQuery, - roaring::RoaringBitmap, -}; +use store::{BlobStore, InMemoryStore, RegistryStore, SearchStore, Store, registry::RegistryQuery}; pub mod archive; pub mod blob; pub mod dav; pub mod document; +pub mod encryption; pub mod index; pub mod quota; pub mod state; diff --git a/crates/common/src/storage/quota.rs b/crates/common/src/storage/quota.rs index f3a732b8..abebe93d 100644 --- a/crates/common/src/storage/quota.rs +++ b/crates/common/src/storage/quota.rs @@ -6,6 +6,7 @@ use crate::{ Server, + auth::AccountCache, storage::{ObjectQuota, TenantQuota}, }; use registry::{ @@ -39,10 +40,13 @@ impl Server { .add_context(|err| err.caused_by(trc::location!())) } - pub async fn has_available_quota(&self, account_id: u32, item_size: u64) -> trc::Result<()> { - let account = self.account(account_id).await.caused_by(trc::location!())?; + pub async fn has_available_quota( + &self, + account: &AccountCache, + item_size: u64, + ) -> trc::Result<()> { if account.quota_disk != 0 { - let used_quota = self.get_used_quota_account(account_id).await? as u64; + let used_quota = self.get_used_quota_account(account.id).await? as u64; if used_quota + item_size > account.quota_disk { return Err(trc::LimitEvent::Quota diff --git a/crates/dav/src/calendar/copy_move.rs b/crates/dav/src/calendar/copy_move.rs index 949be675..16c821ae 100644 --- a/crates/dav/src/calendar/copy_move.rs +++ b/crates/dav/src/calendar/copy_move.rs @@ -1030,7 +1030,10 @@ async fn copy_container( if from_account_id != to_account_id && required_space > 0 { server - .has_available_quota(to_account_id, required_space) + .has_available_quota( + server.account(to_account_id).await?.as_ref(), + required_space, + ) .await?; } diff --git a/crates/dav/src/calendar/update.rs b/crates/dav/src/calendar/update.rs index 8f61a048..b5217a3a 100644 --- a/crates/dav/src/calendar/update.rs +++ b/crates/dav/src/calendar/update.rs @@ -287,7 +287,8 @@ impl CalendarUpdateRequestHandler for Server { let extra_bytes = (bytes.len() as u64).saturating_sub(u32::from(event.inner.size) as u64); if extra_bytes > 0 { - self.has_available_quota(account_id, extra_bytes).await?; + self.has_available_quota(self.account(account_id).await?.as_ref(), extra_bytes) + .await?; } // Prepare write batch @@ -418,8 +419,11 @@ impl CalendarUpdateRequestHandler for Server { // Validate quota if !bytes.is_empty() { - self.has_available_quota(account_id, bytes.len() as u64) - .await?; + self.has_available_quota( + self.account(account_id).await?.as_ref(), + bytes.len() as u64, + ) + .await?; } // Prepare write batch diff --git a/crates/dav/src/card/copy_move.rs b/crates/dav/src/card/copy_move.rs index 820706ab..4945470b 100644 --- a/crates/dav/src/card/copy_move.rs +++ b/crates/dav/src/card/copy_move.rs @@ -977,7 +977,10 @@ async fn copy_container( if from_account_id != to_account_id && required_space > 0 { server - .has_available_quota(to_account_id, required_space) + .has_available_quota( + server.account(to_account_id).await?.as_ref(), + required_space, + ) .await?; } diff --git a/crates/dav/src/card/update.rs b/crates/dav/src/card/update.rs index 5bca1e34..e7f99b60 100644 --- a/crates/dav/src/card/update.rs +++ b/crates/dav/src/card/update.rs @@ -181,7 +181,8 @@ impl CardUpdateRequestHandler for Server { let extra_bytes = (bytes.len() as u64).saturating_sub(u32::from(card.inner.size) as u64); if extra_bytes > 0 { - self.has_available_quota(account_id, extra_bytes).await?; + self.has_available_quota(self.account(account_id).await?.as_ref(), extra_bytes) + .await?; } // Build node @@ -251,8 +252,11 @@ impl CardUpdateRequestHandler for Server { // Validate quota if !bytes.is_empty() { - self.has_available_quota(account_id, bytes.len() as u64) - .await?; + self.has_available_quota( + self.account(account_id).await?.as_ref(), + bytes.len() as u64, + ) + .await?; } // Build node diff --git a/crates/dav/src/file/copy_move.rs b/crates/dav/src/file/copy_move.rs index da93dd1b..78f4a41f 100644 --- a/crates/dav/src/file/copy_move.rs +++ b/crates/dav/src/file/copy_move.rs @@ -240,7 +240,7 @@ impl FileCopyMoveRequestHandler for Server { .subtree(from_resource_name) .map(|a| a.size() as u64) .sum::(); - self.has_available_quota(to_account_id, space_needed) + self.has_available_quota(self.account(to_account_id).await?.as_ref(), space_needed) .await?; } diff --git a/crates/dav/src/file/update.rs b/crates/dav/src/file/update.rs index 06275f08..85fffe97 100644 --- a/crates/dav/src/file/update.rs +++ b/crates/dav/src/file/update.rs @@ -166,7 +166,8 @@ impl FileUpdateRequestHandler for Server { let extra_bytes = (bytes.len() as u64) .saturating_sub(u32::from(node.inner.file.as_ref().unwrap().size) as u64); if extra_bytes > 0 { - self.has_available_quota(account_id, extra_bytes).await?; + self.has_available_quota(self.account(account_id).await?.as_ref(), extra_bytes) + .await?; } // Write blob @@ -242,8 +243,11 @@ impl FileUpdateRequestHandler for Server { // Validate quota if !bytes.is_empty() { - self.has_available_quota(account_id, bytes.len() as u64) - .await?; + self.has_available_quota( + self.account(account_id).await?.as_ref(), + bytes.len() as u64, + ) + .await?; } // Write blob diff --git a/crates/email/src/message/copy.rs b/crates/email/src/message/copy.rs index af8a5253..f64634bd 100644 --- a/crates/email/src/message/copy.rs +++ b/crates/email/src/message/copy.rs @@ -92,7 +92,8 @@ impl EmailCopy for Server { // Check quota let size = metadata.root_part().offset_end; - match self.has_available_quota(to_account_id, size as u64).await { + let to_account = self.account(to_account_id).await?; + match self.has_available_quota(&to_account, size as u64).await { Ok(_) => (), Err(err) => { if err.matches(trc::EventType::Limit(trc::LimitEvent::Quota)) @@ -177,7 +178,7 @@ impl EmailCopy for Server { batch.with_account_id(to_account_id); // Determine thread id - let tenant_id = self.account(to_account_id).await?.tenant_id(); + let tenant_id = to_account.tenant_id(); let thread_id = if let Some(thread_id) = thread_result.thread_id { thread_id } else { diff --git a/crates/email/src/message/crypto.rs b/crates/email/src/message/crypto.rs index 4e20d1fa..dacc299a 100644 --- a/crates/email/src/message/crypto.rs +++ b/crates/email/src/message/crypto.rs @@ -4,12 +4,16 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{borrow::Cow, collections::BTreeSet, fmt::Display, io::Cursor}; +use std::{collections::BTreeSet, io::Cursor}; use aes::cipher::{BlockEncryptMut, KeyIvInit, block_padding::Pkcs7}; +use common::auth::{ + ACCOUNT_FLAG_ENCRYPT_ALGO_AES256, ACCOUNT_FLAG_ENCRYPT_METHOD_PGP, + ACCOUNT_FLAG_ENCRYPT_TRAIN_SPAM_FILTER, EncryptionKeys, +}; use mail_builder::{encoders::base64::base64_encode_mime, mime::make_boundary}; -use mail_parser::{Message, MimeHeaders, PartType, decoders::base64::base64_decode}; +use mail_parser::{Message, MimeHeaders, PartType}; use openpgp::{ parse::Parse, serialize::stream, @@ -26,9 +30,6 @@ use rasn_cms::{ }; use rsa::{Pkcs1v15Encrypt, RsaPublicKey, pkcs1::DecodeRsaPublicKey}; use sequoia_openpgp as openpgp; -use store::{Deserialize, write::Archive}; - -const P: openpgp::policy::StandardPolicy<'static> = openpgp::policy::StandardPolicy::new(); #[derive(Debug)] pub enum EncryptMessageError { @@ -36,90 +37,12 @@ pub enum EncryptMessageError { Error(String), } -#[derive( - rkyv::Serialize, - rkyv::Deserialize, - rkyv::Archive, - Debug, - Clone, - Copy, - serde::Serialize, - serde::Deserialize, -)] -#[rkyv(derive(Clone, Copy))] -pub enum Algorithm { - Aes128, - Aes256, -} - -#[derive( - rkyv::Serialize, - rkyv::Deserialize, - rkyv::Archive, - Debug, - Clone, - Copy, - PartialEq, - Eq, - serde::Serialize, - serde::Deserialize, -)] -pub enum EncryptionMethod { - PGP, - SMIME, -} - -#[derive( - Clone, - rkyv::Serialize, - rkyv::Deserialize, - rkyv::Archive, - Debug, - serde::Serialize, - serde::Deserialize, -)] -pub struct EncryptionParams { - pub certs: Box<[Box<[u8]>]>, - pub flags: u64, -} - -pub const ENCRYPT_TRAIN_SPAM_FILTER: u64 = 1; -pub const ENCRYPT_METHOD_SMIME: u64 = 1 << 1; -pub const ENCRYPT_METHOD_PGP: u64 = 1 << 2; -pub const ENCRYPT_ALGO_AES256: u64 = 1 << 3; -pub const ENCRYPT_ALGO_AES128: u64 = 1 << 4; - -#[derive( - rkyv::Serialize, - rkyv::Deserialize, - rkyv::Archive, - Debug, - serde::Serialize, - serde::Deserialize, - Default, -)] -#[serde(tag = "type")] -#[serde(rename_all = "camelCase")] -pub enum EncryptionType { - PGP { - algo: Algorithm, - certs: String, - allow_spam_training: bool, - }, - SMIME { - algo: Algorithm, - certs: String, - allow_spam_training: bool, - }, - #[default] - Disabled, -} - #[allow(async_fn_in_trait)] pub trait EncryptMessage { async fn encrypt( &self, - params: &ArchivedEncryptionParams, + keys: &EncryptionKeys, + flags: u64, ) -> Result, EncryptMessageError>; fn is_encrypted(&self) -> bool; } @@ -127,7 +50,8 @@ pub trait EncryptMessage { impl EncryptMessage for Message<'_> { async fn encrypt( &self, - params: &ArchivedEncryptionParams, + keys: &EncryptionKeys, + flags: u64, ) -> Result, EncryptMessageError> { let root = self.root_part(); let raw_message = self.raw_message(); @@ -149,263 +73,236 @@ impl EncryptMessage for Message<'_> { inner_message.extend_from_slice(&raw_message[root.raw_body_offset() as usize..]); // Encrypt inner message - match params.method() { - EncryptionMethod::PGP => { - // Prepare encrypted message - let boundary = make_boundary("_"); - outer_message.extend_from_slice( - concat!( - "Content-Type: multipart/encrypted;\r\n\t", - "protocol=\"application/pgp-encrypted\";\r\n\t", - "boundary=\"" - ) - .as_bytes(), - ); - outer_message.extend_from_slice(boundary.as_bytes()); - outer_message.extend_from_slice( - concat!( - "\"\r\n\r\n", - "OpenPGP/MIME message (Automatically encrypted by Stalwart)\r\n\r\n", - "--" - ) - .as_bytes(), - ); - outer_message.extend_from_slice(boundary.as_bytes()); - outer_message.extend_from_slice( - concat!( - "\r\nContent-Type: application/pgp-encrypted\r\n\r\n", - "Version: 1\r\n\r\n--" - ) - .as_bytes(), - ); - outer_message.extend_from_slice(boundary.as_bytes()); - outer_message.extend_from_slice( - concat!( - "\r\nContent-Type: application/octet-stream; name=\"encrypted.asc\"\r\n", - "Content-Disposition: inline; filename=\"encrypted.asc\"\r\n\r\n" - ) - .as_bytes(), - ); + if flags & ACCOUNT_FLAG_ENCRYPT_METHOD_PGP != 0 { + // Prepare encrypted message + let boundary = make_boundary("_"); + outer_message.extend_from_slice( + concat!( + "Content-Type: multipart/encrypted;\r\n\t", + "protocol=\"application/pgp-encrypted\";\r\n\t", + "boundary=\"" + ) + .as_bytes(), + ); + outer_message.extend_from_slice(boundary.as_bytes()); + outer_message.extend_from_slice( + concat!( + "\"\r\n\r\n", + "OpenPGP/MIME message (Automatically encrypted by Stalwart)\r\n\r\n", + "--" + ) + .as_bytes(), + ); + outer_message.extend_from_slice(boundary.as_bytes()); + outer_message.extend_from_slice( + concat!( + "\r\nContent-Type: application/pgp-encrypted\r\n\r\n", + "Version: 1\r\n\r\n--" + ) + .as_bytes(), + ); + outer_message.extend_from_slice(boundary.as_bytes()); + outer_message.extend_from_slice( + concat!( + "\r\nContent-Type: application/octet-stream; name=\"encrypted.asc\"\r\n", + "Content-Disposition: inline; filename=\"encrypted.asc\"\r\n\r\n" + ) + .as_bytes(), + ); - let certs = params - .certs - .iter() - .map(openpgp::Cert::from_bytes) - .collect::, _>>() - .map_err(|err| { - EncryptMessageError::Error(format!( - "Failed to parse OpenPGP public key: {}", - err - )) - })?; - - // Encrypt contents (TODO: use rayon) - let algo = params.algo(); - let encrypted_contents = tokio::task::spawn_blocking(move || { - // Parse public key - let mut keys = Vec::with_capacity(certs.len()); - let policy = openpgp::policy::StandardPolicy::new(); - - for cert in &certs { - for key in cert - .keys() - .with_policy(&policy, None) - .supported() - .alive() - .revoked(false) - .key_flags(KeyFlags::empty().set_transport_encryption()) - { - keys.push(key); - } - } - - // Compose a writer stack corresponding to the output format and - // packet structure we want. - let mut sink = Vec::with_capacity(inner_message.len()); - - // Stream an OpenPGP message. - let message = stream::Armorer::new(stream::Message::new(&mut sink)) - .build() - .map_err(|err| { - EncryptMessageError::Error(format!("Failed to create armorer: {}", err)) - })?; - let message = stream::Encryptor::for_recipients(message, keys) - .symmetric_algo(match algo { - Algorithm::Aes128 => SymmetricAlgorithm::AES128, - Algorithm::Aes256 => SymmetricAlgorithm::AES256, - }) - .build() - .map_err(|err| { - EncryptMessageError::Error(format!( - "Failed to build encryptor: {}", - err - )) - })?; - let mut message = - stream::LiteralWriter::new(message).build().map_err(|err| { - EncryptMessageError::Error(format!( - "Failed to create literal writer: {}", - err - )) - })?; - std::io::copy(&mut Cursor::new(inner_message), &mut message).map_err( - |err| { - EncryptMessageError::Error(format!( - "Failed to encrypt message: {}", - err - )) - }, - )?; - message.finalize().map_err(|err| { - EncryptMessageError::Error(format!("Failed to finalize message: {}", err)) - })?; - - String::from_utf8(sink).map_err(|err| { - EncryptMessageError::Error(format!( - "Failed to convert encrypted message to UTF-8: {}", - err - )) - }) - }) - .await + let certs = keys + .iter() + .map(openpgp::Cert::from_bytes) + .collect::, _>>() .map_err(|err| { - EncryptMessageError::Error(format!("Failed to encrypt message: {}", err)) - })??; - outer_message.extend_from_slice(encrypted_contents.as_bytes()); - outer_message.extend_from_slice(b"\r\n--"); - outer_message.extend_from_slice(boundary.as_bytes()); - outer_message.extend_from_slice(b"--\r\n"); - } - EncryptionMethod::SMIME => { - // Generate random IV - let mut rng = StdRng::from_entropy(); - let mut iv = vec![0u8; 16]; - rng.fill_bytes(&mut iv); - - // Generate random key - let mut key = vec![0u8; params.key_size()]; - rng.fill_bytes(&mut key); - - // Encrypt contents (TODO: use rayon) - let algo = params.algo(); - let (encrypted_contents, key, iv) = tokio::task::spawn_blocking(move || { - (algo.encrypt(&key, &iv, &inner_message), key, iv) - }) - .await - .map_err(|err| { - EncryptMessageError::Error(format!("Failed to encrypt message: {}", err)) + EncryptMessageError::Error(format!( + "Failed to parse OpenPGP public key: {}", + err + )) })?; - // Encrypt key using public keys - #[allow(clippy::mutable_key_type)] - let mut recipient_infos = BTreeSet::new(); - for cert in params.certs.iter() { - let cert = - rasn::der::decode::(cert).map_err(|err| { - EncryptMessageError::Error(format!( - "Failed to parse certificate: {}", - err - )) - })?; + // Encrypt contents (TODO: use rayon) + let encrypted_contents = tokio::task::spawn_blocking(move || { + // Parse public key + let mut keys = Vec::with_capacity(certs.len()); + let policy = openpgp::policy::StandardPolicy::new(); - let public_key = RsaPublicKey::from_pkcs1_der( - cert.tbs_certificate - .subject_public_key_info - .subject_public_key - .as_raw_slice(), - ) + for cert in &certs { + for key in cert + .keys() + .with_policy(&policy, None) + .supported() + .alive() + .revoked(false) + .key_flags(KeyFlags::empty().set_transport_encryption()) + { + keys.push(key); + } + } + + // Compose a writer stack corresponding to the output format and + // packet structure we want. + let mut sink = Vec::with_capacity(inner_message.len()); + + // Stream an OpenPGP message. + let message = stream::Armorer::new(stream::Message::new(&mut sink)) + .build() .map_err(|err| { - EncryptMessageError::Error(format!("Failed to parse public key: {}", err)) + EncryptMessageError::Error(format!("Failed to create armorer: {}", err)) })?; - let encrypted_key = public_key - .encrypt(&mut rng, Pkcs1v15Encrypt, &key[..]) - .map_err(|err| { - EncryptMessageError::Error(format!("Failed to encrypt key: {}", err)) - }) - .unwrap(); + let message = stream::Encryptor::for_recipients(message, keys) + .symmetric_algo(flags.algo()) + .build() + .map_err(|err| { + EncryptMessageError::Error(format!("Failed to build encryptor: {}", err)) + })?; + let mut message = stream::LiteralWriter::new(message).build().map_err(|err| { + EncryptMessageError::Error(format!("Failed to create literal writer: {}", err)) + })?; + std::io::copy(&mut Cursor::new(inner_message), &mut message).map_err(|err| { + EncryptMessageError::Error(format!("Failed to encrypt message: {}", err)) + })?; + message.finalize().map_err(|err| { + EncryptMessageError::Error(format!("Failed to finalize message: {}", err)) + })?; - recipient_infos.insert(RecipientInfo::KeyTransRecipientInfo( - KeyTransRecipientInfo { - version: 0.into(), - rid: RecipientIdentifier::IssuerAndSerialNumber( - IssuerAndSerialNumber { - issuer: cert.tbs_certificate.issuer, - serial_number: cert.tbs_certificate.serial_number, - }, + String::from_utf8(sink).map_err(|err| { + EncryptMessageError::Error(format!( + "Failed to convert encrypted message to UTF-8: {}", + err + )) + }) + }) + .await + .map_err(|err| { + EncryptMessageError::Error(format!("Failed to encrypt message: {}", err)) + })??; + outer_message.extend_from_slice(encrypted_contents.as_bytes()); + outer_message.extend_from_slice(b"\r\n--"); + outer_message.extend_from_slice(boundary.as_bytes()); + outer_message.extend_from_slice(b"--\r\n"); + } else { + // Generate random IV + let mut rng = StdRng::from_entropy(); + let mut iv = vec![0u8; 16]; + rng.fill_bytes(&mut iv); + + // Generate random key + let mut key = vec![0u8; flags.key_size()]; + rng.fill_bytes(&mut key); + + // Encrypt contents (TODO: use rayon) + let (encrypted_contents, key, iv) = tokio::task::spawn_blocking(move || { + (flags.encrypt(&key, &iv, &inner_message), key, iv) + }) + .await + .map_err(|err| { + EncryptMessageError::Error(format!("Failed to encrypt message: {}", err)) + })?; + + // Encrypt key using public keys + #[allow(clippy::mutable_key_type)] + let mut recipient_infos = BTreeSet::new(); + for cert in keys.iter() { + let cert = rasn::der::decode::(cert).map_err(|err| { + EncryptMessageError::Error(format!("Failed to parse certificate: {}", err)) + })?; + + let public_key = RsaPublicKey::from_pkcs1_der( + cert.tbs_certificate + .subject_public_key_info + .subject_public_key + .as_raw_slice(), + ) + .map_err(|err| { + EncryptMessageError::Error(format!("Failed to parse public key: {}", err)) + })?; + let encrypted_key = public_key + .encrypt(&mut rng, Pkcs1v15Encrypt, &key[..]) + .map_err(|err| { + EncryptMessageError::Error(format!("Failed to encrypt key: {}", err)) + }) + .unwrap(); + + recipient_infos.insert(RecipientInfo::KeyTransRecipientInfo( + KeyTransRecipientInfo { + version: 0.into(), + rid: RecipientIdentifier::IssuerAndSerialNumber(IssuerAndSerialNumber { + issuer: cert.tbs_certificate.issuer, + serial_number: cert.tbs_certificate.serial_number, + }), + key_encryption_algorithm: AlgorithmIdentifier { + algorithm: RSA.into(), + parameters: Some( + rasn::der::encode(&()) + .map_err(|err| { + EncryptMessageError::Error(format!( + "Failed to encode RSA algorithm identifier: {}", + err + )) + })? + .into(), ), - key_encryption_algorithm: AlgorithmIdentifier { - algorithm: RSA.into(), + }, + encrypted_key: EncryptedKey::from(encrypted_key), + }, + )); + } + + let pkcs7 = rasn::der::encode(&EncapsulatedContentInfo { + content_type: CONTENT_ENVELOPED_DATA.into(), + content: Some( + rasn::der::encode(&EnvelopedData { + version: 0.into(), + originator_info: None, + recipient_infos, + encrypted_content_info: EncryptedContentInfo { + content_type: CONTENT_DATA.into(), + content_encryption_algorithm: AlgorithmIdentifier { + algorithm: flags.to_algorithm_identifier(), parameters: Some( - rasn::der::encode(&()) + rasn::der::encode(&OctetString::from(iv)) .map_err(|err| { EncryptMessageError::Error(format!( - "Failed to encode RSA algorithm identifier: {}", + "Failed to encode IV: {}", err )) })? .into(), ), }, - encrypted_key: EncryptedKey::from(encrypted_key), + encrypted_content: Some(EncryptedContent::from(encrypted_contents)), }, - )); - } + unprotected_attrs: None, + }) + .map_err(|err| { + EncryptMessageError::Error(format!( + "Failed to encode EnvelopedData: {}", + err + )) + })? + .into(), + ), + }) + .map_err(|err| { + EncryptMessageError::Error(format!("Failed to encode ContentInfo: {}", err)) + })?; - let pkcs7 = rasn::der::encode(&EncapsulatedContentInfo { - content_type: CONTENT_ENVELOPED_DATA.into(), - content: Some( - rasn::der::encode(&EnvelopedData { - version: 0.into(), - originator_info: None, - recipient_infos, - encrypted_content_info: EncryptedContentInfo { - content_type: CONTENT_DATA.into(), - content_encryption_algorithm: AlgorithmIdentifier { - algorithm: params.to_algorithm_identifier(), - parameters: Some( - rasn::der::encode(&OctetString::from(iv)) - .map_err(|err| { - EncryptMessageError::Error(format!( - "Failed to encode IV: {}", - err - )) - })? - .into(), - ), - }, - encrypted_content: Some(EncryptedContent::from(encrypted_contents)), - }, - unprotected_attrs: None, - }) - .map_err(|err| { - EncryptMessageError::Error(format!( - "Failed to encode EnvelopedData: {}", - err - )) - })? - .into(), - ), - }) - .map_err(|err| { - EncryptMessageError::Error(format!("Failed to encode ContentInfo: {}", err)) - })?; - - // Generate message - outer_message.extend_from_slice( - concat!( - "Content-Type: application/pkcs7-mime;\r\n", - "\tname=\"smime.p7m\";\r\n", - "\tsmime-type=enveloped-data\r\n", - "Content-Disposition: attachment;\r\n", - "\tfilename=\"smime.p7m\"\r\n", - "Content-Transfer-Encoding: base64\r\n\r\n" - ) - .as_bytes(), - ); - base64_encode_mime(&pkcs7, &mut outer_message, false).map_err(|err| { - EncryptMessageError::Error(format!("Failed to base64 encode PKCS7: {}", err)) - })?; - } + // Generate message + outer_message.extend_from_slice( + concat!( + "Content-Type: application/pkcs7-mime;\r\n", + "\tname=\"smime.p7m\";\r\n", + "\tsmime-type=enveloped-data\r\n", + "Content-Disposition: attachment;\r\n", + "\tfilename=\"smime.p7m\"\r\n", + "Content-Transfer-Encoding: base64\r\n\r\n" + ) + .as_bytes(), + ); + base64_encode_mime(&pkcs7, &mut outer_message, false).map_err(|err| { + EncryptMessageError::Error(format!("Failed to base64 encode PKCS7: {}", err)) + })?; } Ok(outer_message) @@ -464,25 +361,17 @@ impl EncryptMessage for Message<'_> { } } -impl ArchivedEncryptionParams { - pub fn method(&self) -> EncryptionMethod { - if self.flags & ENCRYPT_METHOD_PGP != 0 { - EncryptionMethod::PGP - } else { - EncryptionMethod::SMIME - } - } - - pub fn algo(&self) -> Algorithm { - if self.flags & ENCRYPT_ALGO_AES256 != 0 { - Algorithm::Aes256 - } else { - Algorithm::Aes128 - } - } +pub trait EncryptionFlags { + fn key_size(&self) -> usize; + fn to_algorithm_identifier(&self) -> ObjectIdentifier; + fn can_train_spam_filter(&self) -> bool; + fn encrypt(&self, key: &[u8], iv: &[u8], contents: &[u8]) -> Vec; + fn algo(&self) -> SymmetricAlgorithm; +} +impl EncryptionFlags for u64 { fn key_size(&self) -> usize { - if self.flags & ENCRYPT_ALGO_AES256 != 0 { + if *self & ACCOUNT_FLAG_ENCRYPT_ALGO_AES256 != 0 { 32 } else { 16 @@ -490,246 +379,32 @@ impl ArchivedEncryptionParams { } fn to_algorithm_identifier(&self) -> ObjectIdentifier { - if self.flags & ENCRYPT_ALGO_AES256 != 0 { + if *self & ACCOUNT_FLAG_ENCRYPT_ALGO_AES256 != 0 { AES256_CBC.into() } else { AES128_CBC.into() } } - pub fn can_train_spam_filter(&self) -> bool { - self.flags & ENCRYPT_TRAIN_SPAM_FILTER != 0 + fn can_train_spam_filter(&self) -> bool { + *self & ACCOUNT_FLAG_ENCRYPT_TRAIN_SPAM_FILTER != 0 } -} -impl Algorithm { fn encrypt(&self, key: &[u8], iv: &[u8], contents: &[u8]) -> Vec { - match self { - Algorithm::Aes128 => cbc::Encryptor::::new(key.into(), iv.into()) - .encrypt_padded_vec_mut::(contents), - Algorithm::Aes256 => cbc::Encryptor::::new(key.into(), iv.into()) - .encrypt_padded_vec_mut::(contents), - } - } -} - -#[allow(clippy::type_complexity)] -pub fn try_parse_certs( - expected_method: EncryptionMethod, - cert: Vec, -) -> Result]>, Cow<'static, str>> { - // Check if it's a PEM file - let (flags, certs) = if let Some(result) = try_parse_pem(&cert)? { - (result.flags, result.certs) - } else if rasn::der::decode::(&cert[..]).is_ok() { - ( - ENCRYPT_METHOD_SMIME, - Box::from_iter([cert.into_boxed_slice()]), - ) - } else if let Ok(cert_) = openpgp::Cert::from_bytes(&cert[..]) { - if !has_pgp_keys(cert_) { - ( - ENCRYPT_METHOD_PGP, - Box::from_iter([cert.into_boxed_slice()]), - ) + if *self & ACCOUNT_FLAG_ENCRYPT_ALGO_AES256 != 0 { + cbc::Encryptor::::new(key.into(), iv.into()) + .encrypt_padded_vec_mut::(contents) } else { - return Err("Could not find any suitable keys in certificate".into()); + cbc::Encryptor::::new(key.into(), iv.into()) + .encrypt_padded_vec_mut::(contents) } - } else { - return Err("Could not find any valid certificates".into()); - }; - - if expected_method.flags() & flags != 0 { - Ok(certs) - } else { - Err("No valid certificates found for the selected encryption".into()) - } -} - -fn has_pgp_keys(cert: openpgp::Cert) -> bool { - cert.keys() - .with_policy(&P, None) - .supported() - .alive() - .revoked(false) - .key_flags(KeyFlags::empty().set_transport_encryption()) - .next() - .is_some() -} - -#[allow(clippy::type_complexity)] -fn try_parse_pem(bytes_: &[u8]) -> Result, Cow<'static, str>> { - if let Some(internal) = std::str::from_utf8(bytes_) - .ok() - .and_then(|cert| cert.strip_prefix("-----STALWART CERTIFICATE-----")) - { - return base64_decode(internal.as_bytes()) - .ok_or(Cow::from("Failed to decode base64")) - .and_then(|bytes| { - Archive::deserialize_owned(bytes) - .and_then(|arch| arch.deserialize::()) - .map_err(|_| Cow::from("Failed to deserialize internal certificate")) - }) - .map(Some); } - let mut bytes = bytes_.iter().enumerate(); - let mut buf = vec![]; - let mut method = None; - let mut certs: Vec> = vec![]; - - loop { - // Find start of PEM block - let mut start_pos = 0; - for (pos, &ch) in bytes.by_ref() { - if ch.is_ascii_whitespace() { - continue; - } else if ch == b'-' { - start_pos = pos; - break; - } else { - return Ok(None); - } - } - - // Find block type - for (_, &ch) in bytes.by_ref() { - match ch { - b'-' => (), - b'\n' => break, - _ => { - if ch.is_ascii() { - buf.push(ch.to_ascii_uppercase()); - } else { - return Ok(None); - } - } - } - } - if buf.is_empty() { - break; - } - - // Find type - let tag = std::str::from_utf8(&buf).unwrap(); - if tag.contains("CERTIFICATE") { - if method.is_some_and(|m| m == EncryptionMethod::PGP) { - return Err("Cannot mix OpenPGP and S/MIME certificates".into()); - } else { - method = Some(EncryptionMethod::SMIME); - } - } else if tag.contains("PGP") { - if method.is_some_and(|m| m == EncryptionMethod::SMIME) { - return Err("Cannot mix OpenPGP and S/MIME certificates".into()); - } else { - method = Some(EncryptionMethod::PGP); - } + fn algo(&self) -> SymmetricAlgorithm { + if *self & ACCOUNT_FLAG_ENCRYPT_ALGO_AES256 != 0 { + SymmetricAlgorithm::AES256 } else { - // Ignore block - let mut found_end = false; - for (_, &ch) in bytes.by_ref() { - if ch == b'-' { - found_end = true; - } else if ch == b'\n' && found_end { - break; - } - } - buf.clear(); - continue; - } - - // Collect base64 - buf.clear(); - let mut found_end = false; - let mut end_pos = 0; - for (pos, &ch) in bytes.by_ref() { - match ch { - b'-' => { - found_end = true; - } - b'\n' => { - if found_end { - end_pos = pos; - break; - } - } - _ => { - if !ch.is_ascii_whitespace() { - buf.push(ch); - } - } - } - } - - // Decode base64 - let cert = base64_decode(&buf) - .ok_or_else(|| Cow::from("Failed to decode base64 certificate."))? - .into_boxed_slice(); - match method.unwrap() { - EncryptionMethod::PGP => match openpgp::Cert::from_bytes(bytes_) { - Ok(cert) => { - if !has_pgp_keys(cert) { - return Err("Could not find any suitable keys in OpenPGP public key".into()); - } - certs.push( - bytes_ - .get(start_pos..end_pos + 1) - .unwrap_or_default() - .into(), - ); - } - Err(err) => { - return Err(format!("Failed to decode OpenPGP public key: {err}").into()); - } - }, - EncryptionMethod::SMIME => { - if let Err(err) = rasn::der::decode::(&cert) { - return Err(format!("Failed to decode X509 certificate: {err}").into()); - } - certs.push(cert); - } - } - buf.clear(); - } - - Ok(method.map(|method| EncryptionParams { - flags: method.flags(), - certs: certs.into_boxed_slice(), - })) -} - -impl EncryptionMethod { - pub fn flags(&self) -> u64 { - match self { - EncryptionMethod::PGP => ENCRYPT_METHOD_PGP, - EncryptionMethod::SMIME => ENCRYPT_METHOD_SMIME, - } - } -} - -impl Algorithm { - pub fn flags(&self) -> u64 { - match self { - Algorithm::Aes128 => ENCRYPT_ALGO_AES128, - Algorithm::Aes256 => ENCRYPT_ALGO_AES256, - } - } -} - -impl Display for EncryptionMethod { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - EncryptionMethod::PGP => write!(f, "OpenPGP"), - EncryptionMethod::SMIME => write!(f, "S/MIME"), - } - } -} - -impl Display for Algorithm { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Algorithm::Aes128 => write!(f, "AES-128"), - Algorithm::Aes256 => write!(f, "AES-256"), + SymmetricAlgorithm::AES128 } } } diff --git a/crates/email/src/message/ingest.rs b/crates/email/src/message/ingest.rs index 0053fe95..1dcb76f9 100644 --- a/crates/email/src/message/ingest.rs +++ b/crates/email/src/message/ingest.rs @@ -9,7 +9,7 @@ use crate::{ cache::{MessageCacheFetch, email::MessageCacheAccess, mailbox::MailboxCacheAccess}, mailbox::{INBOX_ID, JUNK_ID, SENT_ID, TRASH_ID, UidMailbox}, message::{ - crypto::EncryptionParams, + crypto::EncryptionFlags, index::{IndexMessage, extractors::VisitText}, metadata::{MessageData, MessageMetadata}, }, @@ -51,7 +51,7 @@ use types::{ blob::{BlobClass, BlobId}, blob_hash::BlobHash, collection::{Collection, SyncCollection}, - field::{ContactField, EmailField, MailboxField, PrincipalField}, + field::{ContactField, EmailField, MailboxField}, id::Id, keyword::Keyword, special_use::SpecialUse, @@ -151,7 +151,8 @@ impl EmailIngest for Server { let account_id = params.access_token.account_id(); let tenant_id = params.access_token.tenant_id(); let mut raw_message_len = params.raw_message.len() as u64; - self.has_available_quota(account_id, raw_message_len) + let account = self.account(account_id).await.caused_by(trc::location!())?; + self.has_available_quota(&account, raw_message_len) .await .caused_by(trc::location!())?; @@ -372,7 +373,7 @@ impl EmailIngest for Server { .has_permission(Permission::CalendarSchedulingReceive) { let account_info = self - .account_info(account_id) + .build_account_info(account.clone()) .await .caused_by(trc::location!())?; let mut sender = None; @@ -505,21 +506,9 @@ impl EmailIngest for Server { }; let is_encrypted = if do_encrypt && !message.is_encrypted() - && let Some(encrypt_params_) = self - .store() - .get_value::>(ValueKey::property( - account_id, - Collection::Principal, - 0, - PrincipalField::EncryptionKeys, - )) - .await - .caused_by(trc::location!())? + && let Some(encrypt_keys) = &account.encryption_key { - let encrypt_params = encrypt_params_ - .unarchive::() - .caused_by(trc::location!())?; - match message.encrypt(encrypt_params).await { + match message.encrypt(encrypt_keys, account.flags).await { Ok(new_raw_message) => { raw_message = Cow::from(new_raw_message); raw_message_len = raw_message.len() as u64; @@ -535,7 +524,7 @@ impl EmailIngest for Server { })?; // Disable spam training if requested - if !encrypt_params.can_train_spam_filter() { + if !account.flags.can_train_spam_filter() { train_spam = None; } diff --git a/crates/groupware/src/calendar/itip.rs b/crates/groupware/src/calendar/itip.rs index f50abb20..ef9eedbd 100644 --- a/crates/groupware/src/calendar/itip.rs +++ b/crates/groupware/src/calendar/itip.rs @@ -203,7 +203,10 @@ impl ItipIngest for Server { .saturating_sub(event_.inner.size.to_native() as u64); if extra_bytes > 0 && self - .has_available_quota(account_id, extra_bytes) + .has_available_quota( + self.account(account_id).await?.as_ref(), + extra_bytes, + ) .await .is_err() { @@ -309,7 +312,10 @@ impl ItipIngest for Server { // Validate quota if self - .has_available_quota(account_id, itip_message.len() as u64) + .has_available_quota( + self.account(account_id).await?.as_ref(), + itip_message.len() as u64, + ) .await .is_err() { diff --git a/crates/http/src/management/diagnose.rs b/crates/http/src/management/diagnose.rs index 76550483..a972ccd6 100644 --- a/crates/http/src/management/diagnose.rs +++ b/crates/http/src/management/diagnose.rs @@ -4,129 +4,32 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{ - future::Future, - net::{IpAddr, SocketAddr}, - time::{Duration, Instant}, -}; - use common::{ Server, - auth::{AccessToken, oauth::GrantType}, config::smtp::{ queue::MxConfig, resolver::{Policy, Tlsa}, }, - psl, -}; -use http_body_util::{StreamBody, combinators::BoxBody}; -use hyper::{ - Method, StatusCode, - body::{Bytes, Frame}, -}; -use mail_auth::{ - AuthenticatedMessage, DkimResult, DmarcResult, IpLookupStrategy, IprevOutput, IprevResult, - SpfOutput, SpfResult, - dmarc::{self, verify::DmarcParameters}, - mta_sts::TlsRpt, - spf::verify::SpfParameters, }; +use hyper::body::{Bytes, Frame}; +use mail_auth::{IpLookupStrategy, mta_sts::TlsRpt}; use serde::{Deserialize, Serialize}; -use serde_json::json; use smtp::outbound::{ client::{SmtpClient, StartTlsResult}, dane::{dnssec::TlsaLookup, verify::TlsaVerify}, lookup::{DnsLookup, ToNextHop}, mta_sts::{lookup::MtaStsLookup, verify::VerifyPolicy}, }; +use std::{ + net::{IpAddr, SocketAddr}, + time::{Duration, Instant}, +}; use tokio::{io::AsyncWriteExt, sync::mpsc}; -use utils::url_params::UrlParams; - -use http_proto::{request::decode_path_element, *}; - -pub trait TroubleshootApi: Sync + Send { - fn handle_diagnose_api_request( - &self, - req: &HttpRequest, - path: Vec<&str>, - access_token: &AccessToken, - body: Option>, - ) -> impl Future> + Send; -} - -impl TroubleshootApi for Server { - async fn handle_diagnose_api_request( - &self, - req: &HttpRequest, - path: Vec<&str>, - access_token: &AccessToken, - body: Option>, - ) -> trc::Result { - let params = UrlParams::new(req.uri().query()); - let account_id = access_token.account_id(); - - match ( - path.get(1).copied().unwrap_or_default(), - path.get(2).copied(), - req.method(), - ) { - ("token", None, &Method::GET) => { - // Issue a live telemetry token valid for 60 seconds - Ok(JsonResponse::new(json!({ - "data": self.encode_access_token(GrantType::Diagnose, account_id, "web", 60).await?, - })) - .into_http_response()) - } - ("delivery", Some(target), &Method::GET) => { - let timeout = Duration::from_secs( - params - .parse::("timeout") - .filter(|interval| *interval >= 1) - .unwrap_or(30), - ); - - let mut rx = spawn_delivery_diagnose( - self.clone(), - decode_path_element(target).to_lowercase(), - timeout, - ); - - Ok(HttpResponse::new(StatusCode::OK) - .with_content_type("text/event-stream") - .with_cache_control("no-store") - .with_stream_body(BoxBody::new(StreamBody::new(async_stream::stream! { - while let Some(stage) = rx.recv().await { - yield Ok(stage.to_frame()); - } - yield Ok(DeliveryStage::Completed.to_frame()); - })))) - } - ("dmarc", None, &Method::POST) => { - let request = serde_json::from_slice::( - body.as_deref().unwrap_or_default(), - ) - .map_err(|err| { - trc::EventType::Resource(trc::ResourceEvent::BadParameters).from_json_error(err) - })?; - let response = dmarc_diagnose(self, request).await.ok_or_else(|| { - trc::EventType::Resource(trc::ResourceEvent::BadParameters) - .reason("Failed to parse message body") - })?; - - Ok(JsonResponse::new(json!({ - "data": response, - })) - .into_http_response()) - } - _ => Err(trc::ResourceEvent::NotFound.into_err()), - } - } -} #[derive(Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] #[serde(tag = "type")] -enum DeliveryStage { +pub(crate) enum DeliveryStage { MxLookupStart { domain: String, }, @@ -253,7 +156,7 @@ enum DeliveryStage { } #[derive(Debug, Serialize, Deserialize)] -struct MX { +pub(crate) struct MX { pub exchanges: Vec, pub preference: u16, } @@ -267,7 +170,7 @@ pub enum ReportUri { } impl DeliveryStage { - fn to_frame(&self) -> Frame { + pub fn to_frame(&self) -> Frame { let payload = format!( "event: event\ndata: [{}]\n\n", serde_json::to_string(self).unwrap_or_default() @@ -285,7 +188,7 @@ impl ElapsedMs for Instant { self.elapsed().as_millis() as u64 } } -fn spawn_delivery_diagnose( +pub(crate) fn spawn_delivery_diagnose( server: Server, domain_or_email: String, timeout: Duration, @@ -816,388 +719,3 @@ async fn delivery_diagnose( Ok(()) } - -#[derive(Debug, Serialize, Deserialize)] -struct DmarcTroubleshootRequest { - #[serde(rename = "remoteIp")] - remote_ip: IpAddr, - #[serde(rename = "ehloDomain")] - ehlo_domain: String, - #[serde(rename = "mailFrom")] - mail_from: String, - body: Option, -} - -#[derive(Debug, Serialize, Deserialize)] -struct DmarcTroubleshootResponse { - #[serde(rename = "spfEhloDomain")] - spf_ehlo_domain: String, - #[serde(rename = "spfEhloResult")] - spf_ehlo_result: AuthResult, - #[serde(rename = "spfMailFromDomain")] - spf_mail_from_domain: String, - #[serde(rename = "spfMailFromResult")] - spf_mail_from_result: AuthResult, - #[serde(rename = "ipRevResult")] - ip_rev_result: AuthResult, - #[serde(rename = "ipRevPtr")] - ip_rev_ptr: Vec, - #[serde(rename = "dkimResults")] - dkim_results: Vec, - #[serde(rename = "dkimPass")] - dkim_pass: bool, - #[serde(rename = "arcResult")] - arc_result: AuthResult, - #[serde(rename = "dmarcResult")] - dmarc_result: AuthResult, - #[serde(rename = "dmarcPass")] - dmarc_pass: bool, - #[serde(rename = "dmarcPolicy")] - dmarc_policy: DmarcPolicy, - elapsed: u64, -} - -#[derive(Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -#[serde(tag = "type")] -pub enum AuthResult { - Pass, - Fail { details: Option }, - SoftFail { details: Option }, - TempError { details: Option }, - PermError { details: Option }, - Neutral { details: Option }, - None, -} - -#[derive(Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum DmarcPolicy { - None, - Quarantine, - Reject, - Unspecified, -} - -async fn dmarc_diagnose( - server: &Server, - request: DmarcTroubleshootRequest, -) -> Option { - let remote_ip = request.remote_ip; - let ehlo_domain = request.ehlo_domain.to_lowercase(); - let mail_from = request.mail_from.to_lowercase(); - let mail_from_domain = mail_from.rsplit_once('@').map(|(_, domain)| domain); - - let local_host = &server.core.network.server_name; - - let now = Instant::now(); - let ehlo_spf_output = server - .core - .smtp - .resolvers - .dns - .verify_spf( - server - .inner - .cache - .build_auth_parameters(SpfParameters::verify_ehlo( - remote_ip, - &ehlo_domain, - local_host, - )), - ) - .await; - - let iprev = server - .core - .smtp - .resolvers - .dns - .verify_iprev(server.inner.cache.build_auth_parameters(remote_ip)) - .await; - let mail_spf_output = if let Some(mail_from_domain) = mail_from_domain { - server - .core - .smtp - .resolvers - .dns - .check_host(server.inner.cache.build_auth_parameters(SpfParameters::new( - remote_ip, - mail_from_domain, - &ehlo_domain, - local_host, - &mail_from, - ))) - .await - } else { - server - .core - .smtp - .resolvers - .dns - .check_host(server.inner.cache.build_auth_parameters(SpfParameters::new( - remote_ip, - &ehlo_domain, - &ehlo_domain, - local_host, - &format!("postmaster@{ehlo_domain}"), - ))) - .await - }; - - let body = request - .body - .unwrap_or_else(|| format!("From: {mail_from}\r\nSubject: test\r\n\r\ntest")); - let auth_message = AuthenticatedMessage::parse_with_opts(body.as_bytes(), true)?; - - let dkim_output = server - .core - .smtp - .resolvers - .dns - .verify_dkim(server.inner.cache.build_auth_parameters(&auth_message)) - .await; - let dkim_pass = dkim_output - .iter() - .any(|d| matches!(d.result(), DkimResult::Pass)); - - let arc_output = server - .core - .smtp - .resolvers - .dns - .verify_arc(server.inner.cache.build_auth_parameters(&auth_message)) - .await; - - let dmarc_output = server - .core - .smtp - .resolvers - .dns - .verify_dmarc(server.inner.cache.build_auth_parameters(DmarcParameters { - message: &auth_message, - dkim_output: &dkim_output, - rfc5321_mail_from_domain: mail_from_domain.unwrap_or(ehlo_domain.as_str()), - spf_output: &mail_spf_output, - domain_suffix_fn: |domain| psl::domain_str(domain).unwrap_or(domain), - })) - .await; - let dmarc_pass = matches!(dmarc_output.spf_result(), DmarcResult::Pass) - || matches!(dmarc_output.dkim_result(), DmarcResult::Pass); - let dmarc_result = if dmarc_pass { - DmarcResult::Pass - } else if dmarc_output.spf_result() != &DmarcResult::None { - dmarc_output.spf_result().clone() - } else if dmarc_output.dkim_result() != &DmarcResult::None { - dmarc_output.dkim_result().clone() - } else { - DmarcResult::None - }; - - Some(DmarcTroubleshootResponse { - spf_ehlo_domain: ehlo_spf_output.domain().to_string(), - spf_ehlo_result: (&ehlo_spf_output).into(), - spf_mail_from_domain: mail_spf_output.domain().to_string(), - spf_mail_from_result: (&mail_spf_output).into(), - ip_rev_ptr: iprev - .ptr - .as_ref() - .map(|ptr| ptr.iter().map(|s| s.to_string()).collect()) - .unwrap_or_default(), - ip_rev_result: (&iprev).into(), - dkim_pass, - dkim_results: dkim_output - .iter() - .map(|result| result.result().into()) - .collect(), - arc_result: arc_output.result().into(), - dmarc_result: (&dmarc_result).into(), - dmarc_policy: (&dmarc_output.policy()).into(), - dmarc_pass, - elapsed: now.elapsed_ms(), - }) -} - -impl From<&SpfOutput> for AuthResult { - fn from(value: &SpfOutput) -> Self { - match value.result() { - SpfResult::Pass => AuthResult::Pass, - SpfResult::Fail => AuthResult::Fail { - details: value.explanation().map(|e| e.to_string()), - }, - SpfResult::SoftFail => AuthResult::SoftFail { - details: value.explanation().map(|e| e.to_string()), - }, - SpfResult::Neutral => AuthResult::Neutral { - details: value.explanation().map(|e| e.to_string()), - }, - SpfResult::TempError => AuthResult::TempError { - details: value.explanation().map(|e| e.to_string()), - }, - SpfResult::PermError => AuthResult::PermError { - details: value.explanation().map(|e| e.to_string()), - }, - SpfResult::None => AuthResult::None, - } - } -} - -impl From for SpfOutput { - fn from(value: AuthResult) -> Self { - match value { - AuthResult::Pass => SpfOutput::new(String::new()).with_result(SpfResult::Pass), - AuthResult::Fail { .. } => SpfOutput::new(String::new()).with_result(SpfResult::Fail), - AuthResult::SoftFail { .. } => { - SpfOutput::new(String::new()).with_result(SpfResult::SoftFail) - } - AuthResult::Neutral { .. } => { - SpfOutput::new(String::new()).with_result(SpfResult::Neutral) - } - AuthResult::TempError { .. } => { - SpfOutput::new(String::new()).with_result(SpfResult::TempError) - } - AuthResult::PermError { .. } => { - SpfOutput::new(String::new()).with_result(SpfResult::PermError) - } - AuthResult::None => SpfOutput::new(String::new()).with_result(SpfResult::None), - } - } -} - -impl From<&IprevOutput> for AuthResult { - fn from(value: &IprevOutput) -> Self { - match &value.result { - IprevResult::Pass => AuthResult::Pass, - IprevResult::Fail(error) => AuthResult::Fail { - details: error.to_string().into(), - }, - IprevResult::TempError(error) => AuthResult::TempError { - details: error.to_string().into(), - }, - IprevResult::PermError(error) => AuthResult::PermError { - details: error.to_string().into(), - }, - IprevResult::None => AuthResult::None, - } - } -} - -impl From for IprevResult { - fn from(value: AuthResult) -> Self { - match value { - AuthResult::Pass => IprevResult::Pass, - AuthResult::Fail { details } => { - IprevResult::Fail(mail_auth::Error::Io(details.unwrap_or_default())) - } - AuthResult::TempError { details } => { - IprevResult::TempError(mail_auth::Error::Io(details.unwrap_or_default())) - } - AuthResult::PermError { details } => { - IprevResult::PermError(mail_auth::Error::Io(details.unwrap_or_default())) - } - AuthResult::None => IprevResult::None, - _ => IprevResult::None, - } - } -} - -impl From<&DkimResult> for AuthResult { - fn from(value: &DkimResult) -> Self { - match value { - DkimResult::Pass => AuthResult::Pass, - DkimResult::Neutral(error) => AuthResult::Neutral { - details: error.to_string().into(), - }, - DkimResult::Fail(error) => AuthResult::Fail { - details: error.to_string().into(), - }, - DkimResult::PermError(error) => AuthResult::PermError { - details: error.to_string().into(), - }, - DkimResult::TempError(error) => AuthResult::TempError { - details: error.to_string().into(), - }, - DkimResult::None => AuthResult::None, - } - } -} - -impl From for DkimResult { - fn from(value: AuthResult) -> Self { - match value { - AuthResult::Pass => DkimResult::Pass, - AuthResult::Neutral { details } => { - DkimResult::Neutral(mail_auth::Error::Io(details.unwrap_or_default())) - } - AuthResult::Fail { details } => { - DkimResult::Fail(mail_auth::Error::Io(details.unwrap_or_default())) - } - AuthResult::PermError { details } => { - DkimResult::PermError(mail_auth::Error::Io(details.unwrap_or_default())) - } - AuthResult::TempError { details } => { - DkimResult::TempError(mail_auth::Error::Io(details.unwrap_or_default())) - } - _ => DkimResult::None, - } - } -} - -impl From<&DmarcResult> for AuthResult { - fn from(value: &DmarcResult) -> Self { - match value { - DmarcResult::Pass => AuthResult::Pass, - DmarcResult::Fail(error) => AuthResult::Fail { - details: error.to_string().into(), - }, - DmarcResult::TempError(error) => AuthResult::TempError { - details: error.to_string().into(), - }, - DmarcResult::PermError(error) => AuthResult::PermError { - details: error.to_string().into(), - }, - DmarcResult::None => AuthResult::None, - } - } -} - -impl From for DmarcResult { - fn from(value: AuthResult) -> Self { - match value { - AuthResult::Pass => DmarcResult::Pass, - AuthResult::Fail { details } => { - DmarcResult::Fail(mail_auth::Error::Io(details.unwrap_or_default())) - } - AuthResult::TempError { details } => { - DmarcResult::TempError(mail_auth::Error::Io(details.unwrap_or_default())) - } - AuthResult::PermError { details } => { - DmarcResult::PermError(mail_auth::Error::Io(details.unwrap_or_default())) - } - AuthResult::None => DmarcResult::None, - _ => DmarcResult::None, - } - } -} - -impl From<&dmarc::Policy> for DmarcPolicy { - fn from(value: &dmarc::Policy) -> Self { - match value { - dmarc::Policy::None => DmarcPolicy::None, - dmarc::Policy::Quarantine => DmarcPolicy::Quarantine, - dmarc::Policy::Reject => DmarcPolicy::Reject, - dmarc::Policy::Unspecified => DmarcPolicy::Unspecified, - } - } -} - -impl From for dmarc::Policy { - fn from(value: DmarcPolicy) -> Self { - match value { - DmarcPolicy::None => dmarc::Policy::None, - DmarcPolicy::Quarantine => dmarc::Policy::Quarantine, - DmarcPolicy::Reject => dmarc::Policy::Reject, - DmarcPolicy::Unspecified => dmarc::Policy::Unspecified, - } - } -} diff --git a/crates/http/src/management/mod.rs b/crates/http/src/management/mod.rs index a390d8a2..d2dd335a 100644 --- a/crates/http/src/management/mod.rs +++ b/crates/http/src/management/mod.rs @@ -12,13 +12,23 @@ pub mod telemetry; // SPDX-SnippetEnd pub mod diagnose; -use crate::management::diagnose::TroubleshootApi; -use common::{Server, auth::AccessToken}; -use http_proto::{HttpRequest, HttpResponse, HttpSessionData, request::fetch_body}; -use hyper::{StatusCode, header}; +use crate::management::diagnose::{DeliveryStage, spawn_delivery_diagnose}; +use common::{ + Server, + auth::{AccessToken, oauth::GrantType}, +}; +use http_body_util::{StreamBody, combinators::BoxBody}; +use http_proto::{ + HttpRequest, HttpResponse, HttpSessionData, JsonResponse, ToHttpResponse, + request::{decode_path_element, fetch_body}, +}; +use hyper::{Method, StatusCode, header}; use jmap::api::{ToJmapHttpResponse, ToRequestError}; use jmap_proto::error::request::RequestError; use registry::schema::enums::Permission; +use serde_json::json; +use std::time::Duration; +use utils::url_params::UrlParams; pub trait ManagementApi: Sync + Send { fn handle_api_manage_request( @@ -41,37 +51,116 @@ impl ManagementApi for Server { let path = req.uri().path().split('/').skip(2).collect::>(); match path.first().copied().unwrap_or_default() { - "diagnose" => { - // Validate the access token - access_token.enforce_permission(Permission::Troubleshoot)?; + "token" => { + let account_id = access_token.account_id(); + match path.get(1).copied() { + // SPDX-SnippetBegin + // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + // SPDX-License-Identifier: LicenseRef-SEL + #[cfg(feature = "enterprise")] + Some("tracing") if self.core.is_enterprise_edition() => { + // Validate the access token + access_token.enforce_permission(Permission::TracingLive)?; - self.handle_diagnose_api_request(req, path, access_token, body) - .await - } - // SPDX-SnippetBegin - // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - // SPDX-License-Identifier: LicenseRef-SEL - #[cfg(feature = "enterprise")] - "telemetry" => { - // WARNING: TAMPERING WITH THIS FUNCTION IS STRICTLY PROHIBITED - // Any attempt to modify, bypass, or disable this license validation mechanism - // constitutes a severe violation of the Stalwart Enterprise License Agreement. - // Such actions may result in immediate termination of your license, legal action, - // and substantial financial penalties. Stalwart Labs LLC actively monitors for - // unauthorized modifications and will pursue all available legal remedies against - // violators to the fullest extent of the law, including but not limited to claims - // for copyright infringement, breach of contract, and fraud. + // Issue a live telemetry token valid for 60 seconds + Ok(JsonResponse::new(json!({ + "data": self.encode_access_token(GrantType::LiveTracing, account_id, "web", 60).await?, + })) + .into_http_response()) + } + #[cfg(feature = "enterprise")] + Some("metrics") if self.core.is_enterprise_edition() => { + // Validate the access token + access_token.enforce_permission(Permission::MetricsLive)?; - if self.core.is_enterprise_edition() { - use crate::management::telemetry::TelemetryApi; + // Issue a live telemetry token valid for 60 seconds + Ok(JsonResponse::new(json!({ + "data": self.encode_access_token(GrantType::LiveMetrics, account_id, "web", 60).await?, + })) + .into_http_response()) + } + // SPDX-SnippetEnd + Some("delivery") => { + // Validate the access token + access_token.enforce_permission(Permission::Troubleshoot)?; - self.handle_telemetry_api_request(req, path, access_token) - .await - } else { - Err(trc::ResourceEvent::NotFound.ctx(trc::Key::Details, "Enterprise feature")) + // Issue a live telemetry token valid for 60 seconds + Ok(JsonResponse::new(json!({ + "data": self.encode_access_token(GrantType::Diagnose, account_id, "web", 60).await?, + })) + .into_http_response()) + } + Some("tracing") | Some("metrics") => { + Err(trc::ResourceEvent::NotFound + .ctx(trc::Key::Details, "Enterprise feature")) + } + _ => Err(trc::ResourceEvent::NotFound.into_err()), } } - // SPDX-SnippetEnd + "live" => { + let params = UrlParams::new(req.uri().query()); + let account_id = access_token.account_id(); + + match ( + path.get(1).copied().unwrap_or_default(), + path.get(2).copied(), + req.method(), + ) { + ("delivery", Some(target), &Method::GET) => { + // Validate the access token + access_token.enforce_permission(Permission::Troubleshoot)?; + + let timeout = Duration::from_secs( + params + .parse::("timeout") + .filter(|interval| *interval >= 1) + .unwrap_or(30), + ); + + let mut rx = spawn_delivery_diagnose( + self.clone(), + decode_path_element(target).to_lowercase(), + timeout, + ); + + Ok(HttpResponse::new(StatusCode::OK) + .with_content_type("text/event-stream") + .with_cache_control("no-store") + .with_stream_body(BoxBody::new(StreamBody::new( + async_stream::stream! { + while let Some(stage) = rx.recv().await { + yield Ok(stage.to_frame()); + } + yield Ok(DeliveryStage::Completed.to_frame()); + }, + )))) + } + // SPDX-SnippetBegin + // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + // SPDX-License-Identifier: LicenseRef-SEL + #[cfg(feature = "enterprise")] + ("traces", _, &Method::GET) if self.core.is_enterprise_edition() => { + use crate::management::telemetry::TelemetryApi; + + self.handle_telemetry_api_request(req, true, access_token) + .await + } + #[cfg(feature = "enterprise")] + ("metrics", _, &Method::GET) if self.core.is_enterprise_edition() => { + use crate::management::telemetry::TelemetryApi; + + self.handle_telemetry_api_request(req, false, access_token) + .await + } + // SPDX-SnippetEnd + ("traces" | "metrics", _, &Method::GET) => { + Err(trc::ResourceEvent::NotFound + .ctx(trc::Key::Details, "Enterprise feature")) + } + _ => Err(trc::ResourceEvent::NotFound.into_err()), + } + } + _ => Err(trc::ResourceEvent::NotFound.into_err()), } } diff --git a/crates/http/src/management/telemetry.rs b/crates/http/src/management/telemetry.rs index 52529b91..f27e019a 100644 --- a/crates/http/src/management/telemetry.rs +++ b/crates/http/src/management/telemetry.rs @@ -8,19 +8,15 @@ * */ -use common::{ - Server, - auth::{AccessToken, oauth::GrantType}, -}; +use common::{Server, auth::AccessToken}; use http_body_util::{StreamBody, combinators::BoxBody}; use http_proto::*; use hyper::{ - Method, StatusCode, + StatusCode, body::{Bytes, Frame}, }; use mail_parser::DateTime; use registry::schema::enums::Permission; -use serde_json::json; use std::future::Future; use std::{ fmt::Write, @@ -38,7 +34,7 @@ pub trait TelemetryApi: Sync + Send { fn handle_telemetry_api_request( &self, req: &HttpRequest, - path: Vec<&str>, + is_tracing: bool, access_token: &AccessToken, ) -> impl Future> + Send; } @@ -47,48 +43,40 @@ impl TelemetryApi for Server { async fn handle_telemetry_api_request( &self, req: &HttpRequest, - path: Vec<&str>, + is_tracing: bool, access_token: &AccessToken, ) -> trc::Result { let params = UrlParams::new(req.uri().query()); - let account_id = access_token.account_id(); - let todo = "use same format as in JMAP API"; + if is_tracing { + // Validate the access token + access_token.enforce_permission(Permission::TracingLive)?; - match ( - path.get(1).copied().unwrap_or_default(), - path.get(2).copied(), - req.method(), - ) { - ("traces", Some("live"), &Method::GET) => { - // Validate the access token - access_token.enforce_permission(Permission::TracingLive)?; + let mut key_filters = AHashMap::new(); + let mut filter = None; - let mut key_filters = AHashMap::new(); - let mut filter = None; - - for (key, value) in params.into_inner() { - if key == "filter" { - filter = value.into_owned().into(); - } else if let Some(key) = Key::try_parse(key.to_ascii_lowercase().as_str()) { - key_filters.insert(key, value.into_owned()); - } + for (key, value) in params.into_inner() { + if key == "filter" { + filter = value.into_owned().into(); + } else if let Some(key) = Key::try_parse(key.to_ascii_lowercase().as_str()) { + key_filters.insert(key, value.into_owned()); } + } - let (_, mut rx) = SubscriberBuilder::new("live-tracer".to_string()) - .with_interests(Box::new(Bitset::all())) - .with_lossy(false) - .register(); - let throttle = Duration::from_secs(1); - let ping_interval = Duration::from_secs(30); - let ping_payload = Bytes::from(format!( - "event: ping\ndata: {{\"interval\": {}}}\n\n", - ping_interval.as_millis() - )); - let mut last_ping = Instant::now(); - let mut events = Vec::new(); - let mut active_span_ids = AHashSet::new(); + let (_, mut rx) = SubscriberBuilder::new("live-tracer".to_string()) + .with_interests(Box::new(Bitset::all())) + .with_lossy(false) + .register(); + let throttle = Duration::from_secs(1); + let ping_interval = Duration::from_secs(30); + let ping_payload = Bytes::from(format!( + "event: ping\ndata: {{\"interval\": {}}}\n\n", + ping_interval.as_millis() + )); + let mut last_ping = Instant::now(); + let mut events = Vec::new(); + let mut active_span_ids = AHashSet::new(); - Ok(HttpResponse::new(StatusCode::OK) + Ok(HttpResponse::new(StatusCode::OK) .with_content_type("text/event-stream") .with_cache_control("no-store") .with_stream_body(BoxBody::new(StreamBody::new( @@ -185,68 +173,47 @@ impl TelemetryApi for Server { } }, )))) - } - ("live", Some("tracing-token"), &Method::GET) => { - // Validate the access token - access_token.enforce_permission(Permission::TracingLive)?; + } else { + // Validate the access token + access_token.enforce_permission(Permission::MetricsLive)?; - // Issue a live telemetry token valid for 60 seconds - Ok(JsonResponse::new(json!({ - "data": self.encode_access_token(GrantType::LiveTracing, account_id, "web", 60).await?, - })) - .into_http_response()) - } - ("live", Some("metrics-token"), &Method::GET) => { - // Validate the access token - access_token.enforce_permission(Permission::MetricsLive)?; - - // Issue a live telemetry token valid for 60 seconds - Ok(JsonResponse::new(json!({ - "data": self.encode_access_token(GrantType::LiveMetrics, account_id, "web", 60).await?, - })) - .into_http_response()) - } - ("metrics", Some("live"), &Method::GET) => { - // Validate the access token - access_token.enforce_permission(Permission::MetricsLive)?; - - let interval = Duration::from_secs( - params - .parse::("interval") - .filter(|interval| *interval >= 1) - .unwrap_or(30), - ); - let mut event_types = AHashSet::new(); - let mut metric_types = AHashSet::new(); - for metric_name in params.get("metrics").unwrap_or_default().split(',') { - let metric_name = metric_name.trim(); - if !metric_name.is_empty() { - if let Some(event_type) = EventType::parse(metric_name) { - event_types.insert(event_type); - } else if let Some(metric_type) = MetricType::parse(metric_name) { - metric_types.insert(metric_type); - } + let interval = Duration::from_secs( + params + .parse::("interval") + .filter(|interval| *interval >= 1) + .unwrap_or(30), + ); + let mut event_types = AHashSet::new(); + let mut metric_types = AHashSet::new(); + for metric_name in params.get("metrics").unwrap_or_default().split(',') { + let metric_name = metric_name.trim(); + if !metric_name.is_empty() { + if let Some(event_type) = EventType::parse(metric_name) { + event_types.insert(event_type); + } else if let Some(metric_type) = MetricType::parse(metric_name) { + metric_types.insert(metric_type); } } + } - // Refresh expensive metrics - for metric_type in [ - MetricType::QueueCount, - MetricType::UserCount, - MetricType::DomainCount, - ] { - if metric_types.contains(&metric_type) { - let value = match metric_type { - MetricType::QueueCount => self.total_queued_messages().await?, - MetricType::UserCount => self.total_accounts().await? as u64, - MetricType::DomainCount => self.total_domains().await? as u64, - _ => unreachable!(), - }; - Collector::update_gauge(metric_type, value); - } + // Refresh expensive metrics + for metric_type in [ + MetricType::QueueCount, + MetricType::UserCount, + MetricType::DomainCount, + ] { + if metric_types.contains(&metric_type) { + let value = match metric_type { + MetricType::QueueCount => self.total_queued_messages().await?, + MetricType::UserCount => self.total_accounts().await? as u64, + MetricType::DomainCount => self.total_domains().await? as u64, + _ => unreachable!(), + }; + Collector::update_gauge(metric_type, value); } + } - Ok(HttpResponse::new(StatusCode::OK) + Ok(HttpResponse::new(StatusCode::OK) .with_content_type("text/event-stream") .with_cache_control("no-store") .with_stream_body(BoxBody::new(StreamBody::new( @@ -309,8 +276,6 @@ impl TelemetryApi for Server { } }, )))) - } - _ => Err(trc::ResourceEvent::NotFound.into_err()), } } } diff --git a/crates/jmap-proto/src/error/set.rs b/crates/jmap-proto/src/error/set.rs index b60a9ed3..d16f62c0 100644 --- a/crates/jmap-proto/src/error/set.rs +++ b/crates/jmap-proto/src/error/set.rs @@ -201,6 +201,11 @@ impl SetError { self } + pub fn with_object_id_opt(mut self, object_id: Option) -> Self { + self.0.object_id = object_id; + self + } + pub fn with_linked_objects(mut self, linked_objects: Vec) -> Self { self.0.linked_objects = linked_objects; self diff --git a/crates/jmap/src/calendar_event/copy.rs b/crates/jmap/src/calendar_event/copy.rs index 27a9713a..31a4feca 100644 --- a/crates/jmap/src/calendar_event/copy.rs +++ b/crates/jmap/src/calendar_event/copy.rs @@ -169,7 +169,7 @@ impl JmapCalendarEventCopy for Server { &mut batch, access_token, account_id, - account_info.addresses(), + &account_info, false, &can_add_calendars, calendar_event.data.event.into_jscalendar(), diff --git a/crates/jmap/src/calendar_event/set.rs b/crates/jmap/src/calendar_event/set.rs index b96c3791..012d66f5 100644 --- a/crates/jmap/src/calendar_event/set.rs +++ b/crates/jmap/src/calendar_event/set.rs @@ -15,7 +15,10 @@ use calcard::{ jscalendar::{JSCalendar, JSCalendarDateTime, JSCalendarProperty, JSCalendarValue}, }; use chrono::DateTime; -use common::{DavName, DavResources, Server, auth::AccessToken}; +use common::{ + DavName, DavResources, Server, + auth::{AccessToken, AccountInfo}, +}; use groupware::{ DestroyArchive, cache::GroupwareCache, @@ -66,7 +69,7 @@ pub trait CalendarEventSet: Sync + Send { batch: &mut BatchBuilder, access_token: &AccessToken, account_id: u32, - account_emails: &[String], + account_info: &AccountInfo, send_scheduling_messages: bool, can_add_calendars: &Option, js_calendar_event: JSCalendar<'_, Id, BlobId>, @@ -124,7 +127,7 @@ impl CalendarEventSet for Server { &mut batch, access_token, account_id, - account_info.addresses(), + &account_info, send_scheduling_messages, &can_add_calendars, JSCalendar::default(), @@ -388,7 +391,10 @@ impl CalendarEventSet for Server { let extra_bytes = (new_calendar_event.size as u64) .saturating_sub(u32::from(calendar_event.inner.size) as u64); if extra_bytes > 0 { - match self.has_available_quota(account_id, extra_bytes).await { + match self + .has_available_quota(account_info.account(), extra_bytes) + .await + { Ok(_) => {} Err(err) if err.matches(trc::EventType::Limit(trc::LimitEvent::Quota)) => { response.not_updated.append(id, SetError::over_quota()); @@ -512,7 +518,7 @@ impl CalendarEventSet for Server { batch: &mut BatchBuilder, access_token: &AccessToken, account_id: u32, - account_emails: &[String], + account_info: &AccountInfo, send_scheduling_messages: bool, can_add_calendars: &Option, mut js_calendar_group: JSCalendar<'_, Id, BlobId>, @@ -619,11 +625,11 @@ impl CalendarEventSet for Server { let mut itip_messages = None; if send_scheduling_messages && self.core.groupware.itip_enabled - && !account_emails.is_empty() + && !account_info.addresses().is_empty() && access_token.has_permission(Permission::CalendarSchedulingSend) && event.data.event_range_end() > now() as i64 { - match itip_create(&mut event.data.event, account_emails) { + match itip_create(&mut event.data.event, account_info.addresses()) { Ok(messages) => { if messages.iter().map(|r| r.to.len()).sum::() < self.core.groupware.itip_outbound_max_recipients @@ -650,7 +656,10 @@ impl CalendarEventSet for Server { } // Validate quota - match self.has_available_quota(account_id, size as u64).await { + match self + .has_available_quota(account_info.account(), size as u64) + .await + { Ok(_) => {} Err(err) if err.matches(trc::EventType::Limit(trc::LimitEvent::Quota)) => { return Ok(Err(SetError::over_quota())); diff --git a/crates/jmap/src/contact/copy.rs b/crates/jmap/src/contact/copy.rs index 97545e4e..ed921f00 100644 --- a/crates/jmap/src/contact/copy.rs +++ b/crates/jmap/src/contact/copy.rs @@ -22,7 +22,11 @@ use jmap_proto::{ }, types::state::State, }; -use store::{ValueKey, roaring::RoaringBitmap, write::{AlignedBytes, Archive, BatchBuilder}}; +use store::{ + ValueKey, + roaring::RoaringBitmap, + write::{AlignedBytes, Archive, BatchBuilder}, +}; use trc::AddContext; use types::{ acl::Acl, @@ -50,6 +54,7 @@ impl JmapContactCardCopy for Server { ) -> trc::Result> { let account_id = request.account_id.document_id(); let from_account_id = request.from_account_id.document_id(); + let account = self.account(account_id).await.caused_by(trc::location!())?; if account_id == from_account_id { return Err(trc::JmapEvent::InvalidArguments @@ -57,7 +62,11 @@ impl JmapContactCardCopy for Server { .details("From accountId is equal to fromAccountId")); } let cache = self - .fetch_dav_resources(access_token.account_id(), account_id, SyncCollection::AddressBook) + .fetch_dav_resources( + access_token.account_id(), + account_id, + SyncCollection::AddressBook, + ) .await .caused_by(trc::location!())?; let old_state = cache.assert_state(false, &request.if_in_state)?; @@ -71,7 +80,11 @@ impl JmapContactCardCopy for Server { }; let from_cache = self - .fetch_dav_resources(access_token.account_id(), from_account_id, SyncCollection::AddressBook) + .fetch_dav_resources( + access_token.account_id(), + from_account_id, + SyncCollection::AddressBook, + ) .await .caused_by(trc::location!())?; let from_contact_ids = if access_token.is_member(from_account_id) { @@ -134,6 +147,7 @@ impl JmapContactCardCopy for Server { &cache, &mut batch, access_token, + &account, account_id, &can_add_address_books, contact.card.into_jscontact(), diff --git a/crates/jmap/src/contact/set.rs b/crates/jmap/src/contact/set.rs index 89587dcd..0910272d 100644 --- a/crates/jmap/src/contact/set.rs +++ b/crates/jmap/src/contact/set.rs @@ -6,7 +6,10 @@ use crate::contact::assert_is_unique_uid; use calcard::jscontact::{JSContact, JSContactProperty, JSContactValue}; -use common::{DavName, DavResources, Server, auth::AccessToken}; +use common::{ + DavName, DavResources, Server, + auth::{AccessToken, AccountCache}, +}; use groupware::{DestroyArchive, cache::GroupwareCache, contact::ContactCard}; use http_proto::HttpSessionData; use jmap_proto::{ @@ -45,6 +48,7 @@ pub trait ContactCardSet: Sync + Send { cache: &DavResources, batch: &mut BatchBuilder, access_token: &AccessToken, + account: &AccountCache, account_id: u32, can_add_address_books: &Option, js_contact: JSContact<'_, Id, BlobId>, @@ -60,6 +64,7 @@ impl ContactCardSet for Server { _session: &HttpSessionData, ) -> trc::Result> { let account_id = request.account_id.document_id(); + let account = self.account(account_id).await.caused_by(trc::location!())?; let cache = self .fetch_dav_resources( access_token.account_id(), @@ -96,6 +101,7 @@ impl ContactCardSet for Server { &cache, &mut batch, access_token, + &account, account_id, &can_add_address_books, JSContact::default(), @@ -256,7 +262,7 @@ impl ContactCardSet for Server { let extra_bytes = (new_contact_card.size as u64) .saturating_sub(u32::from(contact_card.inner.size) as u64); if extra_bytes > 0 { - match self.has_available_quota(account_id, extra_bytes).await { + match self.has_available_quota(&account, extra_bytes).await { Ok(_) => {} Err(err) if err.matches(trc::EventType::Limit(trc::LimitEvent::Quota)) => { response.not_updated.append(id, SetError::over_quota()); @@ -357,6 +363,7 @@ impl ContactCardSet for Server { cache: &DavResources, batch: &mut BatchBuilder, access_token: &AccessToken, + account: &AccountCache, account_id: u32, can_add_address_books: &Option, mut js_contact: JSContact<'_, Id, BlobId>, @@ -409,7 +416,7 @@ impl ContactCardSet for Server { ), ))); } - match self.has_available_quota(account_id, size as u64).await { + match self.has_available_quota(account, size as u64).await { Ok(_) => {} Err(err) if err.matches(trc::EventType::Limit(trc::LimitEvent::Quota)) => { return Ok(Err(SetError::over_quota())); diff --git a/crates/jmap/src/registry/get.rs b/crates/jmap/src/registry/get.rs index 9c878f6f..cccf659e 100644 --- a/crates/jmap/src/registry/get.rs +++ b/crates/jmap/src/registry/get.rs @@ -8,6 +8,7 @@ use crate::registry::mapping::{ RegistryGetResponse, account::account_get, archived_item::archived_item_get, + dkim::generate_dkim_public_key, log::log_get, queued_message::queued_message_get, report::report_get, @@ -35,6 +36,7 @@ use registry::{ use store::{ahash::AHashSet, registry::RegistryQuery}; use trc::AddContext; use types::id::Id; +use utils::map::vec_map::VecMap; pub trait RegistryGet: Sync + Send { fn registry_get( @@ -234,14 +236,32 @@ impl RegistryGet for Server { continue; }; - let todo = "include account quota if requested"; + let mut extra_properties = VecMap::new(); match &object.inner { ObjectInner::DkimSignature(obj) if get.properties.is_empty() || get.properties.contains(&Property::PublicKey) => { - let todo = "dkim public key"; - todo!() + if let Ok(public_key) = generate_dkim_public_key(obj).await { + extra_properties + .append(Property::PublicKey, JmapValue::Str(public_key.into())); + } + } + ObjectInner::Account(obj) + if get.properties.is_empty() + || get.properties.contains(&Property::UsedDiskQuota) => + { + let quota = self.get_used_quota_account(id.document_id()).await?; + extra_properties + .append(Property::UsedDiskQuota, JmapValue::Number(quota.into())); + } + ObjectInner::Tenant(obj) + if get.properties.is_empty() + || get.properties.contains(&Property::UsedDiskQuota) => + { + let quota = self.get_used_quota_tenant(id.document_id()).await?; + extra_properties + .append(Property::UsedDiskQuota, JmapValue::Number(quota.into())); } ObjectInner::Domain(obj) if get.properties.is_empty() @@ -279,10 +299,7 @@ impl RegistryGet for Server { ObjectType::AccountSettings | ObjectType::Credential => { account_get(get).await.map(|get| get.into_response()) } - ObjectType::Action => { - let todo = "actions"; - todo!() - } + ObjectType::Action => Ok(get.not_found_any().into_response()), } } } diff --git a/crates/jmap/src/registry/mapping/account.rs b/crates/jmap/src/registry/mapping/account.rs index 7b457c8a..773a77d7 100644 --- a/crates/jmap/src/registry/mapping/account.rs +++ b/crates/jmap/src/registry/mapping/account.rs @@ -75,7 +75,7 @@ pub(crate) async fn account_set( let ptr = JsonPointer::new(vec![JsonPointerItem::Key(Key::Property(property))]); if let Err(err) = - account.patch(JsonPointerPatch::new(&ptr).with_create(true), value) + account.patch(JsonPointerPatch::new(&ptr).with_create(false), value) { set.response.not_updated.append(id, err.into()); break 'outer; diff --git a/crates/jmap/src/registry/mapping/action.rs b/crates/jmap/src/registry/mapping/action.rs new file mode 100644 index 00000000..90b1d4e9 --- /dev/null +++ b/crates/jmap/src/registry/mapping/action.rs @@ -0,0 +1,543 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::time::Instant; + +use common::{ + Server, + config::mailstore::spamfilter::SpamFilterAction, + ipc::{BroadcastEvent, QueueEvent, RegistryChange}, + psl, +}; +use jmap_proto::error::set::{SetError, SetErrorType}; +use jmap_tools::{JsonPointer, JsonPointerItem, Key}; +use mail_auth::{ + AuthenticatedMessage, DkimResult, DmarcResult, dmarc::verify::DmarcParameters, + spf::verify::SpfParameters, +}; +use mail_parser::MessageParser; +use registry::{ + jmap::{IntoValue, JsonPointerPatch, RegistryJsonPatch}, + schema::{ + enums::{SpamClassifyParameters, SpamClassifyResult, SpamClassifyTagDisposition}, + prelude::{ObjectType, Property}, + structs::{Action, DmarcTroubleshoot, SpamClassify, SpamClassifyTag}, + }, + types::{ObjectImpl, error::Error}, +}; +use smtp_proto::{MAIL_BODY_7BIT, MAIL_BODY_8BITMIME, MAIL_BODY_BINARYMIME, MAIL_SMTPUTF8}; +use spam_filter::{ + SpamFilterInput, + analysis::{init::SpamFilterInit, score::SpamFilterAnalyzeScore}, +}; +use store::write::now; + +use crate::registry::mapping::RegistrySetResponse; + +pub(crate) async fn action_set( + mut set: RegistrySetResponse<'_>, +) -> trc::Result> { + // Actions cannot be uodated or destroyed, so we fail all updates and destroys. + set.fail_all_update("Actions cannot be updated"); + set.fail_all_destroy("Actions cannot be destroyed"); + + // Process creations + 'outer: for (id, value) in set.create.drain() { + let mut action = Action::default(); + for (key, value) in value.into_expanded_object() { + let Key::Property(prop) = key else { + set.response.not_created.append( + id, + SetError::invalid_properties().with_property(key.into_owned()), + ); + continue 'outer; + }; + let ptr = JsonPointer::new(vec![JsonPointerItem::Key(Key::Property(prop))]); + if let Err(err) = action.patch(JsonPointerPatch::new(&ptr).with_create(true), value) { + set.response.not_created.append(id, err.into()); + continue 'outer; + } + } + + let mut validation_errors = Vec::new(); + if !action.validate(&mut validation_errors) { + set.response.not_created.append( + id, + SetError::new(SetErrorType::ValidationFailed) + .with_validation_errors(validation_errors), + ); + continue 'outer; + } + + match action { + Action::ReloadSettings + | Action::ReloadTlsCertificates + | Action::ReloadLookupStores + | Action::ReloadBlockedIps => { + let object = match action { + Action::ReloadSettings => ObjectType::DataStore, + Action::ReloadTlsCertificates => ObjectType::Certificate, + Action::ReloadLookupStores => ObjectType::StoreLookup, + Action::ReloadBlockedIps => ObjectType::BlockedIp, + _ => unreachable!(), + }; + let result = + Box::pin(set.server.reload_registry(RegistryChange::Reload(object))).await?; + + if !result.has_errors() { + set.server + .cluster_broadcast(BroadcastEvent::RegistryChange(RegistryChange::Reload( + object, + ))) + .await; + set.response.created(id, now()); + } else { + set.response + .not_created + .append(id, map_bootstrap_error(result.errors)); + } + } + Action::InvalidateCaches => { + set.server.invalidate_all_local_caches(); + set.server + .cluster_broadcast(BroadcastEvent::CacheInvalidateAll) + .await; + set.response.created(id, now()); + } + Action::PauseMtaQueue => { + let _ = set + .server + .inner + .ipc + .queue_tx + .send(QueueEvent::Paused(true)) + .await; + set.server + .cluster_broadcast(BroadcastEvent::MtaQueueStatus { is_running: false }) + .await; + set.response.created(id, now()); + } + Action::ResumeMtaQueue => { + let _ = set + .server + .inner + .ipc + .queue_tx + .send(QueueEvent::Paused(false)) + .await; + set.server + .cluster_broadcast(BroadcastEvent::MtaQueueStatus { is_running: true }) + .await; + set.response.created(id, now()); + } + Action::TroubleshootDmarc(troubleshoot) => { + if let Some(result) = dmarc_troubleshoot(set.server, troubleshoot).await { + let mut result = result.into_value(); + result + .as_object_mut() + .unwrap() + .as_mut_vec() + .retain(|(k, _)| { + !matches!( + k, + Key::Property( + Property::Message + | Property::RemoteIp + | Property::EhloDomain + | Property::MailFrom + ) + ) + }); + set.response.created.insert(id, result); + } else { + set.response.not_created.append( + id, + SetError::invalid_properties() + .with_property(Property::Body) + .with_description( + "Failed to parse the message for DMARC troubleshooting".to_string(), + ), + ); + } + } + Action::ClassifySpam(classify) => { + if let Some(result) = classify_spam(set.server, classify).await { + let mut result = result.into_value(); + result + .as_object_mut() + .unwrap() + .as_mut_vec() + .retain(|(k, _)| { + !matches!( + k, + Key::Property( + Property::Message + | Property::RemoteIp + | Property::EhloDomain + | Property::AuthenticatedAs + | Property::IsTls + | Property::EnvFrom + | Property::EnvFromParameters + | Property::EnvRcptTo + ) + ) + }); + set.response.created.insert(id, result); + } else { + set.response.not_created.append( + id, + SetError::invalid_properties() + .with_property(Property::Message) + .with_description( + "Failed to parse the message for spam classification".to_string(), + ), + ); + } + } + } + } + + Ok(set) +} + +async fn classify_spam(server: &Server, mut request: SpamClassify) -> Option { + // Built spam filter input + let message = MessageParser::new() + .parse(request.message.as_bytes()) + .filter(|m| m.root_part().headers().iter().any(|h| !h.name.is_other()))?; + + let remote_ip = request.remote_ip.into_inner(); + let ehlo_domain = request.ehlo_domain.to_lowercase(); + let mail_from = request.env_from.to_lowercase(); + let mail_from_domain = mail_from.rsplit_once('@').map(|(_, domain)| domain); + let local_host = &server.core.network.server_name; + + let spf_ehlo_result = server + .core + .smtp + .resolvers + .dns + .verify_spf( + server + .inner + .cache + .build_auth_parameters(SpfParameters::verify_ehlo( + remote_ip, + &ehlo_domain, + local_host, + )), + ) + .await; + + let iprev_result = server + .core + .smtp + .resolvers + .dns + .verify_iprev(server.inner.cache.build_auth_parameters(remote_ip)) + .await; + + let spf_mail_from_result = if let Some(mail_from_domain) = mail_from_domain { + server + .core + .smtp + .resolvers + .dns + .check_host(server.inner.cache.build_auth_parameters(SpfParameters::new( + remote_ip, + mail_from_domain, + &ehlo_domain, + local_host, + &mail_from, + ))) + .await + } else { + server + .core + .smtp + .resolvers + .dns + .check_host(server.inner.cache.build_auth_parameters(SpfParameters::new( + remote_ip, + &ehlo_domain, + &ehlo_domain, + local_host, + &format!("postmaster@{ehlo_domain}"), + ))) + .await + }; + + let auth_message = AuthenticatedMessage::from_parsed(&message, true); + + let dkim_output = server + .core + .smtp + .resolvers + .dns + .verify_dkim(server.inner.cache.build_auth_parameters(&auth_message)) + .await; + + let arc_output = server + .core + .smtp + .resolvers + .dns + .verify_arc(server.inner.cache.build_auth_parameters(&auth_message)) + .await; + + let dmarc_output = server + .core + .smtp + .resolvers + .dns + .verify_dmarc(server.inner.cache.build_auth_parameters(DmarcParameters { + message: &auth_message, + dkim_output: &dkim_output, + rfc5321_mail_from_domain: mail_from_domain.unwrap_or(ehlo_domain.as_str()), + spf_output: &spf_mail_from_result, + domain_suffix_fn: |domain| psl::domain_str(domain).unwrap_or(domain), + })) + .await; + let dmarc_pass = matches!(dmarc_output.spf_result(), DmarcResult::Pass) + || matches!(dmarc_output.dkim_result(), DmarcResult::Pass); + let dmarc_result = if dmarc_pass { + DmarcResult::Pass + } else if dmarc_output.spf_result() != &DmarcResult::None { + dmarc_output.spf_result().clone() + } else if dmarc_output.dkim_result() != &DmarcResult::None { + dmarc_output.dkim_result().clone() + } else { + DmarcResult::None + }; + let dmarc_policy = dmarc_output.policy(); + + let asn_geo = server.lookup_asn_country(remote_ip).await; + + let input = SpamFilterInput { + message: &message, + span_id: 0, + arc_result: Some(&arc_output), + spf_ehlo_result: Some(&spf_ehlo_result), + spf_mail_from_result: Some(&spf_mail_from_result), + dkim_result: dkim_output.as_slice(), + dmarc_result: Some(&dmarc_result), + dmarc_policy: Some(&dmarc_policy), + iprev_result: Some(&iprev_result), + remote_ip, + ehlo_domain: Some(ehlo_domain.as_str()), + authenticated_as: request.authenticated_as.as_deref(), + asn: asn_geo.asn.as_ref().map(|a| a.id), + country: asn_geo.country.as_ref().map(|c| c.as_str()), + is_tls: request.is_tls, + env_from: &request.env_from, + env_from_flags: match request.env_from_parameters { + SpamClassifyParameters::Bit7 => MAIL_BODY_7BIT, + SpamClassifyParameters::Bit8Mime8BitMIMEMessageContent => MAIL_BODY_BINARYMIME, + SpamClassifyParameters::BinaryMime => MAIL_BODY_8BITMIME, + SpamClassifyParameters::SmtpUtf8 => MAIL_SMTPUTF8, + }, + env_rcpt_to: request.env_rcpt_to.iter().map(String::as_str).collect(), + is_test: true, + is_train: false, + }; + + // Classify + let mut ctx = server.spam_filter_init(input); + let result = server.spam_filter_classify(&mut ctx).await; + + // Build response + request.result = match result { + SpamFilterAction::Allow(result) => { + request.score = (result.score as f64).into(); + if result.is_spam { + SpamClassifyResult::Spam + } else { + SpamClassifyResult::Ham + } + } + SpamFilterAction::Discard => SpamClassifyResult::Discard, + SpamFilterAction::Reject | SpamFilterAction::Disabled => SpamClassifyResult::Reject, + }; + + let mut tags = Vec::with_capacity(ctx.result.tags.len()); + for tag in ctx.result.tags { + let (score, disposition) = match server.core.spam.lists.scores.get(&tag) { + Some(SpamFilterAction::Allow(score)) => (*score, SpamClassifyTagDisposition::Score), + Some(SpamFilterAction::Discard) => (0.0, SpamClassifyTagDisposition::Discard), + _ => (0.0, SpamClassifyTagDisposition::Reject), + }; + tags.push(SpamClassifyTag { + disposition, + name: tag, + score: (score as f64).into(), + }); + } + request.tags = tags.into(); + + Some(request) +} + +async fn dmarc_troubleshoot( + server: &Server, + mut request: DmarcTroubleshoot, +) -> Option { + let remote_ip = request.remote_ip.into_inner(); + let ehlo_domain = request.ehlo_domain.to_lowercase(); + let mail_from = request.mail_from.to_lowercase(); + let mail_from_domain = mail_from.rsplit_once('@').map(|(_, domain)| domain); + + let local_host = &server.core.network.server_name; + + let now = Instant::now(); + let ehlo_spf_output = server + .core + .smtp + .resolvers + .dns + .verify_spf( + server + .inner + .cache + .build_auth_parameters(SpfParameters::verify_ehlo( + remote_ip, + &ehlo_domain, + local_host, + )), + ) + .await; + + let iprev = server + .core + .smtp + .resolvers + .dns + .verify_iprev(server.inner.cache.build_auth_parameters(remote_ip)) + .await; + let mail_spf_output = if let Some(mail_from_domain) = mail_from_domain { + server + .core + .smtp + .resolvers + .dns + .check_host(server.inner.cache.build_auth_parameters(SpfParameters::new( + remote_ip, + mail_from_domain, + &ehlo_domain, + local_host, + &mail_from, + ))) + .await + } else { + server + .core + .smtp + .resolvers + .dns + .check_host(server.inner.cache.build_auth_parameters(SpfParameters::new( + remote_ip, + &ehlo_domain, + &ehlo_domain, + local_host, + &format!("postmaster@{ehlo_domain}"), + ))) + .await + }; + + let body = request + .message + .take() + .unwrap_or_else(|| format!("From: {mail_from}\r\nSubject: test\r\n\r\ntest")); + let auth_message = AuthenticatedMessage::parse_with_opts(body.as_bytes(), true)?; + + let dkim_output = server + .core + .smtp + .resolvers + .dns + .verify_dkim(server.inner.cache.build_auth_parameters(&auth_message)) + .await; + let dkim_pass = dkim_output + .iter() + .any(|d| matches!(d.result(), DkimResult::Pass)); + + let arc_output = server + .core + .smtp + .resolvers + .dns + .verify_arc(server.inner.cache.build_auth_parameters(&auth_message)) + .await; + + let dmarc_output = server + .core + .smtp + .resolvers + .dns + .verify_dmarc(server.inner.cache.build_auth_parameters(DmarcParameters { + message: &auth_message, + dkim_output: &dkim_output, + rfc5321_mail_from_domain: mail_from_domain.unwrap_or(ehlo_domain.as_str()), + spf_output: &mail_spf_output, + domain_suffix_fn: |domain| psl::domain_str(domain).unwrap_or(domain), + })) + .await; + let dmarc_pass = matches!(dmarc_output.spf_result(), DmarcResult::Pass) + || matches!(dmarc_output.dkim_result(), DmarcResult::Pass); + let dmarc_result = if dmarc_pass { + DmarcResult::Pass + } else if dmarc_output.spf_result() != &DmarcResult::None { + dmarc_output.spf_result().clone() + } else if dmarc_output.dkim_result() != &DmarcResult::None { + dmarc_output.dkim_result().clone() + } else { + DmarcResult::None + }; + + request.spf_ehlo_domain = ehlo_spf_output.domain().to_string(); + request.spf_ehlo_result = (&ehlo_spf_output).into(); + request.spf_mail_from_domain = mail_spf_output.domain().to_string(); + request.spf_mail_from_result = (&mail_spf_output).into(); + request.ip_rev_ptr = iprev + .ptr + .as_ref() + .map(|ptr| { + ptr.iter() + .map(|label| label.to_string()) + .collect::>() + }) + .unwrap_or_default() + .into(); + request.ip_rev_result = (&iprev).into(); + request.dkim_pass = dkim_pass; + request.dkim_results = dkim_output + .iter() + .map(|result| result.result().into()) + .collect(); + request.arc_result = arc_output.result().into(); + request.dmarc_result = (&dmarc_result).into(); + request.dmarc_policy = (&dmarc_output.policy()).into(); + request.dmarc_pass = dmarc_pass; + request.elapsed = now.elapsed().into(); + + Some(request) +} + +fn map_bootstrap_error(error: Vec) -> SetError { + match error.into_iter().next().unwrap() { + Error::Validation { object_id, errors } => SetError::new(SetErrorType::ValidationFailed) + .with_validation_errors(errors) + .with_object_id(object_id), + Error::Build { object_id, message } => SetError::new(SetErrorType::ValidationFailed) + .with_description(message) + .with_object_id(object_id), + Error::Internal { object_id, error } => SetError::new(SetErrorType::Forbidden) + .with_description(error.to_string()) + .with_object_id_opt(object_id), + Error::NotFound { object_id } => { + SetError::new(SetErrorType::NotFound).with_object_id(object_id) + } + } +} diff --git a/crates/jmap/src/registry/mapping/dkim.rs b/crates/jmap/src/registry/mapping/dkim.rs new file mode 100644 index 00000000..fb0af70b --- /dev/null +++ b/crates/jmap/src/registry/mapping/dkim.rs @@ -0,0 +1,147 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::registry::mapping::{ + ObjectResponse, RegistrySetResponse, ValidationResult, principal::validate_tenant_quota, +}; +use common::config::smtp::auth::{DkimSigner, rsa_key_parse, simple_pem_parse}; +use jmap_proto::error::set::SetError; +use mail_auth::{common::crypto::Ed25519Key, dkim::generate::DkimKeyPair}; +use mail_builder::encoders::base64::base64_encode; +use pkcs8::Document; +use registry::{ + jmap::IntoValue, + schema::{ + enums::{DkimSignatureType, TenantStorageQuota}, + prelude::{MASKED_PASSWORD, Property}, + structs::{DkimPrivateKey, DkimSignature, SecretTextValue}, + }, +}; +use rsa::pkcs1::DecodeRsaPublicKey; + +pub(crate) async fn validate_dkim_signature( + set: &RegistrySetResponse<'_>, + key: &mut DkimSignature, + old_key: Option<&DkimSignature>, +) -> ValidationResult { + let mut response = if old_key.is_none() { + match validate_tenant_quota(set, TenantStorageQuota::MaxDkimKeys).await? { + Ok(response) => response, + Err(err) => { + return Ok(Err(err)); + } + } + } else { + ObjectResponse::default() + }; + + // Generate private key if requested + let key_type = key.object_type(); + let pk = key.private_key_mut(); + if let Some(old_key) = old_key + && matches!(pk, DkimPrivateKey::Value(value) if value.secret == MASKED_PASSWORD) + { + *pk = old_key.private_key().clone(); + } + if pk == &DkimPrivateKey::Generate { + let private_key = tokio::task::spawn_blocking(move || match key_type { + DkimSignatureType::Dkim1RsaSha256 => { + DkimKeyPair::generate_rsa(2048).map(|key| (key, "RSA PRIVATE KEY")) + } + DkimSignatureType::Dkim1Ed25519Sha256 => { + DkimKeyPair::generate_ed25519().map(|key| (key, "PRIVATE KEY")) + } + }) + .await + .map_err(|err| { + trc::EventType::Server(trc::ServerEvent::ThreadError) + .reason(err) + .caused_by(trc::location!()) + })?; + + match private_key { + Ok((private_key, pk_type)) => { + let mut pem = format!("-----BEGIN {pk_type}-----\n").into_bytes(); + let mut lf_count = 65; + for ch in base64_encode(private_key.private_key()).unwrap_or_default() { + pem.push(ch); + lf_count -= 1; + if lf_count == 0 { + pem.push(b'\n'); + lf_count = 65; + } + } + if lf_count != 65 { + pem.push(b'\n'); + } + pem.extend_from_slice(format!("-----END {pk_type}-----\n").as_bytes()); + + let pk_value = DkimPrivateKey::Value(SecretTextValue { + secret: String::from_utf8(pem).unwrap_or_default(), + }); + + response + .object + .insert(Property::PrivateKey, pk_value.clone().into_value()); + + *pk = pk_value; + } + Err(err) => { + return Ok(Err(SetError::forbidden().with_description(err.to_string()))); + } + } + } + + // Verify signature + match DkimSigner::new("example.com".to_string(), key.clone()).await { + Ok(_) => Ok(Ok(response)), + Err(err) => Ok(Err(SetError::invalid_properties() + .with_description(format!("Failed to build DKIM signature: {err}")))), + } +} + +pub async fn generate_dkim_public_key(key: &DkimSignature) -> trc::Result { + match key { + DkimSignature::Dkim1RsaSha256(key) => key + .private_key + .pem() + .await + .and_then(|pem| rsa_key_parse(pem.as_bytes())) + .and_then(|pk| { + Document::from_pkcs1_der(&pk.public_key()).map_err(|err| { + trc::EventType::Dkim(trc::DkimEvent::BuildError) + .into_err() + .reason(err) + }) + }) + .map(|pk| { + String::from_utf8(base64_encode(pk.as_bytes()).unwrap_or_default()) + .unwrap_or_default() + }), + DkimSignature::Dkim1Ed25519Sha256(key) => key + .private_key + .pem() + .await + .and_then(|pem| { + simple_pem_parse(&pem).ok_or_else(|| { + trc::EventType::Dkim(trc::DkimEvent::BuildError) + .into_err() + .details("Failed to parse private key PEM") + }) + }) + .and_then(|der| { + Ed25519Key::from_pkcs8_maybe_unchecked_der(&der).map_err(|err| { + trc::EventType::Dkim(trc::DkimEvent::BuildError) + .into_err() + .reason(err) + }) + }) + .map(|pk| { + String::from_utf8(base64_encode(&pk.public_key()).unwrap_or_default()) + .unwrap_or_default() + }), + } +} diff --git a/crates/jmap/src/registry/mapping/mod.rs b/crates/jmap/src/registry/mapping/mod.rs index c54a2595..82855ecb 100644 --- a/crates/jmap/src/registry/mapping/mod.rs +++ b/crates/jmap/src/registry/mapping/mod.rs @@ -4,8 +4,6 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::net::IpAddr; - use common::{Server, auth::AccessToken}; use jmap_proto::{ error::set::SetError, @@ -17,12 +15,15 @@ use registry::{ jmap::{JmapValue, RegistryValue}, schema::prelude::{ObjectType, Property}, }; +use std::net::IpAddr; use store::ahash::AHashSet; use types::id::Id; use utils::map::vec_map::VecMap; pub mod account; +pub mod action; pub mod archived_item; +pub mod dkim; pub mod log; pub mod masked_email; pub mod principal; diff --git a/crates/jmap/src/registry/mapping/principal.rs b/crates/jmap/src/registry/mapping/principal.rs index 300ba697..72023cf6 100644 --- a/crates/jmap/src/registry/mapping/principal.rs +++ b/crates/jmap/src/registry/mapping/principal.rs @@ -322,6 +322,7 @@ pub(crate) async fn validate_tenant_quota( TenantStorageQuota::MaxOauthClients => { (ObjectType::OAuthClient, None, "OAuth clients") } + TenantStorageQuota::MaxDkimKeys => (ObjectType::DkimSignature, None, "DKIM keys"), TenantStorageQuota::MaxDiskQuota => unreachable!(), }; let mut query = RegistryQuery::new(object_type).with_tenant(tenant_id.into()); diff --git a/crates/jmap/src/registry/mapping/public_key.rs b/crates/jmap/src/registry/mapping/public_key.rs index 8d5aac24..60b55339 100644 --- a/crates/jmap/src/registry/mapping/public_key.rs +++ b/crates/jmap/src/registry/mapping/public_key.rs @@ -5,29 +5,27 @@ */ use crate::registry::mapping::{ObjectResponse, RegistrySetResponse, ValidationResult}; +use common::storage::encryption::parse_public_key; use jmap_proto::error::set::SetError; -use registry::{ - jmap::JmapValue, - schema::{ - enums::StorageQuota, - prelude::{ObjectType, Property}, - structs::PublicKey, - }, +use registry::schema::{ + enums::StorageQuota, + prelude::{ObjectType, Property}, + structs::PublicKey, }; -use store::{ahash::AHashSet, registry::RegistryQuery}; -use utils::map::vec_map::VecMap; +use store::registry::RegistryQuery; pub(crate) async fn validate_public_key( set: &RegistrySetResponse<'_>, key: &mut PublicKey, old_key: Option<&PublicKey>, - unpatched_properties: VecMap>, ) -> ValidationResult { - let mut response = ObjectResponse::default(); + let response = ObjectResponse::default(); - let todo = "validate key"; - - if old_key.is_none() { + if let Some(old_key) = old_key { + if key.key == old_key.key { + return Ok(Ok(response)); + } + } else { // Validate quotas let num_masked = set .server @@ -46,5 +44,13 @@ pub(crate) async fn validate_public_key( } } - todo!() + match parse_public_key(key) { + Ok(Some(_)) => Ok(Ok(response)), + Ok(None) => Ok(Err(SetError::invalid_properties() + .with_property(Property::Key) + .with_description("No valid public key found."))), + Err(err) => Ok(Err(SetError::invalid_properties() + .with_property(Property::Key) + .with_description(err.into_owned()))), + } } diff --git a/crates/jmap/src/registry/mapping/task.rs b/crates/jmap/src/registry/mapping/task.rs index 6aebf412..526a672d 100644 --- a/crates/jmap/src/registry/mapping/task.rs +++ b/crates/jmap/src/registry/mapping/task.rs @@ -43,7 +43,7 @@ pub(crate) async fn task_set( continue 'outer; }; let ptr = JsonPointer::new(vec![JsonPointerItem::Key(Key::Property(prop))]); - if let Err(err) = task.patch(JsonPointerPatch::new(&ptr).with_create(false), value) { + if let Err(err) = task.patch(JsonPointerPatch::new(&ptr).with_create(true), value) { set.response.not_created.append(id, err.into()); continue 'outer; } diff --git a/crates/jmap/src/registry/set.rs b/crates/jmap/src/registry/set.rs index b8fcfc63..de4801b4 100644 --- a/crates/jmap/src/registry/set.rs +++ b/crates/jmap/src/registry/set.rs @@ -9,7 +9,9 @@ use std::borrow::Cow; use crate::registry::mapping::{ ObjectResponse, RegistrySetResponse, account::account_set, + action::action_set, archived_item::archived_item_set, + dkim::validate_dkim_signature, masked_email::validate_masked_email, principal::{ schedule_account_destruction, validate_account, validate_role, validate_tenant_quota, @@ -37,7 +39,7 @@ use registry::{ OBJ_FILTER_ACCOUNT, OBJ_FILTER_TENANT, OBJ_SINGLETON, Object, ObjectInner, ObjectType, Property, }, - structs::{Account, PublicKey, Role}, + structs::{Account, DkimSignature, PublicKey, Role}, }, types::id::ObjectId, }; @@ -70,6 +72,11 @@ impl RegistrySet for Server { access_token: &AccessToken, session: &HttpSessionData, ) -> trc::Result> { + let todo = "list"; + // locks for expensive tasks should be longer or renewed + // Validate expressions + // Fallback admin password from env or files + // Individual permissions for each object + create/update/destroy let object_flags = object_type.flags(); let is_singleton = (object_flags & OBJ_SINGLETON) != 0; let has_account_id = (object_flags & OBJ_FILTER_ACCOUNT) != 0; @@ -390,13 +397,11 @@ impl RegistrySet for Server { .await? } ObjectInner::PublicKey(key) => { - validate_public_key( - &set, - key, - modification.as_public_key(), - unpatched_properties, - ) - .await? + validate_public_key(&set, key, modification.as_public_key()).await? + } + ObjectInner::DkimSignature(key) => { + validate_dkim_signature(&set, key, modification.as_dkim_signature()) + .await? } ObjectInner::Domain(_) if is_create => { validate_tenant_quota(&set, TenantStorageQuota::MaxDomains).await? @@ -539,10 +544,7 @@ impl RegistrySet for Server { ObjectType::Task => task_set(set).await.map(|set| set.into_response()), - ObjectType::Action => { - let todo = "actions"; - todo!() - } + ObjectType::Action => action_set(set).await.map(|set| set.into_response()), ObjectType::Log | ObjectType::Metric | ObjectType::Trace => { set.fail_all_create("Telemetry objects cannot be created"); @@ -551,15 +553,6 @@ impl RegistrySet for Server { Ok(set.into_response()) } } - - // locks for expensive tasks should be longer or renewed - // management objects for actions (reload, etc)"; - // DkimSignature = Generate keys + Enforce count? - // PublicKey = Validate PK? Store decoded? Enforce count? Update ingest - // Domain = trigger DNIM stuff - // Validate expressions - // Fallback admin password from env or files - // Individual permissions for each object + create/update/destroy } } @@ -684,6 +677,16 @@ impl Modification { }, } } + + fn as_dkim_signature(&self) -> Option<&DkimSignature> { + match self { + Modification::Create(_) => None, + Modification::Update { object, .. } => match &object.inner { + ObjectInner::DkimSignature(key) => Some(key), + _ => None, + }, + } + } } pub(crate) fn map_write_error(err: RegistryWriteResult) -> SetError { diff --git a/crates/jmap/src/sieve/set.rs b/crates/jmap/src/sieve/set.rs index a0ae312a..0db35136 100644 --- a/crates/jmap/src/sieve/set.rs +++ b/crates/jmap/src/sieve/set.rs @@ -5,7 +5,11 @@ */ use crate::{blob::download::BlobDownload, changes::state::StateManager}; -use common::{Server, auth::AccessToken, storage::index::ObjectIndexBuilder}; +use common::{ + Server, + auth::{AccessToken, AccountCache}, + storage::index::ObjectIndexBuilder, +}; use email::sieve::{ ArchivedSieveScript, SieveScript, delete::SieveScriptDelete, ingest::SieveScriptIngest, }; @@ -39,6 +43,7 @@ use types::{ pub struct SetContext<'x> { account_id: u32, access_token: &'x AccessToken, + account_cache: &'x AccountCache, response: SetResponse, } @@ -81,9 +86,11 @@ impl SieveScriptSet for Server { let sieve_ids = self .document_ids(account_id, Collection::SieveScript, SieveField::Name) .await?; + let account = self.account(account_id).await.caused_by(trc::location!())?; let mut ctx = SetContext { account_id, access_token, + account_cache: &account, response: SetResponse::from_request(&request, self.core.jmap.set_max_objects)? .with_state( self.assert_state( @@ -104,7 +111,6 @@ impl SieveScriptSet for Server { } // Process creates - let account = self.account(account_id).await.caused_by(trc::location!())?; let mut batch = BatchBuilder::new(); for (id, object) in request.unwrap_create() { if sieve_ids.len() @@ -471,7 +477,7 @@ impl SieveScriptSet for Server { if let Some(mut bytes) = self.blob_download(&blob_id, ctx.access_token).await? { // Check quota match self - .has_available_quota(ctx.account_id, bytes.len() as u64) + .has_available_quota(ctx.account_cache, bytes.len() as u64) .await { Ok(_) => (), diff --git a/crates/managesieve/src/op/putscript.rs b/crates/managesieve/src/op/putscript.rs index 3647b566..4752c232 100644 --- a/crates/managesieve/src/op/putscript.rs +++ b/crates/managesieve/src/op/putscript.rs @@ -50,7 +50,7 @@ impl Session { let account_id = access_token.account_id(); let account = self.server.account(account_id).await?; self.server - .has_available_quota(account_id, script_bytes.len() as u64) + .has_available_quota(&account, script_bytes.len() as u64) .await .caused_by(trc::location!())?; diff --git a/crates/registry/src/schema/mod.rs b/crates/registry/src/schema/mod.rs index 64c6d24c..3c34d33b 100644 --- a/crates/registry/src/schema/mod.rs +++ b/crates/registry/src/schema/mod.rs @@ -7,7 +7,7 @@ use crate::{ schema::{ enums::{TracingLevel, TracingLevelOpt}, - prelude::{NodeRange, Object, ObjectInner, Property}, + prelude::{Object, ObjectInner, Property}, }, types::EnumImpl, }; @@ -28,12 +28,6 @@ pub mod structs; #[allow(clippy::derivable_impls)] pub mod structs_impl; -impl NodeRange { - pub fn contains(&self, node_id: u64) -> bool { - node_id >= self.from_node_id && node_id <= self.to_node_id - } -} - impl Display for Property { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.as_str()) diff --git a/crates/registry/src/types/duration.rs b/crates/registry/src/types/duration.rs index 206f3978..d057f3a5 100644 --- a/crates/registry/src/types/duration.rs +++ b/crates/registry/src/types/duration.rs @@ -160,3 +160,9 @@ impl IntoValue for Duration { JmapValue::Number((self.0.as_millis() as u64).into()) } } + +impl From for Duration { + fn from(value: std::time::Duration) -> Self { + Duration(value) + } +} diff --git a/crates/registry/src/types/error.rs b/crates/registry/src/types/error.rs index 92ba3013..e1667adc 100644 --- a/crates/registry/src/types/error.rs +++ b/crates/registry/src/types/error.rs @@ -4,7 +4,11 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{jmap::JsonPointerPatch, schema::prelude::Property, types::id::ObjectId}; +use crate::{ + jmap::JsonPointerPatch, + schema::prelude::Property, + types::{EnumImpl, id::ObjectId}, +}; use std::{borrow::Cow, fmt::Display}; #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] @@ -103,6 +107,58 @@ impl Warning { message: message.to_string(), } } + + pub fn log(&self) { + trc::event!( + Registry(trc::RegistryEvent::BuildWarning), + Source = self.object_id.object().as_str(), + Id = self.object_id.id().id(), + Key = self.property.map(|key| key.as_str()), + Reason = self.message.clone(), + ); + } +} + +impl Error { + pub fn log(&self) { + match self { + Error::Validation { object_id, errors } => { + trc::event!( + Registry(trc::RegistryEvent::ValidationError), + Source = object_id.object().as_str(), + Id = object_id.id().id(), + Reason = errors + .iter() + .map(|err| trc::Value::from(err.to_string())) + .collect::>(), + ); + } + Error::Build { object_id, message } => { + trc::event!( + Registry(trc::RegistryEvent::BuildError), + Source = object_id.object().as_str(), + Id = object_id.id().id(), + Reason = message.clone(), + ); + } + Error::Internal { object_id, error } => { + trc::event!( + Registry(trc::RegistryEvent::ReadError), + Source = object_id.as_ref().map(|id| id.object().as_str()), + Id = object_id.as_ref().map(|id| id.id().id()), + CausedBy = error.clone(), + ); + } + Error::NotFound { object_id } => { + trc::event!( + Registry(trc::RegistryEvent::BuildError), + Source = object_id.object().as_str(), + Id = object_id.id().id(), + Reason = "Object not found", + ); + } + } + } } impl Display for ValidationError { diff --git a/crates/registry/src/utils/mod.rs b/crates/registry/src/utils/mod.rs index d3ebca00..a7f6e620 100644 --- a/crates/registry/src/utils/mod.rs +++ b/crates/registry/src/utils/mod.rs @@ -4,7 +4,9 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::schema::prelude::Roles; +use std::borrow::Cow; + +use crate::schema::prelude::{DkimPrivateKey, DkimSignature, Roles}; use types::id::Id; pub mod account; @@ -23,3 +25,30 @@ impl Roles { } } } + +impl DkimSignature { + pub fn private_key(&self) -> &DkimPrivateKey { + match self { + DkimSignature::Dkim1Ed25519Sha256(signature) => &signature.private_key, + DkimSignature::Dkim1RsaSha256(signature) => &signature.private_key, + } + } + + pub fn private_key_mut(&mut self) -> &mut DkimPrivateKey { + match self { + DkimSignature::Dkim1Ed25519Sha256(signature) => &mut signature.private_key, + DkimSignature::Dkim1RsaSha256(signature) => &mut signature.private_key, + } + } +} + +impl DkimPrivateKey { + pub async fn pem(&self) -> trc::Result> { + match self { + DkimPrivateKey::Value(value) => Ok(Cow::Borrowed(value.secret.as_str())), + DkimPrivateKey::File(file) => file.secret().await.map(Cow::Owned), + DkimPrivateKey::Generate => Err("Key is in invalid generate state".to_string()), + } + .map_err(|err| trc::DkimEvent::BuildError.reason(err)) + } +} diff --git a/crates/registry/src/utils/report.rs b/crates/registry/src/utils/report.rs index dbd42cfc..253f142e 100644 --- a/crates/registry/src/utils/report.rs +++ b/crates/registry/src/utils/report.rs @@ -8,7 +8,10 @@ use crate::{ schema::{enums, prelude::UTCDateTime, structs}, types::{ipaddr::IpAddr, list::List}, }; -use mail_auth::report::{tlsrpt::*, *}; +use mail_auth::{ + IprevOutput, IprevResult, SpfOutput, + report::{tlsrpt::*, *}, +}; use std::borrow::Cow; impl From for Alignment { @@ -802,3 +805,125 @@ fn fo_to_failure_reporting_options(fo: &Option) -> Vec for structs::DmarcTroubleshootAuthResult { + fn from(value: &SpfOutput) -> Self { + match value.result() { + mail_auth::SpfResult::Pass => structs::DmarcTroubleshootAuthResult::Pass, + mail_auth::SpfResult::Fail => { + structs::DmarcTroubleshootAuthResult::Fail(structs::DmarcTroubleshootDetails { + details: value.explanation().map(|e| e.to_string()), + }) + } + mail_auth::SpfResult::SoftFail => { + structs::DmarcTroubleshootAuthResult::SoftFail(structs::DmarcTroubleshootDetails { + details: value.explanation().map(|e| e.to_string()), + }) + } + mail_auth::SpfResult::Neutral => { + structs::DmarcTroubleshootAuthResult::Neutral(structs::DmarcTroubleshootDetails { + details: value.explanation().map(|e| e.to_string()), + }) + } + mail_auth::SpfResult::TempError => { + structs::DmarcTroubleshootAuthResult::TempError(structs::DmarcTroubleshootDetails { + details: value.explanation().map(|e| e.to_string()), + }) + } + mail_auth::SpfResult::PermError => { + structs::DmarcTroubleshootAuthResult::PermError(structs::DmarcTroubleshootDetails { + details: value.explanation().map(|e| e.to_string()), + }) + } + mail_auth::SpfResult::None => structs::DmarcTroubleshootAuthResult::None, + } + } +} + +impl From<&IprevOutput> for structs::DmarcTroubleshootAuthResult { + fn from(value: &IprevOutput) -> Self { + match &value.result { + IprevResult::Pass => structs::DmarcTroubleshootAuthResult::Pass, + IprevResult::Fail(error) => { + structs::DmarcTroubleshootAuthResult::Fail(structs::DmarcTroubleshootDetails { + details: error.to_string().into(), + }) + } + IprevResult::TempError(error) => { + structs::DmarcTroubleshootAuthResult::TempError(structs::DmarcTroubleshootDetails { + details: error.to_string().into(), + }) + } + IprevResult::PermError(error) => { + structs::DmarcTroubleshootAuthResult::PermError(structs::DmarcTroubleshootDetails { + details: error.to_string().into(), + }) + } + IprevResult::None => structs::DmarcTroubleshootAuthResult::None, + } + } +} + +impl From<&mail_auth::DkimResult> for structs::DmarcTroubleshootAuthResult { + fn from(value: &mail_auth::DkimResult) -> Self { + match value { + mail_auth::DkimResult::Pass => structs::DmarcTroubleshootAuthResult::Pass, + mail_auth::DkimResult::Neutral(error) => { + structs::DmarcTroubleshootAuthResult::Neutral(structs::DmarcTroubleshootDetails { + details: error.to_string().into(), + }) + } + mail_auth::DkimResult::Fail(error) => { + structs::DmarcTroubleshootAuthResult::Fail(structs::DmarcTroubleshootDetails { + details: error.to_string().into(), + }) + } + mail_auth::DkimResult::PermError(error) => { + structs::DmarcTroubleshootAuthResult::PermError(structs::DmarcTroubleshootDetails { + details: error.to_string().into(), + }) + } + mail_auth::DkimResult::TempError(error) => { + structs::DmarcTroubleshootAuthResult::TempError(structs::DmarcTroubleshootDetails { + details: error.to_string().into(), + }) + } + mail_auth::DkimResult::None => structs::DmarcTroubleshootAuthResult::None, + } + } +} + +impl From<&mail_auth::DmarcResult> for structs::DmarcTroubleshootAuthResult { + fn from(value: &mail_auth::DmarcResult) -> Self { + match value { + mail_auth::DmarcResult::Pass => structs::DmarcTroubleshootAuthResult::Pass, + mail_auth::DmarcResult::Fail(error) => { + structs::DmarcTroubleshootAuthResult::Fail(structs::DmarcTroubleshootDetails { + details: error.to_string().into(), + }) + } + mail_auth::DmarcResult::TempError(error) => { + structs::DmarcTroubleshootAuthResult::TempError(structs::DmarcTroubleshootDetails { + details: error.to_string().into(), + }) + } + mail_auth::DmarcResult::PermError(error) => { + structs::DmarcTroubleshootAuthResult::PermError(structs::DmarcTroubleshootDetails { + details: error.to_string().into(), + }) + } + mail_auth::DmarcResult::None => structs::DmarcTroubleshootAuthResult::None, + } + } +} + +impl From<&mail_auth::dmarc::Policy> for enums::DmarcDisposition { + fn from(value: &mail_auth::dmarc::Policy) -> Self { + match value { + mail_auth::dmarc::Policy::None => enums::DmarcDisposition::None, + mail_auth::dmarc::Policy::Quarantine => enums::DmarcDisposition::Quarantine, + mail_auth::dmarc::Policy::Reject => enums::DmarcDisposition::Reject, + mail_auth::dmarc::Policy::Unspecified => enums::DmarcDisposition::Unspecified, + } + } +} diff --git a/crates/services/src/broadcast/mod.rs b/crates/services/src/broadcast/mod.rs index 016fd9eb..86b8cdc8 100644 --- a/crates/services/src/broadcast/mod.rs +++ b/crates/services/src/broadcast/mod.rs @@ -92,7 +92,7 @@ impl BroadcastBatch> { let _ = serialized.write_leb128(object.to_id()); } }, - BroadcastEvent::CacheInvalidation(items) => { + BroadcastEvent::CacheInvalidate(items) => { serialized.push(7u8); let _ = serialized.write_leb128(items.len()); for item in items { @@ -113,6 +113,16 @@ impl BroadcastBatch> { let _ = serialized.write_leb128(id); } } + BroadcastEvent::CacheInvalidateAll => { + serialized.push(8u8); + } + BroadcastEvent::MtaQueueStatus { is_running } => { + if *is_running { + serialized.push(9u8); + } else { + serialized.push(10u8); + } + } } } serialized @@ -223,9 +233,11 @@ where _ => return Err(()), }); } - Ok(Some(BroadcastEvent::CacheInvalidation(items))) + Ok(Some(BroadcastEvent::CacheInvalidate(items))) } - + 8 => Ok(Some(BroadcastEvent::CacheInvalidateAll)), + 9 => Ok(Some(BroadcastEvent::MtaQueueStatus { is_running: true })), + 10 => Ok(Some(BroadcastEvent::MtaQueueStatus { is_running: false })), _ => Err(()), } } else { diff --git a/crates/services/src/broadcast/subscriber.rs b/crates/services/src/broadcast/subscriber.rs index b6438a40..84c82181 100644 --- a/crates/services/src/broadcast/subscriber.rs +++ b/crates/services/src/broadcast/subscriber.rs @@ -7,7 +7,7 @@ use crate::broadcast::{BROADCAST_TOPIC, BroadcastBatch}; use common::{ BuildServer, Inner, - ipc::{BroadcastEvent, PushEvent, PushNotification, RegistryChange}, + ipc::{BroadcastEvent, PushEvent, PushNotification, QueueEvent, RegistryChange}, }; use registry::types::EnumImpl; use std::{sync::Arc, time::Duration}; @@ -136,17 +136,24 @@ pub fn spawn_broadcast_subscriber(inner: Arc, mut shutdown_rx: watch::Rec ); } } - BroadcastEvent::CacheInvalidation(changes) => { + BroadcastEvent::CacheInvalidate(changes) => { inner.build_server().invalidate_local_caches(&changes).await; } + BroadcastEvent::CacheInvalidateAll => { + inner.build_server().invalidate_all_local_caches(); + } + BroadcastEvent::MtaQueueStatus { is_running } => { + let _ = inner + .ipc + .queue_tx + .send(QueueEvent::Paused(!is_running)) + .await; + } BroadcastEvent::RegistryChange(change) => { - match inner.build_server().reload_registry(change).await { + match Box::pin(inner.build_server().reload_registry(change)).await { Ok(result) => { - if let Some(new_core) = result.new_core { - // Update core - inner.shared_core.store(new_core.into()); - } + result.log(); } Err(err) => { trc::error!( @@ -228,7 +235,7 @@ fn log_event(event: &BroadcastEvent) -> trc::Value { trc::Value::Array(vec!["RegistryReload".into(), object.as_str().into()]) } }, - BroadcastEvent::CacheInvalidation(items) => { + BroadcastEvent::CacheInvalidate(items) => { let mut array = Vec::with_capacity(items.len() + 1); array.push("CacheInvalidation".into()); for item in items { @@ -236,5 +243,13 @@ fn log_event(event: &BroadcastEvent) -> trc::Value { } trc::Value::Array(array) } + BroadcastEvent::CacheInvalidateAll => "CacheInvalidateAll".into(), + BroadcastEvent::MtaQueueStatus { is_running } => { + if *is_running { + "MtaQueueRunning".into() + } else { + "MtaQueuePaused".into() + } + } } } diff --git a/crates/services/src/task_manager/scheduler.rs b/crates/services/src/task_manager/scheduler.rs index 92bdcce3..dcbd5d1f 100644 --- a/crates/services/src/task_manager/scheduler.rs +++ b/crates/services/src/task_manager/scheduler.rs @@ -445,8 +445,10 @@ pub fn spawn_task_scheduler(inner: Arc) { .await { Ok(result) => { - if let Some(new_core) = result.new_core { - if let Some(enterprise) = &new_core.enterprise { + if !result.has_errors() { + if let Some(enterprise) = + server.inner.build_server().core.enterprise.as_ref() + { let renew_in = if enterprise.license.is_near_expiration() { // Something went wrong during renewal, try again in 1 day or 1 hour, // depending on the time left on the license @@ -467,14 +469,13 @@ pub fn spawn_task_scheduler(inner: Arc) { ); } - // Update core - server.inner.shared_core.store(new_core.into()); - server .cluster_broadcast(common::ipc::BroadcastEvent::reload( ObjectType::Enterprise, )) .await; + } else { + result.log(); } } Err(err) => { diff --git a/crates/spam-filter/src/analysis/score.rs b/crates/spam-filter/src/analysis/score.rs index ac00f6a3..3c19cfba 100644 --- a/crates/spam-filter/src/analysis/score.rs +++ b/crates/spam-filter/src/analysis/score.rs @@ -46,6 +46,7 @@ pub struct SpamFilterScore { pub headers: String, pub train_spam: Option, pub score: f32, + pub is_spam: bool, } impl SpamFilterAnalyzeScore for Server { @@ -184,6 +185,7 @@ impl SpamFilterAnalyzeScore for Server { headers, train_spam, score: final_score, + is_spam, }) } } diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index 2de076fe..0062f099 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -52,6 +52,7 @@ memchr = { version = "2.7" } rkyv = { version = "0.8.10", features = ["little_endian"] } compact_str = "0.9.0" rustls_021 = { package = "rustls", version = "0.21", default-features = false, features = ["dangerous_configuration"], optional = true } +gethostname = "1.1.0" [dev-dependencies] tokio = { version = "1.47", features = ["full"] } diff --git a/crates/store/src/build/registry.rs b/crates/store/src/build/registry.rs index 8ef14706..9963773a 100644 --- a/crates/store/src/build/registry.rs +++ b/crates/store/src/build/registry.rs @@ -6,71 +6,23 @@ use crate::{RegistryStore, RegistryStoreInner, Store}; use ahash::AHashSet; -use registry::{ - schema::{ - prelude::ObjectType, - structs::{DataStore, LocalSettings}, - }, - types::{EnumImpl, id::ObjectId}, -}; +use registry::schema::enums::NodeRole; use std::path::PathBuf; -use types::id::Id; use utils::snowflake::SnowflakeIdGenerator; impl RegistryStore { pub async fn init(local: PathBuf) -> Result { - let todo = "environment variables and reading from files"; const ERROR_MSG: &str = "Failed to initialize registry"; - let mut inner = RegistryStoreInner::load(local).await?; - let Some(data_store) = inner - .local_registry - .read() - .get(&ObjectId::new(ObjectType::DataStore, Id::singleton())) - .cloned() - .map(DataStore::from) - else { - return Err(format!( - "{ERROR_MSG}: Missing \"DataStore\" object definition." - )); - }; + let mut inner = RegistryStoreInner::new(local); - let Some(local_settings) = inner - .local_registry - .read() - .get(&ObjectId::new(ObjectType::LocalSettings, Id::singleton())) - .cloned() - .map(LocalSettings::from) - else { - return Err(format!( - "{ERROR_MSG}: Missing \"LocalSettings\" object definition." - )); - }; + // Build store + let store = Store::build(inner.read_data_store().await?).await?; - // Validate local objects - let mut local_objects = - AHashSet::from_iter([ObjectType::DataStore, ObjectType::LocalSettings]); - for object in local_settings.local_registry_object_types { - if let Some(object) = ObjectType::parse(&object) { - local_objects.insert(object); - } else { - return Err(format!( - "{ERROR_MSG}: LocalSettings/localRegistryObjectImpls contains invalid object type: {object}" - )); - } - } - for object_id in inner.local_registry.read().keys() { - if !local_objects.contains(&object_id.object()) { - return Err(format!( - "{ERROR_MSG}: Found object of type {:?} in local registry, but it is not listed in LocalSettings/localRegistryObjectImpls.", - object_id.object().as_str() - )); - } - } + let todo = "obtain node id"; + inner.store = store; + inner.node_id = 0; - inner.local_objects = local_objects; - inner.store = Store::build(data_store).await?; - inner.node_id = local_settings.node_id; if inner.node_id == 0 { return Err(format!( "{ERROR_MSG}: \"LocalSettings\" object has invalid nodeId of 0." @@ -79,4 +31,20 @@ impl RegistryStore { inner.id_generator = SnowflakeIdGenerator::new(); Ok(Self(inner.into())) } + + pub fn recovery_admin(&self) -> Option<&(String, String)> { + self.0.env_recovery_admin.as_ref() + } + + pub fn node_roles(&self) -> &AHashSet { + &self.0.env_node_roles + } + + pub fn node_roles_shard(&self) -> u64 { + self.0.env_node_roles_shard_id + } + + pub fn local_hostname(&self) -> &str { + &self.0.env_hostname + } } diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index 09e3553f..f74cc140 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -12,17 +12,10 @@ pub mod registry; pub mod search; pub mod write; -use ::registry::{ - schema::{ - enums::CompressionAlgo, - prelude::{Object, ObjectType}, - }, - types::id::ObjectId, -}; +use ::registry::schema::enums::{CompressionAlgo, NodeRole}; pub use ahash; pub use blake3; pub use parking_lot; -use parking_lot::RwLock; pub use rand; pub use rkyv; pub use roaring; @@ -213,10 +206,12 @@ pub struct RegistryStore(pub(crate) Arc); pub struct RegistryStoreInner { pub(crate) local_path: PathBuf, - pub(crate) local_registry: RwLock>, - pub(crate) local_objects: AHashSet, pub(crate) store: Store, pub(crate) node_id: u64, + pub(crate) env_recovery_admin: Option<(String, String)>, + pub(crate) env_node_roles: AHashSet, + pub(crate) env_node_roles_shard_id: u64, + pub(crate) env_hostname: String, pub(crate) id_generator: SnowflakeIdGenerator, } diff --git a/crates/store/src/registry/bootstrap.rs b/crates/store/src/registry/bootstrap.rs index c3022042..7ec3444a 100644 --- a/crates/store/src/registry/bootstrap.rs +++ b/crates/store/src/registry/bootstrap.rs @@ -4,18 +4,11 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{ - RegistryStore, Store, - registry::{RegistryObject, RegistryQuery}, -}; -use ahash::AHashSet; +use crate::{RegistryStore, Store, registry::RegistryObject}; use registry::{ - schema::{ - prelude::{Object, ObjectType, Property}, - structs::Node, - }, + schema::prelude::{Object, Property}, types::{ - EnumImpl, ObjectImpl, + ObjectImpl, error::{Error, ValidationError, Warning}, id::ObjectId, }, @@ -28,23 +21,12 @@ pub struct Bootstrap { pub errors: Vec, pub warnings: Vec, pub has_fatal_errors: bool, - pub node: Node, } impl Bootstrap { - pub async fn init(registry: RegistryStore) -> Self { - let mut bp = Self::new(registry); - bp.load_node_settings().await; - bp - } - pub fn new(registry: RegistryStore) -> Self { Self { data_store: registry.0.store.clone(), - node: Node { - node_id: registry.0.node_id, - ..Default::default() - }, registry, errors: Vec::new(), warnings: Vec::new(), @@ -52,45 +34,6 @@ impl Bootstrap { } } - async fn load_node_settings(&mut self) { - let ids = match self - .registry - .query::>( - RegistryQuery::new(ObjectType::Node).equal(Property::NodeId, self.node_id()), - ) - .await - { - Ok(ids) => ids, - Err(err) => { - self.errors.push(Error::Internal { - object_id: None, - error: err, - }); - self.has_fatal_errors = true; - Default::default() - } - }; - let id = ids.into_iter().next(); - if let Some(id) = id - && let Some(node) = self.get_infallible::(Id::new(id)).await - { - self.node = node; - } else { - self.warnings.push(Warning { - object_id: ObjectId::new( - ObjectType::Node, - id.map(Id::new).unwrap_or(Id::singleton()), - ), - property: Some(Property::NodeId), - message: format!( - "No node configuration found for nodeId {}, using defaults.", - self.node_id() - ), - }); - self.node.hostname = "localhost.localdomain".to_string(); - } - } - pub async fn setting>(&mut self) -> trc::Result { let object_id = T::OBJECT.singleton(); @@ -217,61 +160,15 @@ impl Bootstrap { self.registry.0.node_id } - pub fn hostname(&self) -> &str { - &self.node.hostname - } - pub fn log_errors(&self) { for error in &self.errors { - match error { - Error::Validation { object_id, errors } => { - trc::event!( - Registry(trc::RegistryEvent::ValidationError), - Source = object_id.object().as_str(), - Id = object_id.id().id(), - Reason = errors - .iter() - .map(|err| trc::Value::from(err.to_string())) - .collect::>(), - ); - } - Error::Build { object_id, message } => { - trc::event!( - Registry(trc::RegistryEvent::BuildError), - Source = object_id.object().as_str(), - Id = object_id.id().id(), - Reason = message.clone(), - ); - } - Error::Internal { object_id, error } => { - trc::event!( - Registry(trc::RegistryEvent::ReadError), - Source = object_id.as_ref().map(|id| id.object().as_str()), - Id = object_id.as_ref().map(|id| id.id().id()), - CausedBy = error.clone(), - ); - } - Error::NotFound { object_id } => { - trc::event!( - Registry(trc::RegistryEvent::BuildError), - Source = object_id.object().as_str(), - Id = object_id.id().id(), - Reason = "Object not found", - ); - } - } + error.log(); } } pub fn log_warnings(&self) { for warning in &self.warnings { - trc::event!( - Registry(trc::RegistryEvent::BuildWarning), - Source = warning.object_id.object().as_str(), - Id = warning.object_id.id().id(), - Key = warning.property.map(|key| key.as_str()), - Reason = warning.message.clone(), - ); + warning.log(); } } } diff --git a/crates/store/src/registry/get.rs b/crates/store/src/registry/get.rs index 02393cc4..eb84f9c3 100644 --- a/crates/store/src/registry/get.rs +++ b/crates/store/src/registry/get.rs @@ -11,7 +11,7 @@ use crate::{ }; use registry::{ pickle::PickledStream, - schema::prelude::Object, + schema::prelude::{Object, ObjectType}, types::{EnumImpl, ObjectImpl, id::ObjectId}, }; use trc::AddContext; @@ -20,9 +20,7 @@ use utils::codec::leb128::Leb128Reader; impl RegistryStore { pub async fn get(&self, object_id: ObjectId) -> trc::Result> { - if self.0.local_objects.contains(&object_id.object()) { - Ok(self.0.local_registry.read().get(&object_id).cloned()) - } else { + if object_id.object() != ObjectType::DataStore { self.0 .store .get_value::(ValueKey::from(ValueClass::Registry(RegistryClass::Item { @@ -30,6 +28,22 @@ impl RegistryStore { item_id: object_id.id().id(), }))) .await + } else { + self.0 + .read_data_store() + .await + .map(|data_store| { + Some(Object { + inner: data_store.into(), + revision: 0, + }) + }) + .map_err(|err| { + trc::EventType::Registry(trc::RegistryEvent::LocalReadError) + .into_err() + .caused_by(trc::location!()) + .reason(err) + }) } } @@ -42,77 +56,61 @@ impl RegistryStore { pub async fn list>(&self) -> trc::Result>> { let object_type = T::OBJECT; - if self.0.local_objects.contains(&object_type) { - let mut results = Vec::new(); - - for (id, item) in self.0.local_registry.read().iter() { - if id.object() == object_type { - results.push(RegistryObject { - id: *id, - object: T::from(item.clone()), - revision: 0, - }); - } - } - - Ok(results) - } else { - let mut results = Vec::new(); - self.0 - .store - .iterate( - IterateParams::new( - ValueKey::from(ValueClass::Any(AnyClass { - subspace: SUBSPACE_REGISTRY, - key: KeySerializer::new(U16_LEN + 1) - .write(0u8) - .write(object_type.to_id()) - .finalize(), - })), - ValueKey::from(ValueClass::Any(AnyClass { - subspace: SUBSPACE_REGISTRY, - key: KeySerializer::new(U16_LEN + U64_LEN + 1) - .write(0u8) - .write(object_type.to_id()) - .write(u64::MAX) - .finalize(), - })), - ), - |key, value| { - let id = key - .get(U16_LEN + 1..) - .and_then(|key| key.read_leb128::()) - .map(|r| r.0) - .ok_or_else(|| { - trc::EventType::Registry(trc::RegistryEvent::DeserializationError) - .into_err() - .caused_by(trc::location!()) - .details(object_type.as_str()) - .ctx(trc::Key::Key, key) - })?; - let mut stream = PickledStream::new(value); - let object = T::unpickle(&mut stream).ok_or_else(|| { + let mut results = Vec::new(); + self.0 + .store + .iterate( + IterateParams::new( + ValueKey::from(ValueClass::Any(AnyClass { + subspace: SUBSPACE_REGISTRY, + key: KeySerializer::new(U16_LEN + 1) + .write(0u8) + .write(object_type.to_id()) + .finalize(), + })), + ValueKey::from(ValueClass::Any(AnyClass { + subspace: SUBSPACE_REGISTRY, + key: KeySerializer::new(U16_LEN + U64_LEN + 1) + .write(0u8) + .write(object_type.to_id()) + .write(u64::MAX) + .finalize(), + })), + ), + |key, value| { + let id = key + .get(U16_LEN + 1..) + .and_then(|key| key.read_leb128::()) + .map(|r| r.0) + .ok_or_else(|| { trc::EventType::Registry(trc::RegistryEvent::DeserializationError) .into_err() .caused_by(trc::location!()) - .id(id) .details(object_type.as_str()) - .ctx(trc::Key::Value, value) + .ctx(trc::Key::Key, key) })?; + let mut stream = PickledStream::new(value); + let object = T::unpickle(&mut stream).ok_or_else(|| { + trc::EventType::Registry(trc::RegistryEvent::DeserializationError) + .into_err() + .caused_by(trc::location!()) + .id(id) + .details(object_type.as_str()) + .ctx(trc::Key::Value, value) + })?; - results.push(RegistryObject { - id: ObjectId::new(object_type, Id::new(id)), - object, - revision: xxhash_rust::xxh3::xxh3_64(value), - }); + results.push(RegistryObject { + id: ObjectId::new(object_type, Id::new(id)), + object, + revision: xxhash_rust::xxh3::xxh3_64(value), + }); - Ok(true) - }, - ) - .await - .caused_by(trc::location!())?; + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; - Ok(results) - } + Ok(results) } } diff --git a/crates/store/src/registry/local.rs b/crates/store/src/registry/local.rs index ca4c115c..a63685f9 100644 --- a/crates/store/src/registry/local.rs +++ b/crates/store/src/registry/local.rs @@ -5,170 +5,84 @@ */ use crate::{RegistryStore, RegistryStoreInner, Store}; -use ahash::AHashMap; -use parking_lot::RwLock; use registry::{ - schema::prelude::{OBJ_SINGLETON, Object, ObjectInner, ObjectType}, - types::{EnumImpl, id::ObjectId}, + schema::{enums::NodeRole, structs::DataStore}, + types::EnumImpl, }; -use serde_json::{Map, Value, map::Entry}; use std::path::PathBuf; -use types::id::Id; use utils::snowflake::SnowflakeIdGenerator; impl RegistryStoreInner { - pub(crate) async fn load(local_path: PathBuf) -> Result { - let error_msg = format!("Failed to read local registry at {}", local_path.display()); - let contents = tokio::fs::read_to_string(&local_path) - .await - .map_err(|err| format!("{error_msg}: {err}"))?; - let values = serde_json::from_str::(&contents) - .map_err(|err| format!("{error_msg}: {err}"))?; - - let Value::Object(object) = values else { - return Err(format!("{error_msg}: Found invalid JSON structure.")); - }; - - let mut local_registry = AHashMap::new(); - for (key, value) in object.into_iter() { - let object_type = ObjectType::parse(key.as_str()) - .ok_or_else(|| format!("{error_msg}: Unrecognized object {key:?}."))?; - let is_singleton = object_type.flags() & OBJ_SINGLETON != 0; - let Value::Object(object) = value else { - return Err(format!("{error_msg}: Found invalid JSON structure.")); - }; - if !is_singleton { - for (id, value) in object.into_iter() { - let id = id.parse::().map_err(|_| { - format!("{error_msg}: Failed to parse object id {id} for object {key:?}") - })?; - if !matches!(value, Value::Object(_)) { - return Err(format!( - "{error_msg}: Object {key:?} with id {id} is invalid." - )); - } - if local_registry - .insert(ObjectId::new(object_type, Id::new(id)), ObjectInner::deserialize(object_type, value).map_err(|err| { - format!("{error_msg}: Failed to parse object {key:?} with id {id}: {err}") - }).and_then(|inner| { - let obj = Object { inner, revision: 0 }; - let mut errors = Vec::new(); - obj.validate(&mut errors); - if errors.is_empty() { - Ok(obj) - } else { - Err(format!( - "{error_msg}: Validation errors for object {key:?} with id {id}: {}", - errors - .into_iter() - .map(|e| e.to_string()) - .collect::>() - .join("; ") - )) - } - })?) - .is_some() - { - return Err(format!( - "{error_msg}: Object {key:?} with id {id} defined multiple times." - )); - } - } - } else if local_registry - .insert( - ObjectId::new(object_type, Id::singleton()), - ObjectInner::deserialize(object_type, object) - .map_err(|err| { - format!("{error_msg}: Failed to parse object {key:?}: {err}") - }) - .and_then(|inner| { - let obj = Object { inner, revision: 0 }; - let mut errors = Vec::new(); - obj.validate(&mut errors); - if errors.is_empty() { - Ok(obj) - } else { - Err(format!( - "{error_msg}: Validation errors for object {key:?}: {}", - errors - .into_iter() - .map(|e| e.to_string()) - .collect::>() - .join("; ") - )) - } - })?, - ) - .is_some() - { - return Err(format!( - "{error_msg}: Object {key:?} defined multiple times." - )); - } - } - - Ok(RegistryStoreInner { + pub(crate) fn new(local_path: PathBuf) -> Self { + Self { local_path, - local_registry: RwLock::new(local_registry), - local_objects: Default::default(), store: Store::None, id_generator: SnowflakeIdGenerator::new(), node_id: 0, - }) + env_recovery_admin: std::env::var("STALWART_RECOVERY_ACCOUNT") + .ok() + .filter(|a| !a.is_empty()) + .and_then(|a| { + std::env::var("STALWART_RECOVERY_PASS") + .ok() + .filter(|p| !p.is_empty()) + .map(|p| (a, p)) + }), + env_node_roles: std::env::var("STALWART_ROLES") + .ok() + .map(|roles| { + roles + .split(',') + .map(|r| r.trim()) + .filter(|r| !r.is_empty()) + .filter_map(|r| { + let role = NodeRole::parse(r); + if role.is_none() { + eprintln!( + "Invalid node role specified in STALWART_NODE_ROLES: {r}" + ); + } + role + }) + .collect() + }) + .unwrap_or_default(), + env_node_roles_shard_id: std::env::var("STALWART_ROLES_SHARD") + .ok() + .and_then(|id| id.parse::().ok()) + .unwrap_or(1), + env_hostname: std::env::var("STALWART_HOSTNAME") + .ok() + .filter(|h| !h.is_empty()) + .unwrap_or_else(|| gethostname::gethostname().to_string_lossy().into_owned()), + } + } + + pub(crate) async fn read_data_store(&self) -> Result { + tokio::fs::read_to_string(&self.local_path) + .await + .map_err(|err| { + format!( + "Failed to read data store settings at {}: {}", + self.local_path.display(), + err + ) + }) + .and_then(|contents| { + serde_json::from_str::(&contents).map_err(|err| { + format!( + "Failed to parse data store settings at {}: {}", + self.local_path.display(), + err + ) + }) + }) } } impl RegistryStore { - pub async fn write_local_registry(&self) -> trc::Result<()> { - let mut map = Map::new(); - - for (id, value) in self.0.local_registry.read().iter() { - let is_singleton = id.object().flags() & OBJ_SINGLETON != 0; - match map.entry(id.object().as_str().to_string()) { - Entry::Vacant(entry) => { - if is_singleton { - entry.insert(serde_json::to_value(&value.inner).map_err(|err| { - trc::EventType::Registry(trc::RegistryEvent::LocalWriteError) - .into_err() - .caused_by(trc::location!()) - .reason(err) - })?); - } else { - entry.insert( - Map::from_iter([( - id.id().to_string(), - serde_json::to_value(&value.inner).map_err(|err| { - trc::EventType::Registry(trc::RegistryEvent::LocalWriteError) - .into_err() - .caused_by(trc::location!()) - .reason(err) - })?, - )]) - .into(), - ); - } - } - Entry::Occupied(mut entry) => { - if !is_singleton { - if let Value::Object(map) = entry.get_mut() { - map.insert( - id.id().to_string(), - serde_json::to_value(&value.inner).map_err(|err| { - trc::EventType::Registry(trc::RegistryEvent::LocalWriteError) - .into_err() - .caused_by(trc::location!()) - .reason(err) - })?, - ); - } - } else { - debug_assert!(false, "Unexpected double singleton assignment"); - } - } - } - } - - let json_text = serde_json::to_string(&Value::Object(map)).map_err(|err| { + pub async fn write_data_store(&self, data_store: &DataStore) -> trc::Result<()> { + let json_text = serde_json::to_string(data_store).map_err(|err| { trc::EventType::Registry(trc::RegistryEvent::LocalWriteError) .into_err() .caused_by(trc::location!()) diff --git a/crates/store/src/registry/query.rs b/crates/store/src/registry/query.rs index db62b958..0668db11 100644 --- a/crates/store/src/registry/query.rs +++ b/crates/store/src/registry/query.rs @@ -35,20 +35,6 @@ impl RegistryStore { .into_err() .details("Singletons do not support searching")); } - } else if self.0.local_objects.contains(&query.object_type) { - if !query.filters.is_empty() { - trc::event!( - Registry(trc::RegistryEvent::NotSupported), - Details = "Filtering is not supported for local registry" - ); - } - let mut results = T::default(); - for id in self.0.local_registry.read().keys() { - if id.object() == query.object_type { - results.push(id.id().id()); - } - } - return Ok(results); } else if query.filters.is_empty() { return all_ids::(&self.0.store, query.object_type).await; } diff --git a/crates/store/src/registry/write.rs b/crates/store/src/registry/write.rs index aa0fe827..f1a66f12 100644 --- a/crates/store/src/registry/write.rs +++ b/crates/store/src/registry/write.rs @@ -14,8 +14,8 @@ use crate::{ }; use registry::{ schema::prelude::{ - OBJ_FILTER_ACCOUNT, OBJ_FILTER_TENANT, OBJ_SEQ_ID, OBJ_SINGLETON, Object, ObjectType, - Property, + OBJ_FILTER_ACCOUNT, OBJ_FILTER_TENANT, OBJ_SEQ_ID, OBJ_SINGLETON, Object, ObjectInner, + ObjectType, Property, }, types::{ EnumImpl, @@ -159,19 +159,15 @@ impl RegistryStore { } // Write to local registry - if self.0.local_objects.contains(&object_type) { + if let ObjectInner::DataStore(data_store) = &object.inner { if generate_id { return Ok(RegistryWriteResult::NotSupported); } - let id = Id::new(item_id); - self.0 - .local_registry - .write() - .insert(ObjectId::new(object_type, id), object.clone()); + return self - .write_local_registry() + .write_data_store(data_store) .await - .map(|_| RegistryWriteResult::Success(id)); + .map(|_| RegistryWriteResult::Success(Id::singleton())); } // Validate foreign keys @@ -330,17 +326,6 @@ impl RegistryStore { let id = object_id.id(); let item_id = id.id(); - if self.0.local_objects.contains(&object_type) { - let object = ObjectId::new(object_type, id); - return if self.0.local_registry.write().remove(&object).is_some() { - self.write_local_registry() - .await - .map(|_| RegistryWriteResult::Success(id)) - } else { - Ok(RegistryWriteResult::NotFound { object_id: object }) - }; - } - // Fetch object let object = if let Some(object) = object { Cow::Borrowed(object) diff --git a/crates/types/src/field.rs b/crates/types/src/field.rs index 4b475ae0..9ff8dbc8 100644 --- a/crates/types/src/field.rs +++ b/crates/types/src/field.rs @@ -77,7 +77,6 @@ pub enum IdentityField { #[repr(u8)] pub enum PrincipalField { Archive = ARCHIVE_FIELD, - EncryptionKeys = 46, ParticipantIdentities = 45, DefaultCalendarId = 47, DefaultAddressBookId = 48, @@ -157,7 +156,6 @@ impl From for u8 { fn from(value: PrincipalField) -> Self { match value { PrincipalField::ParticipantIdentities => 45, - PrincipalField::EncryptionKeys => 46, PrincipalField::DefaultCalendarId => 47, PrincipalField::DefaultAddressBookId => 48, PrincipalField::ActiveScriptId => 49,