diff --git a/Cargo.lock b/Cargo.lock index dda8c1df..b10abd43 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3585,6 +3585,7 @@ dependencies = [ "rand 0.9.2", "registry", "reqwest", + "rev_lines", "rkyv", "rsa", "serde", diff --git a/crates/common/src/auth/access_token.rs b/crates/common/src/auth/access_token.rs index 05476a83..199c2b71 100644 --- a/crates/common/src/auth/access_token.rs +++ b/crates/common/src/auth/access_token.rs @@ -165,6 +165,7 @@ impl Server { .credentials .into_iter() .filter_map(|(credential_id, credential)| { + let credential = credential.unwrap_properties(); let expires_at = credential .expires_at .map(|v| v.timestamp() as u64) @@ -750,6 +751,7 @@ fn hash_account(account: &Account) -> u64 { account.role_ids.hash(&mut s); hash_permissions(&mut s, &account.permissions); for (credential_id, credential) in &account.credentials { + let credential = credential.as_properties(); credential_id.hash(&mut s); credential.expires_at.hash(&mut s); hash_permissions(&mut s, &credential.permissions); diff --git a/crates/common/src/auth/authentication.rs b/crates/common/src/auth/authentication.rs index 1b9fc88b..35872251 100644 --- a/crates/common/src/auth/authentication.rs +++ b/crates/common/src/auth/authentication.rs @@ -17,8 +17,8 @@ use directory::{ core::secret::{verify_mfa_secret_hash, verify_secret_hash}, }; use registry::schema::{ - enums::{CredentialType, Permission}, - structs, + enums::Permission, + structs::{self, Credential}, }; use std::{net::IpAddr, sync::Arc}; use store::write::now; @@ -305,7 +305,9 @@ impl Server { .and_then(|account| account.into_user()) { // Find credential by credential_id - for (id, credential) in &account.credentials { + for (id, credential_) in &account.credentials { + let credential = credential_.as_properties(); + if *id == credential_id { if !verify_secret_hash(&credential.secret, secret).await? { return Err(trc::AuthEvent::Failed @@ -337,9 +339,9 @@ impl Server { AccountId = account_id, Id = credential_id, SpanId = span_id, - Details = match credential.credential_type { - CredentialType::AppPassword => "Authenticated with app password", - CredentialType::ApiKey => "Authenticated with API key", + Details = match credential_ { + Credential::AppPassword(_) => "Authenticated with app password", + Credential::ApiKey(_) => "Authenticated with API key", } ); diff --git a/crates/email/src/message/ingest.rs b/crates/email/src/message/ingest.rs index 99f407d2..5d953b64 100644 --- a/crates/email/src/message/ingest.rs +++ b/crates/email/src/message/ingest.rs @@ -964,7 +964,7 @@ impl EmailIngest for Server { ObjectId::new(ObjectType::SpamTrainingSample, item_id.into()).serialize(), ) .set( - ValueClass::Registry(RegistryClass::Id { object_id, item_id }), + ValueClass::Registry(RegistryClass::Item { object_id, item_id }), sample, ) .set( @@ -972,7 +972,7 @@ impl EmailIngest for Server { index_id: Property::AccountId.to_id(), object_id, item_id, - key: account_id.serialize(), + key: (account_id as u64).serialize(), }), vec![], ); diff --git a/crates/jmap-proto/src/error/set.rs b/crates/jmap-proto/src/error/set.rs index ba3db253..bffaac33 100644 --- a/crates/jmap-proto/src/error/set.rs +++ b/crates/jmap-proto/src/error/set.rs @@ -195,6 +195,11 @@ impl SetError { Self::new(SetErrorType::WillDestroy).with_description("ID will be destroyed.") } + pub fn singleton() -> Self { + Self::new(SetErrorType::Singleton) + .with_description("Singletons cannot be created or destroyed.") + } + pub fn address_book_has_contents() -> Self { Self::new(SetErrorType::AddressBookHasContents) .with_description("Address book is not empty.") diff --git a/crates/jmap/Cargo.toml b/crates/jmap/Cargo.toml index 62747706..5cc28454 100644 --- a/crates/jmap/Cargo.toml +++ b/crates/jmap/Cargo.toml @@ -53,6 +53,7 @@ rsa = "0.9.2" rkyv = { version = "0.8.10", features = ["little_endian"] } compact_str = "0.9.0" hashify = "0.2" +rev_lines = "0.3.0" [features] test_mode = [] diff --git a/crates/jmap/src/blob/download.rs b/crates/jmap/src/blob/download.rs index 0cc22ab5..76448546 100644 --- a/crates/jmap/src/blob/download.rs +++ b/crates/jmap/src/blob/download.rs @@ -9,6 +9,7 @@ use email::cache::MessageCacheFetch; use email::cache::email::MessageCacheAccess; use email::message::metadata::MessageMetadata; use groupware::cache::GroupwareCache; +use registry::schema::enums::Permission; use std::future::Future; use store::ValueKey; use store::write::{AlignedBytes, Archive}; @@ -100,44 +101,49 @@ impl BlobDownload for Server { blob_id: &BlobId, access_token: &AccessToken, ) -> trc::Result { - Ok(self - .store() - .blob_has_access(&blob_id.hash, &blob_id.class) - .await - .caused_by(trc::location!())? - && match &blob_id.class { - BlobClass::Linked { - account_id, - collection, - document_id, - } => { - if access_token.is_member(*account_id) { - true - } else { - match Collection::from(*collection) { - Collection::Email => self - .get_cached_messages(*account_id) - .await - .caused_by(trc::location!())? - .shared_messages(access_token, Acl::ReadItems) - .contains(*document_id), - collection @ (Collection::FileNode - | Collection::ContactCard - | Collection::CalendarEvent) => self - .fetch_dav_resources( - access_token.account_id(), - *account_id, - SyncCollection::from(collection), - ) - .await - .caused_by(trc::location!())? - .shared_items(access_token, [Acl::ReadItems], true) - .contains(*document_id), - _ => false, + Ok( + (blob_id.class.is_superuser() && access_token.has_permission(Permission::BlobFetch)) + || (self + .store() + .blob_has_access(&blob_id.hash, &blob_id.class) + .await + .caused_by(trc::location!())? + && match &blob_id.class { + BlobClass::Linked { + account_id, + collection, + document_id, + } => { + if access_token.is_member(*account_id) { + true + } else { + match Collection::from(*collection) { + Collection::Email => self + .get_cached_messages(*account_id) + .await + .caused_by(trc::location!())? + .shared_messages(access_token, Acl::ReadItems) + .contains(*document_id), + collection @ (Collection::FileNode + | Collection::ContactCard + | Collection::CalendarEvent) => self + .fetch_dav_resources( + access_token.account_id(), + *account_id, + SyncCollection::from(collection), + ) + .await + .caused_by(trc::location!())? + .shared_items(access_token, [Acl::ReadItems], true) + .contains(*document_id), + _ => false, + } + } } - } - } - BlobClass::Reserved { account_id, .. } => access_token.is_member(*account_id), - }) + BlobClass::Reserved { account_id, .. } => { + access_token.is_member(*account_id) + } + }), + ) } } diff --git a/crates/jmap/src/email/set.rs b/crates/jmap/src/email/set.rs index 9e28b5c4..d9e30289 100644 --- a/crates/jmap/src/email/set.rs +++ b/crates/jmap/src/email/set.rs @@ -105,7 +105,7 @@ impl EmailSet for Server { #[cfg(not(feature = "test_mode"))] { - self.get_access_token(account_id) + self.access_token(account_id) .await .caused_by(trc::location!())? .into() diff --git a/crates/jmap/src/registry/get.rs b/crates/jmap/src/registry/get.rs index 1acb64fe..3e75ed8d 100644 --- a/crates/jmap/src/registry/get.rs +++ b/crates/jmap/src/registry/get.rs @@ -4,6 +4,17 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::registry::mapping::{ + RegistryGetResponse, + account::account_get, + deleted_item::deleted_item_get, + log::log_get, + queued_message::queued_message_get, + report::report_get, + spam_sample::spam_sample_get, + task::task_get, + telemetry::{metric_get, trace_get}, +}; use common::{Server, auth::AccessToken}; use jmap_proto::{ method::get::{GetRequest, GetResponse}, @@ -41,25 +52,38 @@ impl RegistryGet for Server { mut request: GetRequest, access_token: &AccessToken, ) -> trc::Result> { - let ids = request.unwrap_ids(self.core.jmap.get_max_objects)?; - let mut properties = request - .properties - .take() - .map(|p| p.unwrap()) - .unwrap_or_default() - .into_iter() - .filter_map(|prop| prop.try_unwrap()) - .collect::>(); - if !properties.is_empty() { - properties.insert(Property::Id); - } - - let mut response = GetResponse { - account_id: request.account_id.into(), - state: None, - list: vec![], - not_found: vec![], + let object_flags = object_type.flags(); + let is_tenant_filtered = + (object_flags & OBJ_FILTER_TENANT) != 0 && access_token.tenant_id().is_some(); + let is_account_filtered = (object_flags & OBJ_FILTER_ACCOUNT) != 0 + && !access_token.has_permission(Permission::Impersonate); + let mut get = RegistryGetResponse { + access_token, + server: self, + account_id: request.account_id.document_id(), + object_type, + ids: request.unwrap_ids(self.core.jmap.get_max_objects)?, + properties: request + .properties + .take() + .map(|p| p.unwrap()) + .unwrap_or_default() + .into_iter() + .filter_map(|prop| prop.try_unwrap()) + .collect::>(), + response: GetResponse { + account_id: request.account_id.into(), + state: None, + list: vec![], + not_found: vec![], + }, + object_flags, + is_tenant_filtered, + is_account_filtered, }; + if !get.properties.is_empty() { + get.properties.insert(Property::Id); + } match object_type { ObjectType::AcmeProvider @@ -163,34 +187,30 @@ impl RegistryGet for Server { | ObjectType::PublicKey | ObjectType::DkimSignature | ObjectType::Domain => { - let flags = object_type.flags(); - let is_singleton = (flags & OBJ_SINGLETON) != 0; - let is_tenant_filtered = - (flags & OBJ_FILTER_TENANT) != 0 && access_token.tenant_id().is_some(); - let is_account_filtered = (flags & OBJ_FILTER_ACCOUNT) != 0 - && !access_token.has_permission(Permission::Impersonate); + let is_singleton = (get.object_flags & OBJ_SINGLETON) != 0; - let ids = if let Some(ids) = ids { + let ids = if let Some(ids) = get.ids.take() { ids } else { - self.registry() + let mut ids = self + .registry() .query::>( RegistryQuery::new(object_type) .with_tenant(access_token.tenant_id()) - .with_account_opt( - is_account_filtered.then_some(request.account_id.into()), - ), + .with_account_opt(is_account_filtered.then_some(get.account_id)), ) .await .caused_by(trc::location!())? .into_iter() .take(self.core.jmap.get_max_objects) .map(Id::new) - .collect() + .collect::>(); + ids.sort_unstable(); + ids }; - response.list.reserve(ids.len()); + get.response.list.reserve(ids.len()); - 'outer: for id in ids { + for id in ids { let object = if let Some(object) = self .registry() .get(ObjectId::new(object_type, id)) @@ -199,23 +219,23 @@ impl RegistryGet for Server { { object } else if id.is_singleton() && is_singleton { - Object::new(ObjectInner::from(object_type)) + Object::from(object_type) } else { - response.not_found.push(id); + get.not_found(id); continue; }; match &object.inner { ObjectInner::DkimSignature(obj) - if properties.is_empty() - || properties.contains(&Property::PublicKey) => + if get.properties.is_empty() + || get.properties.contains(&Property::PublicKey) => { let todo = "dkim public key"; todo!() } ObjectInner::Domain(obj) - if properties.is_empty() - || properties.contains(&Property::DnsZoneFile) => + if get.properties.is_empty() + || get.properties.contains(&Property::DnsZoneFile) => { let todo = "domain dns zone file"; todo!() @@ -223,61 +243,87 @@ impl RegistryGet for Server { _ => {} } - let todo = "compact pickle"; - let todo = "app passwords, apis and user change pass/OTP"; - - let mut object = object.into_value(); - let object_map = object.as_object_mut().unwrap(); - if is_tenant_filtered && let Some(tenant_id) = access_token.tenant_id() { - let expected_value = - JmapValue::Element(RegistryValue::Id(Id::from(tenant_id))); - for (key, value) in object_map.iter() { - if matches!(key, Key::Property(Property::MemberTenantId)) - && value != &expected_value - { - response.not_found.push(id); - continue 'outer; - } - } - object_map.remove(&Key::Property(Property::MemberTenantId)); - } else if is_account_filtered { - let expected_value = - JmapValue::Element(RegistryValue::Id(request.account_id)); - for (key, value) in object_map.iter() { - if matches!(key, Key::Property(Property::AccountId)) - && value != &expected_value - { - response.not_found.push(id); - continue 'outer; - } - } - object_map.remove(&Key::Property(Property::AccountId)); - } - - object_map.insert_unchecked(Property::Id, RegistryValue::Id(id)); - if !properties.is_empty() { - object_map.as_mut_vec().retain_mut(|(prop, _)| { - prop.as_property() - .is_some_and(|prop| properties.contains(prop)) - }); - } - response.list.push(object); + get.insert(id, object.into_value()); } - } - ObjectType::Log => {} - ObjectType::QueuedMessage => {} - ObjectType::Task => {} - ObjectType::ArfExternalReport => {} - ObjectType::DmarcExternalReport => {} - ObjectType::TlsExternalReport => {} - ObjectType::DeletedItem => {} - ObjectType::Metric => {} - ObjectType::Trace => {} - ObjectType::SpamTrainingSample => {} - ObjectType::DmarcInternalReport => todo!(), - ObjectType::TlsInternalReport => todo!(), - } - Ok(response) + Ok(get.into_response()) + } + ObjectType::QueuedMessage => { + queued_message_get(get).await.map(|get| get.into_response()) + } + ObjectType::Task => task_get(get).await.map(|get| get.into_response()), + + ObjectType::ArfExternalReport + | ObjectType::DmarcExternalReport + | ObjectType::TlsExternalReport + | ObjectType::DmarcInternalReport + | ObjectType::TlsInternalReport => report_get(get).await.map(|get| get.into_response()), + + ObjectType::DeletedItem => deleted_item_get(get).await.map(|get| get.into_response()), + ObjectType::SpamTrainingSample => { + spam_sample_get(get).await.map(|get| get.into_response()) + } + ObjectType::Metric => metric_get(get).await.map(|get| get.into_response()), + ObjectType::Trace => trace_get(get).await.map(|get| get.into_response()), + ObjectType::Log => log_get(get).await.map(|get| get.into_response()), + ObjectType::AccountSettings | ObjectType::Credential => { + account_get(get).await.map(|get| get.into_response()) + } + } + } +} + +impl RegistryGetResponse<'_> { + pub fn insert(&mut self, id: Id, mut object: JmapValue<'static>) { + let object_map = object.as_object_mut().unwrap(); + + if self.is_tenant_filtered + && let Some(tenant_id) = self.access_token.tenant_id() + { + let expected_value = JmapValue::Element(RegistryValue::Id(Id::from(tenant_id))); + for (key, value) in object_map.iter() { + if matches!(key, Key::Property(Property::MemberTenantId)) + && (value != &expected_value + || value + .as_array() + .is_none_or(|arr| !arr.contains(&expected_value))) + { + self.not_found(id); + return; + } + } + object_map.remove(&Key::Property(Property::MemberTenantId)); + } else if self.is_account_filtered { + let expected_value = JmapValue::Element(RegistryValue::Id(self.account_id.into())); + for (key, value) in object_map.iter() { + if matches!(key, Key::Property(Property::AccountId)) && value != &expected_value { + self.not_found(id); + return; + } + } + object_map.remove(&Key::Property(Property::AccountId)); + } + + object_map.insert_unchecked(Property::Id, RegistryValue::Id(id)); + if !self.properties.is_empty() { + object_map.as_mut_vec().retain_mut(|(prop, _)| { + prop.as_property() + .is_some_and(|prop| self.properties.contains(prop)) + }); + } + self.response.list.push(object); + } + + pub fn not_found(&mut self, id: Id) { + self.response.not_found.push(id); + } + + pub fn not_found_any(mut self) -> Self { + self.response.not_found = self.ids.take().unwrap_or_default(); + self + } + + pub fn into_response(self) -> GetResponse { + self.response } } diff --git a/crates/jmap/src/registry/mapping/account.rs b/crates/jmap/src/registry/mapping/account.rs new file mode 100644 index 00000000..c977f717 --- /dev/null +++ b/crates/jmap/src/registry/mapping/account.rs @@ -0,0 +1,87 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::registry::mapping::RegistryGetResponse; +use registry::{ + jmap::IntoValue, + schema::{ + prelude::ObjectType, + structs::{Account, AccountSettings}, + }, +}; +use types::id::Id; + +pub(crate) async fn account_get( + mut get: RegistryGetResponse<'_>, +) -> trc::Result> { + let Some(Account::User(mut account)) = get + .server + .registry() + .object::(get.account_id.into()) + .await? + else { + return Ok(get.not_found_any()); + }; + if get.access_token.tenant_id().is_some_and(|id| { + account + .member_tenant_id + .is_none_or(|aid| aid.document_id() != id) + }) { + return Ok(get.not_found_any()); + } + + match get.object_type { + ObjectType::AccountSettings => { + let mut ids = get + .ids + .take() + .unwrap_or_else(|| vec![Id::singleton()]) + .into_iter(); + + for id in ids.by_ref() { + if id == Id::singleton() { + get.insert( + id, + AccountSettings { + encryption_at_rest: account.encryption_at_rest, + locale: account.locale, + otp_auth: account.otp_auth, + secret: account.secret, + } + .into_value(), + ); + break; + } else { + get.not_found(id); + } + } + + get.response.not_found.extend(ids); + } + ObjectType::Credential => { + let ids = if let Some(ids) = get.ids.take() { + ids + } else { + account + .credentials + .keys() + .map(|id| Id::from(*id)) + .collect::>() + }; + + for id in ids { + if let Some(credential) = account.credentials.remove(&id.document_id()) { + get.insert(id, credential.into_value()); + } else { + get.not_found(id); + } + } + } + _ => unreachable!(), + } + + Ok(get) +} diff --git a/crates/jmap/src/registry/mapping/deleted_item.rs b/crates/jmap/src/registry/mapping/deleted_item.rs new file mode 100644 index 00000000..2a16d2eb --- /dev/null +++ b/crates/jmap/src/registry/mapping/deleted_item.rs @@ -0,0 +1,80 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::registry::mapping::RegistryGetResponse; +use registry::{ + jmap::IntoValue, + schema::{ + prelude::{Object, ObjectInner}, + structs::{DeletedEmail, DeletedFileNode, DeletedItem}, + }, + types::EnumImpl, +}; +use store::{ + ValueKey, + ahash::AHashSet, + registry::RegistryQuery, + write::{RegistryClass, ValueClass}, +}; +use types::{blob::BlobClass, id::Id}; + +pub(crate) async fn deleted_item_get( + mut get: RegistryGetResponse<'_>, +) -> trc::Result> { + let object_id = get.object_type.to_id(); + let ids = if let Some(ids) = get.ids.take() { + ids + } else { + get.server + .registry() + .query::>( + RegistryQuery::new(get.object_type).with_account(get.account_id), + ) + .await? + .into_iter() + .take(get.server.core.jmap.get_max_objects) + .map(Id::from) + .collect() + }; + + for id in ids { + if let Some(mut item) = get + .server + .store() + .get_value::(ValueKey::from(ValueClass::Registry(RegistryClass::Item { + object_id, + item_id: id.id(), + }))) + .await? + { + if get.is_account_filtered + && let ObjectInner::DeletedItem( + DeletedItem::Email(DeletedEmail { + blob_id, + cleanup_at, + .. + }) + | DeletedItem::FileNode(DeletedFileNode { + blob_id, + cleanup_at, + .. + }), + ) = &mut item.inner + { + blob_id.class = BlobClass::Reserved { + account_id: get.account_id, + expires: cleanup_at.timestamp() as u64, + }; + } + + get.insert(id, item.into_value()); + } else { + get.not_found(id); + } + } + + Ok(get) +} diff --git a/crates/jmap/src/registry/mapping/log.rs b/crates/jmap/src/registry/mapping/log.rs new file mode 100644 index 00000000..6d38242c --- /dev/null +++ b/crates/jmap/src/registry/mapping/log.rs @@ -0,0 +1,159 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::registry::mapping::RegistryGetResponse; +use chrono::DateTime; +use registry::{ + jmap::IntoValue, + schema::{enums::TracingLevel, structs::Log}, + types::{EnumImpl, datetime::UTCDateTime}, +}; +use rev_lines::RevLines; +use std::{ + fs::{self, File}, + io, + path::Path, +}; +use store::ahash::AHashSet; +use tokio::sync::oneshot; +use trc::EventType; +use types::id::Id; + +pub(crate) async fn log_get( + mut get: RegistryGetResponse<'_>, +) -> trc::Result> { + let Some(path) = get.server.core.metrics.log_path.clone() else { + return Err(trc::JmapEvent::InvalidArguments + .into_err() + .details("No log tracers configured on the server")); + }; + + let ids = if let Some(ids) = get.ids.take() { + ids.into_iter().map(|id| id.id()).collect::>() + } else { + (0u64..get.server.core.jmap.get_max_objects as u64).collect() + }; + + if !ids.is_empty() { + // TODO: Use worker pool + let (tx, rx) = oneshot::channel(); + tokio::task::spawn_blocking(move || { + let _ = tx.send(read_log_entries(path, ids)); + }); + + rx.await + .map_err(|err| { + trc::EventType::Server(trc::ServerEvent::ThreadError) + .reason(err) + .caused_by(trc::location!()) + })? + .map_err(|err| { + trc::ManageEvent::Error + .reason(err) + .details("Failed to read log files") + .caused_by(trc::location!()) + })? + .into_iter() + .for_each(|(id, log)| { + get.insert(id, log.into_value()); + }); + } + + Ok(get) +} + +fn line_numbers( + path: impl AsRef, + filter: &str, + mut offset: usize, + limit: usize, +) -> io::Result<(usize, Vec)> { + let mut logs = fs::read_dir(path)?.collect::, _>>()?; + let mut total = 0; + + // Sort the entries by file name in reverse order. + logs.sort_by_key(|b| std::cmp::Reverse(b.file_name())); + + let mut entries = Vec::with_capacity(limit); + let mut logs = logs.into_iter(); + let mut current_line = 0u64; + while let Some(log) = logs.next() { + if log.file_type()?.is_file() { + let mut rev_lines = RevLines::new(File::open(log.path())?); + + while let Some(line) = rev_lines.next() { + let line = line.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + + if filter.is_empty() || line.contains(filter) { + total += 1; + if offset == 0 { + entries.push(Id::from(current_line)); + if entries.len() == limit { + if rev_lines.next().is_some() || logs.next().is_some() { + total += limit; + } + + return Ok((total, entries)); + } + } else { + offset -= 1; + } + } + + current_line += 1; + } + } + } + + Ok((total, entries)) +} + +fn read_log_entries(path: impl AsRef, lines: AHashSet) -> io::Result> { + let mut logs = fs::read_dir(path)?.collect::, _>>()?; + + // Sort the entries by file name in reverse order. + logs.sort_by_key(|b| std::cmp::Reverse(b.file_name())); + + let mut entries = Vec::with_capacity(lines.len()); + let mut current_line = 0; + + 'outer: for log in logs.into_iter() { + if log.file_type()?.is_file() { + for line in RevLines::new(File::open(log.path())?) { + let line = line.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + + if lines.contains(¤t_line) + && let Some(log) = log_from_line(&line) + { + entries.push((Id::from(current_line), log)); + + if entries.len() == lines.len() { + break 'outer; + } + } + + current_line += 1; + } + } + } + + Ok(entries) +} + +fn log_from_line(line: &str) -> Option { + let (timestamp, rest) = line.split_once(' ')?; + let timestamp = DateTime::parse_from_rfc3339(timestamp).ok()?; + let (level, rest) = rest.trim().split_once(' ')?; + let (_, rest) = rest.trim().split_once(" (")?; + let (event_id, details) = rest.split_once(")")?; + + Some(Log { + timestamp: UTCDateTime::from_timestamp(timestamp.timestamp()), + level: TracingLevel::parse(&level.to_ascii_uppercase()).unwrap_or(TracingLevel::Info), + event: EventType::parse(event_id)?, + details: details.trim().to_string(), + }) +} diff --git a/crates/jmap/src/registry/mapping/mod.rs b/crates/jmap/src/registry/mapping/mod.rs index 73bfc732..cd475a25 100644 --- a/crates/jmap/src/registry/mapping/mod.rs +++ b/crates/jmap/src/registry/mapping/mod.rs @@ -4,4 +4,51 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use common::{Server, auth::AccessToken}; +use jmap_proto::{ + method::{get::GetResponse, set::SetResponse}, + object::registry::Registry, +}; +use registry::{ + jmap::JmapValue, + schema::prelude::{ObjectType, Property}, +}; +use store::ahash::AHashSet; +use types::id::Id; +use utils::map::vec_map::VecMap; + +pub mod account; +pub mod deleted_item; +pub mod log; pub mod queued_message; +pub mod report; +pub mod spam_sample; +pub mod task; +pub mod telemetry; + +pub(crate) struct RegistryGetResponse<'x> { + pub server: &'x Server, + pub access_token: &'x AccessToken, + pub account_id: u32, + pub ids: Option>, + pub properties: AHashSet, + pub response: GetResponse, + pub object_type: ObjectType, + pub object_flags: u64, + pub is_tenant_filtered: bool, + pub is_account_filtered: bool, +} + +pub(crate) struct RegistrySetResponse<'x> { + pub server: &'x Server, + pub access_token: &'x AccessToken, + pub account_id: u32, + pub create: VecMap>, + pub update: Vec<(Id, JmapValue<'x>)>, + pub destroy: Vec, + pub response: SetResponse, + pub object_type: ObjectType, + pub object_flags: u64, + pub is_tenant_filtered: bool, + pub is_account_filtered: bool, +} diff --git a/crates/jmap/src/registry/mapping/queued_message.rs b/crates/jmap/src/registry/mapping/queued_message.rs index 3a672468..f8aeea25 100644 --- a/crates/jmap/src/registry/mapping/queued_message.rs +++ b/crates/jmap/src/registry/mapping/queued_message.rs @@ -4,8 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::registry::mapping::RegistryGetResponse; use common::{Server, config::smtp::queue::ArchivedQueueExpiry}; use registry::{ + jmap::IntoValue, schema::{ enums::{DeliveryErrorType, MessageFlag, RecipientFlag}, structs::{ @@ -16,16 +18,52 @@ use registry::{ types::{datetime::UTCDateTime, ipaddr::IpAddr}, }; use smtp::queue::{spool::SmtpSpool, *}; -use types::{blob::BlobId, blob_hash::BlobHash}; +use store::{ + IterateParams, U64_LEN, ValueKey, + ahash::AHashSet, + write::{QueueClass, ValueClass, key::DeserializeBigEndian}, +}; +use trc::AddContext; +use types::{blob::BlobId, blob_hash::BlobHash, id::Id}; +use utils::DomainPart; -pub(crate) async fn queued_message_fetch( - server: &Server, - queue_id: u64, -) -> trc::Result> { - let Some(message_archive) = server.read_message_archive(queue_id).await? else { - return Ok(None); +pub(crate) async fn queued_message_get( + mut get: RegistryGetResponse<'_>, +) -> trc::Result> { + let ids = if let Some(ids) = get.ids.take() { + ids + } else { + queued_ids(get.server, get.server.core.jmap.get_max_objects) + .await? + .into_iter() + .map(Id::from) + .collect() }; - let message_in = message_archive.unarchive::()?; + + for id in ids { + let Some(message_archive) = get.server.read_message_archive(id.id()).await? else { + get.not_found(id); + continue; + }; + let message_in = message_archive.unarchive::()?; + if get.access_token.tenant_id().is_some() { + if let Some(domain) = message_in.return_path.try_domain_part() + && let Some(domain) = get.server.domain(domain).await? + && domain.id_tenant == get.access_token.tenant_id() + { + get.insert(id, map_message(message_in).into_value()); + } else { + get.not_found(id); + } + } else { + get.insert(id, map_message(message_in).into_value()); + } + } + + Ok(get) +} + +fn map_message(message_in: &ArchivedMessage) -> QueuedMessage { let mut message_out = QueuedMessage { blob_id: BlobId::new(BlobHash::from(&message_in.blob_hash), Default::default()), created_at: UTCDateTime::from_timestamp(message_in.created.to_native() as i64), @@ -109,7 +147,7 @@ pub(crate) async fn queued_message_fetch( message_out.recipients.push(rcpt_out); } - Ok(Some(message_out)) + message_out } fn map_error_details(err_in: &ArchivedErrorDetails) -> DeliveryError { @@ -164,3 +202,36 @@ fn map_error_details(err_in: &ArchivedErrorDetails) -> DeliveryError { fn build_enhanced_code(esc: &[u8; 3]) -> String { format!("{}.{}.{}", esc[0], esc[1], esc[2]) } + +async fn queued_ids(server: &Server, max_results: usize) -> trc::Result> { + let mut events = AHashSet::with_capacity(8); + + let from_key = ValueKey::from(ValueClass::Queue(QueueClass::MessageEvent( + store::write::QueueEvent { + due: 0, + queue_id: 0, + queue_name: [0; 8], + }, + ))); + let to_key = ValueKey::from(ValueClass::Queue(QueueClass::MessageEvent( + store::write::QueueEvent { + due: u64::MAX, + queue_id: u64::MAX, + queue_name: [u8::MAX; 8], + }, + ))); + + server + .store() + .iterate( + IterateParams::new(from_key, to_key).ascending().no_values(), + |key, _| { + events.insert(key.deserialize_be_u64(U64_LEN)?); + + Ok(events.len() < max_results) + }, + ) + .await + .caused_by(trc::location!()) + .map(|_| events) +} diff --git a/crates/jmap/src/registry/mapping/report.rs b/crates/jmap/src/registry/mapping/report.rs new file mode 100644 index 00000000..94cecbb0 --- /dev/null +++ b/crates/jmap/src/registry/mapping/report.rs @@ -0,0 +1,114 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::registry::mapping::RegistryGetResponse; +use common::Server; +use registry::{ + jmap::IntoValue, + schema::prelude::{Object, ObjectType, Property}, + types::EnumImpl, +}; +use store::{ + IterateParams, U16_LEN, ValueKey, + ahash::AHashSet, + registry::RegistryQuery, + write::{RegistryClass, ValueClass, key::DeserializeBigEndian}, +}; +use trc::AddContext; +use types::id::Id; + +pub(crate) async fn report_get( + mut get: RegistryGetResponse<'_>, +) -> trc::Result> { + let object_id = get.object_type.to_id(); + let ids = if let Some(ids) = get.ids.take() { + ids + } else if matches!( + get.object_type, + ObjectType::DmarcExternalReport + | ObjectType::TlsExternalReport + | ObjectType::ArfExternalReport + ) { + if get.is_tenant_filtered { + get.server.registry().query::>( + RegistryQuery::new(get.object_type).with_tenant(get.access_token.tenant_id()), + ) + } else { + get.server.registry().query::>( + RegistryQuery::new(get.object_type).greater_than(Property::ExpiresAt, 0u64), + ) + } + .await? + .into_iter() + .take(get.server.core.jmap.get_max_objects) + .map(Id::from) + .collect() + } else { + internal_report_ids(get.server, object_id, get.server.core.jmap.get_max_objects).await? + }; + + for id in ids { + if let Some(report) = get + .server + .store() + .get_value::(ValueKey::from(ValueClass::Registry(RegistryClass::Item { + object_id, + item_id: id.id(), + }))) + .await? + { + get.insert(id, report.into_value()); + } else { + get.not_found(id); + } + } + + Ok(get) +} + +async fn internal_report_ids( + server: &Server, + object_id: u16, + max_results: usize, +) -> trc::Result> { + let mut events = Vec::with_capacity(8); + + let from_key = ValueKey::from(ValueClass::Registry(RegistryClass::PrimaryKey { + object_id: object_id.into(), + index_id: Property::Domain.to_id(), + key: vec![], + })); + let to_key = ValueKey::from(ValueClass::Registry(RegistryClass::PrimaryKey { + object_id: object_id.into(), + index_id: Property::Domain.to_id(), + key: vec![ + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + ], + })); + + server + .store() + .iterate( + IterateParams::new(from_key, to_key).ascending(), + |key, value| { + if !value.is_empty() { + events.push(key.deserialize_be_u64(U16_LEN)?.into()); + } + + Ok(events.len() < max_results) + }, + ) + .await + .caused_by(trc::location!()) + .map(|_| events) +} diff --git a/crates/jmap/src/registry/mapping/spam_sample.rs b/crates/jmap/src/registry/mapping/spam_sample.rs new file mode 100644 index 00000000..6866ef96 --- /dev/null +++ b/crates/jmap/src/registry/mapping/spam_sample.rs @@ -0,0 +1,73 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::registry::mapping::RegistryGetResponse; +use registry::{ + jmap::IntoValue, + schema::{ + enums::Permission, + prelude::{Object, ObjectInner, Property}, + }, + types::EnumImpl, +}; +use store::{ + ValueKey, + ahash::AHashSet, + registry::RegistryQuery, + write::{RegistryClass, ValueClass}, +}; +use types::{blob::BlobClass, id::Id}; + +pub(crate) async fn spam_sample_get( + mut get: RegistryGetResponse<'_>, +) -> trc::Result> { + let object_id = get.object_type.to_id(); + let ids = if let Some(ids) = get.ids.take() { + ids + } else { + let query = if get.access_token.has_permission(Permission::Impersonate) { + RegistryQuery::new(get.object_type).greater_than_or_equal(Property::AccountId, 0u64) + } else { + RegistryQuery::new(get.object_type).with_account(get.account_id) + }; + + get.server + .registry() + .query::>(query) + .await? + .into_iter() + .take(get.server.core.jmap.get_max_objects) + .map(Id::from) + .collect() + }; + + for id in ids { + if let Some(mut item) = get + .server + .store() + .get_value::(ValueKey::from(ValueClass::Registry(RegistryClass::Item { + object_id, + item_id: id.id(), + }))) + .await? + { + if get.is_account_filtered + && let ObjectInner::SpamTrainingSample(item) = &mut item.inner + { + item.blob_id.class = BlobClass::Reserved { + account_id: get.account_id, + expires: item.expires_at.timestamp() as u64, + }; + } + + get.insert(id, item.into_value()); + } else { + get.not_found(id); + } + } + + Ok(get) +} diff --git a/crates/jmap/src/registry/mapping/task.rs b/crates/jmap/src/registry/mapping/task.rs new file mode 100644 index 00000000..eb9519bf --- /dev/null +++ b/crates/jmap/src/registry/mapping/task.rs @@ -0,0 +1,68 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::registry::mapping::RegistryGetResponse; +use common::Server; +use registry::{jmap::IntoValue, schema::prelude::Object, types::EnumImpl}; +use store::{ + IterateParams, U64_LEN, ValueKey, + write::{RegistryClass, TaskQueueClass, ValueClass, key::DeserializeBigEndian}, +}; +use trc::AddContext; +use types::id::Id; + +pub(crate) async fn task_get( + mut get: RegistryGetResponse<'_>, +) -> trc::Result> { + let ids = if let Some(ids) = get.ids.take() { + ids + } else { + task_ids(get.server, get.server.core.jmap.get_max_objects).await? + }; + let object_id = get.object_type.to_id(); + + for id in ids { + if let Some(task) = get + .server + .store() + .get_value::(ValueKey::from(ValueClass::Registry(RegistryClass::Item { + object_id, + item_id: id.id(), + }))) + .await? + { + get.insert(id, task.into_value()); + } else { + get.not_found(id); + } + } + + Ok(get) +} + +async fn task_ids(server: &Server, max_results: usize) -> trc::Result> { + let mut events = Vec::with_capacity(8); + + let from_key = ValueKey::from(ValueClass::TaskQueue(TaskQueueClass::Due { id: 0, due: 0 })); + let to_key = ValueKey::from(ValueClass::TaskQueue(TaskQueueClass::Due { + id: u64::MAX, + due: u64::MAX, + })); + + server + .store() + .iterate( + IterateParams::new(from_key, to_key).ascending().no_values(), + |key, _| { + events.push(key.deserialize_be_u64(U64_LEN)?.into()); + + Ok(events.len() < max_results) + }, + ) + .await + .caused_by(trc::location!()) + .map(|_| events) +} diff --git a/crates/jmap/src/registry/mapping/telemetry.rs b/crates/jmap/src/registry/mapping/telemetry.rs new file mode 100644 index 00000000..46109316 --- /dev/null +++ b/crates/jmap/src/registry/mapping/telemetry.rs @@ -0,0 +1,122 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::registry::mapping::RegistryGetResponse; +use common::Server; +use registry::{ + jmap::IntoValue, + schema::prelude::{Object, Property}, + types::datetime::UTCDateTime, +}; +use store::{ + IterateParams, ValueKey, + search::{SearchComparator, SearchField, SearchFilter, SearchQuery}, + write::{SearchIndex, TelemetryClass, ValueClass, key::DeserializeBigEndian, now}, +}; +use trc::AddContext; +use types::id::Id; +use utils::snowflake::SnowflakeIdGenerator; + +pub(crate) async fn trace_get( + mut get: RegistryGetResponse<'_>, +) -> trc::Result> { + let ids = if let Some(ids) = get.ids.take() { + ids + } else { + get.server + .search_store() + .query_global( + SearchQuery::new(SearchIndex::Tracing) + .with_filter(SearchFilter::gt( + SearchField::Id, + SnowflakeIdGenerator::from_timestamp(now() - 86400).unwrap_or_default(), + )) + .with_comparator(SearchComparator::Field { + field: SearchField::Id, + ascending: false, + }), + ) + .await? + .into_iter() + .take(get.server.core.jmap.get_max_objects) + .map(Id::from) + .collect() + }; + + for id in ids { + let item_id = id.id(); + if let Some(trace) = get + .server + .tracing_store() + .get_value::(ValueKey::from(ValueClass::Telemetry(TelemetryClass::Span( + item_id, + )))) + .await? + { + get.insert(id, trace.into_value()); + } else { + get.not_found(id); + } + } + + Ok(get) +} + +pub(crate) async fn metric_get( + mut get: RegistryGetResponse<'_>, +) -> trc::Result> { + let ids = if let Some(ids) = get.ids.take() { + ids + } else { + metric_ids(get.server, get.server.core.jmap.get_max_objects).await? + }; + + for id in ids { + let item_id = id.id(); + if let Some(metric) = get + .server + .metrics_store() + .get_value::(ValueKey::from(ValueClass::Telemetry( + TelemetryClass::Metric(item_id), + ))) + .await? + { + let mut metric = metric.into_value(); + metric.as_object_mut().unwrap().insert_unchecked( + Property::Timestamp, + UTCDateTime::from_timestamp(SnowflakeIdGenerator::to_timestamp(item_id) as i64) + .into_value(), + ); + + get.insert(id, metric); + } else { + get.not_found(id); + } + } + + Ok(get) +} + +async fn metric_ids(server: &Server, max_results: usize) -> trc::Result> { + let mut events = Vec::with_capacity(8); + + let from_key = ValueKey::from(ValueClass::Telemetry(TelemetryClass::Metric(0))); + let to_key = ValueKey::from(ValueClass::Telemetry(TelemetryClass::Metric(u64::MAX))); + + server + .metrics_store() + .iterate( + IterateParams::new(from_key, to_key).ascending().no_values(), + |key, _| { + events.push(key.deserialize_be_u64(0)?.into()); + + Ok(events.len() < max_results) + }, + ) + .await + .caused_by(trc::location!()) + .map(|_| events) +} diff --git a/crates/jmap/src/registry/set.rs b/crates/jmap/src/registry/set.rs index 6224e0fe..9a073cb0 100644 --- a/crates/jmap/src/registry/set.rs +++ b/crates/jmap/src/registry/set.rs @@ -4,12 +4,27 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::registry::mapping::RegistrySetResponse; use common::{Server, auth::AccessToken}; use jmap_proto::{ + error::set::SetError, method::set::{SetRequest, SetResponse}, object::registry::Registry, + request::IntoValid, }; -use registry::schema::prelude::ObjectType; +use jmap_tools::{JsonPointer, JsonPointerItem, Key}; +use registry::{ + jmap::JsonPointerPatch, + schema::{ + enums::Permission, + prelude::{ + OBJ_FILTER_ACCOUNT, OBJ_FILTER_TENANT, OBJ_SINGLETON, Object, ObjectType, Property, + }, + }, + types::id::ObjectId, +}; +use trc::AddContext; +use types::id::Id; pub trait RegistrySet: Sync + Send { fn registry_set( @@ -20,6 +35,11 @@ pub trait RegistrySet: Sync + Send { ) -> impl Future>> + Send; } +enum Modification { + Create(String), + Update(Id), +} + impl RegistrySet for Server { async fn registry_set( &self, @@ -27,109 +47,275 @@ impl RegistrySet for Server { mut request: SetRequest<'_, Registry>, access_token: &AccessToken, ) -> trc::Result> { + let object_flags = object_type.flags(); + let is_singleton = (object_flags & OBJ_SINGLETON) != 0; + let is_tenant_filtered = + (object_flags & OBJ_FILTER_TENANT) != 0 && access_token.tenant_id().is_some(); + let is_account_filtered = (object_flags & OBJ_FILTER_ACCOUNT) != 0 + && !access_token.has_permission(Permission::Impersonate); + + // Build response + let mut response = SetResponse::from_request(&request, self.core.jmap.set_max_objects)?; + + // Initial create validation for singletons + let mut create = request.unwrap_create(); + if is_singleton && !create.is_empty() { + response + .not_created + .extend(create.drain().map(|(id, _)| (id, SetError::singleton()))); + } + + // Initial destroy validation for singletons + let mut destroy = request.unwrap_destroy().into_valid().collect::>(); + if is_singleton && !destroy.is_empty() { + response + .not_destroyed + .extend(destroy.drain(..).map(|id| (id, SetError::singleton()))); + } + + // Update validation for willDestroy + let update = request + .unwrap_update() + .into_valid() + .filter_map(|(id, value)| { + if is_singleton { + if id.is_singleton() { + Some((id, value)) + } else { + response.not_updated.append(id, SetError::not_found()); + None + } + } else if !destroy.contains(&id) { + Some((id, value)) + } else { + response.not_updated.append(id, SetError::will_destroy()); + None + } + }) + .collect::>(); + + let mut set = RegistrySetResponse { + access_token, + server: self, + account_id: request.account_id.document_id(), + object_type, + response, + object_flags, + is_tenant_filtered, + is_account_filtered, + create, + update, + destroy, + }; match object_type { - ObjectType::AcmeProvider => {} - ObjectType::AddressBook => {} - ObjectType::AiModel => {} - ObjectType::Alert => {} - ObjectType::AllowedIp => {} - ObjectType::Application => {} - ObjectType::Asn => {} - ObjectType::Authentication => {} - ObjectType::BlobStore => {} - ObjectType::BlockedIp => {} - ObjectType::Cache => {} - ObjectType::Calendar => {} - ObjectType::CalendarAlarm => {} - ObjectType::CalendarScheduling => {} - ObjectType::Certificate => {} - ObjectType::Coordinator => {} - ObjectType::DataRetention => {} - ObjectType::DataStore => {} - ObjectType::Directory => {} - ObjectType::DkimReportSettings => {} - ObjectType::DmarcReportSettings => {} - ObjectType::DnsResolver => {} - ObjectType::DnsServer => {} - ObjectType::Email => {} - ObjectType::Enterprise => {} - ObjectType::EventTracingLevel => {} - ObjectType::FileStorage => {} - ObjectType::Http => {} - ObjectType::HttpForm => {} - ObjectType::HttpLookup => {} - ObjectType::Imap => {} - ObjectType::InMemoryStore => {} - ObjectType::Jmap => {} - ObjectType::LocalSettings => {} - ObjectType::MemoryLookupKey => {} - ObjectType::MemoryLookupKeyValue => {} - ObjectType::Metrics => {} - ObjectType::MetricsStore => {} - ObjectType::MtaConnectionStrategy => {} - ObjectType::MtaDeliverySchedule => {} - ObjectType::MtaExtensions => {} - ObjectType::MtaHook => {} - ObjectType::MtaInboundSession => {} - ObjectType::MtaInboundThrottle => {} - ObjectType::MtaMilter => {} - ObjectType::MtaOutboundStrategy => {} - ObjectType::MtaOutboundThrottle => {} - ObjectType::MtaQueueQuota => {} - ObjectType::MtaRoute => {} - ObjectType::MtaStageAuth => {} - ObjectType::MtaStageConnect => {} - ObjectType::MtaStageData => {} - ObjectType::MtaStageEhlo => {} - ObjectType::MtaStageMail => {} - ObjectType::MtaStageRcpt => {} - ObjectType::MtaSts => {} - ObjectType::MtaTlsStrategy => {} - ObjectType::MtaVirtualQueue => {} - ObjectType::NetworkListener => {} - ObjectType::Node => {} - ObjectType::NodeRole => {} - ObjectType::NodeShard => {} - ObjectType::OidcProvider => {} - ObjectType::RegistryBundle => {} - ObjectType::ReportSettings => {} - ObjectType::Search => {} - ObjectType::SearchStore => {} - ObjectType::Security => {} - ObjectType::SenderAuth => {} - ObjectType::Sharing => {} - ObjectType::SieveSystemInterpreter => {} - ObjectType::SieveSystemScript => {} - ObjectType::SieveUserInterpreter => {} - ObjectType::SieveUserScript => {} - ObjectType::SpamClassifier => {} - ObjectType::SpamDnsblServer => {} - ObjectType::SpamDnsblSettings => {} - ObjectType::SpamFileExtension => {} - ObjectType::SpamLlm => {} - ObjectType::SpamPyzor => {} - ObjectType::SpamRule => {} - ObjectType::SpamSettings => {} - ObjectType::SpamTag => {} - ObjectType::SpfReportSettings => {} - ObjectType::StoreLookup => {} - ObjectType::TaskManager => {} - ObjectType::TlsReportSettings => {} - ObjectType::Tracer => {} - ObjectType::TracingStore => {} - ObjectType::WebDav => {} - ObjectType::WebHook => {} - ObjectType::Account => {} - ObjectType::DsnReportSettings => {} - ObjectType::MailingList => {} - ObjectType::OAuthClient => {} - ObjectType::Role => {} - ObjectType::Tenant => {} - ObjectType::MaskedEmail => {} - ObjectType::PublicKey => {} - ObjectType::DkimSignature => {} - ObjectType::Domain => {} - ObjectType::Log => {} + ObjectType::AddressBook + | ObjectType::Asn + | ObjectType::Authentication + | ObjectType::BlobStore + | ObjectType::Cache + | ObjectType::Calendar + | ObjectType::CalendarAlarm + | ObjectType::CalendarScheduling + | ObjectType::Coordinator + | ObjectType::DataRetention + | ObjectType::DataStore + | ObjectType::DkimReportSettings + | ObjectType::DmarcReportSettings + | ObjectType::DnsResolver + | ObjectType::Email + | ObjectType::Enterprise + | ObjectType::FileStorage + | ObjectType::Http + | ObjectType::HttpForm + | ObjectType::Imap + | ObjectType::InMemoryStore + | ObjectType::Jmap + | ObjectType::LocalSettings + | ObjectType::Metrics + | ObjectType::MetricsStore + | ObjectType::MtaConnectionStrategy + | ObjectType::MtaExtensions + | ObjectType::MtaInboundSession + | ObjectType::MtaOutboundStrategy + | ObjectType::MtaOutboundThrottle + | ObjectType::MtaStageAuth + | ObjectType::MtaStageConnect + | ObjectType::MtaStageData + | ObjectType::MtaStageEhlo + | ObjectType::MtaStageMail + | ObjectType::MtaStageRcpt + | ObjectType::MtaSts + | ObjectType::OidcProvider + | ObjectType::ReportSettings + | ObjectType::Search + | ObjectType::SearchStore + | ObjectType::Security + | ObjectType::SenderAuth + | ObjectType::Sharing + | ObjectType::SieveSystemInterpreter + | ObjectType::SieveUserInterpreter + | ObjectType::SpamClassifier + | ObjectType::SpamDnsblSettings + | ObjectType::SpamLlm + | ObjectType::SpamPyzor + | ObjectType::SpamSettings + | ObjectType::SpfReportSettings + | ObjectType::TaskManager + | ObjectType::TlsReportSettings + | ObjectType::TracingStore + | ObjectType::WebDav + | ObjectType::DsnReportSettings + | ObjectType::AcmeProvider + | ObjectType::AiModel + | ObjectType::Alert + | ObjectType::AllowedIp + | ObjectType::Application + | ObjectType::BlockedIp + | ObjectType::Certificate + | ObjectType::Directory + | ObjectType::DnsServer + | ObjectType::EventTracingLevel + | ObjectType::HttpLookup + | ObjectType::MemoryLookupKey + | ObjectType::MemoryLookupKeyValue + | ObjectType::MtaVirtualQueue + | ObjectType::MtaQueueQuota + | ObjectType::MtaRoute + | ObjectType::MtaDeliverySchedule + | ObjectType::MtaInboundThrottle + | ObjectType::MtaTlsStrategy + | ObjectType::MtaMilter + | ObjectType::MtaHook + | ObjectType::NetworkListener + | ObjectType::Node + | ObjectType::NodeRole + | ObjectType::NodeShard + | ObjectType::RegistryBundle + | ObjectType::SieveSystemScript + | ObjectType::SieveUserScript + | ObjectType::SpamDnsblServer + | ObjectType::SpamFileExtension + | ObjectType::SpamRule + | ObjectType::SpamTag + | ObjectType::StoreLookup + | ObjectType::Tracer + | ObjectType::WebHook + | ObjectType::PublicKey + | ObjectType::DkimSignature + | ObjectType::MaskedEmail + | ObjectType::Account + | ObjectType::MailingList + | ObjectType::OAuthClient + | ObjectType::Role + | ObjectType::Tenant + | ObjectType::Domain => { + // Bundle modifications together + let mut modifications = Vec::with_capacity(set.create.len() + set.update.len()); + for (id, value) in set.create { + modifications.push(( + Modification::Create(id), + value, + Object::from(set.object_type), + )); + } + for (id, value) in set.update { + if let Some(object) = self + .registry() + .get(ObjectId::new(object_type, id)) + .await + .caused_by(trc::location!())? + { + modifications.push((Modification::Update(id), value, object)); + } else if is_singleton { + modifications.push(( + Modification::Update(id), + value, + Object::from(set.object_type), + )); + } else { + set.response.not_updated.append(id, SetError::not_found()); + } + } + + // Process modifications + 'outer: for (modification, value, mut object) in modifications { + for (key, value) in value.into_expanded_object() { + let ptr = match (key, &modification) { + (Key::Property(prop), _) => { + JsonPointer::new(vec![JsonPointerItem::Key(Key::Property(prop))]) + } + (Key::Borrowed(other), Modification::Update(_)) => { + JsonPointer::parse(other) + } + (Key::Owned(other), Modification::Update(_)) => { + JsonPointer::parse(&other) + } + (key, Modification::Create(_)) => { + set.response.failed( + modification, + SetError::invalid_properties().with_property(key.into_owned()), + ); + continue 'outer; + } + }; + + // Initial validations + let is_create = matches!(modification, Modification::Create(_)); + + // SPDX-SnippetBegin + // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + // SPDX-License-Identifier: LicenseRef-SEL + + #[cfg(feature = "enterprise")] + if is_create + && object_type == ObjectType::Account + && self.core.is_enterprise_edition() + && !self.can_create_account().await? + { + set.response.failed( + modification, + SetError::forbidden().with_description(format!( + "Enterprise licensed account limit reached: {} accounts licensed.", + self.licensed_accounts() + )), + ); + continue 'outer; + } + // SPDX-SnippetEnd + + /* + Principal creation: + + - Add tenantId + - Add default roles on account creation + - Invalidate cache + logo cache + - Validate effective permissions to grant access + + Principal update: + + - Remove tenantId, or return error + - Invalidate cache + logo cache + - Validate effective permissions to grant access + + Principal deletion: + + - Validate tenantId ownership + - Invalidate cache + - Schedule account deletion (if account) + + */ + + // Patch object + if let Err(err) = + object.patch(JsonPointerPatch::new(&ptr).with_create(is_create), value) + { + } + } + } + + // Process destroy + for id in set.destroy {} + } ObjectType::QueuedMessage => {} ObjectType::Task => {} ObjectType::ArfExternalReport => {} @@ -139,12 +325,33 @@ impl RegistrySet for Server { ObjectType::Metric => {} ObjectType::Trace => {} ObjectType::SpamTrainingSample => {} - ObjectType::DmarcInternalReport => todo!(), - ObjectType::TlsInternalReport => todo!(), + ObjectType::DmarcInternalReport => {} + ObjectType::TlsInternalReport => {} + ObjectType::Log => {} + ObjectType::AccountSettings => {} + ObjectType::Credential => {} } let todo = "read only properties"; + let todo = "password encryption"; + let todo = "management objects for actions (reload, etc)"; + // MaskedEmail: Generate masked email + Enforce count + // DkimSignature = Generate keys + Enforce count? + // PublicKey = Validate PK? Store decoded? todo!() } } + +trait SetModification { + fn failed(&mut self, modification: Modification, error: SetError); +} + +impl SetModification for SetResponse { + fn failed(&mut self, modification: Modification, error: SetError) { + match modification { + Modification::Create(id) => self.not_created.append(id, error), + Modification::Update(id) => self.not_updated.append(id, error), + } + } +} diff --git a/crates/registry/src/jmap/mod.rs b/crates/registry/src/jmap/mod.rs index 99d94cbb..630c7123 100644 --- a/crates/registry/src/jmap/mod.rs +++ b/crates/registry/src/jmap/mod.rs @@ -25,11 +25,12 @@ pub enum RegistryValue { IdReference(String), } -#[derive(Debug, Clone)] +#[derive(Clone)] pub struct JsonPointerPatch<'x> { ptr: &'x JsonPointer, pos: usize, validators: &'x [StringValidator], + is_create: bool, } pub trait RegistryJsonPatch: Debug + Default { diff --git a/crates/registry/src/jmap/patch.rs b/crates/registry/src/jmap/patch.rs index 2583a6b2..eb4af1e0 100644 --- a/crates/registry/src/jmap/patch.rs +++ b/crates/registry/src/jmap/patch.rs @@ -26,9 +26,24 @@ impl<'x> JsonPointerPatch<'x> { ptr, pos: 0, validators: &[], + is_create: false, } } + pub fn cloned(&self) -> Self { + Self { + ptr: self.ptr, + pos: 0, + validators: &[], + is_create: false, + } + } + + pub fn with_create(mut self, is_create: bool) -> Self { + self.is_create = is_create; + self + } + pub fn with_validators(mut self, validators: &'x [StringValidator]) -> Self { self.validators = validators; self @@ -63,14 +78,22 @@ impl<'x> JsonPointerPatch<'x> { pub fn assert_eof(&self) -> Result<(), PatchError> { if self.has_next() { - Err(PatchError::new( - JsonPointerPatch::new(self.ptr), - "Invalid JSON Pointer path", - )) + Err(PatchError::new(self.cloned(), "Invalid JSON Pointer path")) } else { Ok(()) } } + + pub fn assert_read_only(self) -> Result { + if self.is_create { + Ok(self) + } else { + Err(PatchError::new( + self.cloned(), + "Cannot modify read-only property", + )) + } + } } impl RegistryJsonPatch for Option { diff --git a/crates/registry/src/schema/mod.rs b/crates/registry/src/schema/mod.rs index 310b7b63..fd649e09 100644 --- a/crates/registry/src/schema/mod.rs +++ b/crates/registry/src/schema/mod.rs @@ -7,7 +7,7 @@ use crate::{ schema::{ enums::{TracingLevel, TracingLevelOpt}, - prelude::{NodeRange, Object, ObjectInner, Property}, + prelude::{Credential, CredentialProperties, NodeRange, Object, ObjectInner, Property}, }, types::EnumImpl, }; @@ -33,6 +33,23 @@ impl NodeRange { node_id >= self.from_node_id && node_id <= self.to_node_id } } + +impl Credential { + pub fn unwrap_properties(self) -> CredentialProperties { + match self { + Credential::AppPassword(credential_properties) => credential_properties, + Credential::ApiKey(credential_properties) => credential_properties, + } + } + + pub fn as_properties(&self) -> &CredentialProperties { + match self { + Credential::AppPassword(credential_properties) => credential_properties, + Credential::ApiKey(credential_properties) => credential_properties, + } + } +} + impl Display for Property { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.as_str()) diff --git a/crates/registry/src/types/datetime.rs b/crates/registry/src/types/datetime.rs index bacb7b55..ca9d78e8 100644 --- a/crates/registry/src/types/datetime.rs +++ b/crates/registry/src/types/datetime.rs @@ -225,7 +225,7 @@ impl Display for UTCDateTime { impl Default for UTCDateTime { fn default() -> Self { - UTCDateTime(i64::MAX) + UTCDateTime::now() } } diff --git a/crates/registry/src/types/string.rs b/crates/registry/src/types/string.rs index d9af552e..51e590b8 100644 --- a/crates/registry/src/types/string.rs +++ b/crates/registry/src/types/string.rs @@ -17,6 +17,7 @@ pub enum StringValidator { Lowercase, Uppercase, Trim, + SecretHash, } pub enum StringValidatorResult { @@ -67,6 +68,13 @@ impl StringValidator { StringValidatorResult::Valid } } + Self::SecretHash => { + if !value.is_empty() && value.len() <= 128 { + StringValidatorResult::Valid + } else { + StringValidatorResult::Invalid("Secret cannot be empty") + } + } } } } diff --git a/crates/services/src/task_manager/index.rs b/crates/services/src/task_manager/index.rs index f586b1e4..5178acf7 100644 --- a/crates/services/src/task_manager/index.rs +++ b/crates/services/src/task_manager/index.rs @@ -692,12 +692,12 @@ async fn delete_email_metadata( index_id: Property::AccountId.to_id(), object_id, item_id, - key: account_id.serialize(), + key: (account_id as u64).serialize(), }), vec![], ) .set( - ValueClass::Registry(RegistryClass::Id { object_id, item_id }), + ValueClass::Registry(RegistryClass::Item { object_id, item_id }), item, ); } diff --git a/crates/smtp/src/queue/spool.rs b/crates/smtp/src/queue/spool.rs index 63698f59..fc982c3b 100644 --- a/crates/smtp/src/queue/spool.rs +++ b/crates/smtp/src/queue/spool.rs @@ -17,7 +17,7 @@ use common::config::smtp::queue::QueueName; use common::ipc::QueueEvent; use common::{KV_LOCK_QUEUE_MESSAGE, Server}; use registry::pickle::Pickle; -use registry::schema::prelude::ObjectType; +use registry::schema::prelude::{ObjectType, Property}; use registry::schema::structs::SpamTrainingSample; use registry::types::EnumImpl; use registry::types::datetime::UTCDateTime; @@ -482,8 +482,17 @@ impl MessageWrapper { ObjectId::new(ObjectType::SpamTrainingSample, item_id.into()).serialize(), ) .set( - ValueClass::Registry(RegistryClass::Id { object_id, item_id }), + ValueClass::Registry(RegistryClass::Item { object_id, item_id }), sample, + ) + .set( + ValueClass::Registry(RegistryClass::Index { + index_id: Property::AccountId.to_id(), + object_id, + item_id, + key: (u32::MAX as u64).serialize(), + }), + vec![], ); trc::event!( diff --git a/crates/spam-filter/src/modules/classifier.rs b/crates/spam-filter/src/modules/classifier.rs index a6e50da7..1088f274 100644 --- a/crates/spam-filter/src/modules/classifier.rs +++ b/crates/spam-filter/src/modules/classifier.rs @@ -188,11 +188,11 @@ impl SpamClassifier for Server { let mut duplicate_samples = Vec::new(); let mut remove_entries = false; let object_id = ObjectType::SpamTrainingSample.to_id(); - let from_key = ValueKey::from(ValueClass::Registry(RegistryClass::Id { + let from_key = ValueKey::from(ValueClass::Registry(RegistryClass::Item { object_id, item_id: trainer.last_id + 1, })); - let to_key = ValueKey::from(ValueClass::Registry(RegistryClass::Id { + let to_key = ValueKey::from(ValueClass::Registry(RegistryClass::Item { object_id, item_id: u64::MAX, })); @@ -564,18 +564,17 @@ impl SpamClassifier for Server { hash: sample.sample.hash, to: BlobLink::Temporary { until }, }) - .clear(ValueClass::Registry(RegistryClass::Id { + .clear(ValueClass::Registry(RegistryClass::Item { object_id, item_id: sample.id, - })); - if sample.sample.account_id != u32::MAX { - batch.clear(ValueClass::Registry(RegistryClass::Index { + })) + .clear(ValueClass::Registry(RegistryClass::Index { index_id: Property::AccountId.to_id(), object_id, item_id: sample.id, - key: sample.sample.account_id.serialize(), + key: (sample.sample.account_id as u64).serialize(), })); - } + if batch.is_large_batch() { self.store() .write(batch.build_all()) diff --git a/crates/store/src/registry/get.rs b/crates/store/src/registry/get.rs index dd9df0ac..02393cc4 100644 --- a/crates/store/src/registry/get.rs +++ b/crates/store/src/registry/get.rs @@ -30,22 +30,6 @@ impl RegistryStore { item_id: object_id.id().id(), }))) .await - .and_then(|v| { - if v.as_ref() - .is_none_or(|v| v.object_type() == object_id.object()) - { - Ok(v) - } else { - Err( - trc::EventType::Registry(trc::RegistryEvent::DeserializationError) - .into_err() - .caused_by(trc::location!()) - .id(object_id.id().id()) - .details(object_id.object().as_str()) - .reason("Object type mismatch"), - ) - } - }) } } diff --git a/crates/store/src/registry/query.rs b/crates/store/src/registry/query.rs index 5a05dfc7..401ec5f1 100644 --- a/crates/store/src/registry/query.rs +++ b/crates/store/src/registry/query.rs @@ -374,11 +374,11 @@ async fn all_ids(store: &Store, object: ObjectType) -> store .iterate( IterateParams::new( - ValueKey::from(ValueClass::Registry(RegistryClass::Id { + ValueKey::from(ValueClass::Registry(RegistryClass::IndexId { object_id, item_id: 0u64, })), - ValueKey::from(ValueClass::Registry(RegistryClass::Id { + ValueKey::from(ValueClass::Registry(RegistryClass::IndexId { object_id, item_id: u64::MAX, })), diff --git a/crates/store/src/registry/write.rs b/crates/store/src/registry/write.rs index b91bbbd3..f92f44b9 100644 --- a/crates/store/src/registry/write.rs +++ b/crates/store/src/registry/write.rs @@ -211,7 +211,7 @@ impl RegistryStore { key: type_filter.serialize(), } } else { - RegistryClass::Id { object_id, item_id } + RegistryClass::IndexId { object_id, item_id } }; if self .0 @@ -317,7 +317,7 @@ impl RegistryStore { // Build batch if write_id { batch.set( - ValueClass::Registry(RegistryClass::Id { object_id, item_id }), + ValueClass::Registry(RegistryClass::IndexId { object_id, item_id }), vec![], ); } @@ -434,7 +434,7 @@ impl RegistryStore { object_id: object_type_id, item_id, })) - .clear(ValueClass::Registry(RegistryClass::Id { + .clear(ValueClass::Registry(RegistryClass::IndexId { object_id: object_type_id, item_id, })) diff --git a/crates/store/src/write/blob.rs b/crates/store/src/write/blob.rs index bdf4ad29..f03adde9 100644 --- a/crates/store/src/write/blob.rs +++ b/crates/store/src/write/blob.rs @@ -167,19 +167,17 @@ impl Store { let item_id = object_id.id().id(); let object_id = object_id.object().to_id(); - if let Some(account_id) = account_id { - batch.clear(ValueClass::Registry(RegistryClass::Index { + batch + .clear(ValueClass::Registry(RegistryClass::Index { index_id: Property::AccountId.to_id(), object_id, item_id, - key: account_id.serialize(), + key: (account_id as u64).serialize(), + })) + .clear(ValueClass::Registry(RegistryClass::Item { + object_id, + item_id, })); - } - - batch.clear(ValueClass::Registry(RegistryClass::Id { - object_id, - item_id, - })); } if !batch.is_empty() { self.write(batch.build_all()) @@ -206,7 +204,7 @@ struct BlobPurgeState { last_hash: BlobHash, last_hash_is_linked: bool, delete_keys: Vec<(Option, BlobOp)>, - delete_registry: Vec<(Option, ObjectId)>, + delete_registry: Vec<(u32, ObjectId)>, now: u64, total_deleted: u64, total_active: u64, @@ -270,10 +268,8 @@ impl BlobPurgeState { }, )); if value.len() == U16_LEN + U64_LEN { - self.delete_registry.push(( - (account_id != u32::MAX).then_some(account_id), - ObjectId::deserialize(value)?, - )); + self.delete_registry + .push((account_id, ObjectId::deserialize(value)?)); } } Ok(()) diff --git a/crates/store/src/write/key.rs b/crates/store/src/write/key.rs index e054b539..3773bc65 100644 --- a/crates/store/src/write/key.rs +++ b/crates/store/src/write/key.rs @@ -304,7 +304,7 @@ impl ValueClass { RegistryClass::Item { object_id, item_id } => { serializer.write(*object_id).write_leb128(*item_id) } - RegistryClass::Id { object_id, item_id } => { + RegistryClass::IndexId { object_id, item_id } => { serializer.write(*object_id).write(*item_id) } RegistryClass::Index { @@ -498,7 +498,7 @@ impl ValueClass { RegistryClass::Reference { .. } => ((U16_LEN + U64_LEN) * 2) + 1, RegistryClass::Index { key, .. } => (U16_LEN * 2) + U64_LEN + key.len() + 1, RegistryClass::PrimaryKey { key, .. } => (U16_LEN * 2) + key.len() + 1, - RegistryClass::Id { .. } => U16_LEN + U64_LEN + 1, + RegistryClass::IndexId { .. } => U16_LEN + U64_LEN + 1, RegistryClass::IdCounter { .. } => U16_LEN + 1, }, ValueClass::Blob(op) => match op { @@ -568,7 +568,9 @@ impl ValueClass { REPORT_INTERNAL_DMARC | REPORT_INTERNAL_TLS => SUBSPACE_REPORT_OUT, _ => SUBSPACE_REGISTRY, }, - RegistryClass::Id { .. } | RegistryClass::Index { .. } => SUBSPACE_REGISTRY_IDX, + RegistryClass::IndexId { .. } | RegistryClass::Index { .. } => { + SUBSPACE_REGISTRY_IDX + } RegistryClass::Reference { .. } | RegistryClass::PrimaryKey { .. } => { SUBSPACE_REGISTRY_PK } diff --git a/crates/store/src/write/mod.rs b/crates/store/src/write/mod.rs index 23f4d3ad..0df4c785 100644 --- a/crates/store/src/write/mod.rs +++ b/crates/store/src/write/mod.rs @@ -265,15 +265,15 @@ pub enum RegistryClass { item_id: u64, key: Vec, }, + IndexId { + object_id: u16, + item_id: u64, + }, PrimaryKey { object_id: Option, index_id: u16, key: Vec, }, - Id { - object_id: u16, - item_id: u64, - }, IdCounter { object_id: u16, }, diff --git a/crates/types/src/blob.rs b/crates/types/src/blob.rs index 374956a4..16619fa5 100644 --- a/crates/types/src/blob.rs +++ b/crates/types/src/blob.rs @@ -32,8 +32,8 @@ pub enum BlobClass { impl Default for BlobClass { fn default() -> Self { BlobClass::Reserved { - account_id: 0, - expires: 0, + account_id: u32::MAX, + expires: u64::MAX, } } } @@ -64,6 +64,10 @@ impl BlobClass { BlobClass::Linked { .. } => true, } } + + pub fn is_superuser(&self) -> bool { + matches!(self, BlobClass::Reserved { account_id, expires } if *account_id == u32::MAX && *expires == u64::MAX) + } } #[derive(Debug, Default, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] diff --git a/crates/utils/src/map/vec_map.rs b/crates/utils/src/map/vec_map.rs index dcd58351..5a05ff5d 100644 --- a/crates/utils/src/map/vec_map.rs +++ b/crates/utils/src/map/vec_map.rs @@ -215,6 +215,16 @@ impl VecMap { cmp => cmp, }); } + + pub fn extend(&mut self, iter: impl IntoIterator) { + for (k, v) in iter { + self.append(k, v); + } + } + + pub fn drain(&mut self) -> impl Iterator + '_ { + self.inner.drain(..).map(|kv| (kv.key, kv.value)) + } } impl VecMap {