From 2a557c58e5b8847b75ecf24c2b404eb07577f437 Mon Sep 17 00:00:00 2001 From: mdecimus <11444311+mdecimus@users.noreply.github.com> Date: Tue, 17 Feb 2026 21:55:28 +0000 Subject: [PATCH] Registry store implementation - part 1 --- crates/common/src/network/security.rs | 18 +- crates/dav/src/common/propfind.rs | 7 +- crates/dav/src/principal/propsearch.rs | 4 +- crates/email/src/message/delete.rs | 10 +- crates/http/src/auth/oauth/registration.rs | 6 +- crates/jmap/src/principal/get.rs | 5 +- crates/jmap/src/principal/query.rs | 13 +- crates/registry/src/jmap.rs | 37 ++- crates/registry/src/schema/prelude.rs | 7 +- crates/registry/src/types/error.rs | 11 +- crates/registry/src/types/index.rs | 30 +- crates/registry/src/types/mod.rs | 13 +- crates/registry/src/types/string.rs | 72 +++++ crates/store/src/dispatch/mod.rs | 1 - crates/store/src/dispatch/registry.rs | 42 --- crates/store/src/lib.rs | 17 +- crates/store/src/registry/bootstrap.rs | 52 +++- crates/store/src/registry/get.rs | 161 +++++++++++ crates/store/src/registry/mod.rs | 12 +- crates/store/src/registry/query.rs | 314 +++++++++++++++++++-- crates/store/src/registry/write.rs | 224 +++++++++++++++ crates/store/src/write/key.rs | 64 ++++- crates/store/src/write/mod.rs | 22 +- crates/trc/src/event/enums.rs | 36 +-- crates/trc/src/event/enums_impl.rs | 248 ++++++++-------- crates/utils/src/lib.rs | 55 ++++ 26 files changed, 1161 insertions(+), 320 deletions(-) create mode 100644 crates/registry/src/types/string.rs delete mode 100644 crates/store/src/dispatch/registry.rs create mode 100644 crates/store/src/registry/get.rs create mode 100644 crates/store/src/registry/write.rs diff --git a/crates/common/src/network/security.rs b/crates/common/src/network/security.rs index 4e6334e7..648a654a 100644 --- a/crates/common/src/network/security.rs +++ b/crates/common/src/network/security.rs @@ -16,10 +16,13 @@ use registry::{ prelude::Object, structs::{self, AllowedIp, BlockedIp, Rate}, }, - types::{datetime::UTCDateTime, id::ObjectId, ipmask::IpAddrOrMask}, + types::{datetime::UTCDateTime, ipmask::IpAddrOrMask}, }; use std::{fmt::Debug, net::IpAddr}; -use store::{registry::bootstrap::Bootstrap, write::now}; +use store::{ + registry::{bootstrap::Bootstrap, write::RegistryWriteResult}, + write::now, +}; use trc::AddContext; use utils::glob::{GlobPattern, MatchType}; @@ -71,7 +74,7 @@ impl Security { if !expired_allows.is_empty() { for (id, _) in &expired_allows { - if let Err(err) = bp.registry.delete(*id).await { + if let Err(err) = bp.registry.delete::(id.id()).await { trc::error!( err.details("Failed to delete expired allowed IP from registry.") .caused_by(trc::location!()) @@ -235,7 +238,7 @@ impl Server { // Write blocked IP to config let now = now() as i64; - let id = self + let RegistryWriteResult::Success(id) = self .registry() .insert(&BlockedIp { address: IpAddrOrMask::from_ip(ip), @@ -249,7 +252,10 @@ impl Server { reason, }) .await - .caused_by(trc::location!())?; + .caused_by(trc::location!())? + else { + return Ok(()); + }; // Increment version self.cluster_broadcast(BroadcastEvent::RegistryChange(RegistryChange::Insert( @@ -311,7 +317,7 @@ impl BlockedIps { if !expired_blocks.is_empty() { for (id, _) in &expired_blocks { - if let Err(err) = bp.registry.delete(*id).await { + if let Err(err) = bp.registry.delete::(id.id()).await { trc::error!( err.details("Failed to delete expired blocked IP from registry.") .caused_by(trc::location!()) diff --git a/crates/dav/src/common/propfind.rs b/crates/dav/src/common/propfind.rs index 7ebfdcd3..b3df6c0b 100644 --- a/crates/dav/src/common/propfind.rs +++ b/crates/dav/src/common/propfind.rs @@ -56,10 +56,7 @@ use groupware::{ }; use http_proto::HttpResponse; use hyper::StatusCode; -use registry::schema::{ - enums::Permission, - prelude::{Object, Property}, -}; +use registry::schema::{enums::Permission, prelude::Object}; use std::sync::Arc; use store::{ ValueKey, @@ -264,7 +261,7 @@ impl PropFindRequestHandler for Server { self.registry() .query::( RegistryQuery::new(Object::Account) - .equal_opt(Property::MemberTenantId, access_token.tenant_id()), + .with_tenant(access_token.tenant_id()), ) .await .caused_by(trc::location!())? diff --git a/crates/dav/src/principal/propsearch.rs b/crates/dav/src/principal/propsearch.rs index 1669bfa5..8a1bab39 100644 --- a/crates/dav/src/principal/propsearch.rs +++ b/crates/dav/src/principal/propsearch.rs @@ -13,7 +13,7 @@ use dav_proto::schema::{ }; use http_proto::HttpResponse; use hyper::StatusCode; -use registry::schema::prelude::{Object, Property}; +use registry::schema::prelude::Object; use store::{registry::RegistryQuery, roaring::RoaringBitmap}; use trc::AddContext; use types::collection::Collection; @@ -50,7 +50,7 @@ impl PrincipalPropSearch for Server { .registry() .query::( RegistryQuery::new(Object::Account) - .equal_opt(Property::MemberTenantId, access_token.tenant_id()) + .with_tenant(access_token.tenant_id()) .text(search_for), ) .await diff --git a/crates/email/src/message/delete.rs b/crates/email/src/message/delete.rs index 03cf010b..f0589412 100644 --- a/crates/email/src/message/delete.rs +++ b/crates/email/src/message/delete.rs @@ -9,7 +9,7 @@ use common::{KV_LOCK_PURGE_ACCOUNT, Server, storage::index::ObjectIndexBuilder}; use groupware::calendar::storage::ItipAutoExpunge; use registry::schema::prelude::Object; use std::future::Future; -use store::rand::prelude::SliceRandom; +use store::ahash::AHashSet; use store::registry::RegistryQuery; use store::write::key::DeserializeBigEndian; use store::write::{IndexPropertyClass, SearchIndex, TaskEpoch, TaskQueueClass, now}; @@ -113,14 +113,12 @@ impl EmailDeletion for Server { async fn purge_accounts(&self, use_roles: bool) { match self .registry() - .query::>(RegistryQuery::new(Object::Account)) + .query::>(RegistryQuery::new(Object::Account)) .await { - Ok(mut account_ids) => { - // Shuffle account ids - account_ids.shuffle(&mut store::rand::rng()); - + Ok(account_ids) => { for account_id in account_ids { + let account_id = account_id as u32; if !use_roles || self .core diff --git a/crates/http/src/auth/oauth/registration.rs b/crates/http/src/auth/oauth/registration.rs index 97508760..ecb5b882 100644 --- a/crates/http/src/auth/oauth/registration.rs +++ b/crates/http/src/auth/oauth/registration.rs @@ -25,6 +25,7 @@ use registry::{ types::datetime::UTCDateTime, }; use store::{ + ahash::AHashSet, rand::{Rng, distr::Alphanumeric, rng}, registry::RegistryQuery, }; @@ -122,11 +123,12 @@ impl ClientRegistrationHandler for Server { // Fetch client registration let found_registration = if let Some(client_id) = self .registry() - .query::>( + .query::>( RegistryQuery::new(Object::OAuthClient).equal(Property::ClientId, client_id), ) .await? - .first() + .iter() + .next() { if let Some(redirect_uri) = redirect_uri { let client = self diff --git a/crates/jmap/src/principal/get.rs b/crates/jmap/src/principal/get.rs index 67a57eb5..c4b90dc1 100644 --- a/crates/jmap/src/principal/get.rs +++ b/crates/jmap/src/principal/get.rs @@ -12,7 +12,7 @@ use jmap_proto::{ types::state::State, }; use jmap_tools::{Key, Map, Value}; -use registry::schema::prelude::{Object, Permission, Property}; +use registry::schema::prelude::{Object, Permission}; use std::future::Future; use store::{registry::RegistryQuery, roaring::RoaringBitmap}; use trc::AddContext; @@ -52,8 +52,7 @@ impl PrincipalGet for Server { let principal_ids = self .registry() .query::( - RegistryQuery::new(Object::Account) - .equal_opt(Property::MemberTenantId, access_token.tenant_id()), + RegistryQuery::new(Object::Account).with_tenant(access_token.tenant_id()), ) .await .caused_by(trc::location!())?; diff --git a/crates/jmap/src/principal/query.rs b/crates/jmap/src/principal/query.rs index 360b24f2..926aff9f 100644 --- a/crates/jmap/src/principal/query.rs +++ b/crates/jmap/src/principal/query.rs @@ -52,8 +52,7 @@ impl PrincipalQuery for Server { let principal_ids = self .registry() .query::( - RegistryQuery::new(Object::Account) - .equal_opt(Property::MemberTenantId, access_token.tenant_id()), + RegistryQuery::new(Object::Account).with_tenant(access_token.tenant_id()), ) .await .caused_by(trc::location!())?; @@ -88,10 +87,7 @@ impl PrincipalQuery for Server { self.registry() .query::( RegistryQuery::new(Object::Account) - .equal_opt( - Property::MemberTenantId, - access_token.tenant_id(), - ) + .with_tenant(access_token.tenant_id()) .text(text), ) .await @@ -113,10 +109,7 @@ impl PrincipalQuery for Server { .query::( RegistryQuery::new(Object::Account) .equal(Property::Type, typ.to_id()) - .equal_opt( - Property::MemberTenantId, - access_token.tenant_id(), - ), + .with_tenant(access_token.tenant_id()), ) .await .caused_by(trc::location!())?, diff --git a/crates/registry/src/jmap.rs b/crates/registry/src/jmap.rs index 0c58a3a7..461770cd 100644 --- a/crates/registry/src/jmap.rs +++ b/crates/registry/src/jmap.rs @@ -6,7 +6,11 @@ use crate::{ schema::prelude::Property, - types::{EnumType, error::PatchError}, + types::{ + EnumType, + error::PatchError, + string::{StringValidator, StringValidatorResult}, + }, }; use jmap_tools::{JsonPointer, JsonPointerItem, Key, Value}; use std::{borrow::Cow, fmt::Debug, str::FromStr}; @@ -24,6 +28,7 @@ pub enum RegistryValue { pub struct JsonPointerPatch<'x> { ptr: &'x JsonPointer, pos: usize, + validators: &'x [StringValidator], } pub trait RegistryJsonPatch: Debug + Default { @@ -51,7 +56,16 @@ pub trait RegistryJsonEnumPatch: Debug { impl<'x> JsonPointerPatch<'x> { pub fn new(ptr: &'x JsonPointer) -> Self { - Self { ptr, pos: 0 } + Self { + ptr, + pos: 0, + validators: &[], + } + } + + pub fn with_validators(mut self, validators: &'x [StringValidator]) -> Self { + self.validators = validators; + self } #[allow(clippy::should_implement_trait)] @@ -215,13 +229,22 @@ impl RegistryJsonPatch for String { value: Value<'_, Property, RegistryValue>, ) -> Result<(), PatchError> { if let Some(value) = value.into_string().filter(|v| !v.is_empty()) { - *self = value.into(); + let mut value = value.into_owned(); + + for validator in pointer.validators { + match validator.validate(&value) { + StringValidatorResult::Valid => {} + StringValidatorResult::Replace(new_value) => value = new_value, + StringValidatorResult::Invalid(err) => { + return Err(PatchError::new(pointer, err)); + } + } + } + + *self = value; pointer.assert_eof() } else { - Err(PatchError::new( - pointer, - "Invalid value for string property (expected non-empty string)", - )) + Err(PatchError::new(pointer, "Invalid value for property.")) } } } diff --git a/crates/registry/src/schema/prelude.rs b/crates/registry/src/schema/prelude.rs index 6ac05ea2..64c3a8b3 100644 --- a/crates/registry/src/schema/prelude.rs +++ b/crates/registry/src/schema/prelude.rs @@ -12,7 +12,6 @@ pub use crate::schema::enums::*; pub use crate::schema::properties::*; pub use crate::schema::structs::*; pub use crate::types::EnumType; -pub use crate::types::ObjectIndex; pub use crate::types::ObjectType; pub use crate::types::datetime::UTCDateTime; pub use crate::types::duration::Duration; @@ -21,6 +20,7 @@ pub use crate::types::index::IndexBuilder; pub use crate::types::ipaddr::IpAddr; pub use crate::types::ipmask::IpAddrOrMask; pub use crate::types::socketaddr::SocketAddr; +pub use crate::types::string::StringValidator; pub use serde::{Deserialize, Serialize}; pub use std::str::FromStr; pub use types::id::Id; @@ -34,3 +34,8 @@ pub struct ExpressionContext<'x> { pub allowed_variables: &'static [ExpressionVariable], pub allowed_constants: &'static [ExpressionConstant], } + +pub const OBJ_SINGLETON: u64 = 1; +pub const OBJ_SEQ_ID: u64 = 1 << 1; +pub const OBJ_FILTER_ACCOUNT: u64 = 1 << 2; +pub const OBJ_FILTER_TENANT: u64 = 1 << 3; diff --git a/crates/registry/src/types/error.rs b/crates/registry/src/types/error.rs index 4fce25de..f98969eb 100644 --- a/crates/registry/src/types/error.rs +++ b/crates/registry/src/types/error.rs @@ -4,11 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{ - jmap::JsonPointerPatch, - schema::prelude::{Object, Property}, - types::id::ObjectId, -}; +use crate::{jmap::JsonPointerPatch, schema::prelude::Property, types::id::ObjectId}; use std::{borrow::Cow, fmt::Display}; #[derive(Debug, Clone, PartialEq, Eq)] @@ -35,11 +31,6 @@ pub enum Error { object_id: Option, error: trc::Error, }, - TypeMismatch { - object_id: ObjectId, - object_type: Object, - expected_type: Object, - }, NotFound { object_id: ObjectId, }, diff --git a/crates/registry/src/types/index.rs b/crates/registry/src/types/index.rs index 6246b4aa..16909db8 100644 --- a/crates/registry/src/types/index.rs +++ b/crates/registry/src/types/index.rs @@ -6,7 +6,7 @@ use crate::{ schema::prelude::{Object, Property}, - types::ipmask::IpAddrOrMask, + types::{id::ObjectId, ipmask::IpAddrOrMask}, }; use ahash::AHashSet; use std::borrow::Cow; @@ -22,21 +22,14 @@ pub enum IndexKey<'x> { property: Property, value: IndexValue<'x>, }, - TextSearch { - property: Property, - value: IndexValue<'x>, - }, Global { property: Property, value_1: IndexValue<'x>, value_2: IndexValue<'x>, }, ForeignKey { - property: Property, - object: Object, - id: u64, + object_id: ObjectId, type_filter: IndexValue<'x>, - tenant_filter: bool, }, } @@ -61,7 +54,6 @@ pub enum IndexValue<'x> { pub struct IndexBuilder<'x> { pub object: Option, - pub tenant_id: Option, pub keys: AHashSet>, } @@ -102,12 +94,12 @@ impl<'x> IndexBuilder<'x> { .chars() .all(|ch| ch.is_lowercase() || !ch.is_alphabetic()) { - self.keys.insert(IndexKey::TextSearch { + self.keys.insert(IndexKey::Search { property, value: IndexValue::Text(Cow::Borrowed(word)), }); } else { - self.keys.insert(IndexKey::TextSearch { + self.keys.insert(IndexKey::Search { property, value: IndexValue::Text(Cow::Owned(word.to_lowercase())), }); @@ -136,21 +128,11 @@ impl<'x> IndexBuilder<'x> { }); } - pub fn foreign_key( - &mut self, - property: Property, - object: Object, - id: Option, - type_filter: Option, - tenant_filter: bool, - ) { + pub fn foreign_key(&mut self, object: Object, id: Option, type_filter: Option) { if let Some(id) = id { self.keys.insert(IndexKey::ForeignKey { - property, - object, - id: id.id(), + object_id: ObjectId::new(object, id.id()), type_filter: type_filter.map(IndexValue::U16).unwrap_or(IndexValue::None), - tenant_filter, }); } } diff --git a/crates/registry/src/types/mod.rs b/crates/registry/src/types/mod.rs index 274fc696..0f68737d 100644 --- a/crates/registry/src/types/mod.rs +++ b/crates/registry/src/types/mod.rs @@ -9,6 +9,7 @@ use crate::{ schema::prelude::Object, types::{error::ValidationError, index::IndexBuilder}, }; +use serde::{Serialize, de::DeserializeOwned}; use std::fmt::Debug; pub mod datetime; @@ -19,6 +20,7 @@ pub mod index; pub mod ipaddr; pub mod ipmask; pub mod socketaddr; +pub mod string; pub trait EnumType: Sized + Debug + PartialEq + Eq { const COUNT: usize; @@ -29,11 +31,12 @@ pub trait EnumType: Sized + Debug + PartialEq + Eq { fn to_id(&self) -> u16; } -pub trait ObjectType: Pickle + Default + Clone + Send + Sync { +pub trait ObjectType: + Pickle + Serialize + DeserializeOwned + Default + Clone + Send + Sync +{ + const FLAGS: u64; + fn object() -> Object; fn validate(&self, errors: &mut Vec) -> bool; -} - -pub trait ObjectIndex<'x>: Send + Sync { - fn index(&'x self, builder: &mut IndexBuilder<'x>); + fn index<'x>(&'x self, builder: &mut IndexBuilder<'x>); } diff --git a/crates/registry/src/types/string.rs b/crates/registry/src/types/string.rs new file mode 100644 index 00000000..d9af552e --- /dev/null +++ b/crates/registry/src/types/string.rs @@ -0,0 +1,72 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::{net::IpAddr, str::FromStr}; +use utils::{sanitize_domain, sanitize_email, sanitize_email_local}; + +#[derive(Debug, Clone)] +pub enum StringValidator { + Email, + EmailLocalPart, + Domain, + Hostname, + RemoveSpaces, + Lowercase, + Uppercase, + Trim, +} + +pub enum StringValidatorResult { + Valid, + Replace(String), + Invalid(&'static str), +} + +impl StringValidator { + pub fn validate(&self, value: &str) -> StringValidatorResult { + match self { + Self::Email => sanitize_email(value) + .map(StringValidatorResult::Replace) + .unwrap_or(StringValidatorResult::Invalid("Invalid email address")), + Self::EmailLocalPart => sanitize_email_local(value) + .map(StringValidatorResult::Replace) + .unwrap_or(StringValidatorResult::Invalid("Invalid email local part")), + Self::Domain => sanitize_domain(value) + .map(StringValidatorResult::Replace) + .unwrap_or(StringValidatorResult::Invalid("Invalid domain name")), + Self::Hostname => IpAddr::from_str(value) + .ok() + .map(|_| StringValidatorResult::Valid) + .or_else(|| sanitize_domain(value).map(StringValidatorResult::Replace)) + .unwrap_or(StringValidatorResult::Invalid( + "Invalid hostname or IP address", + )), + Self::RemoveSpaces => { + if value.chars().any(|c| c.is_whitespace()) { + StringValidatorResult::Replace( + value.chars().filter(|c| !c.is_whitespace()).collect(), + ) + } else { + StringValidatorResult::Valid + } + } + Self::Lowercase => StringValidatorResult::Replace(value.to_lowercase()), + Self::Uppercase => StringValidatorResult::Replace(value.to_uppercase()), + Self::Trim => { + let trimmed = value.trim(); + if trimmed.len() != value.len() { + if !trimmed.is_empty() { + StringValidatorResult::Replace(trimmed.to_string()) + } else { + StringValidatorResult::Invalid("String cannot be empty") + } + } else { + StringValidatorResult::Valid + } + } + } + } +} diff --git a/crates/store/src/dispatch/mod.rs b/crates/store/src/dispatch/mod.rs index 18558640..17228ca2 100644 --- a/crates/store/src/dispatch/mod.rs +++ b/crates/store/src/dispatch/mod.rs @@ -9,7 +9,6 @@ use roaring::RoaringBitmap; pub mod blob; pub mod lookup; -pub mod registry; pub mod search; pub mod store; diff --git a/crates/store/src/dispatch/registry.rs b/crates/store/src/dispatch/registry.rs deleted file mode 100644 index 8be62a65..00000000 --- a/crates/store/src/dispatch/registry.rs +++ /dev/null @@ -1,42 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use crate::{RegistryStore, registry::RegistryObject}; -use registry::{ - schema::prelude::Object, - types::{ObjectType, id::ObjectId}, -}; -use types::id::Id; - -impl RegistryStore { - pub async fn object(&self, id: impl Into) -> trc::Result> { - todo!() - } - - pub async fn singleton(&self) -> trc::Result> { - todo!() - } - - pub async fn insert(&self, object: &T) -> trc::Result { - todo!() - } - - pub async fn update(&self, id: Id, object: &T) -> trc::Result { - todo!() - } - - pub async fn list(&self) -> trc::Result>> { - todo!() - } - - pub async fn delete(&self, id: ObjectId) -> trc::Result<()> { - todo!() - } - - pub async fn count(&self, typ: Object) -> trc::Result { - todo!() - } -} diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index 5e07bcc4..faa7457f 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -12,16 +12,20 @@ pub mod registry; pub mod search; pub mod write; -use ::registry::schema::enums::CompressionAlgo; +use ::registry::{ + schema::{enums::CompressionAlgo, prelude::Object}, + types::id::ObjectId, +}; pub use ahash; pub use blake3; pub use parking_lot; pub use rand; pub use rkyv; pub use roaring; +use types::id::Id; pub use xxhash_rust; -use ahash::AHashMap; +use ahash::{AHashMap, AHashSet}; use backend::{fs::FsStore, http::HttpStore, memory::StaticMemoryStore}; use std::{borrow::Cow, path::PathBuf, sync::Arc}; use write::ValueClass; @@ -190,9 +194,12 @@ pub enum InMemoryStore { } #[derive(Clone)] -pub enum RegistryStore { - Remote(Store), - Local(PathBuf), +pub struct RegistryStore(pub(crate) Arc); + +pub struct RegistryStoreInner { + pub(crate) local_path: PathBuf, + pub(crate) local_objects: AHashMap>, + pub(crate) store: Store, } #[cfg(feature = "sqlite")] diff --git a/crates/store/src/registry/bootstrap.rs b/crates/store/src/registry/bootstrap.rs index 3ad03014..c700ab47 100644 --- a/crates/store/src/registry/bootstrap.rs +++ b/crates/store/src/registry/bootstrap.rs @@ -11,7 +11,7 @@ use registry::{ structs::{LocalSettings, Node}, }, types::{ - ObjectType, + EnumType, ObjectType, error::{Error, ValidationError, Warning}, id::ObjectId, }, @@ -170,10 +170,56 @@ impl Bootstrap { } pub fn log_errors(&self) { - let todo = "implement"; + for error in &self.errors { + match error { + Error::Validation { object_id, errors } => { + trc::event!( + Registry(trc::RegistryEvent::ValidationError), + Source = object_id.object().as_str(), + Id = object_id.id(), + Reason = errors + .iter() + .map(|err| trc::Value::from(err.to_string())) + .collect::>(), + ); + } + Error::Build { object_id, message } => { + trc::event!( + Registry(trc::RegistryEvent::BuildError), + Source = object_id.object().as_str(), + Id = object_id.id(), + Reason = message.clone(), + ); + } + Error::Internal { object_id, error } => { + trc::event!( + Registry(trc::RegistryEvent::ReadError), + Source = object_id.as_ref().map(|id| id.object().as_str()), + Id = object_id.as_ref().map(|id| id.id()), + CausedBy = error.clone(), + ); + } + Error::NotFound { object_id } => { + trc::event!( + Registry(trc::RegistryEvent::BuildError), + Source = object_id.object().as_str(), + Id = object_id.id(), + Reason = "Object not found", + ); + } + } + } } pub fn log_warnings(&self) { - let todo = "implement"; + for warning in &self.warnings { + trc::event!( + Registry(trc::RegistryEvent::BuildWarning), + Source = warning.object_id.object().as_str(), + Id = warning.object_id.id(), + Key = warning.property.map(|key| key.as_str()), + Reason = warning.message.clone(), + ); + } } } diff --git a/crates/store/src/registry/get.rs b/crates/store/src/registry/get.rs new file mode 100644 index 00000000..a3a158d4 --- /dev/null +++ b/crates/store/src/registry/get.rs @@ -0,0 +1,161 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + Deserialize, IterateParams, RegistryStore, SUBSPACE_REGISTRY, U16_LEN, U64_LEN, ValueKey, + registry::{RegistryObject, RegistryQuery}, + write::{AnyClass, RegistryClass, ValueClass, key::KeySerializer}, +}; +use registry::{ + pickle::PickledStream, + schema::prelude::Object, + types::{EnumType, ObjectType, id::ObjectId}, +}; +use roaring::RoaringBitmap; +use trc::AddContext; +use utils::codec::leb128::Leb128Reader; + +impl RegistryStore { + pub async fn object(&self, id: impl Into) -> trc::Result> { + let id = id.into(); + let object = T::object(); + + if let Some(objects) = self.0.local_objects.get(&object) { + let Some(item) = objects.get(&id) else { + return Ok(None); + }; + serde_json::from_value::(item.clone()) + .map(Some) + .map_err(|err| { + trc::EventType::Registry(trc::RegistryEvent::LocalParseError) + .into_err() + .caused_by(trc::location!()) + .id(id) + .details(object.as_str()) + .reason(err) + }) + } else { + let Some(bytes) = self + .0 + .store + .get_value::(ValueKey::from(ValueClass::Registry( + RegistryClass::Item(ObjectId::new(object, id)), + ))) + .await? + else { + return Ok(None); + }; + T::unpickle(&mut PickledStream::new(&bytes.0)) + .ok_or_else(|| { + trc::EventType::Registry(trc::RegistryEvent::DeserializationError) + .into_err() + .caused_by(trc::location!()) + .id(id) + .details(object.as_str()) + .ctx(trc::Key::Value, bytes.0) + }) + .map(Some) + } + } + + pub async fn list(&self) -> trc::Result>> { + let object = T::object(); + + if let Some(objects) = self.0.local_objects.get(&object) { + let mut results = Vec::with_capacity(objects.len()); + + for (id, item) in objects { + let item = serde_json::from_value::(item.clone()).map_err(|err| { + trc::EventType::Registry(trc::RegistryEvent::LocalParseError) + .into_err() + .caused_by(trc::location!()) + .id(*id) + .details(object.as_str()) + .reason(err) + })?; + results.push(RegistryObject { + id: ObjectId::new(object, *id), + object: item, + }); + } + + Ok(results) + } else { + let mut results = Vec::new(); + self.0 + .store + .iterate( + IterateParams::new( + ValueKey::from(ValueClass::Any(AnyClass { + subspace: SUBSPACE_REGISTRY, + key: KeySerializer::new(U16_LEN + 1) + .write(0u8) + .write(object.to_id()) + .finalize(), + })), + ValueKey::from(ValueClass::Any(AnyClass { + subspace: SUBSPACE_REGISTRY, + key: KeySerializer::new(U16_LEN + U64_LEN + 1) + .write(0u8) + .write(object.to_id()) + .write(u64::MAX) + .finalize(), + })), + ), + |key, value| { + let id = key + .get(U16_LEN + 1..) + .and_then(|key| key.read_leb128::()) + .map(|r| r.0) + .ok_or_else(|| { + trc::EventType::Registry(trc::RegistryEvent::DeserializationError) + .into_err() + .caused_by(trc::location!()) + .details(object.as_str()) + .ctx(trc::Key::Key, key) + })?; + let item = + T::unpickle(&mut PickledStream::new(value)).ok_or_else(|| { + trc::EventType::Registry(trc::RegistryEvent::DeserializationError) + .into_err() + .caused_by(trc::location!()) + .id(id) + .details(object.as_str()) + .ctx(trc::Key::Value, value) + })?; + results.push(RegistryObject { + id: ObjectId::new(object, id), + object: item, + }); + + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; + + Ok(results) + } + } + + pub async fn count(&self, object: Object) -> trc::Result { + if let Some(objects) = self.0.local_objects.get(&object) { + Ok(objects.len() as u64) + } else { + self.query::(RegistryQuery::new(object)) + .await + .map(|r| r.len()) + } + } +} + +struct PickledBytes(Vec); + +impl Deserialize for PickledBytes { + fn deserialize(bytes: &[u8]) -> trc::Result { + Ok(Self(bytes.to_vec())) + } +} diff --git a/crates/store/src/registry/mod.rs b/crates/store/src/registry/mod.rs index c533ee3a..fdb7a43a 100644 --- a/crates/store/src/registry/mod.rs +++ b/crates/store/src/registry/mod.rs @@ -5,7 +5,9 @@ */ pub mod bootstrap; +pub mod get; pub mod query; +pub mod write; use registry::{ schema::prelude::{Object, Property}, @@ -20,6 +22,8 @@ pub struct RegistryObject { pub struct RegistryQuery { pub object_type: Object, pub filters: Vec, + pub account_id: Option, + pub tenant_id: Option, } pub struct RegistryFilter { @@ -28,13 +32,13 @@ pub struct RegistryFilter { pub value: RegistryFilterValue, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum RegistryFilterOp { Equal, - NotEqual, GreaterThan, - LessThan, - GreaterThanOrEqual, - LessThanOrEqual, + GreaterEqualThan, + LowerThan, + LowerEqualThan, TextMatch, } diff --git a/crates/store/src/registry/query.rs b/crates/store/src/registry/query.rs index 4ff3940d..8c261a63 100644 --- a/crates/store/src/registry/query.rs +++ b/crates/store/src/registry/query.rs @@ -5,34 +5,171 @@ */ use crate::{ - RegistryStore, + IterateParams, RegistryStore, SUBSPACE_REGISTRY, Store, U16_LEN, U64_LEN, ValueKey, registry::{RegistryFilter, RegistryFilterOp, RegistryFilterValue, RegistryQuery}, + write::{ + AnyClass, RegistryClass, ValueClass, + key::{DeserializeBigEndian, KeySerializer}, + }, }; +use ahash::AHashSet; use registry::{ - schema::prelude::{Object, Property}, - types::EnumType, + schema::prelude::{OBJ_FILTER_ACCOUNT, OBJ_FILTER_TENANT, OBJ_SINGLETON, Object, Property}, + types::{EnumType, id::ObjectId}, }; use roaring::RoaringBitmap; +use std::{borrow::Cow, ops::BitAndAssign}; +use trc::AddContext; impl RegistryStore { pub async fn query(&self, query: RegistryQuery) -> trc::Result { - todo!() + let flags = query.object_type.flags(); + if flags & OBJ_SINGLETON != 0 { + return Err(trc::EventType::Registry(trc::RegistryEvent::NotSupported) + .into_err() + .details("Singletons do not support searching")); + } else if let Some(objects) = self.0.local_objects.get(&query.object_type) { + if !query.filters.is_empty() { + trc::event!( + Registry(trc::RegistryEvent::NotSupported), + Details = "Filtering is not supported for local registry" + ); + } + let mut results = T::default(); + for id in objects.keys() { + results.push(*id); + } + return Ok(results); + } + let mut results = if (flags & OBJ_FILTER_ACCOUNT != 0) + && let Some(account_id) = query.account_id + { + range_to_set::( + &self.0.store, + query.object_type, + Property::AccountId.to_id(), + &account_id.to_be_bytes(), + RegistryFilterOp::Equal, + ) + .await? + } else if (flags & OBJ_FILTER_TENANT != 0) + && let Some(tenant_id) = query.tenant_id + { + range_to_set::( + &self.0.store, + query.object_type, + Property::MemberTenantId.to_id(), + &tenant_id.to_be_bytes(), + RegistryFilterOp::Equal, + ) + .await? + } else { + all_ids::(&self.0.store, query.object_type).await? + }; + + if !results.has_items() { + return Ok(results); + } + + let mut u64_buffer; + let mut u16_buffer; + let mut bool_buffer = [0u8; 1]; + + for filter in query.filters { + if filter.op == RegistryFilterOp::TextMatch { + if let RegistryFilterValue::String(text) = filter.value { + let mut matches = T::default(); + for word in text + .split(|c: char| !c.is_alphanumeric()) + .filter(|s| s.len() > 1) + { + let word = if word + .chars() + .all(|ch| ch.is_lowercase() || !ch.is_alphabetic()) + { + Cow::Borrowed(word) + } else { + Cow::Owned(word.to_lowercase()) + }; + + let result = range_to_set( + &self.0.store, + query.object_type, + filter.property.to_id(), + word.as_bytes(), + RegistryFilterOp::Equal, + ) + .await?; + + if !matches.has_items() { + matches = result; + } else { + matches.intersect(&result); + if !matches.has_items() { + break; + } + } + } + + results.intersect(&matches); + } else { + return Err(trc::EventType::Registry(trc::RegistryEvent::NotSupported) + .into_err() + .details("TextMatch operator only supports string values")); + } + } else { + let result = range_to_set( + &self.0.store, + query.object_type, + filter.property.to_id(), + match &filter.value { + RegistryFilterValue::String(v) => v.as_bytes(), + RegistryFilterValue::U64(v) => { + u64_buffer = v.to_be_bytes(); + &u64_buffer + } + RegistryFilterValue::U16(v) => { + u16_buffer = v.to_be_bytes(); + &u16_buffer + } + RegistryFilterValue::Boolean(v) => { + bool_buffer[0] = *v as u8; + &bool_buffer + } + }, + filter.op, + ) + .await?; + + results.intersect(&result); + } + + if !results.has_items() { + return Ok(results); + } + } + + Ok(results) } } -pub trait RegistryQueryResults: Default { +pub trait RegistryQueryResults: Default + Sized + Sync + Send { fn push(&mut self, id: u64); + fn has_items(&self) -> bool; + fn intersect(&mut self, other: &Self); } -impl RegistryQueryResults for Vec { +impl RegistryQueryResults for AHashSet { fn push(&mut self, id: u64) { - self.push(id); + self.insert(id); } -} -impl RegistryQueryResults for Vec { - fn push(&mut self, id: u64) { - self.push(id as u32); + fn has_items(&self) -> bool { + !self.is_empty() + } + + fn intersect(&mut self, other: &Self) { + self.retain(|id| other.contains(id)); } } @@ -40,6 +177,14 @@ impl RegistryQueryResults for RoaringBitmap { fn push(&mut self, id: u64) { self.insert(id as u32); } + + fn has_items(&self) -> bool { + !self.is_empty() + } + + fn intersect(&mut self, other: &Self) { + self.bitand_assign(other); + } } impl RegistryQuery { @@ -47,9 +192,21 @@ impl RegistryQuery { Self { object_type, filters: Vec::new(), + account_id: None, + tenant_id: None, } } + pub fn with_account(mut self, account_id: u32) -> Self { + self.account_id = Some(account_id); + self + } + + pub fn with_tenant(mut self, tenant_id: Option) -> Self { + self.tenant_id = tenant_id; + self + } + pub fn equal(mut self, property: Property, value: impl Into) -> Self { self.filters.push(RegistryFilter::equal(property, value)); self @@ -66,12 +223,6 @@ impl RegistryQuery { self } - pub fn not_equal(mut self, property: Property, value: impl Into) -> Self { - self.filters - .push(RegistryFilter::not_equal(property, value)); - self - } - pub fn greater_than( mut self, property: Property, @@ -138,14 +289,6 @@ impl RegistryFilter { } } - pub fn not_equal(property: Property, value: impl Into) -> Self { - Self { - property, - op: RegistryFilterOp::NotEqual, - value: value.into(), - } - } - pub fn greater_than(property: Property, value: impl Into) -> Self { Self { property, @@ -157,7 +300,7 @@ impl RegistryFilter { pub fn less_than(property: Property, value: impl Into) -> Self { Self { property, - op: RegistryFilterOp::LessThan, + op: RegistryFilterOp::LowerThan, value: value.into(), } } @@ -168,7 +311,7 @@ impl RegistryFilter { ) -> Self { Self { property, - op: RegistryFilterOp::GreaterThanOrEqual, + op: RegistryFilterOp::GreaterEqualThan, value: value.into(), } } @@ -176,7 +319,7 @@ impl RegistryFilter { pub fn less_than_or_equal(property: Property, value: impl Into) -> Self { Self { property, - op: RegistryFilterOp::LessThanOrEqual, + op: RegistryFilterOp::LowerEqualThan, value: value.into(), } } @@ -211,3 +354,118 @@ impl From for RegistryFilterValue { RegistryFilterValue::U16(value) } } + +async fn all_ids(store: &Store, object: Object) -> trc::Result { + let mut bm = T::default(); + store + .iterate( + IterateParams::new( + ValueKey::from(ValueClass::Registry(RegistryClass::Id { + item_id: ObjectId::new(object, 0u64), + })), + ValueKey::from(ValueClass::Registry(RegistryClass::Id { + item_id: ObjectId::new(object, u64::MAX), + })), + ) + .no_values() + .ascending(), + |key, _| { + bm.push(key.deserialize_be_u64(key.len() - U64_LEN)?); + + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; + Ok(bm) +} + +async fn range_to_set( + store: &Store, + object: Object, + index_id: u16, + match_value: &[u8], + op: RegistryFilterOp, +) -> trc::Result { + let object_id = object.to_id(); + let ((from_value, from_doc_id, from_field), (end_value, end_doc_id, end_field)) = match op { + RegistryFilterOp::LowerThan => ((&[][..], 0, object_id), (match_value, 0, object_id)), + RegistryFilterOp::LowerEqualThan => { + ((&[][..], 0, object_id), (match_value, u64::MAX, object_id)) + } + RegistryFilterOp::GreaterThan => ( + (match_value, u64::MAX, object_id), + (&[][..], u64::MAX, object_id + 1), + ), + RegistryFilterOp::GreaterEqualThan => ( + (match_value, 0, object_id), + (&[][..], u64::MAX, object_id + 1), + ), + RegistryFilterOp::Equal | RegistryFilterOp::TextMatch => ( + (match_value, 0, object_id), + (match_value, u64::MAX, object_id), + ), + }; + + let begin = ValueKey::from(ValueClass::Any(AnyClass { + subspace: SUBSPACE_REGISTRY, + key: KeySerializer::new((U16_LEN * 2) + U64_LEN + 1 + from_value.len()) + .write(2u8) + .write(object_id) + .write(from_field) + .write(from_value) + .write(from_doc_id) + .finalize(), + })); + let end = ValueKey::from(ValueClass::Any(AnyClass { + subspace: SUBSPACE_REGISTRY, + key: KeySerializer::new((U16_LEN * 2) + U64_LEN + 1 + end_value.len()) + .write(2u8) + .write(object_id) + .write(end_field) + .write(end_value) + .write(end_doc_id) + .finalize(), + })); + + let mut bm = T::default(); + let prefix = KeySerializer::new((U16_LEN * 2) + 1) + .write(2u8) + .write(object_id) + .write(index_id) + .finalize(); + let prefix_len = prefix.len(); + + store + .iterate( + IterateParams::new(begin, end).no_values().ascending(), + |key, _| { + if !key.starts_with(&prefix) { + return Ok(false); + } + + let id_pos = key.len() - U64_LEN; + let value = key + .get(prefix_len..id_pos) + .ok_or_else(|| trc::Error::corrupted_key(key, None, trc::location!()))?; + + let matches = match op { + RegistryFilterOp::LowerThan => value < match_value, + RegistryFilterOp::LowerEqualThan => value <= match_value, + RegistryFilterOp::GreaterThan => value > match_value, + RegistryFilterOp::GreaterEqualThan => value >= match_value, + RegistryFilterOp::Equal | RegistryFilterOp::TextMatch => value == match_value, + }; + + if matches { + bm.push(key.deserialize_be_u64(id_pos)?); + } + + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; + + Ok(bm) +} diff --git a/crates/store/src/registry/write.rs b/crates/store/src/registry/write.rs new file mode 100644 index 00000000..85dba827 --- /dev/null +++ b/crates/store/src/registry/write.rs @@ -0,0 +1,224 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{ + IterateParams, RegistryStore, SUBSPACE_REGISTRY, SerializeInfallible, U16_LEN, U64_LEN, + ValueKey, + write::{ + AnyClass, BatchBuilder, RegistryClass, ValueClass, + key::{DeserializeBigEndian, KeySerializer}, + }, +}; +use registry::{ + schema::prelude::{OBJ_SEQ_ID, Object}, + types::{ + EnumType, ObjectType, + error::Error, + id::ObjectId, + index::{IndexBuilder, IndexKey, IndexValue}, + }, +}; +use trc::AddContext; +use types::id::Id; +use utils::codec::leb128::Leb128Reader; + +pub enum RegistryWriteResult { + Success(T), + CannotDelete { + object_id: ObjectId, + linked_objects: Vec, + }, + NotFound { + object_id: ObjectId, + }, +} + +impl RegistryStore { + pub async fn insert(&self, object: &T) -> trc::Result> { + todo!() + } + + pub async fn update( + &self, + id: Id, + object: &T, + ) -> trc::Result> { + todo!() + } + + pub async fn delete(&self, id: u64) -> trc::Result> { + let object_type = T::object(); + let object_id = ObjectId::new(object_type, id); + + let todo = "local registry"; + + // Validate relationships + let mut linked = Vec::new(); + let key = KeySerializer::new(U64_LEN + U16_LEN + 1) + .write(1u8) + .write(object_type.to_id()) + .write(id) + .finalize(); + let prefix_len = key.len(); + let from_key = ValueKey::from(ValueClass::Any(AnyClass { + subspace: SUBSPACE_REGISTRY, + key, + })); + let key = KeySerializer::new((U64_LEN * 2) + U16_LEN + 1) + .write(1u8) + .write(object_type.to_id()) + .write(id) + .write(u64::MAX) + .finalize(); + let to_key = ValueKey::from(ValueClass::Any(AnyClass { + subspace: SUBSPACE_REGISTRY, + key, + })); + self.0 + .store + .iterate( + IterateParams::new(from_key, to_key).no_values().ascending(), + |key, _| { + let object = + Object::from_id(key.deserialize_be_u16(prefix_len)?).ok_or_else(|| { + trc::EventType::Registry(trc::RegistryEvent::DeserializationError) + .into_err() + .caused_by(trc::location!()) + .ctx(trc::Key::Key, key) + })?; + let id = key + .get(prefix_len + U16_LEN..) + .and_then(|key| key.read_leb128::()) + .map(|r| r.0) + .ok_or_else(|| { + trc::EventType::Registry(trc::RegistryEvent::DeserializationError) + .into_err() + .caused_by(trc::location!()) + .details(object.as_str()) + .ctx(trc::Key::Key, key) + })?; + linked.push(ObjectId::new(object, id)); + + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; + + if !linked.is_empty() { + return Ok(RegistryWriteResult::CannotDelete { + object_id: ObjectId::new(object_type, id), + linked_objects: linked, + }); + } + + let Some(object) = self.object::(id).await? else { + return Ok(RegistryWriteResult::NotFound { + object_id: ObjectId::new(object_type, id), + }); + }; + + // Build deletion batch + let mut batch = BatchBuilder::new(); + batch.clear(ValueClass::Registry(RegistryClass::Item(object_id))); + if object_type.flags() & OBJ_SEQ_ID != 0 { + batch.clear(ValueClass::Registry(RegistryClass::Id { + item_id: object_id, + })); + } + let mut index = IndexBuilder::default(); + object.index(&mut index); + batch.registry_index(object_id, index.keys.iter(), false); + + self.0 + .store + .write(batch.build_all()) + .await + .map(|_| RegistryWriteResult::Success(())) + .caused_by(trc::location!()) + } +} + +impl RegistryClass { + fn from_index_key(key: &IndexKey<'_>, item_id: ObjectId) -> Self { + match key { + IndexKey::Unique { property, value } => RegistryClass::Index { + index_id: property.to_id(), + item_id, + key: value.serialize(), + }, + IndexKey::Search { property, value } => RegistryClass::Index { + index_id: property.to_id(), + item_id, + key: value.serialize(), + }, + IndexKey::Global { + property, + value_1, + value_2, + } => RegistryClass::IndexGlobal { + index_id: property.to_id(), + item_id, + key: serialize_composite_key(value_1, value_2), + }, + IndexKey::ForeignKey { object_id, .. } => RegistryClass::Reference { + to: *object_id, + from: item_id, + }, + } + } +} + +impl BatchBuilder { + fn registry_index<'x>( + &mut self, + item_id: ObjectId, + index_keys: impl Iterator>, + is_set: bool, + ) { + for key in index_keys { + if is_set { + self.set( + ValueClass::Registry(RegistryClass::from_index_key(key, item_id)), + vec![], + ); + } else { + self.clear(ValueClass::Registry(RegistryClass::from_index_key( + key, item_id, + ))); + } + } + } +} + +fn serialize_composite_key(value_1: &IndexValue<'_>, value_2: &IndexValue<'_>) -> Vec { + let mut key = value_1.serialize(); + + match value_2 { + IndexValue::Text(text) => key.extend_from_slice(text.as_bytes()), + IndexValue::Bytes(bytes) => key.extend_from_slice(bytes), + IndexValue::U64(num) => key.extend_from_slice(&num.to_be_bytes()), + IndexValue::I64(num) => key.extend_from_slice(&num.to_be_bytes()), + IndexValue::U32(num) => key.extend_from_slice(&num.to_be_bytes()), + IndexValue::U16(num) => key.extend_from_slice(&num.to_be_bytes()), + IndexValue::None => {} + } + key +} + +impl SerializeInfallible for IndexValue<'_> { + fn serialize(&self) -> Vec { + match self { + IndexValue::Text(text) => text.as_bytes().to_vec(), + IndexValue::Bytes(bytes) => bytes.clone(), + IndexValue::U64(num) => num.to_be_bytes().to_vec(), + IndexValue::I64(num) => num.to_be_bytes().to_vec(), + IndexValue::U32(num) => num.to_be_bytes().to_vec(), + IndexValue::U16(num) => num.to_be_bytes().to_vec(), + IndexValue::None => vec![], + } + } +} diff --git a/crates/store/src/write/key.rs b/crates/store/src/write/key.rs index 1c380318..bfab1f4f 100644 --- a/crates/store/src/write/key.rs +++ b/crates/store/src/write/key.rs @@ -19,6 +19,7 @@ use crate::{ BlobLink, IndexPropertyClass, RegistryClass, SearchIndex, SearchIndexId, SearchIndexType, }, }; +use registry::types::EnumType; use std::convert::TryInto; use types::{ blob_hash::BLOB_HASH_LEN, @@ -373,9 +374,43 @@ impl ValueClass { InMemoryClass::Key(key) => serializer.write(key.as_slice()), InMemoryClass::Counter(key) => serializer.write(key.as_slice()), }, - ValueClass::Registry(registry) => { - todo!() - } + ValueClass::Registry(registry) => match registry { + RegistryClass::Item(object_id) => serializer + .write(0u8) + .write(object_id.object().to_id()) + .write_leb128(object_id.id()), + RegistryClass::Reference { to, from } => serializer + .write(1u8) + .write(to.object().to_id()) + .write(to.id()) + .write(from.object().to_id()) + .write_leb128(from.id()), + RegistryClass::Index { + index_id, + item_id, + key, + } => serializer + .write(2u8) + .write(item_id.object().to_id()) + .write(*index_id) + .write(key.as_slice()) + .write(item_id.id()), + RegistryClass::IndexGlobal { + index_id, + item_id, + key, + } => serializer + .write(3u8) + .write(*index_id) + .write(key.as_slice()) + .write(item_id.object().to_id()) + .write(item_id.id()), + RegistryClass::Id { item_id } => serializer + .write(4u8) + .write(item_id.object().to_id()) + .write(item_id.id()), + RegistryClass::IdCounter { object } => serializer.write(object.to_id()), + }, ValueClass::Queue(queue) => match queue { QueueClass::Message(queue_id) => serializer.write(*queue_id), QueueClass::MessageEvent(event) => serializer @@ -562,10 +597,15 @@ impl ValueClass { }, ValueClass::Acl(_) => U32_LEN * 3 + 2, ValueClass::InMemory(InMemoryClass::Counter(v) | InMemoryClass::Key(v)) => v.len(), - ValueClass::Registry(registry) => { - let todo = "implement"; - todo!() - } + ValueClass::Registry(registry) => match registry { + RegistryClass::Item(_) => U16_LEN + U64_LEN + 2, + RegistryClass::Reference { .. } => ((U16_LEN + U64_LEN) * 2) + 2, + RegistryClass::Index { key, .. } | RegistryClass::IndexGlobal { key, .. } => { + (U16_LEN * 2) + U64_LEN + key.len() + 2 + } + RegistryClass::Id { .. } => U16_LEN + U64_LEN + 1, + RegistryClass::IdCounter { .. } => U16_LEN + 1, + }, ValueClass::Blob(op) => match op { BlobOp::Commit { .. } => BLOB_HASH_LEN, BlobOp::Link { to, .. } => { @@ -631,7 +671,7 @@ impl ValueClass { match self { ValueClass::Property(field) => { - if (collection == MAILBOX_COLLECTION && *field == MAILBOX_COUNTER_FIELD) { + if collection == MAILBOX_COLLECTION && *field == MAILBOX_COUNTER_FIELD { SUBSPACE_COUNTER } else { SUBSPACE_PROPERTY @@ -646,7 +686,13 @@ impl ValueClass { SUBSPACE_BLOB_EXTRA } }, - ValueClass::Registry(_) => SUBSPACE_REGISTRY, + ValueClass::Registry(registry) => { + if matches!(registry, RegistryClass::IdCounter { .. }) { + SUBSPACE_COUNTER + } else { + SUBSPACE_REGISTRY + } + } ValueClass::InMemory(lookup) => match lookup { InMemoryClass::Key(_) => SUBSPACE_IN_MEMORY_VALUE, InMemoryClass::Counter(_) => SUBSPACE_IN_MEMORY_COUNTER, diff --git a/crates/store/src/write/mod.rs b/crates/store/src/write/mod.rs index c4915843..f2f22563 100644 --- a/crates/store/src/write/mod.rs +++ b/crates/store/src/write/mod.rs @@ -8,6 +8,7 @@ use self::assert::AssertValue; use crate::backend::MAX_TOKEN_LENGTH; use log::ChangeLogBuilder; use nlp::tokenizers::word::WordTokenizer; +use registry::{schema::prelude::Object, types::id::ObjectId}; use rkyv::util::AlignedVec; use std::{ collections::HashSet, @@ -270,16 +271,27 @@ pub enum InMemoryClass { #[derive(Debug, PartialEq, Clone, Eq, Hash)] pub enum RegistryClass { - Item(u64), - Relation { - from: u64, - to: u64, + Item(ObjectId), + Reference { + to: ObjectId, + from: ObjectId, }, Index { index_id: u16, - item_id: u64, + item_id: ObjectId, key: Vec, }, + IndexGlobal { + index_id: u16, + item_id: ObjectId, + key: Vec, + }, + Id { + item_id: ObjectId, + }, + IdCounter { + object: Object, + }, } #[derive(Debug, PartialEq, Clone, Eq, Hash)] diff --git a/crates/trc/src/event/enums.rs b/crates/trc/src/event/enums.rs index ce04446e..06636cbc 100644 --- a/crates/trc/src/event/enums.rs +++ b/crates/trc/src/event/enums.rs @@ -16,7 +16,6 @@ pub enum EventType { Auth(AuthEvent), Calendar(CalendarEvent), Cluster(ClusterEvent), - Config(ConfigEvent), Dane(DaneEvent), Delivery(DeliveryEvent), Dkim(DkimEvent), @@ -43,6 +42,7 @@ pub enum EventType { Purge(PurgeEvent), PushSubscription(PushSubscriptionEvent), Queue(QueueEvent), + Registry(RegistryEvent), Resource(ResourceEvent), Security(SecurityEvent), Server(ServerEvent), @@ -142,23 +142,6 @@ pub enum ClusterEvent { MessageInvalid = 49, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[repr(u16)] -pub enum ConfigEvent { - ParseError = 62, - BuildError = 54, - MacroError = 60, - WriteError = 65, - FetchError = 58, - DefaultApplied = 56, - MissingSetting = 61, - UnusedSetting = 64, - ParseWarning = 63, - BuildWarning = 55, - ImportExternal = 59, - AlreadyUpToDate = 53, -} - #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum DaneEvent { @@ -611,6 +594,23 @@ pub enum QueueEvent { BackPressure = 48, } +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[repr(u16)] +pub enum RegistryEvent { + LocalReadError = 62, + LocalWriteError = 54, + LocalParseError = 60, + ReadError = 65, + WriteError = 58, + DeserializationError = 56, + BuildError = 61, + BuildWarning = 55, + NotSupported = 64, + ValidationError = 63, + Reserved03 = 59, + Reserved04 = 53, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum ResourceEvent { diff --git a/crates/trc/src/event/enums_impl.rs b/crates/trc/src/event/enums_impl.rs index de7cea16..f4ef4937 100644 --- a/crates/trc/src/event/enums_impl.rs +++ b/crates/trc/src/event/enums_impl.rs @@ -67,18 +67,6 @@ impl EventType { b"cluster.message-received" => EventType::Cluster(ClusterEvent::MessageReceived), b"cluster.message-skipped" => EventType::Cluster(ClusterEvent::MessageSkipped), b"cluster.message-invalid" => EventType::Cluster(ClusterEvent::MessageInvalid), - b"config.parse-error" => EventType::Config(ConfigEvent::ParseError), - b"config.build-error" => EventType::Config(ConfigEvent::BuildError), - b"config.macro-error" => EventType::Config(ConfigEvent::MacroError), - b"config.write-error" => EventType::Config(ConfigEvent::WriteError), - b"config.fetch-error" => EventType::Config(ConfigEvent::FetchError), - b"config.default-applied" => EventType::Config(ConfigEvent::DefaultApplied), - b"config.missing-setting" => EventType::Config(ConfigEvent::MissingSetting), - b"config.unused-setting" => EventType::Config(ConfigEvent::UnusedSetting), - b"config.parse-warning" => EventType::Config(ConfigEvent::ParseWarning), - b"config.build-warning" => EventType::Config(ConfigEvent::BuildWarning), - b"config.import-external" => EventType::Config(ConfigEvent::ImportExternal), - b"config.already-up-to-date" => EventType::Config(ConfigEvent::AlreadyUpToDate), b"dane.authentication-success" => EventType::Dane(DaneEvent::AuthenticationSuccess), b"dane.authentication-failure" => EventType::Dane(DaneEvent::AuthenticationFailure), b"dane.no-certificates-found" => EventType::Dane(DaneEvent::NoCertificatesFound), @@ -401,6 +389,18 @@ impl EventType { b"queue.concurrency-limit-exceeded" => EventType::Queue(QueueEvent::ConcurrencyLimitExceeded), b"queue.quota-exceeded" => EventType::Queue(QueueEvent::QuotaExceeded), b"queue.back-pressure" => EventType::Queue(QueueEvent::BackPressure), + b"registry.local-read-error" => EventType::Registry(RegistryEvent::LocalReadError), + b"registry.local-write-error" => EventType::Registry(RegistryEvent::LocalWriteError), + b"registry.local-parse-error" => EventType::Registry(RegistryEvent::LocalParseError), + b"registry.read-error" => EventType::Registry(RegistryEvent::ReadError), + b"registry.write-error" => EventType::Registry(RegistryEvent::WriteError), + b"registry.deserialization-error" => EventType::Registry(RegistryEvent::DeserializationError), + b"registry.build-error" => EventType::Registry(RegistryEvent::BuildError), + b"registry.build-warning" => EventType::Registry(RegistryEvent::BuildWarning), + b"registry.not-supported" => EventType::Registry(RegistryEvent::NotSupported), + b"registry.validation-error" => EventType::Registry(RegistryEvent::ValidationError), + b"registry.reserved03" => EventType::Registry(RegistryEvent::Reserved03), + b"registry.reserved04" => EventType::Registry(RegistryEvent::Reserved04), b"resource.not-found" => EventType::Resource(ResourceEvent::NotFound), b"resource.bad-parameters" => EventType::Resource(ResourceEvent::BadParameters), b"resource.error" => EventType::Resource(ResourceEvent::Error), @@ -677,18 +677,6 @@ impl EventType { EventType::Cluster(ClusterEvent::MessageReceived) => "cluster.message-received", EventType::Cluster(ClusterEvent::MessageSkipped) => "cluster.message-skipped", EventType::Cluster(ClusterEvent::MessageInvalid) => "cluster.message-invalid", - EventType::Config(ConfigEvent::ParseError) => "config.parse-error", - EventType::Config(ConfigEvent::BuildError) => "config.build-error", - EventType::Config(ConfigEvent::MacroError) => "config.macro-error", - EventType::Config(ConfigEvent::WriteError) => "config.write-error", - EventType::Config(ConfigEvent::FetchError) => "config.fetch-error", - EventType::Config(ConfigEvent::DefaultApplied) => "config.default-applied", - EventType::Config(ConfigEvent::MissingSetting) => "config.missing-setting", - EventType::Config(ConfigEvent::UnusedSetting) => "config.unused-setting", - EventType::Config(ConfigEvent::ParseWarning) => "config.parse-warning", - EventType::Config(ConfigEvent::BuildWarning) => "config.build-warning", - EventType::Config(ConfigEvent::ImportExternal) => "config.import-external", - EventType::Config(ConfigEvent::AlreadyUpToDate) => "config.already-up-to-date", EventType::Dane(DaneEvent::AuthenticationSuccess) => "dane.authentication-success", EventType::Dane(DaneEvent::AuthenticationFailure) => "dane.authentication-failure", EventType::Dane(DaneEvent::NoCertificatesFound) => "dane.no-certificates-found", @@ -1105,6 +1093,20 @@ impl EventType { } EventType::Queue(QueueEvent::QuotaExceeded) => "queue.quota-exceeded", EventType::Queue(QueueEvent::BackPressure) => "queue.back-pressure", + EventType::Registry(RegistryEvent::LocalReadError) => "registry.local-read-error", + EventType::Registry(RegistryEvent::LocalWriteError) => "registry.local-write-error", + EventType::Registry(RegistryEvent::LocalParseError) => "registry.local-parse-error", + EventType::Registry(RegistryEvent::ReadError) => "registry.read-error", + EventType::Registry(RegistryEvent::WriteError) => "registry.write-error", + EventType::Registry(RegistryEvent::DeserializationError) => { + "registry.deserialization-error" + } + EventType::Registry(RegistryEvent::BuildError) => "registry.build-error", + EventType::Registry(RegistryEvent::BuildWarning) => "registry.build-warning", + EventType::Registry(RegistryEvent::NotSupported) => "registry.not-supported", + EventType::Registry(RegistryEvent::ValidationError) => "registry.validation-error", + EventType::Registry(RegistryEvent::Reserved03) => "registry.reserved03", + EventType::Registry(RegistryEvent::Reserved04) => "registry.reserved04", EventType::Resource(ResourceEvent::NotFound) => "resource.not-found", EventType::Resource(ResourceEvent::BadParameters) => "resource.bad-parameters", EventType::Resource(ResourceEvent::Error) => "resource.error", @@ -1386,18 +1388,6 @@ impl EventType { EventType::Cluster(ClusterEvent::MessageReceived) => 46, EventType::Cluster(ClusterEvent::MessageSkipped) => 47, EventType::Cluster(ClusterEvent::MessageInvalid) => 49, - EventType::Config(ConfigEvent::ParseError) => 62, - EventType::Config(ConfigEvent::BuildError) => 54, - EventType::Config(ConfigEvent::MacroError) => 60, - EventType::Config(ConfigEvent::WriteError) => 65, - EventType::Config(ConfigEvent::FetchError) => 58, - EventType::Config(ConfigEvent::DefaultApplied) => 56, - EventType::Config(ConfigEvent::MissingSetting) => 61, - EventType::Config(ConfigEvent::UnusedSetting) => 64, - EventType::Config(ConfigEvent::ParseWarning) => 63, - EventType::Config(ConfigEvent::BuildWarning) => 55, - EventType::Config(ConfigEvent::ImportExternal) => 59, - EventType::Config(ConfigEvent::AlreadyUpToDate) => 53, EventType::Dane(DaneEvent::AuthenticationSuccess) => 67, EventType::Dane(DaneEvent::AuthenticationFailure) => 66, EventType::Dane(DaneEvent::NoCertificatesFound) => 69, @@ -1720,6 +1710,18 @@ impl EventType { EventType::Queue(QueueEvent::ConcurrencyLimitExceeded) => 375, EventType::Queue(QueueEvent::QuotaExceeded) => 383, EventType::Queue(QueueEvent::BackPressure) => 48, + EventType::Registry(RegistryEvent::LocalReadError) => 62, + EventType::Registry(RegistryEvent::LocalWriteError) => 54, + EventType::Registry(RegistryEvent::LocalParseError) => 60, + EventType::Registry(RegistryEvent::ReadError) => 65, + EventType::Registry(RegistryEvent::WriteError) => 58, + EventType::Registry(RegistryEvent::DeserializationError) => 56, + EventType::Registry(RegistryEvent::BuildError) => 61, + EventType::Registry(RegistryEvent::BuildWarning) => 55, + EventType::Registry(RegistryEvent::NotSupported) => 64, + EventType::Registry(RegistryEvent::ValidationError) => 63, + EventType::Registry(RegistryEvent::Reserved03) => 59, + EventType::Registry(RegistryEvent::Reserved04) => 53, EventType::Resource(ResourceEvent::NotFound) => 389, EventType::Resource(ResourceEvent::BadParameters) => 386, EventType::Resource(ResourceEvent::Error) => 388, @@ -1987,18 +1989,6 @@ impl EventType { 46 => Some(EventType::Cluster(ClusterEvent::MessageReceived)), 47 => Some(EventType::Cluster(ClusterEvent::MessageSkipped)), 49 => Some(EventType::Cluster(ClusterEvent::MessageInvalid)), - 62 => Some(EventType::Config(ConfigEvent::ParseError)), - 54 => Some(EventType::Config(ConfigEvent::BuildError)), - 60 => Some(EventType::Config(ConfigEvent::MacroError)), - 65 => Some(EventType::Config(ConfigEvent::WriteError)), - 58 => Some(EventType::Config(ConfigEvent::FetchError)), - 56 => Some(EventType::Config(ConfigEvent::DefaultApplied)), - 61 => Some(EventType::Config(ConfigEvent::MissingSetting)), - 64 => Some(EventType::Config(ConfigEvent::UnusedSetting)), - 63 => Some(EventType::Config(ConfigEvent::ParseWarning)), - 55 => Some(EventType::Config(ConfigEvent::BuildWarning)), - 59 => Some(EventType::Config(ConfigEvent::ImportExternal)), - 53 => Some(EventType::Config(ConfigEvent::AlreadyUpToDate)), 67 => Some(EventType::Dane(DaneEvent::AuthenticationSuccess)), 66 => Some(EventType::Dane(DaneEvent::AuthenticationFailure)), 69 => Some(EventType::Dane(DaneEvent::NoCertificatesFound)), @@ -2357,6 +2347,18 @@ impl EventType { 375 => Some(EventType::Queue(QueueEvent::ConcurrencyLimitExceeded)), 383 => Some(EventType::Queue(QueueEvent::QuotaExceeded)), 48 => Some(EventType::Queue(QueueEvent::BackPressure)), + 62 => Some(EventType::Registry(RegistryEvent::LocalReadError)), + 54 => Some(EventType::Registry(RegistryEvent::LocalWriteError)), + 60 => Some(EventType::Registry(RegistryEvent::LocalParseError)), + 65 => Some(EventType::Registry(RegistryEvent::ReadError)), + 58 => Some(EventType::Registry(RegistryEvent::WriteError)), + 56 => Some(EventType::Registry(RegistryEvent::DeserializationError)), + 61 => Some(EventType::Registry(RegistryEvent::BuildError)), + 55 => Some(EventType::Registry(RegistryEvent::BuildWarning)), + 64 => Some(EventType::Registry(RegistryEvent::NotSupported)), + 63 => Some(EventType::Registry(RegistryEvent::ValidationError)), + 59 => Some(EventType::Registry(RegistryEvent::Reserved03)), + 53 => Some(EventType::Registry(RegistryEvent::Reserved04)), 389 => Some(EventType::Resource(ResourceEvent::NotFound)), 386 => Some(EventType::Resource(ResourceEvent::BadParameters)), 388 => Some(EventType::Resource(ResourceEvent::Error)), @@ -2580,11 +2582,6 @@ impl EventType { EventType::Cluster(ClusterEvent::SubscriberError) => Level::Error, EventType::Cluster(ClusterEvent::PublisherError) => Level::Error, EventType::Cluster(ClusterEvent::MessageInvalid) => Level::Error, - EventType::Config(ConfigEvent::ParseError) => Level::Error, - EventType::Config(ConfigEvent::BuildError) => Level::Error, - EventType::Config(ConfigEvent::MacroError) => Level::Error, - EventType::Config(ConfigEvent::WriteError) => Level::Error, - EventType::Config(ConfigEvent::FetchError) => Level::Error, EventType::Dkim(DkimEvent::BuildError) => Level::Error, EventType::Dns(DnsEvent::BuildError) => Level::Error, EventType::MessageIngest(MessageIngestEvent::Error) => Level::Error, @@ -2593,6 +2590,14 @@ impl EventType { EventType::Network(NetworkEvent::SplitError) => Level::Error, EventType::Network(NetworkEvent::SetOptError) => Level::Error, EventType::Purge(PurgeEvent::Error) => Level::Error, + EventType::Registry(RegistryEvent::LocalReadError) => Level::Error, + EventType::Registry(RegistryEvent::LocalWriteError) => Level::Error, + EventType::Registry(RegistryEvent::LocalParseError) => Level::Error, + EventType::Registry(RegistryEvent::ReadError) => Level::Error, + EventType::Registry(RegistryEvent::WriteError) => Level::Error, + EventType::Registry(RegistryEvent::DeserializationError) => Level::Error, + EventType::Registry(RegistryEvent::BuildError) => Level::Error, + EventType::Registry(RegistryEvent::ValidationError) => Level::Error, EventType::Resource(ResourceEvent::BadParameters) => Level::Error, EventType::Resource(ResourceEvent::Error) => Level::Error, EventType::Server(ServerEvent::StartupError) => Level::Error, @@ -2640,7 +2645,6 @@ impl EventType { EventType::Cluster(ClusterEvent::SubscriberStop) => Level::Info, EventType::Cluster(ClusterEvent::PublisherStart) => Level::Info, EventType::Cluster(ClusterEvent::PublisherStop) => Level::Info, - EventType::Config(ConfigEvent::ImportExternal) => Level::Info, EventType::Dane(DaneEvent::AuthenticationSuccess) => Level::Info, EventType::Dane(DaneEvent::AuthenticationFailure) => Level::Info, EventType::Dane(DaneEvent::NoCertificatesFound) => Level::Info, @@ -2751,6 +2755,7 @@ impl EventType { EventType::Queue(QueueEvent::RateLimitExceeded) => Level::Info, EventType::Queue(QueueEvent::ConcurrencyLimitExceeded) => Level::Info, EventType::Queue(QueueEvent::QuotaExceeded) => Level::Info, + EventType::Registry(RegistryEvent::Reserved03) => Level::Info, EventType::Resource(ResourceEvent::DownloadExternal) => Level::Info, EventType::Resource(ResourceEvent::WebadminUnpacked) => Level::Info, EventType::Security(SecurityEvent::AuthenticationBan) => Level::Info, @@ -2854,8 +2859,6 @@ impl EventType { EventType::Auth(AuthEvent::TooManyAttempts) => Level::Warn, EventType::Calendar(CalendarEvent::AlarmFailed) => Level::Warn, EventType::Cluster(ClusterEvent::SubscriberDisconnected) => Level::Warn, - EventType::Config(ConfigEvent::ParseWarning) => Level::Warn, - EventType::Config(ConfigEvent::BuildWarning) => Level::Warn, EventType::Delivery(DeliveryEvent::MissingOutboundHostname) => Level::Warn, EventType::Delivery(DeliveryEvent::ConcurrencyLimitExceeded) => Level::Warn, EventType::Delivery(DeliveryEvent::RateLimitExceeded) => Level::Warn, @@ -2879,6 +2882,7 @@ impl EventType { EventType::MtaHook(MtaHookEvent::Error) => Level::Warn, EventType::Network(NetworkEvent::ProxyError) => Level::Warn, EventType::Queue(QueueEvent::BackPressure) => Level::Warn, + EventType::Registry(RegistryEvent::BuildWarning) => Level::Warn, EventType::Sieve(SieveEvent::MessageTooLarge) => Level::Warn, EventType::Sieve(SieveEvent::ScriptNotFound) => Level::Warn, EventType::Sieve(SieveEvent::ListNotFound) => Level::Warn, @@ -2969,18 +2973,6 @@ impl EventType { EventType::Cluster(ClusterEvent::MessageReceived) => "PubSub message received", EventType::Cluster(ClusterEvent::MessageSkipped) => "PubSub message skipped", EventType::Cluster(ClusterEvent::MessageInvalid) => "Invalid PubSub message", - EventType::Config(ConfigEvent::ParseError) => "Configuration parse error", - EventType::Config(ConfigEvent::BuildError) => "Configuration build error", - EventType::Config(ConfigEvent::MacroError) => "Configuration macro error", - EventType::Config(ConfigEvent::WriteError) => "Configuration write error", - EventType::Config(ConfigEvent::FetchError) => "Configuration fetch error", - EventType::Config(ConfigEvent::DefaultApplied) => "Default configuration applied", - EventType::Config(ConfigEvent::MissingSetting) => "Missing configuration setting", - EventType::Config(ConfigEvent::UnusedSetting) => "Unused configuration setting", - EventType::Config(ConfigEvent::ParseWarning) => "Configuration parse warning", - EventType::Config(ConfigEvent::BuildWarning) => "Configuration build warning", - EventType::Config(ConfigEvent::ImportExternal) => "Importing external configuration", - EventType::Config(ConfigEvent::AlreadyUpToDate) => "Configuration already up to date", EventType::Dane(DaneEvent::AuthenticationSuccess) => "DANE authentication successful", EventType::Dane(DaneEvent::AuthenticationFailure) => "DANE authentication failed", EventType::Dane(DaneEvent::NoCertificatesFound) => "No certificates found for DANE", @@ -3405,6 +3397,22 @@ impl EventType { EventType::Queue(QueueEvent::ConcurrencyLimitExceeded) => "Concurrency limit exceeded", EventType::Queue(QueueEvent::QuotaExceeded) => "Quota exceeded", EventType::Queue(QueueEvent::BackPressure) => "Queue backpressure detected", + EventType::Registry(RegistryEvent::LocalReadError) => "Local registry read error", + EventType::Registry(RegistryEvent::LocalWriteError) => "Local registry write error", + EventType::Registry(RegistryEvent::LocalParseError) => "Local registry parse error", + EventType::Registry(RegistryEvent::ReadError) => "Registry read error", + EventType::Registry(RegistryEvent::WriteError) => "Registry write error", + EventType::Registry(RegistryEvent::DeserializationError) => { + "Registry deserialization error" + } + EventType::Registry(RegistryEvent::BuildError) => "Configuration build error", + EventType::Registry(RegistryEvent::BuildWarning) => "Configuration build warning", + EventType::Registry(RegistryEvent::NotSupported) => { + "Operation not supported by local registry" + } + EventType::Registry(RegistryEvent::ValidationError) => "Object validation error", + EventType::Registry(RegistryEvent::Reserved03) => "Importing external configuration", + EventType::Registry(RegistryEvent::Reserved04) => "Configuration already up to date", EventType::Resource(ResourceEvent::NotFound) => "Resource not found", EventType::Resource(ResourceEvent::BadParameters) => "Bad resource parameters", EventType::Resource(ResourceEvent::Error) => "Resource error", @@ -3724,38 +3732,6 @@ impl EventType { EventType::Cluster(ClusterEvent::MessageInvalid) => { "An invalid message was received from the PubSub server" } - EventType::Config(ConfigEvent::ParseError) => { - "An error occurred while parsing the configuration" - } - EventType::Config(ConfigEvent::BuildError) => { - "An error occurred while building the configuration" - } - EventType::Config(ConfigEvent::MacroError) => { - "An error occurred with a configuration macro" - } - EventType::Config(ConfigEvent::WriteError) => { - "An error occurred while writing the configuration" - } - EventType::Config(ConfigEvent::FetchError) => { - "An error occurred while fetching the configuration" - } - EventType::Config(ConfigEvent::DefaultApplied) => { - "The default configuration has been applied" - } - EventType::Config(ConfigEvent::MissingSetting) => "A configuration setting is missing", - EventType::Config(ConfigEvent::UnusedSetting) => "A configuration setting is unused", - EventType::Config(ConfigEvent::ParseWarning) => { - "A warning occurred while parsing the configuration" - } - EventType::Config(ConfigEvent::BuildWarning) => { - "A warning occurred while building the configuration" - } - EventType::Config(ConfigEvent::ImportExternal) => { - "An external configuration is being imported" - } - EventType::Config(ConfigEvent::AlreadyUpToDate) => { - "The configuration is already up to date" - } EventType::Dane(DaneEvent::AuthenticationSuccess) => "Successful DANE authentication", EventType::Dane(DaneEvent::AuthenticationFailure) => "Failed DANE authentication", EventType::Dane(DaneEvent::NoCertificatesFound) => { @@ -4342,6 +4318,42 @@ impl EventType { EventType::Queue(QueueEvent::BackPressure) => { "Queue congested, processing can't keep up with incoming message rate" } + EventType::Registry(RegistryEvent::LocalReadError) => { + "An error occurred while reading the local registry file" + } + EventType::Registry(RegistryEvent::LocalWriteError) => { + "An error occurred while writing to the local registry file" + } + EventType::Registry(RegistryEvent::LocalParseError) => { + "An error occurred while parsing the local registry file" + } + EventType::Registry(RegistryEvent::ReadError) => { + "An error occurred while reading the registry file" + } + EventType::Registry(RegistryEvent::WriteError) => { + "An error occurred while writing to the registry file" + } + EventType::Registry(RegistryEvent::DeserializationError) => { + "An error occurred while deserializing a registry entry" + } + EventType::Registry(RegistryEvent::BuildError) => { + "An error occurred while building the configuration from the registry" + } + EventType::Registry(RegistryEvent::BuildWarning) => { + "A warning occurred while building the configuration from the registry" + } + EventType::Registry(RegistryEvent::NotSupported) => { + "The local registry does not support this operation" + } + EventType::Registry(RegistryEvent::ValidationError) => { + "An error occurred while validating a registry object" + } + EventType::Registry(RegistryEvent::Reserved03) => { + "An external configuration is being imported" + } + EventType::Registry(RegistryEvent::Reserved04) => { + "The configuration is already up to date" + } EventType::Resource(ResourceEvent::NotFound) => "The resource was not found", EventType::Resource(ResourceEvent::BadParameters) => "The resource parameters are bad", EventType::Resource(ResourceEvent::Error) => "An error occurred with the resource", @@ -4742,18 +4754,6 @@ impl EventType { EventType::Auth(AuthEvent::TooManyAttempts) => "Too many authentication attempts", EventType::Auth(AuthEvent::ClientRegistration) => "Authentication error", EventType::Auth(AuthEvent::Error) => "Authentication error", - EventType::Config(ConfigEvent::ParseError) => "Configuration error", - EventType::Config(ConfigEvent::BuildError) => "Configuration error", - EventType::Config(ConfigEvent::MacroError) => "Configuration error", - EventType::Config(ConfigEvent::WriteError) => "Configuration error", - EventType::Config(ConfigEvent::FetchError) => "Configuration error", - EventType::Config(ConfigEvent::DefaultApplied) => "Configuration error", - EventType::Config(ConfigEvent::MissingSetting) => "Configuration error", - EventType::Config(ConfigEvent::UnusedSetting) => "Configuration error", - EventType::Config(ConfigEvent::ParseWarning) => "Configuration error", - EventType::Config(ConfigEvent::BuildWarning) => "Configuration error", - EventType::Config(ConfigEvent::ImportExternal) => "Configuration error", - EventType::Config(ConfigEvent::AlreadyUpToDate) => "Configuration error", EventType::Imap(ImapEvent::ConnectionStart) => "IMAP error", EventType::Imap(ImapEvent::ConnectionEnd) => "IMAP error", EventType::Imap(ImapEvent::GetAcl) => "IMAP error", @@ -5071,18 +5071,6 @@ impl EventType { EventType::Cluster(ClusterEvent::MessageReceived), EventType::Cluster(ClusterEvent::MessageSkipped), EventType::Cluster(ClusterEvent::MessageInvalid), - EventType::Config(ConfigEvent::ParseError), - EventType::Config(ConfigEvent::BuildError), - EventType::Config(ConfigEvent::MacroError), - EventType::Config(ConfigEvent::WriteError), - EventType::Config(ConfigEvent::FetchError), - EventType::Config(ConfigEvent::DefaultApplied), - EventType::Config(ConfigEvent::MissingSetting), - EventType::Config(ConfigEvent::UnusedSetting), - EventType::Config(ConfigEvent::ParseWarning), - EventType::Config(ConfigEvent::BuildWarning), - EventType::Config(ConfigEvent::ImportExternal), - EventType::Config(ConfigEvent::AlreadyUpToDate), EventType::Dane(DaneEvent::AuthenticationSuccess), EventType::Dane(DaneEvent::AuthenticationFailure), EventType::Dane(DaneEvent::NoCertificatesFound), @@ -5405,6 +5393,18 @@ impl EventType { EventType::Queue(QueueEvent::ConcurrencyLimitExceeded), EventType::Queue(QueueEvent::QuotaExceeded), EventType::Queue(QueueEvent::BackPressure), + EventType::Registry(RegistryEvent::LocalReadError), + EventType::Registry(RegistryEvent::LocalWriteError), + EventType::Registry(RegistryEvent::LocalParseError), + EventType::Registry(RegistryEvent::ReadError), + EventType::Registry(RegistryEvent::WriteError), + EventType::Registry(RegistryEvent::DeserializationError), + EventType::Registry(RegistryEvent::BuildError), + EventType::Registry(RegistryEvent::BuildWarning), + EventType::Registry(RegistryEvent::NotSupported), + EventType::Registry(RegistryEvent::ValidationError), + EventType::Registry(RegistryEvent::Reserved03), + EventType::Registry(RegistryEvent::Reserved04), EventType::Resource(ResourceEvent::NotFound), EventType::Resource(ResourceEvent::BadParameters), EventType::Resource(ResourceEvent::Error), diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index 1297ef26..6c5f7d8d 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -304,3 +304,58 @@ pub fn sanitize_email(email: &str) -> Option { None } } + +pub fn sanitize_email_local(local: &str) -> Option { + let mut result = String::with_capacity(local.len()); + let mut last_ch = char::from(0); + + for ch in local.chars() { + if !ch.is_whitespace() { + if ch.is_alphanumeric() { + for ch in ch.to_lowercase() { + result.push(ch); + } + } else if result.is_empty() || !last_ch.is_alphanumeric() { + return None; + } + + last_ch = ch; + } + } + + if last_ch.is_alphanumeric() { + Some(result) + } else { + None + } +} + +pub fn sanitize_domain(domain: &str) -> Option { + let mut result = String::with_capacity(domain.len()); + let mut found_dot = false; + let mut last_ch = char::from(0); + + for ch in domain.chars() { + if !ch.is_whitespace() { + if ch == '.' { + found_dot = true; + if !(last_ch.is_alphanumeric() || last_ch == '-' || last_ch == '_') { + return None; + } + } + last_ch = ch; + for ch in ch.to_lowercase() { + result.push(ch); + } + } + } + + if found_dot + && last_ch != '.' + && psl::domain(result.as_bytes()).is_some_and(|d| d.suffix().typ().is_some()) + { + Some(result) + } else { + None + } +}