JMAP Registry API implementation - part 1

crates/common/src/config/server/listener.rs

@@ -52,6 +52,7 @@ impl Listeners {
 
     fn parse_server(&mut self, bp: &mut Bootstrap, listener: RegistryObject<NetworkListener>) {
         let id = listener.id;
+        let revision = listener.revision;
         let listener = listener.object;
 
         // Parse protocol
@@ -139,6 +140,7 @@ impl Listeners {
         self.parsed_listeners.push(RegistryObject {
             id,
             object: listener,
+            revision,
         });
     }
 

crates/common/src/network/acme/mod.rs

@@ -17,7 +17,6 @@ use dns_update::DnsUpdater;
 use registry::schema::structs;
 use rustls::sign::CertifiedKey;
 use std::{fmt::Debug, sync::Arc, time::Duration};
-use store::registry::RegistryObject;
 
 pub struct AcmeProvider {
     pub id: String,

crates/common/src/network/security.rs

@@ -13,7 +13,7 @@ use ahash::AHashSet;
 use registry::{
     schema::{
         enums::BlockReason,
-        prelude::{HashedObject, Object, ObjectType},
+        prelude::{Object, ObjectType},
         structs::{self, AllowedIp, BlockedIp, Rate},
     },
     types::{datetime::UTCDateTime, ipmask::IpAddrOrMask},
@@ -65,10 +65,10 @@ impl Security {
         let mut expired_allows = Vec::new();
         let now = now() as i64;
 
-        for ip in bp.list_infallible::<HashedObject<AllowedIp>>().await {
+        for ip in bp.list_infallible::<AllowedIp>().await {
             let id = ip.id;
-            let revision = ip.object.revision;
+            let revision = ip.revision;
-            let ip = ip.object.object;
+            let ip = ip.object;
 
             if ip.expires_at.as_ref().is_none_or(|ip| ip.timestamp() > now) {
                 if let Some(ip) = ip.address.try_to_ip() {
@@ -328,10 +328,10 @@ impl BlockedIps {
         let mut expired_blocks = Vec::new();
         let now = now() as i64;
 
-        for ip in bp.list_infallible::<HashedObject<BlockedIp>>().await {
+        for ip in bp.list_infallible::<BlockedIp>().await {
             let id = ip.id;
-            let revision = ip.object.revision;
+            let revision = ip.revision;
-            let ip = ip.object.object;
+            let ip = ip.object;
 
             if ip.expires_at.as_ref().is_none_or(|ip| ip.timestamp() > now) {
                 if let Some(ip) = ip.address.try_to_ip() {

crates/jmap/src/registry/get.rs

@@ -9,7 +9,21 @@ use jmap_proto::{
     method::get::{GetRequest, GetResponse},
     object::registry::Registry,
 };
-use registry::schema::prelude::ObjectType;
+use jmap_tools::Key;
+use registry::{
+    jmap::{IntoValue, JmapValue, RegistryValue},
+    schema::{
+        enums::Permission,
+        prelude::{
+            OBJ_FILTER_ACCOUNT, OBJ_FILTER_TENANT, OBJ_SINGLETON, Object, ObjectInner, ObjectType,
+            Property,
+        },
+    },
+    types::id::ObjectId,
+};
+use store::{ahash::AHashSet, registry::RegistryQuery};
+use trc::AddContext;
+use types::id::Id;
 
 pub trait RegistryGet: Sync + Send {
     fn registry_get(
@@ -27,6 +41,239 @@ impl RegistryGet for Server {
         mut request: GetRequest<Registry>,
         access_token: &AccessToken,
     ) -> trc::Result<GetResponse<Registry>> {
-        todo!()
+        let ids = request.unwrap_ids(self.core.jmap.get_max_objects)?;
+        let mut properties = request
+            .properties
+            .take()
+            .map(|p| p.unwrap())
+            .unwrap_or_default()
+            .into_iter()
+            .filter_map(|prop| prop.try_unwrap())
+            .collect::<AHashSet<_>>();
+        if !properties.is_empty() {
+            properties.insert(Property::Id);
+        }
+
+        let mut response = GetResponse {
+            account_id: request.account_id.into(),
+            state: None,
+            list: vec![],
+            not_found: vec![],
+        };
+
+        match object_type {
+            ObjectType::AcmeProvider
+            | ObjectType::AddressBook
+            | ObjectType::AiModel
+            | ObjectType::Alert
+            | ObjectType::AllowedIp
+            | ObjectType::Application
+            | ObjectType::Asn
+            | ObjectType::Authentication
+            | ObjectType::BlobStore
+            | ObjectType::BlockedIp
+            | ObjectType::Cache
+            | ObjectType::Calendar
+            | ObjectType::CalendarAlarm
+            | ObjectType::CalendarScheduling
+            | ObjectType::Certificate
+            | ObjectType::Coordinator
+            | ObjectType::DataRetention
+            | ObjectType::DataStore
+            | ObjectType::Directory
+            | ObjectType::DkimReportSettings
+            | ObjectType::DmarcReportSettings
+            | ObjectType::DnsResolver
+            | ObjectType::DnsServer
+            | ObjectType::Email
+            | ObjectType::Enterprise
+            | ObjectType::EventTracingLevel
+            | ObjectType::FileStorage
+            | ObjectType::Http
+            | ObjectType::HttpForm
+            | ObjectType::HttpLookup
+            | ObjectType::Imap
+            | ObjectType::InMemoryStore
+            | ObjectType::Jmap
+            | ObjectType::LocalSettings
+            | ObjectType::MemoryLookupKey
+            | ObjectType::MemoryLookupKeyValue
+            | ObjectType::Metrics
+            | ObjectType::MetricsStore
+            | ObjectType::MtaConnectionStrategy
+            | ObjectType::MtaDeliverySchedule
+            | ObjectType::MtaExtensions
+            | ObjectType::MtaHook
+            | ObjectType::MtaInboundSession
+            | ObjectType::MtaInboundThrottle
+            | ObjectType::MtaMilter
+            | ObjectType::MtaOutboundStrategy
+            | ObjectType::MtaOutboundThrottle
+            | ObjectType::MtaQueueQuota
+            | ObjectType::MtaRoute
+            | ObjectType::MtaStageAuth
+            | ObjectType::MtaStageConnect
+            | ObjectType::MtaStageData
+            | ObjectType::MtaStageEhlo
+            | ObjectType::MtaStageMail
+            | ObjectType::MtaStageRcpt
+            | ObjectType::MtaSts
+            | ObjectType::MtaTlsStrategy
+            | ObjectType::MtaVirtualQueue
+            | ObjectType::NetworkListener
+            | ObjectType::Node
+            | ObjectType::NodeRole
+            | ObjectType::NodeShard
+            | ObjectType::OidcProvider
+            | ObjectType::RegistryBundle
+            | ObjectType::ReportSettings
+            | ObjectType::Search
+            | ObjectType::SearchStore
+            | ObjectType::Security
+            | ObjectType::SenderAuth
+            | ObjectType::Sharing
+            | ObjectType::SieveSystemInterpreter
+            | ObjectType::SieveSystemScript
+            | ObjectType::SieveUserInterpreter
+            | ObjectType::SieveUserScript
+            | ObjectType::SpamClassifier
+            | ObjectType::SpamDnsblServer
+            | ObjectType::SpamDnsblSettings
+            | ObjectType::SpamFileExtension
+            | ObjectType::SpamLlm
+            | ObjectType::SpamPyzor
+            | ObjectType::SpamRule
+            | ObjectType::SpamSettings
+            | ObjectType::SpamTag
+            | ObjectType::SpfReportSettings
+            | ObjectType::StoreLookup
+            | ObjectType::TlsReportSettings
+            | ObjectType::Tracer
+            | ObjectType::TracingStore
+            | ObjectType::WebDav
+            | ObjectType::WebHook
+            | ObjectType::Account
+            | ObjectType::DsnReportSettings
+            | ObjectType::MailingList
+            | ObjectType::OAuthClient
+            | ObjectType::Role
+            | ObjectType::Tenant
+            | ObjectType::MaskedEmail
+            | ObjectType::PublicKey
+            | ObjectType::DkimSignature
+            | ObjectType::Domain => {
+                let flags = object_type.flags();
+                let is_singleton = (flags & OBJ_SINGLETON) != 0;
+                let is_tenant_filtered =
+                    (flags & OBJ_FILTER_TENANT) != 0 && access_token.tenant_id().is_some();
+                let is_account_filtered = (flags & OBJ_FILTER_ACCOUNT) != 0
+                    && !access_token.has_permission(Permission::Impersonate);
+
+                let ids = if let Some(ids) = ids {
+                    ids
+                } else {
+                    self.registry()
+                        .query::<AHashSet<u64>>(
+                            RegistryQuery::new(object_type)
+                                .with_tenant(access_token.tenant_id())
+                                .with_account_opt(
+                                    is_account_filtered.then_some(request.account_id.into()),
+                                ),
+                        )
+                        .await
+                        .caused_by(trc::location!())?
+                        .into_iter()
+                        .take(self.core.jmap.get_max_objects)
+                        .map(Id::new)
+                        .collect()
+                };
+                response.list.reserve(ids.len());
+
+                'outer: for id in ids {
+                    let object = if let Some(object) = self
+                        .registry()
+                        .get(ObjectId::new(object_type, id))
+                        .await
+                        .caused_by(trc::location!())?
+                    {
+                        object
+                    } else if id.is_singleton() && is_singleton {
+                        Object::new(ObjectInner::from(object_type))
+                    } else {
+                        response.not_found.push(id);
+                        continue;
+                    };
+
+                    match &object.inner {
+                        ObjectInner::DkimSignature(obj)
+                            if properties.is_empty()
+                                || properties.contains(&Property::PublicKey) =>
+                        {
+                            let todo = "dkim public key";
+                            todo!()
+                        }
+                        ObjectInner::Domain(obj)
+                            if properties.is_empty()
+                                || properties.contains(&Property::DnsZoneFile) =>
+                        {
+                            let todo = "domain dns zone file";
+                            todo!()
+                        }
+                        _ => {}
+                    }
+
+                    let mut object = object.into_value();
+                    let object_map = object.as_object_mut().unwrap();
+                    if is_tenant_filtered && let Some(tenant_id) = access_token.tenant_id() {
+                        let expected_value =
+                            JmapValue::Element(RegistryValue::Id(Id::from(tenant_id)));
+                        for (key, value) in object_map.iter() {
+                            if matches!(key, Key::Property(Property::MemberTenantId))
+                                && value != &expected_value
+                            {
+                                response.not_found.push(id);
+                                continue 'outer;
+                            }
+                        }
+                        object_map.remove(&Key::Property(Property::MemberTenantId));
+                    } else if is_account_filtered {
+                        let expected_value =
+                            JmapValue::Element(RegistryValue::Id(request.account_id));
+                        for (key, value) in object_map.iter() {
+                            if matches!(key, Key::Property(Property::AccountId))
+                                && value != &expected_value
+                            {
+                                response.not_found.push(id);
+                                continue 'outer;
+                            }
+                        }
+                        object_map.remove(&Key::Property(Property::AccountId));
+                    }
+
+                    object_map.insert_unchecked(Property::Id, RegistryValue::Id(id));
+                    if !properties.is_empty() {
+                        object_map.as_mut_vec().retain_mut(|(prop, _)| {
+                            prop.as_property()
+                                .is_some_and(|prop| properties.contains(prop))
+                        });
+                    }
+                    response.list.push(object);
+                }
+            }
+            ObjectType::Log => {}
+            ObjectType::QueuedMessage => {}
+
+            // Move to registry
+            ObjectType::Task => {}
+            ObjectType::ArfFeedbackReport => {}
+            ObjectType::DmarcReport => {}
+            ObjectType::TlsReport => {}
+            ObjectType::DeletedItem => {}
+            ObjectType::Metric => {}
+            ObjectType::Trace => {}
+            ObjectType::SpamTrainingSample => {}
+        }
+
+        Ok(response)
     }
 }

crates/jmap/src/registry/set.rs

@@ -27,6 +27,129 @@ impl RegistrySet for Server {
         mut request: SetRequest<'_, Registry>,
         access_token: &AccessToken,
     ) -> trc::Result<SetResponse<Registry>> {
+        match object_type {
+            ObjectType::AcmeProvider => {}
+            ObjectType::AddressBook => {}
+            ObjectType::AiModel => {}
+            ObjectType::Alert => {}
+            ObjectType::AllowedIp => {}
+            ObjectType::Application => {}
+            ObjectType::Asn => {}
+            ObjectType::Authentication => {}
+            ObjectType::BlobStore => {}
+            ObjectType::BlockedIp => {}
+            ObjectType::Cache => {}
+            ObjectType::Calendar => {}
+            ObjectType::CalendarAlarm => {}
+            ObjectType::CalendarScheduling => {}
+            ObjectType::Certificate => {}
+            ObjectType::Coordinator => {}
+            ObjectType::DataRetention => {}
+            ObjectType::DataStore => {}
+            ObjectType::Directory => {}
+            ObjectType::DkimReportSettings => {}
+            ObjectType::DmarcReportSettings => {}
+            ObjectType::DnsResolver => {}
+            ObjectType::DnsServer => {}
+            ObjectType::Email => {}
+            ObjectType::Enterprise => {}
+            ObjectType::EventTracingLevel => {}
+            ObjectType::FileStorage => {}
+            ObjectType::Http => {}
+            ObjectType::HttpForm => {}
+            ObjectType::HttpLookup => {}
+            ObjectType::Imap => {}
+            ObjectType::InMemoryStore => {}
+            ObjectType::Jmap => {}
+            ObjectType::LocalSettings => {}
+            ObjectType::MemoryLookupKey => {}
+            ObjectType::MemoryLookupKeyValue => {}
+            ObjectType::Metrics => {}
+            ObjectType::MetricsStore => {}
+            ObjectType::MtaConnectionStrategy => {}
+            ObjectType::MtaDeliverySchedule => {}
+            ObjectType::MtaExtensions => {}
+            ObjectType::MtaHook => {}
+            ObjectType::MtaInboundSession => {}
+            ObjectType::MtaInboundThrottle => {}
+            ObjectType::MtaMilter => {}
+            ObjectType::MtaOutboundStrategy => {}
+            ObjectType::MtaOutboundThrottle => {}
+            ObjectType::MtaQueueQuota => {}
+            ObjectType::MtaRoute => {}
+            ObjectType::MtaStageAuth => {}
+            ObjectType::MtaStageConnect => {}
+            ObjectType::MtaStageData => {}
+            ObjectType::MtaStageEhlo => {}
+            ObjectType::MtaStageMail => {}
+            ObjectType::MtaStageRcpt => {}
+            ObjectType::MtaSts => {}
+            ObjectType::MtaTlsStrategy => {}
+            ObjectType::MtaVirtualQueue => {}
+            ObjectType::NetworkListener => {}
+            ObjectType::Node => {}
+            ObjectType::NodeRole => {}
+            ObjectType::NodeShard => {}
+            ObjectType::OidcProvider => {}
+            ObjectType::RegistryBundle => {}
+            ObjectType::ReportSettings => {}
+            ObjectType::Search => {}
+            ObjectType::SearchStore => {}
+            ObjectType::Security => {}
+            ObjectType::SenderAuth => {}
+            ObjectType::Sharing => {}
+            ObjectType::SieveSystemInterpreter => {}
+            ObjectType::SieveSystemScript => {}
+            ObjectType::SieveUserInterpreter => {}
+            ObjectType::SieveUserScript => {}
+            ObjectType::SpamClassifier => {}
+            ObjectType::SpamDnsblServer => {}
+            ObjectType::SpamDnsblSettings => {}
+            ObjectType::SpamFileExtension => {}
+            ObjectType::SpamLlm => {}
+            ObjectType::SpamPyzor => {}
+            ObjectType::SpamRule => {}
+            ObjectType::SpamSettings => {}
+            ObjectType::SpamTag => {}
+            ObjectType::SpfReportSettings => {}
+            ObjectType::StoreLookup => {}
+            ObjectType::TlsReportSettings => {}
+            ObjectType::Tracer => {}
+            ObjectType::TracingStore => {}
+            ObjectType::WebDav => {}
+            ObjectType::WebHook => {}
+
+            // Tenant filtered
+            ObjectType::Account => {}
+            ObjectType::DsnReportSettings => {}
+            ObjectType::MailingList => {}
+            ObjectType::OAuthClient => {}
+            ObjectType::Role => {}
+            ObjectType::Tenant => {}
+
+            // Account filtered
+            ObjectType::MaskedEmail => {}
+            ObjectType::PublicKey => {}
+
+            // Special
+            ObjectType::DkimSignature => {}
+            ObjectType::Domain => {}
+            ObjectType::Log => {}
+            ObjectType::QueuedMessage => {}
+            ObjectType::Task => {}
+
+            // Move to registry?
+            ObjectType::ArfFeedbackReport => {}
+            ObjectType::DmarcReport => {}
+            ObjectType::TlsReport => {}
+            ObjectType::DeletedItem => {}
+            ObjectType::Metric => {}
+            ObjectType::Trace => {}
+            ObjectType::SpamTrainingSample => {}
+        }
+
+        let todo = "read only properties";
+
         todo!()
     }
 }

crates/registry/src/jmap/mod.rs

@@ -0,0 +1,60 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::{
+    schema::prelude::Property,
+    types::{error::PatchError, string::StringValidator},
+};
+use jmap_tools::{JsonPointer, Value};
+use std::fmt::Debug;
+use types::{blob::BlobId, id::Id};
+
+pub mod patch;
+pub mod properties;
+pub mod ser;
+
+pub type JmapValue<'x> = Value<'x, Property, RegistryValue>;
+
+#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub enum RegistryValue {
+    Id(Id),
+    BlobId(BlobId),
+    IdReference(String),
+}
+
+#[derive(Debug, Clone)]
+pub struct JsonPointerPatch<'x> {
+    ptr: &'x JsonPointer<Property>,
+    pos: usize,
+    validators: &'x [StringValidator],
+}
+
+pub trait RegistryJsonPatch: Debug + Default {
+    fn patch(
+        &mut self,
+        pointer: JsonPointerPatch<'_>,
+        value: JmapValue<'_>,
+    ) -> Result<(), PatchError>;
+}
+pub trait RegistryJsonPropertyPatch: Debug + Default {
+    fn patch_property(
+        &mut self,
+        pointer: JsonPointerPatch<'_>,
+        value: JmapValue<'_>,
+    ) -> Result<(), PatchError>;
+}
+
+pub trait RegistryJsonEnumPatch: Debug {
+    fn patch(
+        &mut self,
+        pointer: JsonPointerPatch<'_>,
+        value: JmapValue<'_>,
+    ) -> Result<(), PatchError>;
+}
+
+pub trait IntoValue {
+    fn into_value(self) -> JmapValue<'static>;
+}

crates/registry/src/jmap/patch.rs

@@ -5,6 +5,10 @@
  */
 
 use crate::{
+    jmap::{
+        JsonPointerPatch, RegistryJsonEnumPatch, RegistryJsonPatch, RegistryJsonPropertyPatch,
+        RegistryValue,
+    },
     schema::prelude::Property,
     types::{
         EnumImpl,
@@ -13,47 +17,9 @@ use crate::{
     },
 };
 use jmap_tools::{JsonPointer, JsonPointerItem, Key, Value};
-use std::{borrow::Cow, fmt::Debug, str::FromStr};
+use std::fmt::Debug;
-use types::{blob::BlobId, id::Id};
 use utils::map::vec_map::VecMap;
 
-#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
-pub enum RegistryValue {
-    Id(Id),
-    BlobId(BlobId),
-    IdReference(String),
-}
-
-#[derive(Debug, Clone)]
-pub struct JsonPointerPatch<'x> {
-    ptr: &'x JsonPointer<Property>,
-    pos: usize,
-    validators: &'x [StringValidator],
-}
-
-pub trait RegistryJsonPatch: Debug + Default {
-    fn patch(
-        &mut self,
-        pointer: JsonPointerPatch<'_>,
-        value: Value<'_, Property, RegistryValue>,
-    ) -> Result<(), PatchError>;
-}
-pub trait RegistryJsonPropertyPatch: Debug + Default {
-    fn patch_property(
-        &mut self,
-        pointer: JsonPointerPatch<'_>,
-        value: Value<'_, Property, RegistryValue>,
-    ) -> Result<(), PatchError>;
-}
-
-pub trait RegistryJsonEnumPatch: Debug {
-    fn patch(
-        &mut self,
-        pointer: JsonPointerPatch<'_>,
-        value: Value<'_, Property, RegistryValue>,
-    ) -> Result<(), PatchError>;
-}
-
 impl<'x> JsonPointerPatch<'x> {
     pub fn new(ptr: &'x JsonPointer<Property>) -> Self {
         Self {
@@ -107,86 +73,6 @@ impl<'x> JsonPointerPatch<'x> {
     }
 }
 
-impl jmap_tools::Property for Property {
-    fn try_parse(_: Option<&Key<'_, Self>>, value: &str) -> Option<Self> {
-        Property::parse(value)
-    }
-
-    fn to_cow(&self) -> Cow<'static, str> {
-        self.as_str().into()
-    }
-}
-
-impl FromStr for Property {
-    type Err = ();
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        Property::parse(s).ok_or(())
-    }
-}
-
-impl jmap_tools::Element for RegistryValue {
-    type Property = Property;
-
-    fn try_parse<P>(key: &Key<'_, Self::Property>, value: &str) -> Option<Self> {
-        if let Key::Property(prop) = key {
-            match prop {
-                Property::Id
-                | Property::MemberGroupIds
-                | Property::MemberTenantId
-                | Property::RoleIds
-                | Property::DnsServerId
-                | Property::DirectoryId
-                | Property::DomainId
-                | Property::AccountId
-                | Property::DefaultDomainId
-                | Property::DefaultUserRoleIds
-                | Property::DefaultGroupRoleIds
-                | Property::DefaultTenantRoleIds
-                | Property::QueueId
-                | Property::ModelId
-                | Property::AcmeProviderId => {
-                    if let Some(reference) = value.strip_prefix('#') {
-                        Some(RegistryValue::IdReference(reference.to_string()))
-                    } else {
-                        Id::from_str(value).map(RegistryValue::Id).ok()
-                    }
-                }
-                Property::BlobId => {
-                    if let Some(reference) = value.strip_prefix('#') {
-                        Some(RegistryValue::IdReference(reference.to_string()))
-                    } else {
-                        BlobId::from_str(value).map(RegistryValue::BlobId).ok()
-                    }
-                }
-                _ => None,
-            }
-        } else {
-            None
-        }
-    }
-
-    fn to_cow(&self) -> Cow<'static, str> {
-        match self {
-            RegistryValue::Id(id) => id.to_string().into(),
-            RegistryValue::BlobId(blob_id) => blob_id.to_string().into(),
-            RegistryValue::IdReference(r) => format!("#{r}").into(),
-        }
-    }
-}
-
-impl From<Id> for RegistryValue {
-    fn from(id: Id) -> Self {
-        RegistryValue::Id(id)
-    }
-}
-
-impl From<BlobId> for RegistryValue {
-    fn from(id: BlobId) -> Self {
-        RegistryValue::BlobId(id)
-    }
-}
-
 impl<T: RegistryJsonPatch> RegistryJsonPatch for Option<T> {
     fn patch(
         &mut self,

crates/registry/src/jmap/properties.rs

@@ -0,0 +1,90 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::{jmap::RegistryValue, schema::prelude::Property, types::EnumImpl};
+use jmap_tools::Key;
+use std::{borrow::Cow, str::FromStr};
+use types::{blob::BlobId, id::Id};
+
+impl jmap_tools::Property for Property {
+    fn try_parse(_: Option<&Key<'_, Self>>, value: &str) -> Option<Self> {
+        Property::parse(value)
+    }
+
+    fn to_cow(&self) -> Cow<'static, str> {
+        self.as_str().into()
+    }
+}
+
+impl FromStr for Property {
+    type Err = ();
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Property::parse(s).ok_or(())
+    }
+}
+
+impl jmap_tools::Element for RegistryValue {
+    type Property = Property;
+
+    fn try_parse<P>(key: &Key<'_, Self::Property>, value: &str) -> Option<Self> {
+        if let Key::Property(prop) = key {
+            match prop {
+                Property::Id
+                | Property::MemberGroupIds
+                | Property::MemberTenantId
+                | Property::RoleIds
+                | Property::DnsServerId
+                | Property::DirectoryId
+                | Property::DomainId
+                | Property::AccountId
+                | Property::DefaultDomainId
+                | Property::DefaultUserRoleIds
+                | Property::DefaultGroupRoleIds
+                | Property::DefaultTenantRoleIds
+                | Property::QueueId
+                | Property::ModelId
+                | Property::AcmeProviderId => {
+                    if let Some(reference) = value.strip_prefix('#') {
+                        Some(RegistryValue::IdReference(reference.to_string()))
+                    } else {
+                        Id::from_str(value).map(RegistryValue::Id).ok()
+                    }
+                }
+                Property::BlobId => {
+                    if let Some(reference) = value.strip_prefix('#') {
+                        Some(RegistryValue::IdReference(reference.to_string()))
+                    } else {
+                        BlobId::from_str(value).map(RegistryValue::BlobId).ok()
+                    }
+                }
+                _ => None,
+            }
+        } else {
+            None
+        }
+    }
+
+    fn to_cow(&self) -> Cow<'static, str> {
+        match self {
+            RegistryValue::Id(id) => id.to_string().into(),
+            RegistryValue::BlobId(blob_id) => blob_id.to_string().into(),
+            RegistryValue::IdReference(r) => format!("#{r}").into(),
+        }
+    }
+}
+
+impl From<Id> for RegistryValue {
+    fn from(id: Id) -> Self {
+        RegistryValue::Id(id)
+    }
+}
+
+impl From<BlobId> for RegistryValue {
+    fn from(id: BlobId) -> Self {
+        RegistryValue::BlobId(id)
+    }
+}

crates/registry/src/jmap/ser.rs

@@ -0,0 +1,102 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::{
+    jmap::{IntoValue, JmapValue},
+    schema::prelude::Property,
+    types::EnumImpl,
+};
+use jmap_tools::Key;
+use std::fmt::Debug;
+use utils::map::vec_map::VecMap;
+
+impl<T: IntoValue> IntoValue for Option<T> {
+    fn into_value(self) -> JmapValue<'static> {
+        match self {
+            Some(value) => value.into_value(),
+            None => JmapValue::Null,
+        }
+    }
+}
+
+impl IntoValue for String {
+    fn into_value(self) -> JmapValue<'static> {
+        JmapValue::Str(self.into())
+    }
+}
+
+impl IntoValue for bool {
+    fn into_value(self) -> JmapValue<'static> {
+        JmapValue::Bool(self)
+    }
+}
+
+impl IntoValue for u64 {
+    fn into_value(self) -> JmapValue<'static> {
+        JmapValue::Number(self.into())
+    }
+}
+
+impl IntoValue for i64 {
+    fn into_value(self) -> JmapValue<'static> {
+        JmapValue::Number(self.into())
+    }
+}
+
+impl IntoValue for f64 {
+    fn into_value(self) -> JmapValue<'static> {
+        JmapValue::Number(self.into())
+    }
+}
+
+impl<T: EnumImpl> IntoValue for T {
+    fn into_value(self) -> JmapValue<'static> {
+        JmapValue::Str(self.as_str().into())
+    }
+}
+
+trait MapKey: Sized + PartialEq + Eq + Debug {
+    fn to_key(self) -> Key<'static, Property>;
+}
+
+impl MapKey for String {
+    fn to_key(self) -> Key<'static, Property> {
+        Key::Owned(self)
+    }
+}
+
+impl MapKey for u32 {
+    fn to_key(self) -> Key<'static, Property> {
+        Key::Owned(self.to_string())
+    }
+}
+
+impl<T: EnumImpl> MapKey for T {
+    fn to_key(self) -> Key<'static, Property> {
+        Key::Borrowed(self.as_str())
+    }
+}
+
+impl<K: MapKey, V: IntoValue> IntoValue for VecMap<K, V> {
+    fn into_value(self) -> JmapValue<'static> {
+        let mut map = jmap_tools::Map::with_capacity(self.len());
+        for (k, v) in self {
+            map.insert_unchecked(k.to_key(), v.into_value());
+        }
+        JmapValue::Object(map)
+    }
+}
+
+impl<V: IntoValue> IntoValue for Vec<V> {
+    fn into_value(self) -> JmapValue<'static> {
+        let mut array = Vec::with_capacity(self.len());
+        for v in self {
+            array.push(v.into_value());
+        }
+
+        JmapValue::Array(array)
+    }
+}

crates/registry/src/schema/mod.rs

@@ -5,15 +5,14 @@
  */
 
 use crate::{
-    pickle::Pickle,
     schema::{
         enums::{TracingLevel, TracingLevelOpt},
         prelude::{
-            Account, Duration, GroupAccount, HashedObject, HttpAuth, NodeRange, Object,
+            Account, Duration, GroupAccount, HttpAuth, NodeRange, Object, ObjectInner, Property,
-            ObjectInner, ObjectType, Property, UserAccount,
+            UserAccount,
         },
     },
-    types::{EnumImpl, ObjectImpl},
+    types::EnumImpl,
 };
 use std::{cmp::Ordering, fmt::Display};
 use trc::TOTAL_EVENT_COUNT;
@@ -239,52 +238,8 @@ impl<T: Into<ObjectInner>> From<T> for Object {
     }
 }
 
-impl<T: ObjectImpl + From<Object>> From<Object> for HashedObject<T> {
+impl Object {
-    fn from(value: Object) -> Self {
+    pub fn new(inner: ObjectInner) -> Self {
-        HashedObject {
+        Object { inner, revision: 0 }
-            revision: value.revision,
-            object: T::from(value),
-        }
-    }
-}
-
-impl<T: ObjectImpl> ObjectImpl for HashedObject<T> {
-    const FLAGS: u64 = T::FLAGS;
-    const OBJECT: ObjectType = T::OBJECT;
-
-    fn validate(&self, errors: &mut Vec<prelude::ValidationError>) -> bool {
-        self.object.validate(errors)
-    }
-
-    fn index<'x>(&'x self, builder: &mut prelude::IndexBuilder<'x>) {
-        self.object.index(builder)
-    }
-}
-
-impl<T: ObjectImpl> Pickle for HashedObject<T> {
-    fn pickle(&self, out: &mut Vec<u8>) {
-        T::OBJECT.pickle(out);
-        self.object.pickle(out);
-        (xxhash_rust::xxh3::xxh3_64(out) as u32).pickle(out);
-    }
-
-    fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option<Self> {
-        let _ = u16::unpickle(stream)?;
-        Some(Self {
-            object: T::unpickle(stream)?,
-            revision: u32::unpickle(stream)?,
-        })
-    }
-}
-
-impl<'de, T: ObjectImpl> serde::Deserialize<'de> for HashedObject<T> {
-    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
-    where
-        D: serde::Deserializer<'de>,
-    {
-        T::deserialize(deserializer).map(|object| Self {
-            object,
-            revision: 0,
-        })
     }
 }

crates/registry/src/schema/prelude.rs

@@ -3,9 +3,11 @@
  *
  * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
  */
+pub use crate::jmap::IntoValue;
+pub use crate::jmap::JmapValue;
 pub use crate::jmap::{
     JsonPointerPatch, RegistryJsonEnumPatch, RegistryJsonPatch, RegistryJsonPropertyPatch,
-    object_type,
+    patch::object_type,
 };
 pub use crate::pickle::Pickle;
 pub use crate::schema::enums::*;
@@ -32,12 +34,6 @@ pub struct Object {
     pub revision: u32,
 }
 
-#[derive(Debug, Clone, Default, Serialize)]
-pub struct HashedObject<T: ObjectImpl> {
-    pub object: T,
-    pub revision: u32,
-}
-
 #[derive(Debug)]
 pub struct ExpressionContext<'x> {
     pub expr: &'x Expression,

crates/registry/src/types/datetime.rs

@@ -5,7 +5,7 @@
  */
 
 use crate::{
-    jmap::{JsonPointerPatch, RegistryJsonPatch},
+    jmap::{IntoValue, JmapValue, JsonPointerPatch, RegistryJsonPatch},
     pickle::{Pickle, PickledStream},
     types::error::PatchError,
 };
@@ -270,7 +270,7 @@ impl RegistryJsonPatch for UTCDateTime {
     fn patch(
         &mut self,
         mut pointer: JsonPointerPatch<'_>,
-        value: jmap_tools::Value<'_, crate::schema::prelude::Property, crate::jmap::RegistryValue>,
+        value: JmapValue<'_>,
     ) -> Result<(), PatchError> {
         match (value, pointer.next()) {
             (jmap_tools::Value::Str(value), None) => {
@@ -292,6 +292,12 @@ impl RegistryJsonPatch for UTCDateTime {
     }
 }
 
+impl IntoValue for UTCDateTime {
+    fn into_value(self) -> JmapValue<'static> {
+        JmapValue::Str(self.to_string().into())
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use std::str::FromStr;

crates/registry/src/types/duration.rs

@@ -5,7 +5,7 @@
  */
 
 use crate::{
-    jmap::{JsonPointerPatch, RegistryJsonPatch},
+    jmap::{IntoValue, JmapValue, JsonPointerPatch, RegistryJsonPatch},
     pickle::{Pickle, PickledStream},
     types::error::PatchError,
 };
@@ -141,7 +141,7 @@ impl RegistryJsonPatch for Duration {
     fn patch(
         &mut self,
         mut pointer: JsonPointerPatch<'_>,
-        value: jmap_tools::Value<'_, crate::schema::prelude::Property, crate::jmap::RegistryValue>,
+        value: JmapValue<'_>,
     ) -> Result<(), PatchError> {
         match (value, pointer.next()) {
             (jmap_tools::Value::Number(value), None) => {
@@ -156,3 +156,9 @@ impl RegistryJsonPatch for Duration {
         }
     }
 }
+
+impl IntoValue for Duration {
+    fn into_value(self) -> JmapValue<'static> {
+        JmapValue::Number((self.0.as_millis() as u64).into())
+    }
+}

crates/registry/src/types/id.rs

@@ -5,7 +5,7 @@
  */
 
 use crate::{
-    jmap::{JsonPointerPatch, RegistryJsonPatch, RegistryValue},
+    jmap::{IntoValue, JmapValue, JsonPointerPatch, RegistryJsonPatch, RegistryValue},
     pickle::{Pickle, PickledStream},
     schema::prelude::ObjectType,
     types::{EnumImpl, error::PatchError},
@@ -80,7 +80,7 @@ impl RegistryJsonPatch for Id {
     fn patch(
         &mut self,
         mut pointer: JsonPointerPatch<'_>,
-        value: jmap_tools::Value<'_, crate::schema::prelude::Property, crate::jmap::RegistryValue>,
+        value: JmapValue<'_>,
     ) -> Result<(), PatchError> {
         match (value, pointer.next()) {
             (jmap_tools::Value::Element(RegistryValue::Id(value)), None) => {
@@ -107,7 +107,7 @@ impl RegistryJsonPatch for BlobId {
     fn patch(
         &mut self,
         mut pointer: JsonPointerPatch<'_>,
-        value: jmap_tools::Value<'_, crate::schema::prelude::Property, crate::jmap::RegistryValue>,
+        value: JmapValue<'_>,
     ) -> Result<(), PatchError> {
         match (value, pointer.next()) {
             (jmap_tools::Value::Element(RegistryValue::BlobId(value)), None) => {
@@ -132,3 +132,15 @@ impl RegistryJsonPatch for BlobId {
         }
     }
 }
+
+impl IntoValue for Id {
+    fn into_value(self) -> JmapValue<'static> {
+        JmapValue::Element(RegistryValue::Id(self))
+    }
+}
+
+impl IntoValue for BlobId {
+    fn into_value(self) -> JmapValue<'static> {
+        JmapValue::Element(RegistryValue::BlobId(self))
+    }
+}

crates/registry/src/types/ipaddr.rs

@@ -7,7 +7,7 @@
 use std::{fmt::Display, net::Ipv4Addr, str::FromStr};
 
 use crate::{
-    jmap::{JsonPointerPatch, RegistryJsonPatch},
+    jmap::{IntoValue, JmapValue, JsonPointerPatch, RegistryJsonPatch},
     pickle::{Pickle, PickledStream},
     types::error::PatchError,
 };
@@ -122,7 +122,7 @@ impl RegistryJsonPatch for IpAddr {
     fn patch(
         &mut self,
         mut pointer: JsonPointerPatch<'_>,
-        value: jmap_tools::Value<'_, crate::schema::prelude::Property, crate::jmap::RegistryValue>,
+        value: JmapValue<'_>,
     ) -> Result<(), PatchError> {
         match (value, pointer.next()) {
             (jmap_tools::Value::Str(value), None) => {
@@ -143,3 +143,9 @@ impl RegistryJsonPatch for IpAddr {
         }
     }
 }
+
+impl IntoValue for IpAddr {
+    fn into_value(self) -> JmapValue<'static> {
+        JmapValue::Str(self.to_string().into())
+    }
+}

crates/registry/src/types/ipmask.rs

@@ -5,7 +5,7 @@
  */
 
 use crate::{
-    jmap::{JsonPointerPatch, RegistryJsonPatch},
+    jmap::{IntoValue, JmapValue, JsonPointerPatch, RegistryJsonPatch},
     pickle::{Pickle, PickledStream},
     types::error::PatchError,
 };
@@ -251,7 +251,7 @@ impl RegistryJsonPatch for IpAddrOrMask {
     fn patch(
         &mut self,
         mut pointer: JsonPointerPatch<'_>,
-        value: jmap_tools::Value<'_, crate::schema::prelude::Property, crate::jmap::RegistryValue>,
+        value: JmapValue<'_>,
     ) -> Result<(), PatchError> {
         match (value, pointer.next()) {
             (jmap_tools::Value::Str(value), None) => {
@@ -273,6 +273,12 @@ impl RegistryJsonPatch for IpAddrOrMask {
     }
 }
 
+impl IntoValue for IpAddrOrMask {
+    fn into_value(self) -> JmapValue<'static> {
+        JmapValue::Str(self.to_string().into())
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

crates/registry/src/types/socketaddr.rs

@@ -7,7 +7,7 @@
 use std::{fmt::Display, str::FromStr};
 
 use crate::{
-    jmap::{JsonPointerPatch, RegistryJsonPatch},
+    jmap::{IntoValue, JmapValue, JsonPointerPatch, RegistryJsonPatch},
     pickle::{Pickle, PickledStream},
     types::error::PatchError,
 };
@@ -91,7 +91,7 @@ impl RegistryJsonPatch for SocketAddr {
     fn patch(
         &mut self,
         mut pointer: JsonPointerPatch<'_>,
-        value: jmap_tools::Value<'_, crate::schema::prelude::Property, crate::jmap::RegistryValue>,
+        value: JmapValue<'_>,
     ) -> Result<(), PatchError> {
         match (value, pointer.next()) {
             (jmap_tools::Value::Str(value), None) => {
@@ -112,3 +112,9 @@ impl RegistryJsonPatch for SocketAddr {
         }
     }
 }
+
+impl IntoValue for SocketAddr {
+    fn into_value(self) -> JmapValue<'static> {
+        JmapValue::Str(self.to_string().into())
+    }
+}

crates/services/src/housekeeper/mod.rs

@@ -395,6 +395,7 @@ pub fn spawn_housekeeper(inner: Arc<Inner>, mut rx: mpsc::Receiver<HousekeeperEv
                                     Housekeeper(trc::HousekeeperEvent::Run),
                                     Type = "purge_data_store"
                                 );
+                                let todo = "make sure all store types are purged, in memory, metrics, tracing, etc";
 
                                 queue.schedule(
                                     Instant::now()

crates/store/src/registry/get.rs

@@ -10,7 +10,7 @@ use crate::{
     write::{AnyClass, RegistryClass, ValueClass, key::KeySerializer},
 };
 use registry::{
-    pickle::PickledStream,
+    pickle::{Pickle, PickledStream},
     schema::prelude::Object,
     types::{EnumImpl, ObjectImpl, id::ObjectId},
 };
@@ -56,16 +56,17 @@ impl RegistryStore {
     }
 
     pub async fn list<T: ObjectImpl + From<Object>>(&self) -> trc::Result<Vec<RegistryObject<T>>> {
-        let object = T::OBJECT;
+        let object_type = T::OBJECT;
 
-        if self.0.local_objects.contains(&object) {
+        if self.0.local_objects.contains(&object_type) {
             let mut results = Vec::new();
 
             for (id, item) in self.0.local_registry.read().iter() {
-                if id.object() == object {
+                if id.object() == object_type {
                     results.push(RegistryObject {
                         id: *id,
                         object: T::from(item.clone()),
+                        revision: 0,
                     });
                 }
             }
@@ -81,14 +82,14 @@ impl RegistryStore {
                             subspace: SUBSPACE_REGISTRY,
                             key: KeySerializer::new(U16_LEN + 1)
                                 .write(0u8)
-                                .write(object.to_id())
+                                .write(object_type.to_id())
                                 .finalize(),
                         })),
                         ValueKey::from(ValueClass::Any(AnyClass {
                             subspace: SUBSPACE_REGISTRY,
                             key: KeySerializer::new(U16_LEN + U64_LEN + 1)
                                 .write(0u8)
-                                .write(object.to_id())
+                                .write(object_type.to_id())
                                 .write(u64::MAX)
                                 .finalize(),
                         })),
@@ -102,23 +103,26 @@ impl RegistryStore {
                                 trc::EventType::Registry(trc::RegistryEvent::DeserializationError)
                                     .into_err()
                                     .caused_by(trc::location!())
-                                    .details(object.as_str())
+                                    .details(object_type.as_str())
                                     .ctx(trc::Key::Key, key)
                             })?;
-                        let item = T::unpickle(&mut PickledStream::new(
+                        let mut stream = PickledStream::new(value);
-                            value.get(U16_LEN..).unwrap_or_default(),
+                        let _ = u16::unpickle(&mut stream);
-                        ))
+                        let (object, revision) = T::unpickle(&mut stream)
-                        .ok_or_else(|| {
+                            .and_then(|item| u32::unpickle(&mut stream).map(|rev| (item, rev)))
-                            trc::EventType::Registry(trc::RegistryEvent::DeserializationError)
+                            .ok_or_else(|| {
-                                .into_err()
+                                trc::EventType::Registry(trc::RegistryEvent::DeserializationError)
-                                .caused_by(trc::location!())
+                                    .into_err()
-                                .id(id)
+                                    .caused_by(trc::location!())
-                                .details(object.as_str())
+                                    .id(id)
-                                .ctx(trc::Key::Value, value)
+                                    .details(object_type.as_str())
-                        })?;
+                                    .ctx(trc::Key::Value, value)
+                            })?;
+
                         results.push(RegistryObject {
-                            id: ObjectId::new(object, Id::new(id)),
+                            id: ObjectId::new(object_type, Id::new(id)),
-                            object: item,
+                            object,
+                            revision,
                         });
 
                         Ok(true)

crates/store/src/registry/mod.rs

@@ -18,6 +18,7 @@ use registry::{
 pub struct RegistryObject<T: ObjectImpl> {
     pub id: ObjectId,
     pub object: T,
+    pub revision: u32,
 }
 
 pub struct RegistryQuery {

crates/store/src/registry/query.rs

@@ -211,6 +211,11 @@ impl RegistryQuery {
         self
     }
 
+    pub fn with_account_opt(mut self, account_id: Option<u32>) -> Self {
+        self.account_id = account_id;
+        self
+    }
+
     pub fn with_tenant(mut self, tenant_id: Option<u32>) -> Self {
         self.tenant_id = tenant_id;
         self