filip/Stalwart
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

JMAP Registry API implementation - part 5

Browse Source
This commit is contained in:
mdecimus
2026-02-27 20:10:24 +01:00
parent e24b595a16
commit 07d5748f6b
35 changed files with 1460 additions and 316 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Cargo.lock generated
View File

@@ -3585,6 +3585,7 @@ dependencies = [
"rand 0.9.2",
"registry",
"reqwest",
"rev_lines",
"rkyv",
"rsa",
"serde",

2
crates/common/src/auth/access_token.rs
View File

@@ -165,6 +165,7 @@ impl Server {
.credentials
.into_iter()
.filter_map(|(credential_id, credential)| {
let credential = credential.unwrap_properties();
let expires_at = credential
.expires_at
.map(|v| v.timestamp() as u64)
@@ -750,6 +751,7 @@ fn hash_account(account: &Account) -> u64 {
account.role_ids.hash(&mut s);
hash_permissions(&mut s, &account.permissions);
for (credential_id, credential) in &account.credentials {
let credential = credential.as_properties();
credential_id.hash(&mut s);
credential.expires_at.hash(&mut s);
hash_permissions(&mut s, &credential.permissions);

14
crates/common/src/auth/authentication.rs
View File

@@ -17,8 +17,8 @@ use directory::{
core::secret::{verify_mfa_secret_hash, verify_secret_hash},
};
use registry::schema::{
enums::{CredentialType, Permission},
structs,
enums::Permission,
structs::{self, Credential},
};
use std::{net::IpAddr, sync::Arc};
use store::write::now;
@@ -305,7 +305,9 @@ impl Server {
.and_then(|account| account.into_user())
{
// Find credential by credential_id
for (id, credential) in &account.credentials {
for (id, credential_) in &account.credentials {
let credential = credential_.as_properties();
if *id == credential_id {
if !verify_secret_hash(&credential.secret, secret).await? {
return Err(trc::AuthEvent::Failed
@@ -337,9 +339,9 @@ impl Server {
AccountId = account_id,
Id = credential_id,
SpanId = span_id,
Details = match credential.credential_type {
CredentialType::AppPassword => "Authenticated with app password",
CredentialType::ApiKey => "Authenticated with API key",
Details = match credential_ {
Credential::AppPassword(_) => "Authenticated with app password",
Credential::ApiKey(_) => "Authenticated with API key",
}
);

4
crates/email/src/message/ingest.rs
View File

@@ -964,7 +964,7 @@ impl EmailIngest for Server {
ObjectId::new(ObjectType::SpamTrainingSample, item_id.into()).serialize(),
)
.set(
ValueClass::Registry(RegistryClass::Id { object_id, item_id }),
ValueClass::Registry(RegistryClass::Item { object_id, item_id }),
sample,
)
.set(
@@ -972,7 +972,7 @@ impl EmailIngest for Server {
index_id: Property::AccountId.to_id(),
object_id,
item_id,
key: account_id.serialize(),
key: (account_id as u64).serialize(),
}),
vec![],
);

5
crates/jmap-proto/src/error/set.rs
View File

@@ -195,6 +195,11 @@ impl<T: Property> SetError<T> {
Self::new(SetErrorType::WillDestroy).with_description("ID will be destroyed.")
}
pub fn singleton() -> Self {
Self::new(SetErrorType::Singleton)
.with_description("Singletons cannot be created or destroyed.")
}
pub fn address_book_has_contents() -> Self {
Self::new(SetErrorType::AddressBookHasContents)
.with_description("Address book is not empty.")

1
crates/jmap/Cargo.toml
View File

@@ -53,6 +53,7 @@ rsa = "0.9.2"
rkyv = { version = "0.8.10", features = ["little_endian"] }
compact_str = "0.9.0"
hashify = "0.2"
rev_lines = "0.3.0"
[features]
test_mode = []

82
crates/jmap/src/blob/download.rs
View File

@@ -9,6 +9,7 @@ use email::cache::MessageCacheFetch;
use email::cache::email::MessageCacheAccess;
use email::message::metadata::MessageMetadata;
use groupware::cache::GroupwareCache;
use registry::schema::enums::Permission;
use std::future::Future;
use store::ValueKey;
use store::write::{AlignedBytes, Archive};
@@ -100,44 +101,49 @@ impl BlobDownload for Server {
blob_id: &BlobId,
access_token: &AccessToken,
) -> trc::Result<bool> {
Ok(self
.store()
.blob_has_access(&blob_id.hash, &blob_id.class)
.await
.caused_by(trc::location!())?
&& match &blob_id.class {
BlobClass::Linked {
account_id,
collection,
document_id,
} => {
if access_token.is_member(*account_id) {
true
} else {
match Collection::from(*collection) {
Collection::Email => self
.get_cached_messages(*account_id)
.await
.caused_by(trc::location!())?
.shared_messages(access_token, Acl::ReadItems)
.contains(*document_id),
collection @ (Collection::FileNode
| Collection::ContactCard
| Collection::CalendarEvent) => self
.fetch_dav_resources(
access_token.account_id(),
*account_id,
SyncCollection::from(collection),
)
.await
.caused_by(trc::location!())?
.shared_items(access_token, [Acl::ReadItems], true)
.contains(*document_id),
_ => false,
Ok(
(blob_id.class.is_superuser() && access_token.has_permission(Permission::BlobFetch))
|| (self
.store()
.blob_has_access(&blob_id.hash, &blob_id.class)
.await
.caused_by(trc::location!())?
&& match &blob_id.class {
BlobClass::Linked {
account_id,
collection,
document_id,
} => {
if access_token.is_member(*account_id) {
true
} else {
match Collection::from(*collection) {
Collection::Email => self
.get_cached_messages(*account_id)
.await
.caused_by(trc::location!())?
.shared_messages(access_token, Acl::ReadItems)
.contains(*document_id),
collection @ (Collection::FileNode
| Collection::ContactCard
| Collection::CalendarEvent) => self
.fetch_dav_resources(
access_token.account_id(),
*account_id,
SyncCollection::from(collection),
)
.await
.caused_by(trc::location!())?
.shared_items(access_token, [Acl::ReadItems], true)
.contains(*document_id),
_ => false,
}
}
}
}
}
BlobClass::Reserved { account_id, .. } => access_token.is_member(*account_id),
})
BlobClass::Reserved { account_id, .. } => {
access_token.is_member(*account_id)
}
}),
)
}
}

2
crates/jmap/src/email/set.rs
View File

@@ -105,7 +105,7 @@ impl EmailSet for Server {
#[cfg(not(feature = "test_mode"))]
{
self.get_access_token(account_id)
self.access_token(account_id)
.await
.caused_by(trc::location!())?
.into()

230
crates/jmap/src/registry/get.rs
View File

@@ -4,6 +4,17 @@
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use crate::registry::mapping::{
RegistryGetResponse,
account::account_get,
deleted_item::deleted_item_get,
log::log_get,
queued_message::queued_message_get,
report::report_get,
spam_sample::spam_sample_get,
task::task_get,
telemetry::{metric_get, trace_get},
};
use common::{Server, auth::AccessToken};
use jmap_proto::{
method::get::{GetRequest, GetResponse},
@@ -41,25 +52,38 @@ impl RegistryGet for Server {
mut request: GetRequest<Registry>,
access_token: &AccessToken,
) -> trc::Result<GetResponse<Registry>> {
let ids = request.unwrap_ids(self.core.jmap.get_max_objects)?;
let mut properties = request
.properties
.take()
.map(|p| p.unwrap())
.unwrap_or_default()
.into_iter()
.filter_map(|prop| prop.try_unwrap())
.collect::<AHashSet<_>>();
if !properties.is_empty() {
properties.insert(Property::Id);
}
let mut response = GetResponse {
account_id: request.account_id.into(),
state: None,
list: vec![],
not_found: vec![],
let object_flags = object_type.flags();
let is_tenant_filtered =
(object_flags & OBJ_FILTER_TENANT) != 0 && access_token.tenant_id().is_some();
let is_account_filtered = (object_flags & OBJ_FILTER_ACCOUNT) != 0
&& !access_token.has_permission(Permission::Impersonate);
let mut get = RegistryGetResponse {
access_token,
server: self,
account_id: request.account_id.document_id(),
object_type,
ids: request.unwrap_ids(self.core.jmap.get_max_objects)?,
properties: request
.properties
.take()
.map(|p| p.unwrap())
.unwrap_or_default()
.into_iter()
.filter_map(|prop| prop.try_unwrap())
.collect::<AHashSet<_>>(),
response: GetResponse {
account_id: request.account_id.into(),
state: None,
list: vec![],
not_found: vec![],
},
object_flags,
is_tenant_filtered,
is_account_filtered,
};
if !get.properties.is_empty() {
get.properties.insert(Property::Id);
}
match object_type {
ObjectType::AcmeProvider
@@ -163,34 +187,30 @@ impl RegistryGet for Server {
| ObjectType::PublicKey
| ObjectType::DkimSignature
| ObjectType::Domain => {
let flags = object_type.flags();
let is_singleton = (flags & OBJ_SINGLETON) != 0;
let is_tenant_filtered =
(flags & OBJ_FILTER_TENANT) != 0 && access_token.tenant_id().is_some();
let is_account_filtered = (flags & OBJ_FILTER_ACCOUNT) != 0
&& !access_token.has_permission(Permission::Impersonate);
let is_singleton = (get.object_flags & OBJ_SINGLETON) != 0;
let ids = if let Some(ids) = ids {
let ids = if let Some(ids) = get.ids.take() {
ids
} else {
self.registry()
let mut ids = self
.registry()
.query::<AHashSet<u64>>(
RegistryQuery::new(object_type)
.with_tenant(access_token.tenant_id())
.with_account_opt(
is_account_filtered.then_some(request.account_id.into()),
),
.with_account_opt(is_account_filtered.then_some(get.account_id)),
)
.await
.caused_by(trc::location!())?
.into_iter()
.take(self.core.jmap.get_max_objects)
.map(Id::new)
.collect()
.collect::<Vec<_>>();
ids.sort_unstable();
ids
};
response.list.reserve(ids.len());
get.response.list.reserve(ids.len());
'outer: for id in ids {
for id in ids {
let object = if let Some(object) = self
.registry()
.get(ObjectId::new(object_type, id))
@@ -199,23 +219,23 @@ impl RegistryGet for Server {
{
object
} else if id.is_singleton() && is_singleton {
Object::new(ObjectInner::from(object_type))
Object::from(object_type)
} else {
response.not_found.push(id);
get.not_found(id);
continue;
};
match &object.inner {
ObjectInner::DkimSignature(obj)
if properties.is_empty()
|| properties.contains(&Property::PublicKey) =>
if get.properties.is_empty()
|| get.properties.contains(&Property::PublicKey) =>
{
let todo = "dkim public key";
todo!()
}
ObjectInner::Domain(obj)
if properties.is_empty()
|| properties.contains(&Property::DnsZoneFile) =>
if get.properties.is_empty()
|| get.properties.contains(&Property::DnsZoneFile) =>
{
let todo = "domain dns zone file";
todo!()
@@ -223,61 +243,87 @@ impl RegistryGet for Server {
_ => {}
}
let todo = "compact pickle";
let todo = "app passwords, apis and user change pass/OTP";
let mut object = object.into_value();
let object_map = object.as_object_mut().unwrap();
if is_tenant_filtered && let Some(tenant_id) = access_token.tenant_id() {
let expected_value =
JmapValue::Element(RegistryValue::Id(Id::from(tenant_id)));
for (key, value) in object_map.iter() {
if matches!(key, Key::Property(Property::MemberTenantId))
&& value != &expected_value
{
response.not_found.push(id);
continue 'outer;
}
}
object_map.remove(&Key::Property(Property::MemberTenantId));
} else if is_account_filtered {
let expected_value =
JmapValue::Element(RegistryValue::Id(request.account_id));
for (key, value) in object_map.iter() {
if matches!(key, Key::Property(Property::AccountId))
&& value != &expected_value
{
response.not_found.push(id);
continue 'outer;
}
}
object_map.remove(&Key::Property(Property::AccountId));
}
object_map.insert_unchecked(Property::Id, RegistryValue::Id(id));
if !properties.is_empty() {
object_map.as_mut_vec().retain_mut(|(prop, _)| {
prop.as_property()
.is_some_and(|prop| properties.contains(prop))
});
}
response.list.push(object);
get.insert(id, object.into_value());
}
}
ObjectType::Log => {}
ObjectType::QueuedMessage => {}
ObjectType::Task => {}
ObjectType::ArfExternalReport => {}
ObjectType::DmarcExternalReport => {}
ObjectType::TlsExternalReport => {}
ObjectType::DeletedItem => {}
ObjectType::Metric => {}
ObjectType::Trace => {}
ObjectType::SpamTrainingSample => {}
ObjectType::DmarcInternalReport => todo!(),
ObjectType::TlsInternalReport => todo!(),
}
Ok(response)
Ok(get.into_response())
}
ObjectType::QueuedMessage => {
queued_message_get(get).await.map(|get| get.into_response())
}
ObjectType::Task => task_get(get).await.map(|get| get.into_response()),
ObjectType::ArfExternalReport
| ObjectType::DmarcExternalReport
| ObjectType::TlsExternalReport
| ObjectType::DmarcInternalReport
| ObjectType::TlsInternalReport => report_get(get).await.map(|get| get.into_response()),
ObjectType::DeletedItem => deleted_item_get(get).await.map(|get| get.into_response()),
ObjectType::SpamTrainingSample => {
spam_sample_get(get).await.map(|get| get.into_response())
}
ObjectType::Metric => metric_get(get).await.map(|get| get.into_response()),
ObjectType::Trace => trace_get(get).await.map(|get| get.into_response()),
ObjectType::Log => log_get(get).await.map(|get| get.into_response()),
ObjectType::AccountSettings | ObjectType::Credential => {
account_get(get).await.map(|get| get.into_response())
}
}
}
}
impl RegistryGetResponse<'_> {
pub fn insert(&mut self, id: Id, mut object: JmapValue<'static>) {
let object_map = object.as_object_mut().unwrap();
if self.is_tenant_filtered
&& let Some(tenant_id) = self.access_token.tenant_id()
{
let expected_value = JmapValue::Element(RegistryValue::Id(Id::from(tenant_id)));
for (key, value) in object_map.iter() {
if matches!(key, Key::Property(Property::MemberTenantId))
&& (value != &expected_value
|| value
.as_array()
.is_none_or(|arr| !arr.contains(&expected_value)))
{
self.not_found(id);
return;
}
}
object_map.remove(&Key::Property(Property::MemberTenantId));
} else if self.is_account_filtered {
let expected_value = JmapValue::Element(RegistryValue::Id(self.account_id.into()));
for (key, value) in object_map.iter() {
if matches!(key, Key::Property(Property::AccountId)) && value != &expected_value {
self.not_found(id);
return;
}
}
object_map.remove(&Key::Property(Property::AccountId));
}
object_map.insert_unchecked(Property::Id, RegistryValue::Id(id));
if !self.properties.is_empty() {
object_map.as_mut_vec().retain_mut(|(prop, _)| {
prop.as_property()
.is_some_and(|prop| self.properties.contains(prop))
});
}
self.response.list.push(object);
}
pub fn not_found(&mut self, id: Id) {
self.response.not_found.push(id);
}
pub fn not_found_any(mut self) -> Self {
self.response.not_found = self.ids.take().unwrap_or_default();
self
}
pub fn into_response(self) -> GetResponse<Registry> {
self.response
}
}

87
crates/jmap/src/registry/mapping/account.rs Normal file
View File

@@ -0,0 +1,87 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use crate::registry::mapping::RegistryGetResponse;
use registry::{
jmap::IntoValue,
schema::{
prelude::ObjectType,
structs::{Account, AccountSettings},
},
};
use types::id::Id;
pub(crate) async fn account_get(
mut get: RegistryGetResponse<'_>,
) -> trc::Result<RegistryGetResponse<'_>> {
let Some(Account::User(mut account)) = get
.server
.registry()
.object::<Account>(get.account_id.into())
.await?
else {
return Ok(get.not_found_any());
};
if get.access_token.tenant_id().is_some_and(|id| {
account
.member_tenant_id
.is_none_or(|aid| aid.document_id() != id)
}) {
return Ok(get.not_found_any());
}
match get.object_type {
ObjectType::AccountSettings => {
let mut ids = get
.ids
.take()
.unwrap_or_else(|| vec![Id::singleton()])
.into_iter();
for id in ids.by_ref() {
if id == Id::singleton() {
get.insert(
id,
AccountSettings {
encryption_at_rest: account.encryption_at_rest,
locale: account.locale,
otp_auth: account.otp_auth,
secret: account.secret,
}
.into_value(),
);
break;
} else {
get.not_found(id);
}
}
get.response.not_found.extend(ids);
}
ObjectType::Credential => {
let ids = if let Some(ids) = get.ids.take() {
ids
} else {
account
.credentials
.keys()
.map(|id| Id::from(*id))
.collect::<Vec<_>>()
};
for id in ids {
if let Some(credential) = account.credentials.remove(&id.document_id()) {
get.insert(id, credential.into_value());
} else {
get.not_found(id);
}
}
}
_ => unreachable!(),
}
Ok(get)
}

80
crates/jmap/src/registry/mapping/deleted_item.rs Normal file
View File

@@ -0,0 +1,80 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use crate::registry::mapping::RegistryGetResponse;
use registry::{
jmap::IntoValue,
schema::{
prelude::{Object, ObjectInner},
structs::{DeletedEmail, DeletedFileNode, DeletedItem},
},
types::EnumImpl,
};
use store::{
ValueKey,
ahash::AHashSet,
registry::RegistryQuery,
write::{RegistryClass, ValueClass},
};
use types::{blob::BlobClass, id::Id};
pub(crate) async fn deleted_item_get(
mut get: RegistryGetResponse<'_>,
) -> trc::Result<RegistryGetResponse<'_>> {
let object_id = get.object_type.to_id();
let ids = if let Some(ids) = get.ids.take() {
ids
} else {
get.server
.registry()
.query::<AHashSet<u64>>(
RegistryQuery::new(get.object_type).with_account(get.account_id),
)
.await?
.into_iter()
.take(get.server.core.jmap.get_max_objects)
.map(Id::from)
.collect()
};
for id in ids {
if let Some(mut item) = get
.server
.store()
.get_value::<Object>(ValueKey::from(ValueClass::Registry(RegistryClass::Item {
object_id,
item_id: id.id(),
})))
.await?
{
if get.is_account_filtered
&& let ObjectInner::DeletedItem(
DeletedItem::Email(DeletedEmail {
blob_id,
cleanup_at,
..
})
| DeletedItem::FileNode(DeletedFileNode {
blob_id,
cleanup_at,
..
}),
) = &mut item.inner
{
blob_id.class = BlobClass::Reserved {
account_id: get.account_id,
expires: cleanup_at.timestamp() as u64,
};
}
get.insert(id, item.into_value());
} else {
get.not_found(id);
}
}
Ok(get)
}

159
crates/jmap/src/registry/mapping/log.rs Normal file
View File

@@ -0,0 +1,159 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use crate::registry::mapping::RegistryGetResponse;
use chrono::DateTime;
use registry::{
jmap::IntoValue,
schema::{enums::TracingLevel, structs::Log},
types::{EnumImpl, datetime::UTCDateTime},
};
use rev_lines::RevLines;
use std::{
fs::{self, File},
io,
path::Path,
};
use store::ahash::AHashSet;
use tokio::sync::oneshot;
use trc::EventType;
use types::id::Id;
pub(crate) async fn log_get(
mut get: RegistryGetResponse<'_>,
) -> trc::Result<RegistryGetResponse<'_>> {
let Some(path) = get.server.core.metrics.log_path.clone() else {
return Err(trc::JmapEvent::InvalidArguments
.into_err()
.details("No log tracers configured on the server"));
};
let ids = if let Some(ids) = get.ids.take() {
ids.into_iter().map(|id| id.id()).collect::<AHashSet<_>>()
} else {
(0u64..get.server.core.jmap.get_max_objects as u64).collect()
};
if !ids.is_empty() {
// TODO: Use worker pool
let (tx, rx) = oneshot::channel();
tokio::task::spawn_blocking(move || {
let _ = tx.send(read_log_entries(path, ids));
});
rx.await
.map_err(|err| {
trc::EventType::Server(trc::ServerEvent::ThreadError)
.reason(err)
.caused_by(trc::location!())
})?
.map_err(|err| {
trc::ManageEvent::Error
.reason(err)
.details("Failed to read log files")
.caused_by(trc::location!())
})?
.into_iter()
.for_each(|(id, log)| {
get.insert(id, log.into_value());
});
}
Ok(get)
}
fn line_numbers(
path: impl AsRef<Path>,
filter: &str,
mut offset: usize,
limit: usize,
) -> io::Result<(usize, Vec<Id>)> {
let mut logs = fs::read_dir(path)?.collect::<Result<Vec<_>, _>>()?;
let mut total = 0;
// Sort the entries by file name in reverse order.
logs.sort_by_key(|b| std::cmp::Reverse(b.file_name()));
let mut entries = Vec::with_capacity(limit);
let mut logs = logs.into_iter();
let mut current_line = 0u64;
while let Some(log) = logs.next() {
if log.file_type()?.is_file() {
let mut rev_lines = RevLines::new(File::open(log.path())?);
while let Some(line) = rev_lines.next() {
let line = line.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
if filter.is_empty() || line.contains(filter) {
total += 1;
if offset == 0 {
entries.push(Id::from(current_line));
if entries.len() == limit {
if rev_lines.next().is_some() || logs.next().is_some() {
total += limit;
}
return Ok((total, entries));
}
} else {
offset -= 1;
}
}
current_line += 1;
}
}
}
Ok((total, entries))
}
fn read_log_entries(path: impl AsRef<Path>, lines: AHashSet<u64>) -> io::Result<Vec<(Id, Log)>> {
let mut logs = fs::read_dir(path)?.collect::<Result<Vec<_>, _>>()?;
// Sort the entries by file name in reverse order.
logs.sort_by_key(|b| std::cmp::Reverse(b.file_name()));
let mut entries = Vec::with_capacity(lines.len());
let mut current_line = 0;
'outer: for log in logs.into_iter() {
if log.file_type()?.is_file() {
for line in RevLines::new(File::open(log.path())?) {
let line = line.map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
if lines.contains(&current_line)
&& let Some(log) = log_from_line(&line)
{
entries.push((Id::from(current_line), log));
if entries.len() == lines.len() {
break 'outer;
}
}
current_line += 1;
}
}
}
Ok(entries)
}
fn log_from_line(line: &str) -> Option<Log> {
let (timestamp, rest) = line.split_once(' ')?;
let timestamp = DateTime::parse_from_rfc3339(timestamp).ok()?;
let (level, rest) = rest.trim().split_once(' ')?;
let (_, rest) = rest.trim().split_once(" (")?;
let (event_id, details) = rest.split_once(")")?;
Some(Log {
timestamp: UTCDateTime::from_timestamp(timestamp.timestamp()),
level: TracingLevel::parse(&level.to_ascii_uppercase()).unwrap_or(TracingLevel::Info),
event: EventType::parse(event_id)?,
details: details.trim().to_string(),
})
}

47
crates/jmap/src/registry/mapping/mod.rs
View File

@@ -4,4 +4,51 @@
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use common::{Server, auth::AccessToken};
use jmap_proto::{
method::{get::GetResponse, set::SetResponse},
object::registry::Registry,
};
use registry::{
jmap::JmapValue,
schema::prelude::{ObjectType, Property},
};
use store::ahash::AHashSet;
use types::id::Id;
use utils::map::vec_map::VecMap;
pub mod account;
pub mod deleted_item;
pub mod log;
pub mod queued_message;
pub mod report;
pub mod spam_sample;
pub mod task;
pub mod telemetry;
pub(crate) struct RegistryGetResponse<'x> {
pub server: &'x Server,
pub access_token: &'x AccessToken,
pub account_id: u32,
pub ids: Option<Vec<Id>>,
pub properties: AHashSet<Property>,
pub response: GetResponse<Registry>,
pub object_type: ObjectType,
pub object_flags: u64,
pub is_tenant_filtered: bool,
pub is_account_filtered: bool,
}
pub(crate) struct RegistrySetResponse<'x> {
pub server: &'x Server,
pub access_token: &'x AccessToken,
pub account_id: u32,
pub create: VecMap<String, JmapValue<'x>>,
pub update: Vec<(Id, JmapValue<'x>)>,
pub destroy: Vec<Id>,
pub response: SetResponse<Registry>,
pub object_type: ObjectType,
pub object_flags: u64,
pub is_tenant_filtered: bool,
pub is_account_filtered: bool,
}

89
crates/jmap/src/registry/mapping/queued_message.rs
View File

@@ -4,8 +4,10 @@
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use crate::registry::mapping::RegistryGetResponse;
use common::{Server, config::smtp::queue::ArchivedQueueExpiry};
use registry::{
jmap::IntoValue,
schema::{
enums::{DeliveryErrorType, MessageFlag, RecipientFlag},
structs::{
@@ -16,16 +18,52 @@ use registry::{
types::{datetime::UTCDateTime, ipaddr::IpAddr},
};
use smtp::queue::{spool::SmtpSpool, *};
use types::{blob::BlobId, blob_hash::BlobHash};
use store::{
IterateParams, U64_LEN, ValueKey,
ahash::AHashSet,
write::{QueueClass, ValueClass, key::DeserializeBigEndian},
};
use trc::AddContext;
use types::{blob::BlobId, blob_hash::BlobHash, id::Id};
use utils::DomainPart;
pub(crate) async fn queued_message_fetch(
server: &Server,
queue_id: u64,
) -> trc::Result<Option<QueuedMessage>> {
let Some(message_archive) = server.read_message_archive(queue_id).await? else {
return Ok(None);
pub(crate) async fn queued_message_get(
mut get: RegistryGetResponse<'_>,
) -> trc::Result<RegistryGetResponse<'_>> {
let ids = if let Some(ids) = get.ids.take() {
ids
} else {
queued_ids(get.server, get.server.core.jmap.get_max_objects)
.await?
.into_iter()
.map(Id::from)
.collect()
};
let message_in = message_archive.unarchive::<Message>()?;
for id in ids {
let Some(message_archive) = get.server.read_message_archive(id.id()).await? else {
get.not_found(id);
continue;
};
let message_in = message_archive.unarchive::<Message>()?;
if get.access_token.tenant_id().is_some() {
if let Some(domain) = message_in.return_path.try_domain_part()
&& let Some(domain) = get.server.domain(domain).await?
&& domain.id_tenant == get.access_token.tenant_id()
{
get.insert(id, map_message(message_in).into_value());
} else {
get.not_found(id);
}
} else {
get.insert(id, map_message(message_in).into_value());
}
}
Ok(get)
}
fn map_message(message_in: &ArchivedMessage) -> QueuedMessage {
let mut message_out = QueuedMessage {
blob_id: BlobId::new(BlobHash::from(&message_in.blob_hash), Default::default()),
created_at: UTCDateTime::from_timestamp(message_in.created.to_native() as i64),
@@ -109,7 +147,7 @@ pub(crate) async fn queued_message_fetch(
message_out.recipients.push(rcpt_out);
}
Ok(Some(message_out))
message_out
}
fn map_error_details(err_in: &ArchivedErrorDetails) -> DeliveryError {
@@ -164,3 +202,36 @@ fn map_error_details(err_in: &ArchivedErrorDetails) -> DeliveryError {
fn build_enhanced_code(esc: &[u8; 3]) -> String {
format!("{}.{}.{}", esc[0], esc[1], esc[2])
}
async fn queued_ids(server: &Server, max_results: usize) -> trc::Result<AHashSet<u64>> {
let mut events = AHashSet::with_capacity(8);
let from_key = ValueKey::from(ValueClass::Queue(QueueClass::MessageEvent(
store::write::QueueEvent {
due: 0,
queue_id: 0,
queue_name: [0; 8],
},
)));
let to_key = ValueKey::from(ValueClass::Queue(QueueClass::MessageEvent(
store::write::QueueEvent {
due: u64::MAX,
queue_id: u64::MAX,
queue_name: [u8::MAX; 8],
},
)));
server
.store()
.iterate(
IterateParams::new(from_key, to_key).ascending().no_values(),
|key, _| {
events.insert(key.deserialize_be_u64(U64_LEN)?);
Ok(events.len() < max_results)
},
)
.await
.caused_by(trc::location!())
.map(|_| events)
}

114
crates/jmap/src/registry/mapping/report.rs Normal file
View File

@@ -0,0 +1,114 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use crate::registry::mapping::RegistryGetResponse;
use common::Server;
use registry::{
jmap::IntoValue,
schema::prelude::{Object, ObjectType, Property},
types::EnumImpl,
};
use store::{
IterateParams, U16_LEN, ValueKey,
ahash::AHashSet,
registry::RegistryQuery,
write::{RegistryClass, ValueClass, key::DeserializeBigEndian},
};
use trc::AddContext;
use types::id::Id;
pub(crate) async fn report_get(
mut get: RegistryGetResponse<'_>,
) -> trc::Result<RegistryGetResponse<'_>> {
let object_id = get.object_type.to_id();
let ids = if let Some(ids) = get.ids.take() {
ids
} else if matches!(
get.object_type,
ObjectType::DmarcExternalReport
| ObjectType::TlsExternalReport
| ObjectType::ArfExternalReport
) {
if get.is_tenant_filtered {
get.server.registry().query::<AHashSet<u64>>(
RegistryQuery::new(get.object_type).with_tenant(get.access_token.tenant_id()),
)
} else {
get.server.registry().query::<AHashSet<u64>>(
RegistryQuery::new(get.object_type).greater_than(Property::ExpiresAt, 0u64),
)
}
.await?
.into_iter()
.take(get.server.core.jmap.get_max_objects)
.map(Id::from)
.collect()
} else {
internal_report_ids(get.server, object_id, get.server.core.jmap.get_max_objects).await?
};
for id in ids {
if let Some(report) = get
.server
.store()
.get_value::<Object>(ValueKey::from(ValueClass::Registry(RegistryClass::Item {
object_id,
item_id: id.id(),
})))
.await?
{
get.insert(id, report.into_value());
} else {
get.not_found(id);
}
}
Ok(get)
}
async fn internal_report_ids(
server: &Server,
object_id: u16,
max_results: usize,
) -> trc::Result<Vec<Id>> {
let mut events = Vec::with_capacity(8);
let from_key = ValueKey::from(ValueClass::Registry(RegistryClass::PrimaryKey {
object_id: object_id.into(),
index_id: Property::Domain.to_id(),
key: vec![],
}));
let to_key = ValueKey::from(ValueClass::Registry(RegistryClass::PrimaryKey {
object_id: object_id.into(),
index_id: Property::Domain.to_id(),
key: vec![
u8::MAX,
u8::MAX,
u8::MAX,
u8::MAX,
u8::MAX,
u8::MAX,
u8::MAX,
u8::MAX,
],
}));
server
.store()
.iterate(
IterateParams::new(from_key, to_key).ascending(),
|key, value| {
if !value.is_empty() {
events.push(key.deserialize_be_u64(U16_LEN)?.into());
}
Ok(events.len() < max_results)
},
)
.await
.caused_by(trc::location!())
.map(|_| events)
}

73
crates/jmap/src/registry/mapping/spam_sample.rs Normal file
View File

@@ -0,0 +1,73 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use crate::registry::mapping::RegistryGetResponse;
use registry::{
jmap::IntoValue,
schema::{
enums::Permission,
prelude::{Object, ObjectInner, Property},
},
types::EnumImpl,
};
use store::{
ValueKey,
ahash::AHashSet,
registry::RegistryQuery,
write::{RegistryClass, ValueClass},
};
use types::{blob::BlobClass, id::Id};
pub(crate) async fn spam_sample_get(
mut get: RegistryGetResponse<'_>,
) -> trc::Result<RegistryGetResponse<'_>> {
let object_id = get.object_type.to_id();
let ids = if let Some(ids) = get.ids.take() {
ids
} else {
let query = if get.access_token.has_permission(Permission::Impersonate) {
RegistryQuery::new(get.object_type).greater_than_or_equal(Property::AccountId, 0u64)
} else {
RegistryQuery::new(get.object_type).with_account(get.account_id)
};
get.server
.registry()
.query::<AHashSet<u64>>(query)
.await?
.into_iter()
.take(get.server.core.jmap.get_max_objects)
.map(Id::from)
.collect()
};
for id in ids {
if let Some(mut item) = get
.server
.store()
.get_value::<Object>(ValueKey::from(ValueClass::Registry(RegistryClass::Item {
object_id,
item_id: id.id(),
})))
.await?
{
if get.is_account_filtered
&& let ObjectInner::SpamTrainingSample(item) = &mut item.inner
{
item.blob_id.class = BlobClass::Reserved {
account_id: get.account_id,
expires: item.expires_at.timestamp() as u64,
};
}
get.insert(id, item.into_value());
} else {
get.not_found(id);
}
}
Ok(get)
}

68
crates/jmap/src/registry/mapping/task.rs Normal file
View File

@@ -0,0 +1,68 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use crate::registry::mapping::RegistryGetResponse;
use common::Server;
use registry::{jmap::IntoValue, schema::prelude::Object, types::EnumImpl};
use store::{
IterateParams, U64_LEN, ValueKey,
write::{RegistryClass, TaskQueueClass, ValueClass, key::DeserializeBigEndian},
};
use trc::AddContext;
use types::id::Id;
pub(crate) async fn task_get(
mut get: RegistryGetResponse<'_>,
) -> trc::Result<RegistryGetResponse<'_>> {
let ids = if let Some(ids) = get.ids.take() {
ids
} else {
task_ids(get.server, get.server.core.jmap.get_max_objects).await?
};
let object_id = get.object_type.to_id();
for id in ids {
if let Some(task) = get
.server
.store()
.get_value::<Object>(ValueKey::from(ValueClass::Registry(RegistryClass::Item {
object_id,
item_id: id.id(),
})))
.await?
{
get.insert(id, task.into_value());
} else {
get.not_found(id);
}
}
Ok(get)
}
async fn task_ids(server: &Server, max_results: usize) -> trc::Result<Vec<Id>> {
let mut events = Vec::with_capacity(8);
let from_key = ValueKey::from(ValueClass::TaskQueue(TaskQueueClass::Due { id: 0, due: 0 }));
let to_key = ValueKey::from(ValueClass::TaskQueue(TaskQueueClass::Due {
id: u64::MAX,
due: u64::MAX,
}));
server
.store()
.iterate(
IterateParams::new(from_key, to_key).ascending().no_values(),
|key, _| {
events.push(key.deserialize_be_u64(U64_LEN)?.into());
Ok(events.len() < max_results)
},
)
.await
.caused_by(trc::location!())
.map(|_| events)
}

122
crates/jmap/src/registry/mapping/telemetry.rs Normal file
View File

@@ -0,0 +1,122 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
*
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use crate::registry::mapping::RegistryGetResponse;
use common::Server;
use registry::{
jmap::IntoValue,
schema::prelude::{Object, Property},
types::datetime::UTCDateTime,
};
use store::{
IterateParams, ValueKey,
search::{SearchComparator, SearchField, SearchFilter, SearchQuery},
write::{SearchIndex, TelemetryClass, ValueClass, key::DeserializeBigEndian, now},
};
use trc::AddContext;
use types::id::Id;
use utils::snowflake::SnowflakeIdGenerator;
pub(crate) async fn trace_get(
mut get: RegistryGetResponse<'_>,
) -> trc::Result<RegistryGetResponse<'_>> {
let ids = if let Some(ids) = get.ids.take() {
ids
} else {
get.server
.search_store()
.query_global(
SearchQuery::new(SearchIndex::Tracing)
.with_filter(SearchFilter::gt(
SearchField::Id,
SnowflakeIdGenerator::from_timestamp(now() - 86400).unwrap_or_default(),
))
.with_comparator(SearchComparator::Field {
field: SearchField::Id,
ascending: false,
}),
)
.await?
.into_iter()
.take(get.server.core.jmap.get_max_objects)
.map(Id::from)
.collect()
};
for id in ids {
let item_id = id.id();
if let Some(trace) = get
.server
.tracing_store()
.get_value::<Object>(ValueKey::from(ValueClass::Telemetry(TelemetryClass::Span(
item_id,
))))
.await?
{
get.insert(id, trace.into_value());
} else {
get.not_found(id);
}
}
Ok(get)
}
pub(crate) async fn metric_get(
mut get: RegistryGetResponse<'_>,
) -> trc::Result<RegistryGetResponse<'_>> {
let ids = if let Some(ids) = get.ids.take() {
ids
} else {
metric_ids(get.server, get.server.core.jmap.get_max_objects).await?
};
for id in ids {
let item_id = id.id();
if let Some(metric) = get
.server
.metrics_store()
.get_value::<Object>(ValueKey::from(ValueClass::Telemetry(
TelemetryClass::Metric(item_id),
)))
.await?
{
let mut metric = metric.into_value();
metric.as_object_mut().unwrap().insert_unchecked(
Property::Timestamp,
UTCDateTime::from_timestamp(SnowflakeIdGenerator::to_timestamp(item_id) as i64)
.into_value(),
);
get.insert(id, metric);
} else {
get.not_found(id);
}
}
Ok(get)
}
async fn metric_ids(server: &Server, max_results: usize) -> trc::Result<Vec<Id>> {
let mut events = Vec::with_capacity(8);
let from_key = ValueKey::from(ValueClass::Telemetry(TelemetryClass::Metric(0)));
let to_key = ValueKey::from(ValueClass::Telemetry(TelemetryClass::Metric(u64::MAX)));
server
.metrics_store()
.iterate(
IterateParams::new(from_key, to_key).ascending().no_values(),
|key, _| {
events.push(key.deserialize_be_u64(0)?.into());
Ok(events.len() < max_results)
},
)
.await
.caused_by(trc::location!())
.map(|_| events)
}

417
crates/jmap/src/registry/set.rs
View File

@@ -4,12 +4,27 @@
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use crate::registry::mapping::RegistrySetResponse;
use common::{Server, auth::AccessToken};
use jmap_proto::{
error::set::SetError,
method::set::{SetRequest, SetResponse},
object::registry::Registry,
request::IntoValid,
};
use registry::schema::prelude::ObjectType;
use jmap_tools::{JsonPointer, JsonPointerItem, Key};
use registry::{
jmap::JsonPointerPatch,
schema::{
enums::Permission,
prelude::{
OBJ_FILTER_ACCOUNT, OBJ_FILTER_TENANT, OBJ_SINGLETON, Object, ObjectType, Property,
},
},
types::id::ObjectId,
};
use trc::AddContext;
use types::id::Id;
pub trait RegistrySet: Sync + Send {
fn registry_set(
@@ -20,6 +35,11 @@ pub trait RegistrySet: Sync + Send {
) -> impl Future<Output = trc::Result<SetResponse<Registry>>> + Send;
}
enum Modification {
Create(String),
Update(Id),
}
impl RegistrySet for Server {
async fn registry_set(
&self,
@@ -27,109 +47,275 @@ impl RegistrySet for Server {
mut request: SetRequest<'_, Registry>,
access_token: &AccessToken,
) -> trc::Result<SetResponse<Registry>> {
let object_flags = object_type.flags();
let is_singleton = (object_flags & OBJ_SINGLETON) != 0;
let is_tenant_filtered =
(object_flags & OBJ_FILTER_TENANT) != 0 && access_token.tenant_id().is_some();
let is_account_filtered = (object_flags & OBJ_FILTER_ACCOUNT) != 0
&& !access_token.has_permission(Permission::Impersonate);
// Build response
let mut response = SetResponse::from_request(&request, self.core.jmap.set_max_objects)?;
// Initial create validation for singletons
let mut create = request.unwrap_create();
if is_singleton && !create.is_empty() {
response
.not_created
.extend(create.drain().map(|(id, _)| (id, SetError::singleton())));
}
// Initial destroy validation for singletons
let mut destroy = request.unwrap_destroy().into_valid().collect::<Vec<_>>();
if is_singleton && !destroy.is_empty() {
response
.not_destroyed
.extend(destroy.drain(..).map(|id| (id, SetError::singleton())));
}
// Update validation for willDestroy
let update = request
.unwrap_update()
.into_valid()
.filter_map(|(id, value)| {
if is_singleton {
if id.is_singleton() {
Some((id, value))
} else {
response.not_updated.append(id, SetError::not_found());
None
}
} else if !destroy.contains(&id) {
Some((id, value))
} else {
response.not_updated.append(id, SetError::will_destroy());
None
}
})
.collect::<Vec<_>>();
let mut set = RegistrySetResponse {
access_token,
server: self,
account_id: request.account_id.document_id(),
object_type,
response,
object_flags,
is_tenant_filtered,
is_account_filtered,
create,
update,
destroy,
};
match object_type {
ObjectType::AcmeProvider => {}
ObjectType::AddressBook => {}
ObjectType::AiModel => {}
ObjectType::Alert => {}
ObjectType::AllowedIp => {}
ObjectType::Application => {}
ObjectType::Asn => {}
ObjectType::Authentication => {}
ObjectType::BlobStore => {}
ObjectType::BlockedIp => {}
ObjectType::Cache => {}
ObjectType::Calendar => {}
ObjectType::CalendarAlarm => {}
ObjectType::CalendarScheduling => {}
ObjectType::Certificate => {}
ObjectType::Coordinator => {}
ObjectType::DataRetention => {}
ObjectType::DataStore => {}
ObjectType::Directory => {}
ObjectType::DkimReportSettings => {}
ObjectType::DmarcReportSettings => {}
ObjectType::DnsResolver => {}
ObjectType::DnsServer => {}
ObjectType::Email => {}
ObjectType::Enterprise => {}
ObjectType::EventTracingLevel => {}
ObjectType::FileStorage => {}
ObjectType::Http => {}
ObjectType::HttpForm => {}
ObjectType::HttpLookup => {}
ObjectType::Imap => {}
ObjectType::InMemoryStore => {}
ObjectType::Jmap => {}
ObjectType::LocalSettings => {}
ObjectType::MemoryLookupKey => {}
ObjectType::MemoryLookupKeyValue => {}
ObjectType::Metrics => {}
ObjectType::MetricsStore => {}
ObjectType::MtaConnectionStrategy => {}
ObjectType::MtaDeliverySchedule => {}
ObjectType::MtaExtensions => {}
ObjectType::MtaHook => {}
ObjectType::MtaInboundSession => {}
ObjectType::MtaInboundThrottle => {}
ObjectType::MtaMilter => {}
ObjectType::MtaOutboundStrategy => {}
ObjectType::MtaOutboundThrottle => {}
ObjectType::MtaQueueQuota => {}
ObjectType::MtaRoute => {}
ObjectType::MtaStageAuth => {}
ObjectType::MtaStageConnect => {}
ObjectType::MtaStageData => {}
ObjectType::MtaStageEhlo => {}
ObjectType::MtaStageMail => {}
ObjectType::MtaStageRcpt => {}
ObjectType::MtaSts => {}
ObjectType::MtaTlsStrategy => {}
ObjectType::MtaVirtualQueue => {}
ObjectType::NetworkListener => {}
ObjectType::Node => {}
ObjectType::NodeRole => {}
ObjectType::NodeShard => {}
ObjectType::OidcProvider => {}
ObjectType::RegistryBundle => {}
ObjectType::ReportSettings => {}
ObjectType::Search => {}
ObjectType::SearchStore => {}
ObjectType::Security => {}
ObjectType::SenderAuth => {}
ObjectType::Sharing => {}
ObjectType::SieveSystemInterpreter => {}
ObjectType::SieveSystemScript => {}
ObjectType::SieveUserInterpreter => {}
ObjectType::SieveUserScript => {}
ObjectType::SpamClassifier => {}
ObjectType::SpamDnsblServer => {}
ObjectType::SpamDnsblSettings => {}
ObjectType::SpamFileExtension => {}
ObjectType::SpamLlm => {}
ObjectType::SpamPyzor => {}
ObjectType::SpamRule => {}
ObjectType::SpamSettings => {}
ObjectType::SpamTag => {}
ObjectType::SpfReportSettings => {}
ObjectType::StoreLookup => {}
ObjectType::TaskManager => {}
ObjectType::TlsReportSettings => {}
ObjectType::Tracer => {}
ObjectType::TracingStore => {}
ObjectType::WebDav => {}
ObjectType::WebHook => {}
ObjectType::Account => {}
ObjectType::DsnReportSettings => {}
ObjectType::MailingList => {}
ObjectType::OAuthClient => {}
ObjectType::Role => {}
ObjectType::Tenant => {}
ObjectType::MaskedEmail => {}
ObjectType::PublicKey => {}
ObjectType::DkimSignature => {}
ObjectType::Domain => {}
ObjectType::Log => {}
ObjectType::AddressBook
| ObjectType::Asn
| ObjectType::Authentication
| ObjectType::BlobStore
| ObjectType::Cache
| ObjectType::Calendar
| ObjectType::CalendarAlarm
| ObjectType::CalendarScheduling
| ObjectType::Coordinator
| ObjectType::DataRetention
| ObjectType::DataStore
| ObjectType::DkimReportSettings
| ObjectType::DmarcReportSettings
| ObjectType::DnsResolver
| ObjectType::Email
| ObjectType::Enterprise
| ObjectType::FileStorage
| ObjectType::Http
| ObjectType::HttpForm
| ObjectType::Imap
| ObjectType::InMemoryStore
| ObjectType::Jmap
| ObjectType::LocalSettings
| ObjectType::Metrics
| ObjectType::MetricsStore
| ObjectType::MtaConnectionStrategy
| ObjectType::MtaExtensions
| ObjectType::MtaInboundSession
| ObjectType::MtaOutboundStrategy
| ObjectType::MtaOutboundThrottle
| ObjectType::MtaStageAuth
| ObjectType::MtaStageConnect
| ObjectType::MtaStageData
| ObjectType::MtaStageEhlo
| ObjectType::MtaStageMail
| ObjectType::MtaStageRcpt
| ObjectType::MtaSts
| ObjectType::OidcProvider
| ObjectType::ReportSettings
| ObjectType::Search
| ObjectType::SearchStore
| ObjectType::Security
| ObjectType::SenderAuth
| ObjectType::Sharing
| ObjectType::SieveSystemInterpreter
| ObjectType::SieveUserInterpreter
| ObjectType::SpamClassifier
| ObjectType::SpamDnsblSettings
| ObjectType::SpamLlm
| ObjectType::SpamPyzor
| ObjectType::SpamSettings
| ObjectType::SpfReportSettings
| ObjectType::TaskManager
| ObjectType::TlsReportSettings
| ObjectType::TracingStore
| ObjectType::WebDav
| ObjectType::DsnReportSettings
| ObjectType::AcmeProvider
| ObjectType::AiModel
| ObjectType::Alert
| ObjectType::AllowedIp
| ObjectType::Application
| ObjectType::BlockedIp
| ObjectType::Certificate
| ObjectType::Directory
| ObjectType::DnsServer
| ObjectType::EventTracingLevel
| ObjectType::HttpLookup
| ObjectType::MemoryLookupKey
| ObjectType::MemoryLookupKeyValue
| ObjectType::MtaVirtualQueue
| ObjectType::MtaQueueQuota
| ObjectType::MtaRoute
| ObjectType::MtaDeliverySchedule
| ObjectType::MtaInboundThrottle
| ObjectType::MtaTlsStrategy
| ObjectType::MtaMilter
| ObjectType::MtaHook
| ObjectType::NetworkListener
| ObjectType::Node
| ObjectType::NodeRole
| ObjectType::NodeShard
| ObjectType::RegistryBundle
| ObjectType::SieveSystemScript
| ObjectType::SieveUserScript
| ObjectType::SpamDnsblServer
| ObjectType::SpamFileExtension
| ObjectType::SpamRule
| ObjectType::SpamTag
| ObjectType::StoreLookup
| ObjectType::Tracer
| ObjectType::WebHook
| ObjectType::PublicKey
| ObjectType::DkimSignature
| ObjectType::MaskedEmail
| ObjectType::Account
| ObjectType::MailingList
| ObjectType::OAuthClient
| ObjectType::Role
| ObjectType::Tenant
| ObjectType::Domain => {
// Bundle modifications together
let mut modifications = Vec::with_capacity(set.create.len() + set.update.len());
for (id, value) in set.create {
modifications.push((
Modification::Create(id),
value,
Object::from(set.object_type),
));
}
for (id, value) in set.update {
if let Some(object) = self
.registry()
.get(ObjectId::new(object_type, id))
.await
.caused_by(trc::location!())?
{
modifications.push((Modification::Update(id), value, object));
} else if is_singleton {
modifications.push((
Modification::Update(id),
value,
Object::from(set.object_type),
));
} else {
set.response.not_updated.append(id, SetError::not_found());
}
}
// Process modifications
'outer: for (modification, value, mut object) in modifications {
for (key, value) in value.into_expanded_object() {
let ptr = match (key, &modification) {
(Key::Property(prop), _) => {
JsonPointer::new(vec![JsonPointerItem::Key(Key::Property(prop))])
}
(Key::Borrowed(other), Modification::Update(_)) => {
JsonPointer::parse(other)
}
(Key::Owned(other), Modification::Update(_)) => {
JsonPointer::parse(&other)
}
(key, Modification::Create(_)) => {
set.response.failed(
modification,
SetError::invalid_properties().with_property(key.into_owned()),
);
continue 'outer;
}
};
// Initial validations
let is_create = matches!(modification, Modification::Create(_));
// SPDX-SnippetBegin
// SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
// SPDX-License-Identifier: LicenseRef-SEL
#[cfg(feature = "enterprise")]
if is_create
&& object_type == ObjectType::Account
&& self.core.is_enterprise_edition()
&& !self.can_create_account().await?
{
set.response.failed(
modification,
SetError::forbidden().with_description(format!(
"Enterprise licensed account limit reached: {} accounts licensed.",
self.licensed_accounts()
)),
);
continue 'outer;
}
// SPDX-SnippetEnd
/*
Principal creation:
- Add tenantId
- Add default roles on account creation
- Invalidate cache + logo cache
- Validate effective permissions to grant access
Principal update:
- Remove tenantId, or return error
- Invalidate cache + logo cache
- Validate effective permissions to grant access
Principal deletion:
- Validate tenantId ownership
- Invalidate cache
- Schedule account deletion (if account)
*/
// Patch object
if let Err(err) =
object.patch(JsonPointerPatch::new(&ptr).with_create(is_create), value)
{
}
}
}
// Process destroy
for id in set.destroy {}
}
ObjectType::QueuedMessage => {}
ObjectType::Task => {}
ObjectType::ArfExternalReport => {}
@@ -139,12 +325,33 @@ impl RegistrySet for Server {
ObjectType::Metric => {}
ObjectType::Trace => {}
ObjectType::SpamTrainingSample => {}
ObjectType::DmarcInternalReport => todo!(),
ObjectType::TlsInternalReport => todo!(),
ObjectType::DmarcInternalReport => {}
ObjectType::TlsInternalReport => {}
ObjectType::Log => {}
ObjectType::AccountSettings => {}
ObjectType::Credential => {}
}
let todo = "read only properties";
let todo = "password encryption";
let todo = "management objects for actions (reload, etc)";
// MaskedEmail: Generate masked email + Enforce count
// DkimSignature = Generate keys + Enforce count?
// PublicKey = Validate PK? Store decoded?
todo!()
}
}
trait SetModification {
fn failed(&mut self, modification: Modification, error: SetError<Property>);
}
impl SetModification for SetResponse<Registry> {
fn failed(&mut self, modification: Modification, error: SetError<Property>) {
match modification {
Modification::Create(id) => self.not_created.append(id, error),
Modification::Update(id) => self.not_updated.append(id, error),
}
}
}

3
crates/registry/src/jmap/mod.rs
View File

@@ -25,11 +25,12 @@ pub enum RegistryValue {
IdReference(String),
}
#[derive(Debug, Clone)]
#[derive(Clone)]
pub struct JsonPointerPatch<'x> {
ptr: &'x JsonPointer<Property>,
pos: usize,
validators: &'x [StringValidator],
is_create: bool,
}
pub trait RegistryJsonPatch: Debug + Default {

31
crates/registry/src/jmap/patch.rs
View File

@@ -26,9 +26,24 @@ impl<'x> JsonPointerPatch<'x> {
ptr,
pos: 0,
validators: &[],
is_create: false,
}
}
pub fn cloned(&self) -> Self {
Self {
ptr: self.ptr,
pos: 0,
validators: &[],
is_create: false,
}
}
pub fn with_create(mut self, is_create: bool) -> Self {
self.is_create = is_create;
self
}
pub fn with_validators(mut self, validators: &'x [StringValidator]) -> Self {
self.validators = validators;
self
@@ -63,14 +78,22 @@ impl<'x> JsonPointerPatch<'x> {
pub fn assert_eof(&self) -> Result<(), PatchError> {
if self.has_next() {
Err(PatchError::new(
JsonPointerPatch::new(self.ptr),
"Invalid JSON Pointer path",
))
Err(PatchError::new(self.cloned(), "Invalid JSON Pointer path"))
} else {
Ok(())
}
}
pub fn assert_read_only(self) -> Result<Self, PatchError> {
if self.is_create {
Ok(self)
} else {
Err(PatchError::new(
self.cloned(),
"Cannot modify read-only property",
))
}
}
}
impl<T: RegistryJsonPatch> RegistryJsonPatch for Option<T> {

19
crates/registry/src/schema/mod.rs
View File

@@ -7,7 +7,7 @@
use crate::{
schema::{
enums::{TracingLevel, TracingLevelOpt},
prelude::{NodeRange, Object, ObjectInner, Property},
prelude::{Credential, CredentialProperties, NodeRange, Object, ObjectInner, Property},
},
types::EnumImpl,
};
@@ -33,6 +33,23 @@ impl NodeRange {
node_id >= self.from_node_id && node_id <= self.to_node_id
}
}
impl Credential {
pub fn unwrap_properties(self) -> CredentialProperties {
match self {
Credential::AppPassword(credential_properties) => credential_properties,
Credential::ApiKey(credential_properties) => credential_properties,
}
}
pub fn as_properties(&self) -> &CredentialProperties {
match self {
Credential::AppPassword(credential_properties) => credential_properties,
Credential::ApiKey(credential_properties) => credential_properties,
}
}
}
impl Display for Property {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.as_str())

2
crates/registry/src/types/datetime.rs
View File

@@ -225,7 +225,7 @@ impl Display for UTCDateTime {
impl Default for UTCDateTime {
fn default() -> Self {
UTCDateTime(i64::MAX)
UTCDateTime::now()
}
}

8
crates/registry/src/types/string.rs
View File

@@ -17,6 +17,7 @@ pub enum StringValidator {
Lowercase,
Uppercase,
Trim,
SecretHash,
}
pub enum StringValidatorResult {
@@ -67,6 +68,13 @@ impl StringValidator {
StringValidatorResult::Valid
}
}
Self::SecretHash => {
if !value.is_empty() && value.len() <= 128 {
StringValidatorResult::Valid
} else {
StringValidatorResult::Invalid("Secret cannot be empty")
}
}
}
}
}

4
crates/services/src/task_manager/index.rs
View File

@@ -692,12 +692,12 @@ async fn delete_email_metadata(
index_id: Property::AccountId.to_id(),
object_id,
item_id,
key: account_id.serialize(),
key: (account_id as u64).serialize(),
}),
vec![],
)
.set(
ValueClass::Registry(RegistryClass::Id { object_id, item_id }),
ValueClass::Registry(RegistryClass::Item { object_id, item_id }),
item,
);
}

13
crates/smtp/src/queue/spool.rs
View File

@@ -17,7 +17,7 @@ use common::config::smtp::queue::QueueName;
use common::ipc::QueueEvent;
use common::{KV_LOCK_QUEUE_MESSAGE, Server};
use registry::pickle::Pickle;
use registry::schema::prelude::ObjectType;
use registry::schema::prelude::{ObjectType, Property};
use registry::schema::structs::SpamTrainingSample;
use registry::types::EnumImpl;
use registry::types::datetime::UTCDateTime;
@@ -482,8 +482,17 @@ impl MessageWrapper {
ObjectId::new(ObjectType::SpamTrainingSample, item_id.into()).serialize(),
)
.set(
ValueClass::Registry(RegistryClass::Id { object_id, item_id }),
ValueClass::Registry(RegistryClass::Item { object_id, item_id }),
sample,
)
.set(
ValueClass::Registry(RegistryClass::Index {
index_id: Property::AccountId.to_id(),
object_id,
item_id,
key: (u32::MAX as u64).serialize(),
}),
vec![],
);
trc::event!(

15
crates/spam-filter/src/modules/classifier.rs
View File

@@ -188,11 +188,11 @@ impl SpamClassifier for Server {
let mut duplicate_samples = Vec::new();
let mut remove_entries = false;
let object_id = ObjectType::SpamTrainingSample.to_id();
let from_key = ValueKey::from(ValueClass::Registry(RegistryClass::Id {
let from_key = ValueKey::from(ValueClass::Registry(RegistryClass::Item {
object_id,
item_id: trainer.last_id + 1,
}));
let to_key = ValueKey::from(ValueClass::Registry(RegistryClass::Id {
let to_key = ValueKey::from(ValueClass::Registry(RegistryClass::Item {
object_id,
item_id: u64::MAX,
}));
@@ -564,18 +564,17 @@ impl SpamClassifier for Server {
hash: sample.sample.hash,
to: BlobLink::Temporary { until },
})
.clear(ValueClass::Registry(RegistryClass::Id {
.clear(ValueClass::Registry(RegistryClass::Item {
object_id,
item_id: sample.id,
}));
if sample.sample.account_id != u32::MAX {
batch.clear(ValueClass::Registry(RegistryClass::Index {
}))
.clear(ValueClass::Registry(RegistryClass::Index {
index_id: Property::AccountId.to_id(),
object_id,
item_id: sample.id,
key: sample.sample.account_id.serialize(),
key: (sample.sample.account_id as u64).serialize(),
}));
}
if batch.is_large_batch() {
self.store()
.write(batch.build_all())

16
crates/store/src/registry/get.rs
View File

@@ -30,22 +30,6 @@ impl RegistryStore {
item_id: object_id.id().id(),
})))
.await
.and_then(|v| {
if v.as_ref()
.is_none_or(|v| v.object_type() == object_id.object())
{
Ok(v)
} else {
Err(
trc::EventType::Registry(trc::RegistryEvent::DeserializationError)
.into_err()
.caused_by(trc::location!())
.id(object_id.id().id())
.details(object_id.object().as_str())
.reason("Object type mismatch"),
)
}
})
}
}

4
crates/store/src/registry/query.rs
View File

@@ -374,11 +374,11 @@ async fn all_ids<T: RegistryQueryResults>(store: &Store, object: ObjectType) ->
store
.iterate(
IterateParams::new(
ValueKey::from(ValueClass::Registry(RegistryClass::Id {
ValueKey::from(ValueClass::Registry(RegistryClass::IndexId {
object_id,
item_id: 0u64,
})),
ValueKey::from(ValueClass::Registry(RegistryClass::Id {
ValueKey::from(ValueClass::Registry(RegistryClass::IndexId {
object_id,
item_id: u64::MAX,
})),

6
crates/store/src/registry/write.rs
View File

@@ -211,7 +211,7 @@ impl RegistryStore {
key: type_filter.serialize(),
}
} else {
RegistryClass::Id { object_id, item_id }
RegistryClass::IndexId { object_id, item_id }
};
if self
.0
@@ -317,7 +317,7 @@ impl RegistryStore {
// Build batch
if write_id {
batch.set(
ValueClass::Registry(RegistryClass::Id { object_id, item_id }),
ValueClass::Registry(RegistryClass::IndexId { object_id, item_id }),
vec![],
);
}
@@ -434,7 +434,7 @@ impl RegistryStore {
object_id: object_type_id,
item_id,
}))
.clear(ValueClass::Registry(RegistryClass::Id {
.clear(ValueClass::Registry(RegistryClass::IndexId {
object_id: object_type_id,
item_id,
}))

24
crates/store/src/write/blob.rs
View File

@@ -167,19 +167,17 @@ impl Store {
let item_id = object_id.id().id();
let object_id = object_id.object().to_id();
if let Some(account_id) = account_id {
batch.clear(ValueClass::Registry(RegistryClass::Index {
batch
.clear(ValueClass::Registry(RegistryClass::Index {
index_id: Property::AccountId.to_id(),
object_id,
item_id,
key: account_id.serialize(),
key: (account_id as u64).serialize(),
}))
.clear(ValueClass::Registry(RegistryClass::Item {
object_id,
item_id,
}));
}
batch.clear(ValueClass::Registry(RegistryClass::Id {
object_id,
item_id,
}));
}
if !batch.is_empty() {
self.write(batch.build_all())
@@ -206,7 +204,7 @@ struct BlobPurgeState {
last_hash: BlobHash,
last_hash_is_linked: bool,
delete_keys: Vec<(Option<u32>, BlobOp)>,
delete_registry: Vec<(Option<u32>, ObjectId)>,
delete_registry: Vec<(u32, ObjectId)>,
now: u64,
total_deleted: u64,
total_active: u64,
@@ -270,10 +268,8 @@ impl BlobPurgeState {
},
));
if value.len() == U16_LEN + U64_LEN {
self.delete_registry.push((
(account_id != u32::MAX).then_some(account_id),
ObjectId::deserialize(value)?,
));
self.delete_registry
.push((account_id, ObjectId::deserialize(value)?));
}
}
Ok(())

8
crates/store/src/write/key.rs
View File

@@ -304,7 +304,7 @@ impl ValueClass {
RegistryClass::Item { object_id, item_id } => {
serializer.write(*object_id).write_leb128(*item_id)
}
RegistryClass::Id { object_id, item_id } => {
RegistryClass::IndexId { object_id, item_id } => {
serializer.write(*object_id).write(*item_id)
}
RegistryClass::Index {
@@ -498,7 +498,7 @@ impl ValueClass {
RegistryClass::Reference { .. } => ((U16_LEN + U64_LEN) * 2) + 1,
RegistryClass::Index { key, .. } => (U16_LEN * 2) + U64_LEN + key.len() + 1,
RegistryClass::PrimaryKey { key, .. } => (U16_LEN * 2) + key.len() + 1,
RegistryClass::Id { .. } => U16_LEN + U64_LEN + 1,
RegistryClass::IndexId { .. } => U16_LEN + U64_LEN + 1,
RegistryClass::IdCounter { .. } => U16_LEN + 1,
},
ValueClass::Blob(op) => match op {
@@ -568,7 +568,9 @@ impl ValueClass {
REPORT_INTERNAL_DMARC | REPORT_INTERNAL_TLS => SUBSPACE_REPORT_OUT,
_ => SUBSPACE_REGISTRY,
},
RegistryClass::Id { .. } | RegistryClass::Index { .. } => SUBSPACE_REGISTRY_IDX,
RegistryClass::IndexId { .. } | RegistryClass::Index { .. } => {
SUBSPACE_REGISTRY_IDX
}
RegistryClass::Reference { .. } | RegistryClass::PrimaryKey { .. } => {
SUBSPACE_REGISTRY_PK
}

8
crates/store/src/write/mod.rs
View File

@@ -265,15 +265,15 @@ pub enum RegistryClass {
item_id: u64,
key: Vec<u8>,
},
IndexId {
object_id: u16,
item_id: u64,
},
PrimaryKey {
object_id: Option<u16>,
index_id: u16,
key: Vec<u8>,
},
Id {
object_id: u16,
item_id: u64,
},
IdCounter {
object_id: u16,
},

8
crates/types/src/blob.rs
View File

@@ -32,8 +32,8 @@ pub enum BlobClass {
impl Default for BlobClass {
fn default() -> Self {
BlobClass::Reserved {
account_id: 0,
expires: 0,
account_id: u32::MAX,
expires: u64::MAX,
}
}
}
@@ -64,6 +64,10 @@ impl BlobClass {
BlobClass::Linked { .. } => true,
}
}
pub fn is_superuser(&self) -> bool {
matches!(self, BlobClass::Reserved { account_id, expires } if *account_id == u32::MAX && *expires == u64::MAX)
}
}
#[derive(Debug, Default, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]

10
crates/utils/src/map/vec_map.rs
View File

@@ -215,6 +215,16 @@ impl<K: Eq + PartialEq, V> VecMap<K, V> {
cmp => cmp,
});
}
pub fn extend(&mut self, iter: impl IntoIterator<Item = (K, V)>) {
for (k, v) in iter {
self.append(k, v);
}
}
pub fn drain(&mut self) -> impl Iterator<Item = (K, V)> + '_ {
self.inner.drain(..).map(|kv| (kv.key, kv.value))
}
}
impl<K: Eq + PartialEq, V: Default> VecMap<K, V> {