filip/Stalwart
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

JMAP Registry API implementation - part 2

Browse Source
This commit is contained in:
mdecimus
2026-02-24 17:19:44 +01:00
parent ec148c5550
commit 2fea9a46b0
61 changed files with 1746 additions and 1992 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
crates/common/src/config/inner.rs
View File

@@ -50,7 +50,8 @@ impl Data {
// Build and test snowflake id generator
let node_id = bp.node_id();
let id_generator = SnowflakeIdGenerator::with_node_id(node_id);
SnowflakeIdGenerator::set_node_id(node_id);
let id_generator = SnowflakeIdGenerator::new();
if !id_generator.is_valid() {
panic!("Invalid system time, panicking to avoid data corruption");
}

6
crates/common/src/config/network.rs
View File

@@ -13,8 +13,8 @@ use ahash::AHashMap;
use registry::{
schema::{
enums::NodeShardType,
prelude::{Object, ObjectType},
structs::{self, Asn, HttpForm, NodeRole, NodeShard, Rate},
prelude::ObjectType,
structs::{self, Asn, HttpForm, NodeRole, NodeShard, Rate, TaskManager},
},
types::EnumImpl,
};
@@ -30,6 +30,7 @@ pub struct Network {
pub http: Http,
pub contact_form: Option<ContactForm>,
pub asn_geo_lookup: AsnGeoLookupConfig,
pub task_manager: TaskManager,
}
#[derive(Clone)]
@@ -155,6 +156,7 @@ impl Network {
asn_geo_lookup: AsnGeoLookupConfig::parse(bp).await.unwrap_or_default(),
roles: ClusterRoles::default(),
http: Http::parse(bp).await,
task_manager: bp.setting_infallible::<TaskManager>().await,
};
// Process ranges

2
crates/common/src/config/server/listener.rs
View File

@@ -30,7 +30,7 @@ impl Listeners {
pub async fn parse(bp: &mut Bootstrap) -> Self {
// Parse ACME managers
let mut servers = Listeners {
span_id_gen: Arc::new(SnowflakeIdGenerator::with_node_id(bp.node_id())),
span_id_gen: Arc::new(SnowflakeIdGenerator::new()),
..Default::default()
};

1
crates/common/src/enterprise/mod.rs
View File

@@ -12,7 +12,6 @@ pub mod alerts;
pub mod config;
pub mod license;
pub mod llm;
pub mod undelete;
use crate::{
Core, Server, config::groupware::CalendarTemplateVariable, expr::Expression,

113
crates/common/src/enterprise/undelete.rs
View File

@@ -1,113 +0,0 @@
/*
* SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
*
* SPDX-License-Identifier: LicenseRef-SEL
*
* This file is subject to the Stalwart Enterprise License Agreement (SEL) and
* is NOT open source software.
*
*/
use crate::Core;
use store::{
Deserialize, IterateParams, U32_LEN, U64_LEN, ValueKey,
write::{AlignedBytes, Archive, BlobOp, ValueClass, key::DeserializeBigEndian, now},
};
use trc::AddContext;
use types::blob_hash::{BLOB_HASH_LEN, BlobHash};
pub struct DeletedBlob {
pub hash: BlobHash,
pub expires_at: u64,
pub item: DeletedItem,
}
#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)]
pub struct DeletedItem {
pub typ: DeletedItemType,
pub size: u32,
pub deleted_at: u64,
}
#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)]
pub enum DeletedItemType {
Email {
from: Box<str>,
subject: Box<str>,
received_at: u64,
},
FileNode {
name: Box<str>,
},
CalendarEvent {
title: Box<str>,
start_time: u64,
},
ContactCard {
name: Box<str>,
},
SieveScript {
name: Box<str>,
},
}
impl Core {
pub async fn list_deleted(&self, account_id: u32) -> trc::Result<Vec<DeletedBlob>> {
let from_key = ValueKey {
account_id,
collection: 0,
document_id: 0,
class: ValueClass::Blob(BlobOp::Undelete {
hash: BlobHash::default(),
until: 0,
}),
};
let to_key = ValueKey {
account_id,
collection: 0,
document_id: u32::MAX,
class: ValueClass::Blob(BlobOp::Undelete {
hash: BlobHash::new_max(),
until: u64::MAX,
}),
};
let now = now();
let mut results = Vec::new();
self.storage
.data
.iterate(
IterateParams::new(from_key, to_key).ascending(),
|key, value| {
let expires_at = key.deserialize_be_u64(key.len() - U64_LEN)?;
if expires_at > now {
let item = <Archive<AlignedBytes> as Deserialize>::deserialize(value)
.and_then(|bytes| bytes.deserialize::<DeletedItem>())
.add_context(|ctx| ctx.ctx(trc::Key::Key, key))?;
results.push(DeletedBlob {
hash: BlobHash::try_from_hash_slice(
key.get(U32_LEN + 1..U32_LEN + 1 + BLOB_HASH_LEN)
.ok_or_else(|| {
trc::Error::corrupted_key(
key,
value.into(),
trc::location!(),
)
})?,
)
.unwrap(),
expires_at,
item,
});
}
Ok(true)
},
)
.await
.caused_by(trc::location!())?;
Ok(results)
}
}

1
crates/common/src/lib.rs
View File

@@ -120,6 +120,7 @@ pub const KV_RATE_LIMIT_CONTACT: u8 = 7;
pub const KV_RATE_LIMIT_HTTP_AUTHENTICATED: u8 = 8;
pub const KV_RATE_LIMIT_HTTP_ANONYMOUS: u8 = 9;
pub const KV_RATE_LIMIT_IMAP: u8 = 10;
pub const KV_QUOTA_BLOB: u8 = 11;
pub const KV_GREYLIST: u8 = 16;
pub const KV_LOCK_PURGE_ACCOUNT: u8 = 20;
pub const KV_LOCK_QUEUE_MESSAGE: u8 = 21;

2
crates/common/src/manager/backup.rs
View File

@@ -312,7 +312,7 @@ impl Family {
SUBSPACE_COUNTER,
SUBSPACE_PROPERTY,
],
Family::Blob => &[SUBSPACE_BLOBS, SUBSPACE_BLOB_EXTRA, SUBSPACE_BLOB_LINK],
Family::Blob => &[SUBSPACE_BLOBS, SUBSPACE_BLOB_LINK],
Family::Registry => &[SUBSPACE_REGISTRY],
Family::Changelog => &[SUBSPACE_LOGS],
Family::Queue => &[SUBSPACE_QUEUE_MESSAGE, SUBSPACE_QUEUE_EVENT],

67
crates/common/src/storage/blob.rs
View File

@@ -4,13 +4,14 @@
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
*/
use crate::Server;
use crate::{KV_QUOTA_BLOB, Server};
use mail_parser::{
Encoding,
decoders::{base64::base64_decode, quoted_printable::quoted_printable_decode},
};
use store::{
SerializeInfallible,
U32_LEN, U64_LEN,
dispatch::lookup::KeyValue,
write::{BatchBuilder, BlobLink, BlobOp, now},
};
use trc::AddContext;
@@ -19,7 +20,46 @@ use types::{
blob_hash::BlobHash,
};
const COUNT_BYTES: u32 = 20;
const COUNT_SHIFT: u32 = 64 - COUNT_BYTES;
const SIZE_MASK: u64 = (1u64 << COUNT_SHIFT) - 1;
impl Server {
pub async fn blob_has_quota(&self, account_id: u32, bytes: usize) -> trc::Result<bool> {
if self.core.jmap.upload_tmp_quota_size > 0 || self.core.jmap.upload_tmp_quota_amount > 0 {
let now = now();
let range_start = now / self.core.jmap.upload_tmp_ttl;
let range_end =
(range_start * self.core.jmap.upload_tmp_ttl) + self.core.jmap.upload_tmp_ttl;
let expires_in = range_end - now;
let mut bucket = Vec::with_capacity(U32_LEN + U64_LEN + 1);
bucket.push(KV_QUOTA_BLOB);
bucket.extend_from_slice(account_id.to_be_bytes().as_slice());
bucket.extend_from_slice(range_start.to_be_bytes().as_slice());
self.in_memory_store()
.counter_incr(
KeyValue::new(bucket, 1i64 << COUNT_SHIFT | bytes as i64).expires(expires_in),
true,
)
.await
.caused_by(trc::location!())
.map(|v| {
let v = v as u64;
let count = v >> COUNT_SHIFT;
let size = v & SIZE_MASK;
(self.core.jmap.upload_tmp_quota_amount == 0
|| count <= self.core.jmap.upload_tmp_quota_amount as u64)
&& (self.core.jmap.upload_tmp_quota_size == 0
|| size <= self.core.jmap.upload_tmp_quota_size as u64)
})
} else {
Ok(true)
}
}
#[allow(clippy::blocks_in_conditions)]
pub async fn put_jmap_blob(&self, account_id: u32, data: &[u8]) -> trc::Result<BlobId> {
// First reserve the hash
@@ -27,22 +67,13 @@ impl Server {
let mut batch = BatchBuilder::new();
let until = now() + self.core.jmap.upload_tmp_ttl;
batch
.with_account_id(account_id)
.set(
BlobOp::Link {
hash: hash.clone(),
to: BlobLink::Temporary { until },
},
vec![BlobLink::QUOTA_LINK],
)
.set(
BlobOp::Quota {
hash: hash.clone(),
until,
},
(data.len() as u32).serialize(),
);
batch.with_account_id(account_id).set(
BlobOp::Link {
hash: hash.clone(),
to: BlobLink::Temporary { until },
},
vec![],
);
self.core
.storage

57
crates/common/src/storage/index.rs
View File

@@ -5,6 +5,10 @@
*/
use crate::{auth::AccountTenantIds, sharing::notification::ShareNotification};
use registry::schema::{
enums::IndexDocumentType,
structs::{Task, TaskIndexDocument, TaskStatus},
};
use rkyv::{
option::ArchivedOption,
primitive::{ArchivedU32, ArchivedU64},
@@ -15,7 +19,7 @@ use store::{
Serialize, SerializeInfallible,
write::{
Archive, Archiver, BatchBuilder, BlobLink, BlobOp, IntoOperations, Params, SearchIndex,
TaskEpoch, TaskQueueClass, ValueClass,
ValueClass,
},
};
use types::{
@@ -418,14 +422,23 @@ fn build_index(
}
}
IndexValue::SearchIndex { index, .. } => {
batch.set(
ValueClass::TaskQueue(TaskQueueClass::UpdateIndex {
due: TaskEpoch::now().with_random_sequence_id(),
index,
is_insert: set,
}),
vec![],
);
let task = TaskIndexDocument {
account_id: batch.last_account_id().unwrap().into(),
document_id: batch.last_document_id().unwrap().into(),
document_type: match index {
SearchIndex::Email => IndexDocumentType::Email,
SearchIndex::Calendar => IndexDocumentType::Calendar,
SearchIndex::Contacts => IndexDocumentType::Contacts,
SearchIndex::File => IndexDocumentType::File,
SearchIndex::Tracing | SearchIndex::InMemory => unreachable!(),
},
status: TaskStatus::now(),
};
batch.schedule_task(if set {
Task::IndexDocument(task)
} else {
Task::UnindexDocument(task)
});
}
IndexValue::Property { field, value } => {
if !value.is_none() {
@@ -456,9 +469,8 @@ fn build_index(
let object_account_id = batch.last_account_id().unwrap_or_default();
let object_type = batch.last_collection().unwrap_or(Collection::None);
let object_id = batch.last_document_id().unwrap_or_default();
let notification_id = SnowflakeIdGenerator::from_sequence_and_node_id(
let notification_id = SnowflakeIdGenerator::from_sequence_id(
object_type as u64 ^ object_account_id as u64,
None,
)
.unwrap_or_default();
@@ -559,14 +571,18 @@ fn merge_index(
}
}
(IndexValue::SearchIndex { index, .. }, IndexValue::SearchIndex { .. }) => {
batch.set(
ValueClass::TaskQueue(TaskQueueClass::UpdateIndex {
due: TaskEpoch::now().with_random_sequence_id(),
index,
is_insert: true,
}),
vec![],
);
batch.schedule_task(Task::IndexDocument(TaskIndexDocument {
account_id: batch.last_account_id().unwrap().into(),
document_id: batch.last_document_id().unwrap().into(),
document_type: match index {
SearchIndex::Email => IndexDocumentType::Email,
SearchIndex::Calendar => IndexDocumentType::Calendar,
SearchIndex::Contacts => IndexDocumentType::Contacts,
SearchIndex::File => IndexDocumentType::File,
SearchIndex::Tracing | SearchIndex::InMemory => unreachable!(),
},
status: TaskStatus::now(),
}));
}
(
IndexValue::Property {
@@ -614,9 +630,8 @@ fn merge_index(
let object_account_id = batch.last_account_id().unwrap_or_default();
let object_type = batch.last_collection().unwrap_or(Collection::None);
let object_id = batch.last_document_id().unwrap_or_default();
let notification_id = SnowflakeIdGenerator::from_sequence_and_node_id(
let notification_id = SnowflakeIdGenerator::from_sequence_id(
object_type as u64 ^ object_account_id as u64,
None,
)
.unwrap_or_default();

17
crates/common/src/telemetry/tracers/store.rs
View File

@@ -10,11 +10,12 @@
use crate::config::telemetry::StoreTracer;
use ahash::{AHashMap, AHashSet};
use registry::schema::structs::{Task, TaskIndexTrace, TaskStatus};
use std::{collections::HashSet, future::Future, time::Duration};
use store::{
Deserialize, SearchStore, Store, ValueKey,
search::{IndexDocument, SearchField, SearchFilter, SearchQuery, TracingSearchField},
write::{BatchBuilder, SearchIndex, TaskEpoch, TaskQueueClass, TelemetryClass, ValueClass},
write::{BatchBuilder, SearchIndex, TelemetryClass, ValueClass},
};
use trc::{
AddContext, AuthEvent, Event, EventDetails, EventType, Key, MessageIngestEvent,
@@ -61,16 +62,10 @@ pub(crate) fn spawn_store_tracer(builder: SubscriberBuilder, settings: StoreTrac
events.len() + 2,
),
)
.with_account_id((span_id >> 32) as u32) // TODO: This is hacky, improve
.with_document(span_id as u32)
.set(
ValueClass::TaskQueue(TaskQueueClass::UpdateIndex {
due: TaskEpoch::now(),
index: SearchIndex::Tracing,
is_insert: true,
}),
vec![],
);
.schedule_task(Task::IndexTrace(TaskIndexTrace {
status: TaskStatus::now(),
trace_id: span_id.into(),
}));
}
}
}