diff --git a/Cargo.lock b/Cargo.lock index c3d66d91..e2ed1c29 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2592,7 +2592,10 @@ dependencies = [ "hashify", "nlp", "percent-encoding", + "registry", "rkyv", + "serde", + "serde_json", "store", "tokio", "trc", diff --git a/crates/common/src/config/inner.rs b/crates/common/src/config/inner.rs index 1cafcb2c..524b2256 100644 --- a/crates/common/src/config/inner.rs +++ b/crates/common/src/config/inner.rs @@ -50,7 +50,8 @@ impl Data { // Build and test snowflake id generator let node_id = bp.node_id(); - let id_generator = SnowflakeIdGenerator::with_node_id(node_id); + SnowflakeIdGenerator::set_node_id(node_id); + let id_generator = SnowflakeIdGenerator::new(); if !id_generator.is_valid() { panic!("Invalid system time, panicking to avoid data corruption"); } diff --git a/crates/common/src/config/network.rs b/crates/common/src/config/network.rs index 09a1859e..32cf5368 100644 --- a/crates/common/src/config/network.rs +++ b/crates/common/src/config/network.rs @@ -13,8 +13,8 @@ use ahash::AHashMap; use registry::{ schema::{ enums::NodeShardType, - prelude::{Object, ObjectType}, - structs::{self, Asn, HttpForm, NodeRole, NodeShard, Rate}, + prelude::ObjectType, + structs::{self, Asn, HttpForm, NodeRole, NodeShard, Rate, TaskManager}, }, types::EnumImpl, }; @@ -30,6 +30,7 @@ pub struct Network { pub http: Http, pub contact_form: Option, pub asn_geo_lookup: AsnGeoLookupConfig, + pub task_manager: TaskManager, } #[derive(Clone)] @@ -155,6 +156,7 @@ impl Network { asn_geo_lookup: AsnGeoLookupConfig::parse(bp).await.unwrap_or_default(), roles: ClusterRoles::default(), http: Http::parse(bp).await, + task_manager: bp.setting_infallible::().await, }; // Process ranges diff --git a/crates/common/src/config/server/listener.rs b/crates/common/src/config/server/listener.rs index e3827aa2..1554adfe 100644 --- a/crates/common/src/config/server/listener.rs +++ b/crates/common/src/config/server/listener.rs @@ -30,7 +30,7 @@ impl Listeners { pub async fn parse(bp: &mut Bootstrap) -> Self { // Parse ACME managers let mut servers = Listeners { - span_id_gen: Arc::new(SnowflakeIdGenerator::with_node_id(bp.node_id())), + span_id_gen: Arc::new(SnowflakeIdGenerator::new()), ..Default::default() }; diff --git a/crates/common/src/enterprise/mod.rs b/crates/common/src/enterprise/mod.rs index 712a46e2..dd9eb9cc 100644 --- a/crates/common/src/enterprise/mod.rs +++ b/crates/common/src/enterprise/mod.rs @@ -12,7 +12,6 @@ pub mod alerts; pub mod config; pub mod license; pub mod llm; -pub mod undelete; use crate::{ Core, Server, config::groupware::CalendarTemplateVariable, expr::Expression, diff --git a/crates/common/src/enterprise/undelete.rs b/crates/common/src/enterprise/undelete.rs deleted file mode 100644 index 28281bf3..00000000 --- a/crates/common/src/enterprise/undelete.rs +++ /dev/null @@ -1,113 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: LicenseRef-SEL - * - * This file is subject to the Stalwart Enterprise License Agreement (SEL) and - * is NOT open source software. - * - */ - -use crate::Core; -use store::{ - Deserialize, IterateParams, U32_LEN, U64_LEN, ValueKey, - write::{AlignedBytes, Archive, BlobOp, ValueClass, key::DeserializeBigEndian, now}, -}; -use trc::AddContext; -use types::blob_hash::{BLOB_HASH_LEN, BlobHash}; - -pub struct DeletedBlob { - pub hash: BlobHash, - pub expires_at: u64, - pub item: DeletedItem, -} - -#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)] -pub struct DeletedItem { - pub typ: DeletedItemType, - pub size: u32, - pub deleted_at: u64, -} - -#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)] -pub enum DeletedItemType { - Email { - from: Box, - subject: Box, - received_at: u64, - }, - FileNode { - name: Box, - }, - CalendarEvent { - title: Box, - start_time: u64, - }, - ContactCard { - name: Box, - }, - SieveScript { - name: Box, - }, -} - -impl Core { - pub async fn list_deleted(&self, account_id: u32) -> trc::Result> { - let from_key = ValueKey { - account_id, - collection: 0, - document_id: 0, - class: ValueClass::Blob(BlobOp::Undelete { - hash: BlobHash::default(), - until: 0, - }), - }; - let to_key = ValueKey { - account_id, - collection: 0, - document_id: u32::MAX, - class: ValueClass::Blob(BlobOp::Undelete { - hash: BlobHash::new_max(), - until: u64::MAX, - }), - }; - - let now = now(); - let mut results = Vec::new(); - - self.storage - .data - .iterate( - IterateParams::new(from_key, to_key).ascending(), - |key, value| { - let expires_at = key.deserialize_be_u64(key.len() - U64_LEN)?; - if expires_at > now { - let item = as Deserialize>::deserialize(value) - .and_then(|bytes| bytes.deserialize::()) - .add_context(|ctx| ctx.ctx(trc::Key::Key, key))?; - - results.push(DeletedBlob { - hash: BlobHash::try_from_hash_slice( - key.get(U32_LEN + 1..U32_LEN + 1 + BLOB_HASH_LEN) - .ok_or_else(|| { - trc::Error::corrupted_key( - key, - value.into(), - trc::location!(), - ) - })?, - ) - .unwrap(), - expires_at, - item, - }); - } - Ok(true) - }, - ) - .await - .caused_by(trc::location!())?; - - Ok(results) - } -} diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 8ecf7df6..9197dd26 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -120,6 +120,7 @@ pub const KV_RATE_LIMIT_CONTACT: u8 = 7; pub const KV_RATE_LIMIT_HTTP_AUTHENTICATED: u8 = 8; pub const KV_RATE_LIMIT_HTTP_ANONYMOUS: u8 = 9; pub const KV_RATE_LIMIT_IMAP: u8 = 10; +pub const KV_QUOTA_BLOB: u8 = 11; pub const KV_GREYLIST: u8 = 16; pub const KV_LOCK_PURGE_ACCOUNT: u8 = 20; pub const KV_LOCK_QUEUE_MESSAGE: u8 = 21; diff --git a/crates/common/src/manager/backup.rs b/crates/common/src/manager/backup.rs index cbcbb656..8d5d9c75 100644 --- a/crates/common/src/manager/backup.rs +++ b/crates/common/src/manager/backup.rs @@ -312,7 +312,7 @@ impl Family { SUBSPACE_COUNTER, SUBSPACE_PROPERTY, ], - Family::Blob => &[SUBSPACE_BLOBS, SUBSPACE_BLOB_EXTRA, SUBSPACE_BLOB_LINK], + Family::Blob => &[SUBSPACE_BLOBS, SUBSPACE_BLOB_LINK], Family::Registry => &[SUBSPACE_REGISTRY], Family::Changelog => &[SUBSPACE_LOGS], Family::Queue => &[SUBSPACE_QUEUE_MESSAGE, SUBSPACE_QUEUE_EVENT], diff --git a/crates/common/src/storage/blob.rs b/crates/common/src/storage/blob.rs index bde449ab..86a6f34d 100644 --- a/crates/common/src/storage/blob.rs +++ b/crates/common/src/storage/blob.rs @@ -4,13 +4,14 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::Server; +use crate::{KV_QUOTA_BLOB, Server}; use mail_parser::{ Encoding, decoders::{base64::base64_decode, quoted_printable::quoted_printable_decode}, }; use store::{ - SerializeInfallible, + U32_LEN, U64_LEN, + dispatch::lookup::KeyValue, write::{BatchBuilder, BlobLink, BlobOp, now}, }; use trc::AddContext; @@ -19,7 +20,46 @@ use types::{ blob_hash::BlobHash, }; +const COUNT_BYTES: u32 = 20; +const COUNT_SHIFT: u32 = 64 - COUNT_BYTES; +const SIZE_MASK: u64 = (1u64 << COUNT_SHIFT) - 1; + impl Server { + pub async fn blob_has_quota(&self, account_id: u32, bytes: usize) -> trc::Result { + if self.core.jmap.upload_tmp_quota_size > 0 || self.core.jmap.upload_tmp_quota_amount > 0 { + let now = now(); + let range_start = now / self.core.jmap.upload_tmp_ttl; + let range_end = + (range_start * self.core.jmap.upload_tmp_ttl) + self.core.jmap.upload_tmp_ttl; + let expires_in = range_end - now; + + let mut bucket = Vec::with_capacity(U32_LEN + U64_LEN + 1); + bucket.push(KV_QUOTA_BLOB); + bucket.extend_from_slice(account_id.to_be_bytes().as_slice()); + bucket.extend_from_slice(range_start.to_be_bytes().as_slice()); + + self.in_memory_store() + .counter_incr( + KeyValue::new(bucket, 1i64 << COUNT_SHIFT | bytes as i64).expires(expires_in), + true, + ) + .await + .caused_by(trc::location!()) + .map(|v| { + let v = v as u64; + let count = v >> COUNT_SHIFT; + let size = v & SIZE_MASK; + + (self.core.jmap.upload_tmp_quota_amount == 0 + || count <= self.core.jmap.upload_tmp_quota_amount as u64) + && (self.core.jmap.upload_tmp_quota_size == 0 + || size <= self.core.jmap.upload_tmp_quota_size as u64) + }) + } else { + Ok(true) + } + } + #[allow(clippy::blocks_in_conditions)] pub async fn put_jmap_blob(&self, account_id: u32, data: &[u8]) -> trc::Result { // First reserve the hash @@ -27,22 +67,13 @@ impl Server { let mut batch = BatchBuilder::new(); let until = now() + self.core.jmap.upload_tmp_ttl; - batch - .with_account_id(account_id) - .set( - BlobOp::Link { - hash: hash.clone(), - to: BlobLink::Temporary { until }, - }, - vec![BlobLink::QUOTA_LINK], - ) - .set( - BlobOp::Quota { - hash: hash.clone(), - until, - }, - (data.len() as u32).serialize(), - ); + batch.with_account_id(account_id).set( + BlobOp::Link { + hash: hash.clone(), + to: BlobLink::Temporary { until }, + }, + vec![], + ); self.core .storage diff --git a/crates/common/src/storage/index.rs b/crates/common/src/storage/index.rs index bd4e205c..95c95936 100644 --- a/crates/common/src/storage/index.rs +++ b/crates/common/src/storage/index.rs @@ -5,6 +5,10 @@ */ use crate::{auth::AccountTenantIds, sharing::notification::ShareNotification}; +use registry::schema::{ + enums::IndexDocumentType, + structs::{Task, TaskIndexDocument, TaskStatus}, +}; use rkyv::{ option::ArchivedOption, primitive::{ArchivedU32, ArchivedU64}, @@ -15,7 +19,7 @@ use store::{ Serialize, SerializeInfallible, write::{ Archive, Archiver, BatchBuilder, BlobLink, BlobOp, IntoOperations, Params, SearchIndex, - TaskEpoch, TaskQueueClass, ValueClass, + ValueClass, }, }; use types::{ @@ -418,14 +422,23 @@ fn build_index( } } IndexValue::SearchIndex { index, .. } => { - batch.set( - ValueClass::TaskQueue(TaskQueueClass::UpdateIndex { - due: TaskEpoch::now().with_random_sequence_id(), - index, - is_insert: set, - }), - vec![], - ); + let task = TaskIndexDocument { + account_id: batch.last_account_id().unwrap().into(), + document_id: batch.last_document_id().unwrap().into(), + document_type: match index { + SearchIndex::Email => IndexDocumentType::Email, + SearchIndex::Calendar => IndexDocumentType::Calendar, + SearchIndex::Contacts => IndexDocumentType::Contacts, + SearchIndex::File => IndexDocumentType::File, + SearchIndex::Tracing | SearchIndex::InMemory => unreachable!(), + }, + status: TaskStatus::now(), + }; + batch.schedule_task(if set { + Task::IndexDocument(task) + } else { + Task::UnindexDocument(task) + }); } IndexValue::Property { field, value } => { if !value.is_none() { @@ -456,9 +469,8 @@ fn build_index( let object_account_id = batch.last_account_id().unwrap_or_default(); let object_type = batch.last_collection().unwrap_or(Collection::None); let object_id = batch.last_document_id().unwrap_or_default(); - let notification_id = SnowflakeIdGenerator::from_sequence_and_node_id( + let notification_id = SnowflakeIdGenerator::from_sequence_id( object_type as u64 ^ object_account_id as u64, - None, ) .unwrap_or_default(); @@ -559,14 +571,18 @@ fn merge_index( } } (IndexValue::SearchIndex { index, .. }, IndexValue::SearchIndex { .. }) => { - batch.set( - ValueClass::TaskQueue(TaskQueueClass::UpdateIndex { - due: TaskEpoch::now().with_random_sequence_id(), - index, - is_insert: true, - }), - vec![], - ); + batch.schedule_task(Task::IndexDocument(TaskIndexDocument { + account_id: batch.last_account_id().unwrap().into(), + document_id: batch.last_document_id().unwrap().into(), + document_type: match index { + SearchIndex::Email => IndexDocumentType::Email, + SearchIndex::Calendar => IndexDocumentType::Calendar, + SearchIndex::Contacts => IndexDocumentType::Contacts, + SearchIndex::File => IndexDocumentType::File, + SearchIndex::Tracing | SearchIndex::InMemory => unreachable!(), + }, + status: TaskStatus::now(), + })); } ( IndexValue::Property { @@ -614,9 +630,8 @@ fn merge_index( let object_account_id = batch.last_account_id().unwrap_or_default(); let object_type = batch.last_collection().unwrap_or(Collection::None); let object_id = batch.last_document_id().unwrap_or_default(); - let notification_id = SnowflakeIdGenerator::from_sequence_and_node_id( + let notification_id = SnowflakeIdGenerator::from_sequence_id( object_type as u64 ^ object_account_id as u64, - None, ) .unwrap_or_default(); diff --git a/crates/common/src/telemetry/tracers/store.rs b/crates/common/src/telemetry/tracers/store.rs index b75a5f28..9118ebe7 100644 --- a/crates/common/src/telemetry/tracers/store.rs +++ b/crates/common/src/telemetry/tracers/store.rs @@ -10,11 +10,12 @@ use crate::config::telemetry::StoreTracer; use ahash::{AHashMap, AHashSet}; +use registry::schema::structs::{Task, TaskIndexTrace, TaskStatus}; use std::{collections::HashSet, future::Future, time::Duration}; use store::{ Deserialize, SearchStore, Store, ValueKey, search::{IndexDocument, SearchField, SearchFilter, SearchQuery, TracingSearchField}, - write::{BatchBuilder, SearchIndex, TaskEpoch, TaskQueueClass, TelemetryClass, ValueClass}, + write::{BatchBuilder, SearchIndex, TelemetryClass, ValueClass}, }; use trc::{ AddContext, AuthEvent, Event, EventDetails, EventType, Key, MessageIngestEvent, @@ -61,16 +62,10 @@ pub(crate) fn spawn_store_tracer(builder: SubscriberBuilder, settings: StoreTrac events.len() + 2, ), ) - .with_account_id((span_id >> 32) as u32) // TODO: This is hacky, improve - .with_document(span_id as u32) - .set( - ValueClass::TaskQueue(TaskQueueClass::UpdateIndex { - due: TaskEpoch::now(), - index: SearchIndex::Tracing, - is_insert: true, - }), - vec![], - ); + .schedule_task(Task::IndexTrace(TaskIndexTrace { + status: TaskStatus::now(), + trace_id: span_id.into(), + })); } } } diff --git a/crates/email/src/mailbox/destroy.rs b/crates/email/src/mailbox/destroy.rs index 09fca5b2..9e57b85e 100644 --- a/crates/email/src/mailbox/destroy.rs +++ b/crates/email/src/mailbox/destroy.rs @@ -12,15 +12,15 @@ use crate::{ use common::{ Server, auth::AccessToken, sharing::EffectiveAcl, storage::index::ObjectIndexBuilder, }; -use store::{ - SerializeInfallible, - roaring::RoaringBitmap, - write::{BatchBuilder, SearchIndex, TaskEpoch, TaskQueueClass, ValueClass}, +use registry::schema::{ + enums::IndexDocumentType, + structs::{Task, TaskIndexDocument, TaskStatus}, }; use store::{ ValueKey, write::{AlignedBytes, Archive}, }; +use store::{roaring::RoaringBitmap, write::BatchBuilder}; use trc::AddContext; use types::{ acl::Acl, @@ -123,14 +123,12 @@ impl MailboxDestroy for Server { .with_current(prev_message_data), ) .caused_by(trc::location!())? - .set( - ValueClass::TaskQueue(TaskQueueClass::UpdateIndex { - index: SearchIndex::Email, - due: TaskEpoch::now(), - is_insert: false, - }), - 0u64.serialize(), - ) + .schedule_task(Task::UnindexDocument(TaskIndexDocument { + account_id: account_id.into(), + document_id: message_id.into(), + document_type: IndexDocumentType::Email, + status: TaskStatus::now(), + })) .commit_point(); } else { let new_message_data = MessageData { diff --git a/crates/email/src/message/copy.rs b/crates/email/src/message/copy.rs index 0cb3f3c6..cf98ad75 100644 --- a/crates/email/src/message/copy.rs +++ b/crates/email/src/message/copy.rs @@ -12,7 +12,7 @@ use crate::{ mailbox::UidMailbox, message::{ index::extractors::VisitTextArchived, - ingest::{MergeThreadIds, ThreadInfo}, + ingest::ThreadInfo, metadata::{ MESSAGE_HAS_ATTACHMENT, MESSAGE_RECEIVED_MASK, MetadataHeaderName, MetadataHeaderValue, }, @@ -20,9 +20,11 @@ use crate::{ }; use common::{Server, storage::index::ObjectIndexBuilder}; use mail_parser::parsers::fields::thread::thread_name; -use store::write::{ - BatchBuilder, IndexPropertyClass, SearchIndex, TaskEpoch, TaskQueueClass, ValueClass, +use registry::schema::{ + enums::IndexDocumentType, + structs::{Task, TaskIndexDocument, TaskMergeThreads, TaskStatus}, }; +use store::write::{BatchBuilder, IndexPropertyClass, ValueClass}; use store::{ ValueKey, write::{AlignedBytes, Archive}, @@ -203,23 +205,26 @@ impl EmailCopy for Server { }), ThreadInfo::serialize(thread_id, &message_ids), ) - .set( - ValueClass::TaskQueue(TaskQueueClass::UpdateIndex { - index: SearchIndex::Email, - due: TaskEpoch::now(), - is_insert: true, - }), - vec![], - ); + .schedule_task(Task::IndexDocument(TaskIndexDocument { + account_id: to_account_id.into(), + document_id: document_id.into(), + document_type: IndexDocumentType::Email, + status: TaskStatus::now(), + })); // Merge threads if necessary - if let Some(merge_threads) = MergeThreadIds::new(thread_result).serialize() { - batch.set( - ValueClass::TaskQueue(TaskQueueClass::MergeThreads { - due: TaskEpoch::now(), - }), - merge_threads, - ); + if !thread_result.merge_ids.is_empty() { + batch.schedule_task(Task::MergeThreads(TaskMergeThreads { + account_id: to_account_id.into(), + document_id: document_id.into(), + status: TaskStatus::now(), + thread_ids: thread_result + .merge_ids + .into_iter() + .map(|id| id.into()) + .collect(), + thread_hash: thread_result.thread_hash.to_string(), + })); } metadata diff --git a/crates/email/src/message/delete.rs b/crates/email/src/message/delete.rs index 4d061990..89467db5 100644 --- a/crates/email/src/message/delete.rs +++ b/crates/email/src/message/delete.rs @@ -7,13 +7,15 @@ use super::metadata::MessageData; use common::{KV_LOCK_PURGE_ACCOUNT, Server, storage::index::ObjectIndexBuilder}; use groupware::calendar::storage::ItipAutoExpunge; +use registry::schema::enums::IndexDocumentType; use registry::schema::prelude::ObjectType; +use registry::schema::structs::{Task, TaskIndexDocument, TaskStatus}; use std::future::Future; use store::ahash::AHashSet; use store::registry::RegistryQuery; use store::write::key::DeserializeBigEndian; -use store::write::{IndexPropertyClass, SearchIndex, TaskEpoch, TaskQueueClass, now}; -use store::{IterateParams, SerializeInfallible, U32_LEN, U64_LEN, ValueKey}; +use store::write::{IndexPropertyClass, now}; +use store::{IterateParams, U32_LEN, U64_LEN, ValueKey}; use store::{ roaring::RoaringBitmap, write::{BatchBuilder, ValueClass}, @@ -83,14 +85,12 @@ impl EmailDeletion for Server { .with_current(metadata), ) .caused_by(trc::location!())? - .set( - ValueClass::TaskQueue(TaskQueueClass::UpdateIndex { - index: SearchIndex::Email, - due: TaskEpoch::now(), - is_insert: false, - }), - 0u64.serialize(), - ) + .schedule_task(Task::UnindexDocument(TaskIndexDocument { + account_id: account_id.into(), + document_id: document_id.into(), + document_type: IndexDocumentType::Email, + status: TaskStatus::now(), + })) .commit_point(); deleted_ids.insert(document_id); diff --git a/crates/email/src/message/ingest.rs b/crates/email/src/message/ingest.rs index 10f18ae3..99f407d2 100644 --- a/crates/email/src/message/ingest.rs +++ b/crates/email/src/message/ingest.rs @@ -23,17 +23,28 @@ use mail_parser::{ DateTime, Header, HeaderName, HeaderValue, Message, MessageParser, MimeHeaders, PartType, parsers::fields::thread::thread_name, }; -use registry::schema::prelude::Permission; -use std::{borrow::Cow, cmp::Ordering, fmt::Write, time::Instant}; -use std::{future::Future, hash::Hasher}; -use store::write::{AlignedBytes, Archive}; -use store::{ - IndexKeyPrefix, IterateParams, U32_LEN, ValueKey, - ahash::{AHashMap, AHashSet}, - write::{ - AssignedId, AssignedIds, BatchBuilder, BlobLink, BlobOp, IndexPropertyClass, SearchIndex, - TaskEpoch, TaskQueueClass, ValueClass, key::DeserializeBigEndian, now, +use registry::{ + pickle::Pickle, + schema::{ + enums::IndexDocumentType, + prelude::{ObjectType, Permission, Property}, + structs::{SpamTrainingSample, Task, TaskIndexDocument, TaskMergeThreads, TaskStatus}, }, + types::{EnumImpl, datetime::UTCDateTime, id::ObjectId}, +}; +use std::future::Future; +use std::{borrow::Cow, cmp::Ordering, fmt::Write, time::Instant}; +use store::{ + IndexKeyPrefix, IterateParams, SerializeInfallible, U32_LEN, ValueKey, + ahash::AHashMap, + write::{ + AssignedId, AssignedIds, BatchBuilder, BlobLink, BlobOp, IndexPropertyClass, ValueClass, + key::DeserializeBigEndian, now, + }, +}; +use store::{ + write::{AlignedBytes, Archive, RegistryClass}, + xxhash_rust, }; use trc::{AddContext, MessageIngestEvent, SpamEvent}; use types::{ @@ -41,10 +52,11 @@ use types::{ blob_hash::BlobHash, collection::{Collection, SyncCollection}, field::{ContactField, EmailField, MailboxField, PrincipalField}, + id::Id, keyword::Keyword, special_use::SpecialUse, }; -use utils::{cheeky_hash::CheekyHash, sanitize_email}; +use utils::{cheeky_hash::CheekyHash, sanitize_email, snowflake::SnowflakeIdGenerator}; #[derive(Default)] pub struct IngestedEmail { @@ -109,10 +121,15 @@ pub trait EmailIngest: Sync + Send { is_spam: bool, span_id: u64, ) -> impl Future> + Send; + + #[allow(clippy::too_many_arguments)] fn add_spam_sample( &self, + account_id: u32, batch: &mut BatchBuilder, hash: BlobHash, + from: String, + subject: String, is_spam: bool, hold_sample: bool, span_id: u64, @@ -603,6 +620,25 @@ impl EmailIngest for Server { size: (message.raw_message.len() + extra_headers.len()) as u32, }; + // Request spam training + if let Some(learn_spam) = train_spam { + self.add_spam_sample( + account_id, + &mut batch, + params.blob_hash.unwrap_or(&blob_hash).clone(), + message + .from() + .and_then(|s| s.first()) + .and_then(|s| s.address()) + .unwrap_or_default() + .to_string(), + thread_name(message.subject().unwrap_or_default()).to_string(), + learn_spam, + !is_encrypted, + params.session_id, + ); + } + batch .with_collection(Collection::Email) .with_document(document_id) @@ -623,38 +659,30 @@ impl EmailIngest for Server { }), ThreadInfo::serialize(thread_id, &message_ids), ) - .set( - ValueClass::TaskQueue(TaskQueueClass::UpdateIndex { - index: SearchIndex::Email, - due: TaskEpoch::now(), - is_insert: true, - }), - vec![], - ); + .schedule_task(Task::IndexDocument(TaskIndexDocument { + account_id: account_id.into(), + document_id: document_id.into(), + document_type: IndexDocumentType::Email, + status: TaskStatus::now(), + })); if let Some(blob_hold) = blob_hold { batch.clear(blob_hold); } // Merge threads if necessary - if let Some(merge_threads) = MergeThreadIds::new(thread_result).serialize() { - batch.set( - ValueClass::TaskQueue(TaskQueueClass::MergeThreads { - due: TaskEpoch::now(), - }), - merge_threads, - ); - } - - // Request spam training - if let Some(learn_spam) = train_spam { - self.add_spam_sample( - &mut batch, - params.blob_hash.unwrap_or(&blob_hash).clone(), - learn_spam, - !is_encrypted, - params.session_id, - ); + if !thread_result.merge_ids.is_empty() { + batch.schedule_task(Task::MergeThreads(TaskMergeThreads { + account_id: account_id.into(), + document_id: document_id.into(), + status: TaskStatus::now(), + thread_ids: thread_result + .merge_ids + .into_iter() + .map(|id| id.into()) + .collect(), + thread_hash: thread_result.thread_hash.to_string(), + })); } // Add iTIP responses to batch @@ -876,9 +904,14 @@ impl EmailIngest for Server { let metadata = archive .to_unarchived::() .caused_by(trc::location!())?; + let part = metadata.inner.root_part(); + self.add_spam_sample( + account_id, batch, (&metadata.inner.blob_hash).into(), + part.from().unwrap_or_default().to_string(), + thread_name(part.subject().unwrap_or_default()).to_string(), is_spam, true, span_id, @@ -890,8 +923,11 @@ impl EmailIngest for Server { fn add_spam_sample( &self, + account_id: u32, batch: &mut BatchBuilder, hash: BlobHash, + from: String, + subject: String, is_spam: bool, hold_sample: bool, span_id: u64, @@ -903,17 +939,42 @@ impl EmailIngest for Server { dt.second = 0; let until = dt.to_timestamp() as u64 + config.hold_samples_for; + let sample = SpamTrainingSample { + account_id: Some(Id::from(account_id)), + blob_id: BlobId::new(hash.clone(), BlobClass::default()), + delete_after_use: !hold_sample, + expires_at: UTCDateTime::from_timestamp(until as i64), + from, + is_spam, + subject, + } + .to_pickled_vec(); + + let object_id = ObjectType::SpamTrainingSample.to_id(); + let item_id = SnowflakeIdGenerator::from_sequence_id(xxhash_rust::xxh3::xxh3_64( + sample.as_slice(), + )) + .unwrap_or_default(); batch .set( BlobOp::Link { - hash: hash.clone(), + hash, to: BlobLink::Temporary { until }, }, - vec![BlobLink::SPAM_SAMPLE_LINK], + ObjectId::new(ObjectType::SpamTrainingSample, item_id.into()).serialize(), ) .set( - BlobOp::SpamSample { hash, until }, - vec![u8::from(is_spam), u8::from(hold_sample)], + ValueClass::Registry(RegistryClass::Id { object_id, item_id }), + sample, + ) + .set( + ValueClass::Registry(RegistryClass::Index { + index_id: Property::AccountId.to_id(), + object_id, + item_id, + key: account_id.serialize(), + }), + vec![], ); trc::event!( @@ -955,65 +1016,6 @@ impl IngestSource<'_> { } } -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct MergeThreadIds { - pub thread_hash: CheekyHash, - pub merge_ids: T, -} - -impl MergeThreadIds> { - pub(crate) fn new(thread_result: ThreadResult) -> Self { - Self { - thread_hash: thread_result.thread_hash, - merge_ids: thread_result.merge_ids, - } - } - - pub(crate) fn serialize(&self) -> Option> { - if !self.merge_ids.is_empty() { - let mut buf = - Vec::with_capacity(self.thread_hash.len() + self.merge_ids.len() * U32_LEN); - buf.extend_from_slice(self.thread_hash.as_bytes()); - for id in &self.merge_ids { - buf.extend_from_slice(&id.to_be_bytes()); - } - Some(buf) - } else { - None - } - } -} - -impl MergeThreadIds> { - pub fn deserialize(bytes: &[u8]) -> Option { - if !bytes.is_empty() { - let thread_hash = CheekyHash::deserialize(bytes)?; - let mut merge_ids = - AHashSet::with_capacity(((bytes.len() - thread_hash.len()) / U32_LEN) + 1); - let mut start_offset = thread_hash.len(); - - while let Some(id_bytes) = bytes.get(start_offset..start_offset + U32_LEN) { - merge_ids.insert(u32::from_be_bytes(id_bytes.try_into().ok()?)); - start_offset += U32_LEN; - } - - Some(Self { - thread_hash, - merge_ids, - }) - } else { - None - } - } -} - -impl std::hash::Hash for MergeThreadIds> { - fn hash(&self, state: &mut H) { - self.thread_hash.hash(state); - self.merge_ids.len().hash(state); - } -} - pub struct ThreadInfo; impl ThreadInfo { diff --git a/crates/email/src/message/metadata.rs b/crates/email/src/message/metadata.rs index 8e09a4b2..de55e88b 100644 --- a/crates/email/src/message/metadata.rs +++ b/crates/email/src/message/metadata.rs @@ -630,6 +630,12 @@ impl ArchivedMessageMetadataPart { }) } + pub fn from(&self) -> Option<&str> { + self.header_value(&MetadataHeaderName::From) + .and_then(|header| header.as_single_address()) + .and_then(|addr| addr.address.as_deref()) + } + pub fn subject(&self) -> Option<&str> { self.header_value(&MetadataHeaderName::Subject) .and_then(|header| header.as_text()) diff --git a/crates/groupware/Cargo.toml b/crates/groupware/Cargo.toml index c4a655bd..4ab1142e 100644 --- a/crates/groupware/Cargo.toml +++ b/crates/groupware/Cargo.toml @@ -11,6 +11,7 @@ types = { path = "../types" } trc = { path = "../trc" } nlp = { path = "../nlp" } directory = { path = "../directory" } +registry = { path = "../registry" } calcard = { version = "0.3", features = ["rkyv"] } hashify = "0.2" tokio = { version = "1.47", features = ["net", "macros"] } @@ -19,6 +20,8 @@ percent-encoding = "2.3.1" compact_str = "0.9.0" ahash = { version = "0.8" } chrono = "0.4.40" +serde = { version = "1.0", features = ["derive"]} +serde_json = "1.0" [features] test_mode = [] diff --git a/crates/groupware/src/calendar/storage.rs b/crates/groupware/src/calendar/storage.rs index 7ae5a799..63822dd2 100644 --- a/crates/groupware/src/calendar/storage.rs +++ b/crates/groupware/src/calendar/storage.rs @@ -21,20 +21,24 @@ use common::{ auth::{AccountInfo, AccountTenantIds}, storage::index::ObjectIndexBuilder, }; +use registry::{ + pickle::Pickle, + schema::structs::{Task, TaskCalendarAlarmEmail, TaskCalendarAlarmNotification, TaskStatus}, + types::{EnumImpl, datetime::UTCDateTime}, +}; use store::{ - IterateParams, U16_LEN, U32_LEN, U64_LEN, ValueKey, + IterateParams, SerializeInfallible, U32_LEN, ValueKey, roaring::RoaringBitmap, write::{ - AlignedBytes, Archive, BatchBuilder, IndexPropertyClass, TaskEpoch, TaskQueueClass, - ValueClass, - key::{DeserializeBigEndian, KeySerializer}, - now, + AlignedBytes, Archive, BatchBuilder, IndexPropertyClass, Operation, TaskQueueClass, + ValueClass, ValueOp, key::DeserializeBigEndian, now, }, }; use trc::AddContext; use types::{ collection::{Collection, VanishedCollection}, field::CalendarNotificationField, + id::Id, }; pub trait ItipAutoExpunge: Sync + Send { @@ -526,52 +530,70 @@ impl DestroyArchive> { } impl CalendarAlarm { - pub fn write_task(&self, batch: &mut BatchBuilder) { - match &self.typ { + pub fn build_write_ops(&self, account_id: u32, document_id: u32) -> [Operation; 2] { + let task = match &self.typ { CalendarAlarmType::Email { event_start, event_start_tz, event_end, event_end_tz, - } => { - batch.set( - ValueClass::TaskQueue(TaskQueueClass::SendAlarm { - due: TaskEpoch::new(self.alarm_time as u64), - event_id: self.event_id, - alarm_id: self.alarm_id, - is_email_alert: true, - }), - KeySerializer::new((U64_LEN * 2) + (U16_LEN * 2)) - .write(*event_start as u64) - .write(*event_end as u64) - .write(*event_start_tz) - .write(*event_end_tz) - .finalize(), - ); - } + } => Task::CalendarAlarmEmail(TaskCalendarAlarmEmail { + account_id: account_id.into(), + document_id: document_id.into(), + alarm_id: self.alarm_id.into(), + event_id: self.event_id.into(), + event_end: UTCDateTime::from_timestamp(*event_end), + event_end_tz: (*event_end_tz).into(), + event_start: UTCDateTime::from_timestamp(*event_start), + event_start_tz: (*event_start_tz).into(), + status: TaskStatus::at(self.alarm_time), + }), CalendarAlarmType::Display { recurrence_id } => { - batch.set( - ValueClass::TaskQueue(TaskQueueClass::SendAlarm { - due: TaskEpoch::new(self.alarm_time as u64), - event_id: self.event_id, - alarm_id: self.alarm_id, - is_email_alert: false, - }), - KeySerializer::new(U64_LEN) - .write(recurrence_id.unwrap_or_default() as u64) - .finalize(), - ); + Task::CalendarAlarmNotification(TaskCalendarAlarmNotification { + account_id: account_id.into(), + document_id: document_id.into(), + alarm_id: self.alarm_id.into(), + event_id: self.event_id.into(), + recurrence_id: *recurrence_id, + status: TaskStatus::at(self.alarm_time), + }) } + }; + let id = Id::from_parts(account_id, document_id).id(); + [ + Operation::Value { + class: ValueClass::TaskQueue(TaskQueueClass::Due { + id, + due: self.alarm_time as u64, + }), + op: ValueOp::Set(task.object_type().to_id().serialize()), + }, + Operation::Value { + class: ValueClass::TaskQueue(TaskQueueClass::Task { id }), + op: ValueOp::Set(task.to_pickled_vec()), + }, + ] + } + + pub fn write_task(&self, batch: &mut BatchBuilder) { + let account_id = batch.last_account_id().unwrap(); + let document_id = batch.last_document_id().unwrap(); + + for op in self.build_write_ops(account_id, document_id) { + batch.any_op(op); } } pub fn delete_task(&self, batch: &mut BatchBuilder) { - batch.clear(ValueClass::TaskQueue(TaskQueueClass::SendAlarm { - due: TaskEpoch::new(self.alarm_time as u64), - event_id: self.event_id, - alarm_id: self.alarm_id, - is_email_alert: matches!(self.typ, CalendarAlarmType::Email { .. }), - })); + let account_id = batch.last_account_id().unwrap(); + let document_id = batch.last_document_id().unwrap(); + let id = Id::from_parts(account_id, document_id).id(); + batch + .clear(ValueClass::TaskQueue(TaskQueueClass::Task { id })) + .clear(ValueClass::TaskQueue(TaskQueueClass::Due { + id, + due: self.alarm_time as u64, + })); } } diff --git a/crates/groupware/src/scheduling/itip.rs b/crates/groupware/src/scheduling/itip.rs index 7b1b4c03..9f0dfce9 100644 --- a/crates/groupware/src/scheduling/itip.rs +++ b/crates/groupware/src/scheduling/itip.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::scheduling::{ArchivedItipSummary, ItipMessage, ItipMessages}; +use crate::scheduling::{ItipMessage, ItipMessages, ItipSummary}; use calcard::{ common::{IanaString, PartialDateTime}, icalendar::{ @@ -14,11 +14,10 @@ use calcard::{ }, }; use common::PROD_ID; -use store::{ - Serialize, - write::{Archiver, BatchBuilder, TaskEpoch, TaskQueueClass, ValueClass}, +use registry::schema::structs::{ + Task, TaskCalendarItipContents, TaskCalendarItipMessage, TaskStatus, }; -use trc::AddContext; +use store::write::BatchBuilder; pub(crate) fn itip_build_envelope(method: ICalendarMethod) -> ICalendarComponent { ICalendarComponent { @@ -273,28 +272,26 @@ pub(crate) fn can_attendee_modify_property( impl ItipMessages { pub fn new(messages: Vec>) -> Self { ItipMessages { - messages: messages.into_iter().map(|m| m.into()).collect(), + messages: messages + .into_iter() + .map(|m| TaskCalendarItipContents { + from: m.from, + i_calendar_data: m.message.to_string(), + is_from_organizer: m.from_organizer, + summary: serde_json::to_string(&m.summary).unwrap_or_default(), + to: m.to, + }) + .collect(), } } pub fn queue(self, batch: &mut BatchBuilder) -> trc::Result<()> { - let due = TaskEpoch::now().with_random_sequence_id(); - batch.set( - ValueClass::TaskQueue(TaskQueueClass::SendImip { - due, - is_payload: false, - }), - vec![], - ); - batch.set( - ValueClass::TaskQueue(TaskQueueClass::SendImip { - due, - is_payload: true, - }), - Archiver::new(self) - .serialize() - .caused_by(trc::location!())?, - ); + batch.schedule_task(Task::CalendarItipMessage(TaskCalendarItipMessage { + account_id: batch.last_account_id().unwrap().into(), + document_id: batch.last_document_id().unwrap().into(), + messages: self.messages, + status: TaskStatus::now(), + })); Ok(()) } @@ -312,13 +309,13 @@ impl From> for ItipMessage { } } -impl ArchivedItipSummary { +impl ItipSummary { pub fn method(&self) -> &str { match self { - ArchivedItipSummary::Invite(_) => ICalendarMethod::Request.as_str(), - ArchivedItipSummary::Update { method, .. } => method.as_str(), - ArchivedItipSummary::Cancel(_) => ICalendarMethod::Cancel.as_str(), - ArchivedItipSummary::Rsvp { .. } => ICalendarMethod::Reply.as_str(), + ItipSummary::Invite(_) => ICalendarMethod::Request.as_str(), + ItipSummary::Update { method, .. } => method.as_str(), + ItipSummary::Cancel(_) => ICalendarMethod::Cancel.as_str(), + ItipSummary::Rsvp { .. } => ICalendarMethod::Reply.as_str(), } } } diff --git a/crates/groupware/src/scheduling/mod.rs b/crates/groupware/src/scheduling/mod.rs index 2e40031b..12a8f536 100644 --- a/crates/groupware/src/scheduling/mod.rs +++ b/crates/groupware/src/scheduling/mod.rs @@ -14,6 +14,7 @@ use calcard::{ ICalendarStatus, ICalendarUserTypes, ICalendarValue, Uri, }, }; +use registry::schema::structs::TaskCalendarItipContents; use std::{fmt::Display, hash::Hash}; pub mod attendee; @@ -144,7 +145,7 @@ pub enum ItipError { AutoAddDisabled, } -#[derive(Debug, rkyv::Archive, rkyv::Deserialize, rkyv::Serialize)] +#[derive(Debug)] pub struct ItipMessage { pub from: String, pub from_organizer: bool, @@ -153,7 +154,7 @@ pub struct ItipMessage { pub message: T, } -#[derive(Debug, Clone, rkyv::Archive, rkyv::Deserialize, rkyv::Serialize)] +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub enum ItipSummary { Invite(Vec), Update { @@ -168,13 +169,14 @@ pub enum ItipSummary { }, } -#[derive(Debug, Clone, Hash, PartialEq, Eq, rkyv::Archive, rkyv::Deserialize, rkyv::Serialize)] +#[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct ItipField { pub name: ICalendarProperty, pub value: ItipValue, } -#[derive(Debug, Clone, Hash, PartialEq, Eq, rkyv::Archive, rkyv::Deserialize, rkyv::Serialize)] +#[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(tag = "type", content = "value")] pub enum ItipValue { Text(String), Time(ItipTime), @@ -182,22 +184,21 @@ pub enum ItipValue { Participants(Vec), } -#[derive(Debug, Clone, Hash, PartialEq, Eq, rkyv::Archive, rkyv::Deserialize, rkyv::Serialize)] +#[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct ItipTime { pub start: i64, pub tz_id: u16, } -#[derive(Debug, Clone, Hash, PartialEq, Eq, rkyv::Archive, rkyv::Deserialize, rkyv::Serialize)] +#[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct ItipParticipant { pub email: String, pub name: Option, pub is_organizer: bool, } -#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize)] pub struct ItipMessages { - pub messages: Vec>, + pub messages: Vec, } impl Attendee<'_> { diff --git a/crates/imap/src/op/expunge.rs b/crates/imap/src/op/expunge.rs index bc5c7f5d..44993fb5 100644 --- a/crates/imap/src/op/expunge.rs +++ b/crates/imap/src/op/expunge.rs @@ -17,13 +17,12 @@ use imap_proto::{ parser::parse_sequence_set, receiver::{Request, Token}, }; -use registry::schema::enums::Permission; -use std::{sync::Arc, time::Instant}; -use store::{ - SerializeInfallible, - roaring::RoaringBitmap, - write::{BatchBuilder, SearchIndex, TaskEpoch, TaskQueueClass, ValueClass}, +use registry::schema::{ + enums::{IndexDocumentType, Permission}, + structs::{Task, TaskIndexDocument, TaskStatus}, }; +use std::{sync::Arc, time::Instant}; +use store::{roaring::RoaringBitmap, write::BatchBuilder}; use trc::AddContext; use types::{ acl::Acl, @@ -198,14 +197,12 @@ impl SessionData { .with_current(metadata), ) .caused_by(trc::location!())? - .set( - ValueClass::TaskQueue(TaskQueueClass::UpdateIndex { - index: SearchIndex::Email, - due: TaskEpoch::now(), - is_insert: false, - }), - 0u64.serialize(), - ) + .schedule_task(Task::UnindexDocument(TaskIndexDocument { + account_id: account_id.into(), + document_id: document_id.into(), + document_type: IndexDocumentType::Email, + status: TaskStatus::now(), + })) .commit_point(); } else { // Untag message from this mailbox and remove Deleted flag diff --git a/crates/jmap/src/blob/copy.rs b/crates/jmap/src/blob/copy.rs index 7a80fd0c..98a48b8d 100644 --- a/crates/jmap/src/blob/copy.rs +++ b/crates/jmap/src/blob/copy.rs @@ -6,12 +6,12 @@ use super::download::BlobDownload; use common::{Server, auth::AccessToken}; -use registry::schema::enums::Permission; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::copy::{CopyBlobRequest, CopyBlobResponse}, request::IntoValid, }; +use registry::schema::enums::Permission; use std::future::Future; use store::write::{BatchBuilder, BlobLink, BlobOp, now}; use trc::AddContext; @@ -43,19 +43,11 @@ impl BlobCopy for Server { for blob_id in request.blob_ids.into_valid() { if self.has_access_blob(&blob_id, access_token).await? { // Enforce quota - let used = self - .core - .storage - .data - .blob_quota(account_id) - .await - .caused_by(trc::location!())?; - - if ((self.core.jmap.upload_tmp_quota_size > 0 - && used.bytes >= self.core.jmap.upload_tmp_quota_size) - || (self.core.jmap.upload_tmp_quota_amount > 0 - && used.count + 1 > self.core.jmap.upload_tmp_quota_amount)) - && !access_token.has_permission(Permission::UnlimitedUploads) + if !access_token.has_permission(Permission::UnlimitedUploads) + && !self + .blob_has_quota(account_id, 1) + .await + .caused_by(trc::location!())? { response.not_copied.append( blob_id, diff --git a/crates/jmap/src/blob/upload.rs b/crates/jmap/src/blob/upload.rs index b569f1ad..24e98c0c 100644 --- a/crates/jmap/src/blob/upload.rs +++ b/crates/jmap/src/blob/upload.rs @@ -164,19 +164,11 @@ impl BlobUpload for Server { } // Enforce quota - let used = self - .core - .storage - .data - .blob_quota(account_id) - .await - .caused_by(trc::location!())?; - - if ((self.core.jmap.upload_tmp_quota_size > 0 - && used.bytes + data.len() > self.core.jmap.upload_tmp_quota_size) - || (self.core.jmap.upload_tmp_quota_amount > 0 - && used.count + 1 > self.core.jmap.upload_tmp_quota_amount)) - && !access_token.has_permission(Permission::UnlimitedUploads) + if !access_token.has_permission(Permission::UnlimitedUploads) + && !self + .blob_has_quota(account_id, data.len()) + .await + .caused_by(trc::location!())? { response.not_created.append( create_id, @@ -224,19 +216,11 @@ impl BlobUpload for Server { } // Enforce quota - let used = self - .core - .storage - .data - .blob_quota(account_id.document_id()) - .await - .caused_by(trc::location!())?; - - if ((self.core.jmap.upload_tmp_quota_size > 0 - && used.bytes + data.len() > self.core.jmap.upload_tmp_quota_size) - || (self.core.jmap.upload_tmp_quota_amount > 0 - && used.count + 1 > self.core.jmap.upload_tmp_quota_amount)) - && !access_token.has_permission(Permission::UnlimitedUploads) + if !access_token.has_permission(Permission::UnlimitedUploads) + && !self + .blob_has_quota(account_id.document_id(), data.len()) + .await + .caused_by(trc::location!())? { let err = Err(trc::LimitEvent::BlobQuota .into_err() diff --git a/crates/jmap/src/registry/get.rs b/crates/jmap/src/registry/get.rs index edb675fe..b596375a 100644 --- a/crates/jmap/src/registry/get.rs +++ b/crates/jmap/src/registry/get.rs @@ -147,6 +147,7 @@ impl RegistryGet for Server { | ObjectType::SpamTag | ObjectType::SpfReportSettings | ObjectType::StoreLookup + | ObjectType::TaskManager | ObjectType::TlsReportSettings | ObjectType::Tracer | ObjectType::TracingStore @@ -222,6 +223,9 @@ impl RegistryGet for Server { _ => {} } + let todo = "compact pickle"; + let todo = "app passwords, apis and user change pass/OTP"; + let mut object = object.into_value(); let object_map = object.as_object_mut().unwrap(); if is_tenant_filtered && let Some(tenant_id) = access_token.tenant_id() { @@ -262,8 +266,6 @@ impl RegistryGet for Server { } ObjectType::Log => {} ObjectType::QueuedMessage => {} - - // Move to registry ObjectType::Task => {} ObjectType::ArfFeedbackReport => {} ObjectType::DmarcReport => {} diff --git a/crates/jmap/src/registry/set.rs b/crates/jmap/src/registry/set.rs index c3f14a46..09845844 100644 --- a/crates/jmap/src/registry/set.rs +++ b/crates/jmap/src/registry/set.rs @@ -113,6 +113,7 @@ impl RegistrySet for Server { ObjectType::SpamTag => {} ObjectType::SpfReportSettings => {} ObjectType::StoreLookup => {} + ObjectType::TaskManager => {} ObjectType::TlsReportSettings => {} ObjectType::Tracer => {} ObjectType::TracingStore => {} diff --git a/crates/registry/src/pickle.rs b/crates/registry/src/pickle.rs index 45935b03..f06c3222 100644 --- a/crates/registry/src/pickle.rs +++ b/crates/registry/src/pickle.rs @@ -12,6 +12,11 @@ use std::collections::HashMap; pub trait Pickle: Sized { fn pickle(&self, out: &mut Vec); fn unpickle(stream: &mut PickledStream<'_>) -> Option; + fn to_pickled_vec(&self) -> Vec { + let mut out = Vec::with_capacity(256); + self.pickle(&mut out); + out + } } pub struct PickledStream<'x> { diff --git a/crates/registry/src/schema/mod.rs b/crates/registry/src/schema/mod.rs index ed31285a..7f6e2d48 100644 --- a/crates/registry/src/schema/mod.rs +++ b/crates/registry/src/schema/mod.rs @@ -5,11 +5,12 @@ */ use crate::{ + pickle::Pickle, schema::{ enums::{TracingLevel, TracingLevelOpt}, prelude::{ Account, Duration, GroupAccount, HttpAuth, NodeRange, Object, ObjectInner, Property, - UserAccount, + Task, TaskStatus, TaskStatusPending, UTCDateTime, UserAccount, }, }, types::EnumImpl, @@ -146,6 +147,65 @@ impl HttpAuth { } } +impl Task { + pub fn set_status(&mut self, status: TaskStatus) { + match self { + Task::IndexDocument(task) => task.status = status, + Task::UnindexDocument(task) => task.status = status, + Task::IndexTrace(task) => task.status = status, + Task::CalendarAlarmEmail(task) => task.status = status, + Task::CalendarAlarmNotification(task) => task.status = status, + Task::CalendarItipMessage(task) => task.status = status, + Task::MergeThreads(task) => task.status = status, + } + } + + pub fn status(&self) -> &TaskStatus { + match self { + Task::IndexDocument(task) => &task.status, + Task::UnindexDocument(task) => &task.status, + Task::IndexTrace(task) => &task.status, + Task::CalendarAlarmEmail(task) => &task.status, + Task::CalendarAlarmNotification(task) => &task.status, + Task::CalendarItipMessage(task) => &task.status, + Task::MergeThreads(task) => &task.status, + } + } + + pub fn attempt_number(&self) -> u64 { + match self.status() { + TaskStatus::Pending(_) => 0, + TaskStatus::Retry(status) => status.attempt_number, + TaskStatus::Failed(status) => status.failed_attempt_number, + } + } + + pub fn due_timestamp(&self) -> u64 { + match self.status() { + TaskStatus::Pending(status) => status.due.timestamp() as u64, + TaskStatus::Retry(status) => status.due.timestamp() as u64, + TaskStatus::Failed(_) => u64::MAX, + } + } +} + +impl TaskStatus { + pub fn now() -> Self { + let now = UTCDateTime::now(); + TaskStatus::Pending(TaskStatusPending { + created_at: now, + due: now, + }) + } + + pub fn at(timestamp: i64) -> Self { + TaskStatus::Pending(TaskStatusPending { + due: UTCDateTime::from_timestamp(timestamp), + created_at: UTCDateTime::now(), + }) + } +} + impl Display for Property { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}", self.as_str()) @@ -243,3 +303,13 @@ impl Object { Object { inner, revision: 0 } } } + +impl Pickle for Object { + fn pickle(&self, out: &mut Vec) { + Object::pickle(self, out); + } + + fn unpickle(stream: &mut crate::pickle::PickledStream<'_>) -> Option { + Object::unpickle(stream) + } +} diff --git a/crates/registry/src/schema/prelude.rs b/crates/registry/src/schema/prelude.rs index d0fb446e..6f1f4e49 100644 --- a/crates/registry/src/schema/prelude.rs +++ b/crates/registry/src/schema/prelude.rs @@ -25,6 +25,7 @@ pub use crate::types::socketaddr::SocketAddr; pub use crate::types::string::StringValidator; pub use serde::{Deserialize, Serialize}; pub use std::str::FromStr; +pub use types::blob::BlobId; pub use types::id::Id; pub use utils::map::vec_map::VecMap; diff --git a/crates/registry/src/types/datetime.rs b/crates/registry/src/types/datetime.rs index a5c5a52c..bacb7b55 100644 --- a/crates/registry/src/types/datetime.rs +++ b/crates/registry/src/types/datetime.rs @@ -11,7 +11,7 @@ use crate::{ }; use std::{fmt::Display, str::FromStr, time::SystemTime}; -#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] #[repr(transparent)] pub struct UTCDateTime(i64); diff --git a/crates/registry/src/types/id.rs b/crates/registry/src/types/id.rs index 997bb683..69eb6113 100644 --- a/crates/registry/src/types/id.rs +++ b/crates/registry/src/types/id.rs @@ -11,7 +11,11 @@ use crate::{ types::{EnumImpl, error::PatchError}, }; use std::{fmt::Display, str::FromStr}; -use types::{blob::BlobId, id::Id}; +use types::{ + blob::{BlobClass, BlobId}, + blob_hash::{BLOB_HASH_LEN, BlobHash}, + id::Id, +}; #[derive(Debug, PartialEq, Clone, Copy, Eq, Hash)] pub struct ObjectId { @@ -69,13 +73,31 @@ impl Pickle for Id { fn unpickle(data: &mut PickledStream<'_>) -> Option { let mut arr = [0u8; std::mem::size_of::()]; - arr.copy_from_slice(data.read_bytes(8)?); + arr.copy_from_slice(data.read_bytes(std::mem::size_of::())?); let id = u64::from_be_bytes(arr); Some(Id::new(id)) } } +impl Pickle for BlobId { + fn pickle(&self, out: &mut Vec) { + out.extend_from_slice(self.hash.as_slice()); + } + + fn unpickle(stream: &mut PickledStream<'_>) -> Option { + stream.read_bytes(BLOB_HASH_LEN).map(|bytes| { + BlobId::new( + BlobHash::try_from_hash_slice(bytes).unwrap(), + BlobClass::Reserved { + account_id: 0, + expires: 0, + }, + ) + }) + } +} + impl RegistryJsonPatch for Id { fn patch( &mut self, diff --git a/crates/services/src/housekeeper/mod.rs b/crates/services/src/housekeeper/mod.rs index 2a20311f..16b69c72 100644 --- a/crates/services/src/housekeeper/mod.rs +++ b/crates/services/src/housekeeper/mod.rs @@ -396,6 +396,7 @@ pub fn spawn_housekeeper(inner: Arc, mut rx: mpsc::Receiver impl Future + Send; + + fn send_email_alarm( + &self, + task: &TaskCalendarAlarmEmail, server_instance: Arc, - ) -> impl Future + Send; + ) -> impl Future + Send; } impl SendAlarmTask for Server { - async fn send_alarm( - &self, - account_id: u32, - document_id: u32, - alarm: &CalendarAlarm, - server_instance: Arc, - ) -> bool { - match &alarm.typ { - CalendarAlarmType::Display { .. } => { - match send_display_alarm(self, account_id, document_id, alarm).await { - Ok(result) => result, - Err(err) => { - trc::error!( - err.account_id(account_id) - .document_id(document_id) - .caused_by(trc::location!()) - .details("Failed to process e-mail alarm") - ); - false - } - } + async fn send_display_alarm(&self, task: &TaskCalendarAlarmNotification) -> TaskResult { + match send_display_alarm(self, task).await { + Ok(result) => result, + Err(err) => { + let result = TaskResult::temporary(err.to_string()); + trc::error!( + err.account_id(task.account_id.document_id()) + .document_id(task.document_id.document_id()) + .caused_by(trc::location!()) + .details("Failed to process e-mail alarm") + ); + result } - CalendarAlarmType::Email { .. } => { - match send_email_alarm(self, account_id, document_id, alarm, server_instance).await - { - Ok(result) => result, - Err(err) => { - trc::error!( - err.account_id(account_id) - .document_id(document_id) - .caused_by(trc::location!()) - .details("Failed to process e-mail alarm") - ); - false - } - } + } + } + + async fn send_email_alarm( + &self, + task: &TaskCalendarAlarmEmail, + server_instance: Arc, + ) -> TaskResult { + match send_email_alarm(self, task, server_instance).await { + Ok(result) => result, + Err(err) => { + let result = TaskResult::temporary(err.to_string()); + trc::error!( + err.account_id(task.account_id.document_id()) + .document_id(task.document_id.document_id()) + .caused_by(trc::location!()) + .details("Failed to process e-mail alarm") + ); + result } } } @@ -93,12 +97,12 @@ impl SendAlarmTask for Server { async fn send_email_alarm( server: &Server, - account_id: u32, - document_id: u32, - alarm: &CalendarAlarm, + task: &TaskCalendarAlarmEmail, server_instance: Arc, -) -> trc::Result { +) -> trc::Result { // Obtain access token + let account_id = task.account_id.document_id(); + let document_id = task.document_id.document_id(); let access_token = server .access_token(account_id) .await @@ -112,7 +116,7 @@ async fn send_email_alarm( AccountId = account_id, DocumentId = document_id, ); - return Ok(true); + return Ok(TaskResult::Success); } let account_info = server .account_info(account_id) @@ -126,7 +130,7 @@ async fn send_email_alarm( AccountId = account_id, DocumentId = document_id, ); - return Ok(true); + return Ok(TaskResult::Success); } // Fetch event @@ -147,7 +151,7 @@ async fn send_email_alarm( DocumentId = document_id, ); - return Ok(true); + return Ok(TaskResult::Success); }; // Unarchive event @@ -159,18 +163,8 @@ async fn send_email_alarm( let account_main_email = account_info.name(); let account_main_domain = account_main_email.rsplit('@').next().unwrap_or("localhost"); let logo_cid = format!("logo.{}@{account_main_domain}", now()); - let Some(tpl) = build_template( - server, - &account_info, - account_id, - document_id, - alarm, - event, - &logo_cid, - ) - .await? - else { - return Ok(true); + let Some(tpl) = build_template(server, &account_info, task, event, &logo_cid).await? else { + return Ok(TaskResult::Success); }; let txt_body = html_to_text(&tpl.body); @@ -310,20 +304,20 @@ async fn send_email_alarm( DocumentId = document_id, CausedBy = trc::location!(), ); - return Ok(false); + return Ok(TaskResult::temporary("Thread join error")); } } - write_next_alarm(server, account_id, document_id, event).await + build_next_alarm(server, account_id, document_id, event) } async fn send_display_alarm( server: &Server, - account_id: u32, - document_id: u32, - alarm: &CalendarAlarm, -) -> trc::Result { + task: &TaskCalendarAlarmNotification, +) -> trc::Result { // Fetch event + let account_id = task.account_id.document_id(); + let document_id = task.document_id.document_id(); let Some(event_) = server .store() .get_value::>(ValueKey::archive( @@ -341,7 +335,7 @@ async fn send_display_alarm( DocumentId = document_id, ); - return Ok(true); + return Ok(TaskResult::Success); }; // Unarchive event @@ -349,10 +343,7 @@ async fn send_display_alarm( .unarchive::() .caused_by(trc::location!())?; - let recurrence_id = match &alarm.typ { - CalendarAlarmType::Display { recurrence_id } => *recurrence_id, - _ => None, - }; + let recurrence_id = task.recurrence_id; let ical = &event.data.event; server @@ -363,7 +354,7 @@ async fn send_display_alarm( uid: ical.uids().next().unwrap_or_default().to_string(), alert_id: ical .components - .get(alarm.alarm_id as usize) + .get(task.alarm_id as usize) .and_then(|c| c.property(&ICalendarProperty::Jsid)) .and_then(|v| v.values.first()) .and_then(|v| v.as_text()) @@ -372,11 +363,11 @@ async fn send_display_alarm( format!( "k{}", ical.components - .get(alarm.event_id as usize) + .get(task.event_id as usize) .and_then(|c| c .component_ids .iter() - .position(|id| id.to_native() == alarm.alarm_id as u32)) + .position(|id| id.to_native() == task.alarm_id as u32)) .unwrap_or_default() + 1 ) @@ -384,15 +375,15 @@ async fn send_display_alarm( })) .await; - write_next_alarm(server, account_id, document_id, event).await + build_next_alarm(server, account_id, document_id, event) } -async fn write_next_alarm( +fn build_next_alarm( server: &Server, account_id: u32, document_id: u32, event: &ArchivedCalendarEvent, -) -> trc::Result { +) -> trc::Result { // Find next alarm time and write to task queue let now = now() as i64; if let Some(next_alarm) = @@ -416,21 +407,12 @@ async fn write_next_alarm( } }) { - let mut batch = BatchBuilder::new(); - batch - .with_account_id(account_id) - .with_collection(Collection::CalendarEvent) - .with_document(document_id); - next_alarm.write_task(&mut batch); - server - .store() - .write(batch.build_all()) - .await - .caused_by(trc::location!())?; - server.notify_task_queue(); + Ok(TaskResult::Update( + next_alarm.build_write_ops(account_id, document_id), + )) + } else { + Ok(TaskResult::Success) } - - Ok(true) } struct Details { @@ -442,12 +424,12 @@ struct Details { async fn build_template( server: &Server, account_info: &AccountInfo, - account_id: u32, - document_id: u32, - alarm: &CalendarAlarm, + alarm: &TaskCalendarAlarmEmail, event: &ArchivedCalendarEvent, logo_cid: &str, ) -> trc::Result> { + let account_id = alarm.account_id.document_id(); + let document_id = alarm.document_id.document_id(); let (Some(event_component), Some(alarm_component)) = ( event.data.event.components.get(alarm.event_id as usize), event.data.event.components.get(alarm.alarm_id as usize), @@ -577,15 +559,10 @@ async fn build_template( let template = &server.core.groupware.alarms_template; let locale = i18n::locale_or_default(account_info.locale().as_str()); let chrono_locale = Locale::from_str(account_info.locale().as_str()).unwrap_or(Locale::en_US); - let (event_start, event_start_tz, event_end, event_end_tz) = match alarm.typ { - CalendarAlarmType::Email { - event_start, - event_start_tz, - event_end, - event_end_tz, - } => (event_start, event_start_tz, event_end, event_end_tz), - CalendarAlarmType::Display { .. } => unreachable!(), - }; + let event_start = alarm.event_start.timestamp(); + let event_end = alarm.event_end.timestamp(); + let event_start_tz = alarm.event_start_tz as u16; + let event_end_tz = alarm.event_end_tz as u16; let start = format!( "{} ({})", diff --git a/crates/services/src/task_manager/imip.rs b/crates/services/src/task_manager/imip.rs index df3542bd..31f19471 100644 --- a/crates/services/src/task_manager/imip.rs +++ b/crates/services/src/task_manager/imip.rs @@ -4,12 +4,12 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::task_manager::TaskResult; use calcard::{ common::timezone::Tz, icalendar::{ - ArchivedICalendarDay, ArchivedICalendarFrequency, ArchivedICalendarMonth, - ArchivedICalendarParticipationStatus, ArchivedICalendarRecurrenceRule, - ArchivedICalendarWeekday, ICalendarParticipationStatus, ICalendarProperty, + ICalendarDay, ICalendarFrequency, ICalendarMonth, ICalendarParticipationStatus, + ICalendarProperty, ICalendarRecurrenceRule, ICalendarWeekday, }, }; use chrono::{DateTime, Locale}; @@ -22,7 +22,7 @@ use common::{ }; use groupware::{ calendar::itip::ItipIngest, - scheduling::{ArchivedItipSummary, ArchivedItipValue, ItipMessages}, + scheduling::{ItipSummary, ItipValue}, }; use mail_builder::{ MessageBuilder, @@ -30,47 +30,39 @@ use mail_builder::{ mime::{BodyPart, MimePart}, }; use mail_parser::decoders::html::html_to_text; -use registry::types::EnumImpl; +use registry::{schema::structs::TaskCalendarItipMessage, types::EnumImpl}; use smtp::core::{Session, SessionData}; use smtp_proto::{MailFrom, RcptTo}; use std::{str::FromStr, sync::Arc, time::Duration}; -use store::{ - ValueKey, - ahash::AHashMap, - rkyv::rend::{i16_le, i32_le}, - write::{AlignedBytes, Archive, TaskEpoch, TaskQueueClass, ValueClass, now}, -}; +use store::{ahash::AHashMap, write::now}; use trc::AddContext; use utils::template::{Variable, Variables}; -pub trait SendImipTask: Sync + Send { +pub(crate) trait SendImipTask: Sync + Send { fn send_imip( &self, - account_id: u32, - document_id: u32, - due: TaskEpoch, + task: &TaskCalendarItipMessage, server_instance: Arc, - ) -> impl Future + Send; + ) -> impl Future + Send; } impl SendImipTask for Server { async fn send_imip( &self, - account_id: u32, - document_id: u32, - due: TaskEpoch, + task: &TaskCalendarItipMessage, server_instance: Arc, - ) -> bool { - match send_imip(self, account_id, document_id, due, server_instance).await { + ) -> TaskResult { + match send_imip(self, task, server_instance).await { Ok(result) => result, Err(err) => { + let result = TaskResult::temporary(err.to_string()); trc::error!( - err.account_id(account_id) - .document_id(document_id) + err.account_id(task.account_id.document_id()) + .document_id(task.document_id.document_id()) .caused_by(trc::location!()) - .details("Failed to process alarm") + .details("Failed to send iMIP message") ); - false + result } } } @@ -78,38 +70,12 @@ impl SendImipTask for Server { async fn send_imip( server: &Server, - account_id: u32, - document_id: u32, - due: TaskEpoch, + imip: &TaskCalendarItipMessage, server_instance: Arc, -) -> trc::Result { +) -> trc::Result { // Obtain iMIP payload - let Some(archive) = server - .store() - .get_value::>(ValueKey { - account_id, - collection: 0, - document_id, - class: ValueClass::TaskQueue(TaskQueueClass::SendImip { - due, - is_payload: true, - }), - }) - .await - .caused_by(trc::location!())? - else { - trc::event!( - Calendar(trc::CalendarEvent::ItipMessageError), - AccountId = account_id, - DocumentId = document_id, - Reason = "Missing iMIP payload", - ); - return Ok(true); - }; - - let imip = archive - .unarchive::() - .caused_by(trc::location!())?; + let account_id = imip.account_id.document_id(); + let document_id = imip.document_id.document_id(); let sender_domain = imip .messages @@ -150,6 +116,12 @@ async fn send_imip( .caused_by(trc::location!())?; for itip_message in imip.messages.iter() { + let Ok(summary) = serde_json::from_str::(&itip_message.summary) else { + return Ok(TaskResult::permanent( + "Failed to parse iMIP message summary.", + )); + }; + for recipient in itip_message.to.iter() { // Build template let tpl = build_itip_template( @@ -159,7 +131,7 @@ async fn send_imip( document_id, itip_message.from.as_str(), recipient.as_str(), - &itip_message.summary, + &summary, &logo_cid, ) .await; @@ -202,9 +174,9 @@ async fn send_imip( ), MimePart::new( ContentType::new("text/calendar") - .attribute("method", itip_message.summary.method()) + .attribute("method", summary.method()) .attribute("charset", "utf-8"), - BodyPart::Text(itip_message.message.as_str().into()), + BodyPart::Text(itip_message.i_calendar_data.as_str().into()), ) .attachment("event.ics"), ]), @@ -298,7 +270,7 @@ async fn send_imip( } } - Ok(true) + Ok(TaskResult::Success) } pub struct Details { @@ -314,7 +286,7 @@ pub async fn build_itip_template( document_id: u32, from: &str, to: &str, - summary: &ArchivedItipSummary, + summary: &ItipSummary, logo_cid: &str, ) -> Details { // SPDX-SnippetBegin @@ -336,12 +308,12 @@ pub async fn build_itip_template( let mut variables = Variables::new(); let mut subject; let (fields, old_fields) = match summary { - ArchivedItipSummary::Invite(fields) => { + ItipSummary::Invite(fields) => { subject = format!("{}: ", locale.calendar_invitation); (fields, None) } - ArchivedItipSummary::Update { + ItipSummary::Update { current, previous, .. } => { subject = format!("{}: ", locale.calendar_updated_invitation); @@ -352,7 +324,7 @@ pub async fn build_itip_template( variables.insert_single(CalendarTemplateVariable::Color, "info".to_string()); (current, Some(previous)) } - ArchivedItipSummary::Cancel(fields) => { + ItipSummary::Cancel(fields) => { subject = format!("{}: ", locale.calendar_cancelled); variables.insert_single( CalendarTemplateVariable::Header, @@ -361,9 +333,9 @@ pub async fn build_itip_template( variables.insert_single(CalendarTemplateVariable::Color, "danger".to_string()); (fields, None) } - ArchivedItipSummary::Rsvp { part_stat, current } => { + ItipSummary::Rsvp { part_stat, current } => { let (color, value) = match part_stat { - ArchivedICalendarParticipationStatus::Accepted => { + ICalendarParticipationStatus::Accepted => { subject = format!("{}: ", locale.calendar_accepted); ( @@ -371,21 +343,21 @@ pub async fn build_itip_template( locale.calendar_participant_accepted.replace("$name", from), ) } - ArchivedICalendarParticipationStatus::Declined => { + ICalendarParticipationStatus::Declined => { subject = format!("{}: ", locale.calendar_declined); ( "danger", locale.calendar_participant_declined.replace("$name", from), ) } - ArchivedICalendarParticipationStatus::Tentative => { + ICalendarParticipationStatus::Tentative => { subject = format!("{}: ", locale.calendar_tentative); ( "warning", locale.calendar_participant_tentative.replace("$name", from), ) } - ArchivedICalendarParticipationStatus::Delegated => { + ICalendarParticipationStatus::Delegated => { subject = format!("{}: ", locale.calendar_delegated); ( "warning", @@ -479,7 +451,7 @@ pub async fn build_itip_template( if let Some(guests) = fields .iter() .find(|e| e.name == ICalendarProperty::Attendee) - && let ArchivedItipValue::Participants(guests) = &guests.value + && let ItipValue::Participants(guests) = &guests.value { variables.insert_single( CalendarTemplateVariable::AttendeesTitle, @@ -498,12 +470,7 @@ pub async fn build_itip_template( locale.calendar_organizer.to_string() } } else { - guest - .name - .as_ref() - .map(|n| n.as_str()) - .unwrap_or_default() - .to_string() + guest.name.as_deref().unwrap_or_default().to_string() }, ), (CalendarTemplateVariable::Value, guest.email.to_string()), @@ -513,10 +480,8 @@ pub async fn build_itip_template( } // Add RSVP buttons - if matches!( - summary, - ArchivedItipSummary::Invite(_) | ArchivedItipSummary::Update { .. } - ) && let Some(rsvp_url) = server.http_rsvp_url(account_id, document_id, to).await + if matches!(summary, ItipSummary::Invite(_) | ItipSummary::Update { .. }) + && let Some(rsvp_url) = server.http_rsvp_url(account_id, document_id, to).await { variables.insert_single( CalendarTemplateVariable::Rsvp, @@ -574,16 +539,16 @@ pub async fn build_itip_template( } } -fn format_field(value: &ArchivedItipValue, template: &str, chrono_locale: Locale) -> String { +fn format_field(value: &ItipValue, template: &str, chrono_locale: Locale) -> String { match value { - ArchivedItipValue::Text(text) => text.to_string(), - ArchivedItipValue::Time(time) => { + ItipValue::Text(text) => text.to_string(), + ItipValue::Time(time) => { use chrono::TimeZone; - let tz = Tz::from_id(time.tz_id.to_native()).unwrap_or(Tz::UTC); + let tz = Tz::from_id(time.tz_id).unwrap_or(Tz::UTC); format!( "{} ({})", tz.from_utc_datetime( - &DateTime::from_timestamp(time.start.to_native(), 0) + &DateTime::from_timestamp(time.start, 0) .unwrap_or_default() .naive_local() ) @@ -591,8 +556,8 @@ fn format_field(value: &ArchivedItipValue, template: &str, chrono_locale: Locale tz.name().unwrap_or_default() ) } - ArchivedItipValue::Rrule(rrule) => RecurrenceFormatter.format(rrule), - ArchivedItipValue::Participants(_) => String::new(), // Handled separately + ItipValue::Rrule(rrule) => RecurrenceFormatter.format(rrule), + ItipValue::Participants(_) => String::new(), // Handled separately } } @@ -600,14 +565,11 @@ fn format_field(value: &ArchivedItipValue, template: &str, chrono_locale: Locale pub struct RecurrenceFormatter; impl RecurrenceFormatter { - pub fn format(&self, rule: &ArchivedICalendarRecurrenceRule) -> String { + pub fn format(&self, rule: &ICalendarRecurrenceRule) -> String { let mut parts = Vec::new(); // Format frequency and interval - let freq_part = self.format_frequency( - &rule.freq, - rule.interval.as_ref().map(|i| i.to_native()).unwrap_or(1), - ); + let freq_part = self.format_frequency(&rule.freq, rule.interval.unwrap_or(1)); parts.push(freq_part); // Format day constraints @@ -657,15 +619,15 @@ impl RecurrenceFormatter { parts.join(" ") } - fn format_frequency(&self, freq: &ArchivedICalendarFrequency, interval: u16) -> String { + fn format_frequency(&self, freq: &ICalendarFrequency, interval: u16) -> String { let (singular, plural) = match freq { - ArchivedICalendarFrequency::Daily => ("day", "days"), - ArchivedICalendarFrequency::Weekly => ("week", "weeks"), - ArchivedICalendarFrequency::Monthly => ("month", "months"), - ArchivedICalendarFrequency::Yearly => ("year", "years"), - ArchivedICalendarFrequency::Hourly => ("hour", "hours"), - ArchivedICalendarFrequency::Minutely => ("minute", "minutes"), - ArchivedICalendarFrequency::Secondly => ("second", "seconds"), + ICalendarFrequency::Daily => ("day", "days"), + ICalendarFrequency::Weekly => ("week", "weeks"), + ICalendarFrequency::Monthly => ("month", "months"), + ICalendarFrequency::Yearly => ("year", "years"), + ICalendarFrequency::Hourly => ("hour", "hours"), + ICalendarFrequency::Minutely => ("minute", "minutes"), + ICalendarFrequency::Secondly => ("second", "seconds"), }; if interval == 1 { @@ -675,24 +637,24 @@ impl RecurrenceFormatter { } } - fn format_by_day(&self, days: &[ArchivedICalendarDay]) -> String { + fn format_by_day(&self, days: &[ICalendarDay]) -> String { let day_names: Vec = days.iter().map(|day| self.format_day(day)).collect(); format!("on {}", self.format_list(&day_names)) } - fn format_day(&self, day: &ArchivedICalendarDay) -> String { + fn format_day(&self, day: &ICalendarDay) -> String { let day_name = match day.weekday { - ArchivedICalendarWeekday::Monday => "Monday", - ArchivedICalendarWeekday::Tuesday => "Tuesday", - ArchivedICalendarWeekday::Wednesday => "Wednesday", - ArchivedICalendarWeekday::Thursday => "Thursday", - ArchivedICalendarWeekday::Friday => "Friday", - ArchivedICalendarWeekday::Saturday => "Saturday", - ArchivedICalendarWeekday::Sunday => "Sunday", + ICalendarWeekday::Monday => "Monday", + ICalendarWeekday::Tuesday => "Tuesday", + ICalendarWeekday::Wednesday => "Wednesday", + ICalendarWeekday::Thursday => "Thursday", + ICalendarWeekday::Friday => "Friday", + ICalendarWeekday::Saturday => "Saturday", + ICalendarWeekday::Sunday => "Sunday", }; - if let Some(occurrence) = day.ordwk.as_ref().map(|o| o.to_native()) { + if let Some(occurrence) = day.ordwk { if occurrence > 0 { format!("the {} {}", self.ordinal(occurrence as u32), day_name) } else { @@ -759,7 +721,7 @@ impl RecurrenceFormatter { format!("on the {}", self.format_list(&day_strings)) } - fn format_months(&self, months: &[ArchivedICalendarMonth]) -> String { + fn format_months(&self, months: &[ICalendarMonth]) -> String { let month_names: Vec = months .iter() .map(|month| self.month_name(month.month())) @@ -768,7 +730,7 @@ impl RecurrenceFormatter { format!("in {}", self.format_list(&month_names)) } - fn format_year_days(&self, days: &[i16_le]) -> String { + fn format_year_days(&self, days: &[i16]) -> String { let day_strings: Vec = days .iter() .map(|&day| { @@ -798,12 +760,12 @@ impl RecurrenceFormatter { format!("in {}", self.format_list(&week_strings)) } - fn format_set_positions(&self, positions: &[i32_le]) -> String { + fn format_set_positions(&self, positions: &[i32]) -> String { let pos_strings: Vec = positions .iter() .map(|&pos| { if pos > 0 { - self.ordinal(pos.to_native() as u32) + self.ordinal(pos as u32) } else { format!("{} from the end", self.ordinal((-pos) as u32)) } diff --git a/crates/services/src/task_manager/index.rs b/crates/services/src/task_manager/index.rs index f3bc6988..46d74444 100644 --- a/crates/services/src/task_manager/index.rs +++ b/crates/services/src/task_manager/index.rs @@ -4,21 +4,29 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::task_manager::{IndexAction, Task}; +use crate::task_manager::{Task, TaskDetails, TaskResult}; use common::Server; use email::{cache::MessageCacheFetch, message::metadata::MessageMetadata}; use groupware::{cache::GroupwareCache, calendar::CalendarEvent, contact::ContactCard}; -use registry::schema::prelude::{ObjectType, Property}; +use registry::{ + schema::{ + enums::IndexDocumentType, + prelude::{ObjectType, Property}, + structs::{TaskIndexDocument, TaskIndexTrace, TaskStatus}, + }, + types::EnumImpl, +}; use std::cmp::Ordering; use store::{ - IterateParams, SerializeInfallible, ValueKey, + IterateParams, ValueKey, ahash::AHashMap, + rand::{self, Rng}, registry::RegistryQuery, roaring::RoaringBitmap, search::{IndexDocument, SearchField, SearchFilter, SearchQuery}, write::{ - AlignedBytes, Archive, BatchBuilder, SearchIndex, TaskEpoch, TaskQueueClass, - TelemetryClass, ValueClass, key::DeserializeBigEndian, + AlignedBytes, Archive, BatchBuilder, SearchIndex, TelemetryClass, ValueClass, + key::DeserializeBigEndian, now, }, }; use trc::{AddContext, TaskQueueEvent}; @@ -29,10 +37,7 @@ use types::{ }; pub(crate) trait SearchIndexTask: Sync + Send { - fn index( - &self, - tasks: &[Task], - ) -> impl Future> + Send; + fn index(&self, tasks: &[TaskDetails]) -> impl Future> + Send; } pub trait ReindexIndexTask: Sync + Send { @@ -52,22 +57,15 @@ enum TaskType { Delete, } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum TaskStatus { - Success, - Failed, - Ignored, -} - #[derive(Debug)] pub(crate) struct IndexTaskResult { - index: SearchIndex, + index: IndexDocumentType, task_type: TaskType, - status: TaskStatus, + pub result: TaskResult, } impl SearchIndexTask for Server { - async fn index(&self, tasks: &[Task]) -> Vec { + async fn index(&self, tasks: &[TaskDetails]) -> Vec { let mut results: Vec = Vec::with_capacity(tasks.len()); let mut batch = BatchBuilder::new(); let mut document_insertions = Vec::new(); @@ -75,101 +73,135 @@ impl SearchIndexTask for Server { std::array::from_fn(|_| AHashMap::new()); for task in tasks { - if task.action.is_insert { - let document = match task.action.index { - SearchIndex::Email => { - build_email_document(self, task.account_id, task.document_id).await - } - SearchIndex::Calendar => { - build_calendar_document(self, task.account_id, task.document_id).await - } - SearchIndex::Contacts => { - build_contact_document(self, task.account_id, task.document_id).await - } - SearchIndex::File => { - // File indexing not implemented yet - continue; - } - SearchIndex::Tracing => { - build_tracing_span_document(self, task.account_id, task.document_id).await - } - SearchIndex::InMemory => unreachable!(), - }; + match &task.task { + Task::IndexDocument(task) => { + let account_id = task.account_id.document_id(); + let document_id = task.document_id.document_id(); - let result = match document { - Ok(Some(doc)) if !doc.is_empty() => { - document_insertions.push(doc); - TaskStatus::Success - } - Err(err) => { - trc::error!( - err.account_id(task.account_id) - .document_id(task.document_id) - .caused_by(trc::location!()) - .ctx(trc::Key::Collection, task.action.index.name()) - .details("Failed to build document for indexing") - ); - TaskStatus::Failed - } - _ => { - trc::event!( - TaskQueue(TaskQueueEvent::TaskIgnored), - Collection = task.action.index.name(), - Reason = "Nothing to index", - AccountId = task.account_id, - DocumentId = task.document_id, - ); - TaskStatus::Ignored - } - }; - - results.push(IndexTaskResult { - task_type: TaskType::Insert, - index: task.action.index, - status: result, - }); - } else { - let idx = match task.action.index { - SearchIndex::Email => { - if let Err(err) = delete_email_metadata( - self, - &mut batch, - task.account_id, - task.document_id, - ) - .await - { - trc::error!( - err.account_id(task.account_id) - .document_id(task.document_id) - .caused_by(trc::location!()) - .details("Failed to delete email metadata from index") - ); - results.push(IndexTaskResult { - task_type: TaskType::Delete, - index: task.action.index, - status: TaskStatus::Failed, - }); + let document = match task.document_type { + IndexDocumentType::Email => { + build_email_document(self, account_id, document_id).await + } + IndexDocumentType::Calendar => { + build_calendar_document(self, account_id, document_id).await + } + IndexDocumentType::Contacts => { + build_contact_document(self, account_id, document_id).await + } + IndexDocumentType::File => { + // File indexing not implemented yet continue; } - 0 - } - SearchIndex::Calendar => 1, - SearchIndex::Contacts => 2, - SearchIndex::File => 3, - SearchIndex::Tracing | SearchIndex::InMemory => unreachable!(), - }; + }; - document_deletions[idx] - .entry(task.account_id) - .or_default() - .push(task.document_id); + let result = match document { + Ok(Some(doc)) if !doc.is_empty() => { + document_insertions.push(doc); + TaskResult::Success + } + Err(err) => { + let result = TaskResult::temporary(err.to_string()); + trc::error!( + err.account_id(account_id) + .document_id(document_id) + .caused_by(trc::location!()) + .ctx(trc::Key::Collection, task.document_type.as_str()) + .details("Failed to build document for indexing") + ); + result + } + _ => { + trc::event!( + TaskQueue(TaskQueueEvent::TaskIgnored), + Collection = task.document_type.as_str(), + Reason = "Nothing to index", + AccountId = account_id, + DocumentId = document_id, + ); + TaskResult::Ignored + } + }; - results.push(IndexTaskResult { - task_type: TaskType::Delete, - index: task.action.index, - status: TaskStatus::Success, - }); + results.push(IndexTaskResult { + task_type: TaskType::Insert, + index: task.document_type, + result, + }); + } + Task::IndexTrace(task) => { + let result = match build_tracing_span_document(self, task.trace_id.id()).await { + Ok(Some(doc)) if !doc.is_empty() => { + document_insertions.push(doc); + TaskResult::Success + } + Err(err) => { + let result = TaskResult::temporary(err.to_string()); + trc::error!( + err.id(task.trace_id.id()) + .caused_by(trc::location!()) + .details("Failed to build document for indexing") + ); + result + } + _ => { + trc::event!( + TaskQueue(TaskQueueEvent::TaskIgnored), + Reason = "Nothing to index", + Id = task.trace_id.id(), + ); + TaskResult::Ignored + } + }; + + results.push(IndexTaskResult { + task_type: TaskType::Insert, + index: IndexDocumentType::File, // use File index for tracing spans to avoid creating a new index type + result, + }); + } + Task::UnindexDocument(task) => { + let account_id = task.account_id.document_id(); + let document_id = task.document_id.document_id(); + let idx = match task.document_type { + IndexDocumentType::Email => { + if let Err(err) = + delete_email_metadata(self, &mut batch, account_id, document_id) + .await + { + trc::error!( + err.account_id(account_id) + .document_id(document_id) + .caused_by(trc::location!()) + .details("Failed to delete email metadata from index") + ); + results.push(IndexTaskResult { + task_type: TaskType::Delete, + index: task.document_type, + result: TaskResult::temporary( + "Failed to delete email metadata from index", + ), + }); + continue; + } + 0 + } + IndexDocumentType::Calendar => 1, + IndexDocumentType::Contacts => 2, + IndexDocumentType::File => 3, + }; + + document_deletions[idx] + .entry(account_id) + .or_default() + .push(document_id); + + results.push(IndexTaskResult { + task_type: TaskType::Delete, + index: task.document_type, + result: TaskResult::Success, + }); + } + _ => unreachable!(), } } @@ -183,10 +215,11 @@ impl SearchIndexTask for Server { ); for r in results.iter_mut() { if r.task_type == TaskType::Delete - && r.status == TaskStatus::Success - && r.index == SearchIndex::Email + && r.result == TaskResult::Success + && r.index == IndexDocumentType::Email { - r.status = TaskStatus::Failed; + r.result = + TaskResult::temporary("Failed to commit index deletions to data store"); } } return results; @@ -201,8 +234,8 @@ impl SearchIndexTask for Server { .details("Failed to index documents") ); for r in results.iter_mut() { - if r.task_type == TaskType::Insert && r.status == TaskStatus::Success { - r.status = TaskStatus::Failed; + if r.task_type == TaskType::Insert && r.result == TaskResult::Success { + r.result = TaskResult::temporary("Failed to index documents"); } } return results; @@ -256,8 +289,8 @@ impl SearchIndexTask for Server { .ctx(trc::Key::Collection, index.name()) ); for r in results.iter_mut() { - if r.task_type == TaskType::Delete && r.status == TaskStatus::Success { - r.status = TaskStatus::Failed; + if r.task_type == TaskType::Delete && r.result == TaskResult::Success { + r.result = TaskResult::temporary("Failed to delete documents from index"); } } return results; @@ -286,15 +319,12 @@ impl ReindexIndexTask for Server { .await .caused_by(trc::location!())? }; - let due = TaskEpoch::now(); + let now = now() as i64; match index { SearchIndex::Email => { for account_id in accounts { let mut batch = BatchBuilder::new(); - batch - .with_account_id(account_id) - .with_collection(Collection::Email); for document_id in self .get_cached_messages(account_id) @@ -305,21 +335,16 @@ impl ReindexIndexTask for Server { .iter() .map(|v| v.document_id) { - batch.with_document(document_id).set( - ValueClass::TaskQueue(TaskQueueClass::UpdateIndex { - due, - index: SearchIndex::Email, - is_insert: true, - }), - 0u64.serialize(), - ); + batch.schedule_task(Task::IndexDocument(TaskIndexDocument { + account_id: account_id.into(), + document_id: document_id.into(), + document_type: IndexDocumentType::Email, + status: TaskStatus::at(now + rand::rng().random_range(0..=300)), + })); if batch.len() >= 2000 { self.core.storage.data.write(batch.build_all()).await?; batch = BatchBuilder::new(); - batch - .with_account_id(account_id) - .with_collection(Collection::Email); } } @@ -343,22 +368,22 @@ impl ReindexIndexTask for Server { .await .caused_by(trc::location!())?; let mut batch = BatchBuilder::new(); - batch.with_account_id(account_id); for document_id in cache.document_ids(false) { - batch.with_document(document_id).set( - ValueClass::TaskQueue(TaskQueueClass::UpdateIndex { - due, - index, - is_insert: true, - }), - 0u64.serialize(), - ); + batch.schedule_task(Task::IndexDocument(TaskIndexDocument { + account_id: account_id.into(), + document_id: document_id.into(), + document_type: if index == SearchIndex::Calendar { + IndexDocumentType::Calendar + } else { + IndexDocumentType::Contacts + }, + status: TaskStatus::at(now + rand::rng().random_range(0..=300)), + })); if batch.len() >= 2000 { self.core.storage.data.write(batch.build_all()).await?; batch = BatchBuilder::new(); - batch.with_account_id(account_id); } } @@ -372,54 +397,13 @@ impl ReindexIndexTask for Server { // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC // SPDX-License-Identifier: LicenseRef-SEL - #[cfg(feature = "enterprise")] - if let Some(store) = self - .core - .enterprise - .as_ref() - .and_then(|e| e.trace_store.as_ref()) - { - let mut spans = Vec::new(); - store - .store - .iterate( - IterateParams::new( - ValueKey::from(ValueClass::Telemetry(TelemetryClass::Span { - span_id: 0, - })), - ValueKey::from(ValueClass::Telemetry(TelemetryClass::Span { - span_id: u64::MAX, - })), - ) - .no_values(), - |key, _| { - spans.push(key.deserialize_be_u64(0)?); - Ok(true) - }, - ) - .await - .caused_by(trc::location!())?; - - let mut batch = BatchBuilder::new(); - for span_id in spans { - batch - .with_account_id((span_id >> 32) as u32) // TODO: This is hacky, improve - .with_document(span_id as u32) - .set( - ValueClass::TaskQueue(TaskQueueClass::UpdateIndex { - due: TaskEpoch::now(), - index: SearchIndex::Tracing, - is_insert: true, - }), - vec![], - ); - if batch.len() >= 2000 { - self.core.storage.data.write(batch.build_all()).await?; - batch = BatchBuilder::new(); - } - } - - if !batch.is_empty() { + let mut batch = BatchBuilder::new(); + for span_id in spans { + batch.schedule_task(Task::IndexTrace(TaskIndexTrace { + trace_id: span_id.into(), + status: TaskStatus::at(now + rand::rng().random_range(0..=300)), + })); + if batch.len() >= 2000 { self.core.storage.data.write(batch.build_all()).await?; } } @@ -556,8 +540,7 @@ async fn build_contact_document( #[cfg(feature = "enterprise")] async fn build_tracing_span_document( server: &Server, - account_id: u32, - document_id: u32, + span_id: u64, ) -> trc::Result> { use common::telemetry::tracers::store::{TracingStore, build_span_document}; @@ -573,7 +556,6 @@ async fn build_tracing_span_document( return Ok(None); }; - let span_id = ((account_id as u64) << 32) | document_id as u64; let span = server.tracing_store().get_span(span_id).await?; if !span.is_empty() { @@ -586,11 +568,7 @@ async fn build_tracing_span_document( // SPDX-SnippetEnd #[cfg(not(feature = "enterprise"))] -async fn build_tracing_span_document( - _: &Server, - _: u32, - _: u32, -) -> trc::Result> { +async fn build_tracing_span_document(_: &Server, _: u64) -> trc::Result> { Ok(None) } @@ -627,7 +605,6 @@ async fn delete_email_metadata( // Hold blob for undeletion #[cfg(feature = "enterprise")] { - use common::enterprise::undelete::DeletedItemType; use email::message::metadata::ArchivedMetadataHeaderName; if let Some(undelete_retention) = server @@ -636,20 +613,27 @@ async fn delete_email_metadata( .as_ref() .and_then(|e| e.undelete_retention.as_ref()) { - use common::enterprise::undelete::DeletedItem; use email::message::metadata::MESSAGE_RECEIVED_MASK; - use store::{ - Serialize, - write::{Archiver, BlobLink, BlobOp, now}, + use registry::{ + pickle::Pickle, + schema::structs::{DeletedEmail, DeletedItem}, + types::{datetime::UTCDateTime, id::ObjectId}, }; + use store::{ + SerializeInfallible, + write::{BlobLink, BlobOp, RegistryClass, now}, + xxhash_rust, + }; + use types::blob::BlobId; + use utils::snowflake::SnowflakeIdGenerator; let root_part = metadata.root_part(); - let from: Option> = root_part.headers.iter().find_map(|h| { + let from: Option = root_part.headers.iter().find_map(|h| { if let ArchivedMetadataHeaderName::From = &h.name { h.value.as_single_address().and_then(|addr| { match (addr.address.as_ref(), addr.name.as_ref()) { (Some(address), Some(name)) => { - Some(format!("{} <{}>", name, address).into_boxed_str()) + Some(format!("{} <{}>", name, address)) } (Some(address), None) => Some(address.as_ref().into()), (None, Some(name)) => Some(name.as_ref().into()), @@ -660,7 +644,7 @@ async fn delete_email_metadata( None } }); - let subject: Option> = root_part.headers.iter().rev().find_map(|h| { + let subject: Option = root_part.headers.iter().rev().find_map(|h| { if let ArchivedMetadataHeaderName::Subject = &h.name { h.value.as_text().map(Into::into) } else { @@ -670,31 +654,46 @@ async fn delete_email_metadata( let now = now(); let until = now + undelete_retention.as_secs(); let blob_hash = BlobHash::from(&metadata.blob_hash); + + let item = DeletedItem::Email(DeletedEmail { + account_id: account_id.into(), + blob_id: BlobId::new(blob_hash.clone(), Default::default()), + cleanup_at: UTCDateTime::from_timestamp(until as i64), + deleted_at: UTCDateTime::now(), + from: from.unwrap_or_default(), + received_at: UTCDateTime::from_timestamp( + (metadata.rcvd_attach.to_native() & MESSAGE_RECEIVED_MASK) as i64, + ), + subject: subject.unwrap_or_default(), + size: root_part.offset_end.to_native() as u64, + }) + .to_pickled_vec(); + let object_id = ObjectType::DeletedItem.to_id(); + let item_id = SnowflakeIdGenerator::from_sequence_id( + xxhash_rust::xxh3::xxh3_64(item.as_slice()), + ) + .unwrap_or_default(); + batch .set( BlobOp::Link { - hash: blob_hash.clone(), + hash: blob_hash, to: BlobLink::Temporary { until }, }, - vec![BlobLink::UNDELETE_LINK], + ObjectId::new(ObjectType::DeletedItem, item_id.into()).serialize(), ) .set( - BlobOp::Undelete { - hash: blob_hash, - until, - }, - Archiver::new(DeletedItem { - typ: DeletedItemType::Email { - from: from.unwrap_or_default(), - subject: subject.unwrap_or_default(), - received_at: metadata.rcvd_attach.to_native() - & MESSAGE_RECEIVED_MASK, - }, - size: root_part.offset_end.to_native(), - deleted_at: now, - }) - .serialize() - .caused_by(trc::location!())?, + ValueClass::Registry(RegistryClass::Index { + index_id: Property::AccountId.to_id(), + object_id, + item_id, + key: account_id.serialize(), + }), + vec![], + ) + .set( + ValueClass::Registry(RegistryClass::Id { object_id, item_id }), + item, ); } } @@ -713,9 +712,3 @@ async fn delete_email_metadata( Ok(()) } - -impl IndexTaskResult { - pub fn is_done(&self) -> bool { - self.status != TaskStatus::Failed - } -} diff --git a/crates/services/src/task_manager/lock.rs b/crates/services/src/task_manager/lock.rs index d1210388..78b75468 100644 --- a/crates/services/src/task_manager/lock.rs +++ b/crates/services/src/task_manager/lock.rs @@ -7,266 +7,47 @@ use crate::task_manager::*; pub(crate) trait TaskLockManager: Sync + Send { - fn try_lock_task( - &self, - account_id: u32, - document_id: u32, - lock_key: Vec, - lock_expiry: u64, - ) -> impl Future + Send; - fn remove_index_lock(&self, lock_key: Vec) -> impl Future + Send; + fn try_lock_task(&self, task: u64) -> impl Future + Send; + fn remove_index_lock(&self, id: u64) -> impl Future + Send; } impl TaskLockManager for Server { - async fn try_lock_task( - &self, - account_id: u32, - document_id: u32, - lock_key: Vec, - lock_expiry: u64, - ) -> bool { + async fn try_lock_task(&self, id: u64) -> bool { match self .in_memory_store() - .try_lock(KV_LOCK_TASK, &lock_key, lock_expiry) + .try_lock(KV_LOCK_TASK, &id.to_be_bytes(), DEFAULT_LOCK_EXPIRY) .await { Ok(result) => { if !result { trc::event!( TaskQueue(TaskQueueEvent::TaskLocked), - AccountId = account_id, - DocumentId = document_id, - Expires = trc::Value::Timestamp(now() + lock_expiry), + Id = id, + Details = "Task details not available", + Expires = trc::Value::Timestamp(now() + DEFAULT_LOCK_EXPIRY), ); } result } Err(err) => { - trc::error!( - err.account_id(account_id) - .document_id(document_id) - .details("Failed to lock task") - ); + trc::error!(err.id(id).details("Failed to lock task")); false } } } - async fn remove_index_lock(&self, lock_key: Vec) { + async fn remove_index_lock(&self, id: u64) { if let Err(err) = self .in_memory_store() - .remove_lock(KV_LOCK_TASK, &lock_key) + .remove_lock(KV_LOCK_TASK, &id.to_be_bytes()) .await { trc::error!( err.details("Failed to unlock task") - .ctx(trc::Key::Key, lock_key) + .ctx(trc::Key::Id, id) .caused_by(trc::location!()) ); } } } - -pub(crate) trait TaskLock { - fn account_id(&self) -> u32; - fn document_id(&self) -> u32; - fn lock_key(&self) -> Vec; - fn lock_expiry(&self) -> u64; - fn value_classes(&self) -> impl Iterator; -} - -impl TaskLock for Task { - fn account_id(&self) -> u32 { - self.account_id - } - - fn document_id(&self) -> u32 { - self.document_id - } - - fn lock_key(&self) -> Vec { - KeySerializer::new((U32_LEN * 2) + U64_LEN + 2) - .write(0u8) - .write(self.due.inner()) - .write_leb128(self.account_id) - .write_leb128(self.document_id) - .write(self.action.index.to_u8()) - .finalize() - } - - fn lock_expiry(&self) -> u64 { - INDEX_EXPIRY - } - - fn value_classes(&self) -> impl Iterator { - std::iter::once(ValueClass::TaskQueue(TaskQueueClass::UpdateIndex { - due: self.due, - index: self.action.index, - is_insert: self.action.is_insert, - })) - } -} - -impl TaskLock for Task { - fn account_id(&self) -> u32 { - self.account_id - } - - fn document_id(&self) -> u32 { - self.document_id - } - - fn lock_key(&self) -> Vec { - KeySerializer::new((U32_LEN * 2) + U64_LEN + 1) - .write(2u8) - .write(self.due.inner()) - .write_leb128(self.account_id) - .write_leb128(self.document_id) - .finalize() - } - - fn lock_expiry(&self) -> u64 { - ALARM_EXPIRY - } - - fn value_classes(&self) -> impl Iterator { - std::iter::once(ValueClass::TaskQueue(TaskQueueClass::SendAlarm { - event_id: self.action.event_id, - alarm_id: self.action.alarm_id, - due: self.due, - is_email_alert: matches!(self.action.typ, CalendarAlarmType::Email { .. }), - })) - } -} - -impl TaskLock for Task { - fn account_id(&self) -> u32 { - self.account_id - } - - fn document_id(&self) -> u32 { - self.document_id - } - - fn lock_key(&self) -> Vec { - KeySerializer::new((U32_LEN * 2) + U64_LEN + 1) - .write(3u8) - .write(self.due.inner()) - .write_leb128(self.account_id) - .write_leb128(self.document_id) - .finalize() - } - - fn lock_expiry(&self) -> u64 { - ALARM_EXPIRY - } - - fn value_classes(&self) -> impl Iterator { - [ - ValueClass::TaskQueue(TaskQueueClass::SendImip { - due: self.due, - is_payload: false, - }), - ValueClass::TaskQueue(TaskQueueClass::SendImip { - due: self.due, - is_payload: true, - }), - ] - .into_iter() - } -} - -impl TaskLock for Task>> { - fn account_id(&self) -> u32 { - self.account_id - } - - fn document_id(&self) -> u32 { - self.document_id - } - - fn lock_key(&self) -> Vec { - KeySerializer::new((U32_LEN * 2) + U64_LEN + 1) - .write(4u8) - .write(self.due.inner()) - .write_leb128(self.account_id) - .write_leb128(self.document_id) - .finalize() - } - - fn lock_expiry(&self) -> u64 { - ALARM_EXPIRY - } - - fn value_classes(&self) -> impl Iterator { - std::iter::once(ValueClass::TaskQueue(TaskQueueClass::MergeThreads { - due: self.due, - })) - } -} - -impl Task { - pub(crate) fn lock_expiry(&self) -> u64 { - match &self.action { - TaskAction::UpdateIndex(_) => INDEX_EXPIRY, - TaskAction::SendAlarm(_) => ALARM_EXPIRY, - _ => ALARM_EXPIRY, - } - } - - pub fn deserialize(key: &[u8], value: &[u8]) -> trc::Result { - let document_id = key.deserialize_be_u32(U64_LEN + U32_LEN + 1)?; - - Ok(Task { - due: TaskEpoch::from_inner(key.deserialize_be_u64(0)?), - account_id: key.deserialize_be_u32(U64_LEN)?, - document_id, - action: match key.get(U64_LEN + U32_LEN) { - Some(v @ (7 | 8)) => TaskAction::UpdateIndex(IndexAction { - index: key - .last() - .copied() - .and_then(SearchIndex::try_from_u8) - .ok_or_else(|| trc::Error::corrupted_key(key, None, trc::location!()))?, - is_insert: *v == 7, - }), - Some(3) => TaskAction::SendAlarm(CalendarAlarm { - event_id: key.deserialize_be_u16(U64_LEN + U32_LEN + U32_LEN + 1)?, - alarm_id: key.deserialize_be_u16(U64_LEN + U32_LEN + U32_LEN + U16_LEN + 1)?, - alarm_time: 0, - typ: CalendarAlarmType::Email { - event_start: value.deserialize_be_u64(0)? as i64, - event_end: value.deserialize_be_u64(U64_LEN)? as i64, - event_start_tz: value.deserialize_be_u16(U64_LEN * 2)?, - event_end_tz: value.deserialize_be_u16((U64_LEN * 2) + U16_LEN)?, - }, - }), - Some(6) => { - let recurrence_id = value.deserialize_be_u64(0)? as i64; - - TaskAction::SendAlarm(CalendarAlarm { - event_id: key.deserialize_be_u16(U64_LEN + U32_LEN + U32_LEN + 1)?, - alarm_id: key - .deserialize_be_u16(U64_LEN + U32_LEN + U32_LEN + U16_LEN + 1)?, - alarm_time: 0, - typ: CalendarAlarmType::Display { - recurrence_id: if recurrence_id != 0 { - Some(recurrence_id) - } else { - None - }, - }, - }) - } - Some(4) => TaskAction::SendImip, - Some(9) => { - TaskAction::MergeThreads(MergeThreadIds::deserialize(value).ok_or_else( - || trc::Error::corrupted_key(key, value.into(), trc::location!()), - )?) - } - _ => return Err(trc::Error::corrupted_key(key, None, trc::location!())), - }, - }) - } -} diff --git a/crates/services/src/task_manager/merge_threads.rs b/crates/services/src/task_manager/merge_threads.rs index 4ff1a9eb..af8251ed 100644 --- a/crates/services/src/task_manager/merge_threads.rs +++ b/crates/services/src/task_manager/merge_threads.rs @@ -4,12 +4,11 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::task_manager::TaskResult; use common::{Server, storage::index::ObjectIndexBuilder}; -use email::message::{ - ingest::{MergeThreadIds, ThreadMerge}, - metadata::MessageData, -}; -use std::time::Duration; +use email::message::{ingest::ThreadMerge, metadata::MessageData}; +use registry::schema::structs::TaskMergeThreads; +use std::{str::FromStr, time::Duration}; use store::{ IndexKeyPrefix, IterateParams, U32_LEN, ValueKey, ahash::{AHashMap, AHashSet}, @@ -24,31 +23,25 @@ use types::{ collection::{Collection, SyncCollection}, field::EmailField, }; +use utils::cheeky_hash::CheekyHash; const MAX_RETRIES: usize = 5; -pub trait MergeThreadsTask: Sync + Send { - fn merge_threads( - &self, - account_id: u32, - threads: &MergeThreadIds>, - ) -> impl Future + Send; +pub(crate) trait MergeThreadsTask: Sync + Send { + fn merge_threads(&self, threads: &TaskMergeThreads) -> impl Future + Send; } impl MergeThreadsTask for Server { - async fn merge_threads( - &self, - account_id: u32, - threads: &MergeThreadIds>, - ) -> bool { - match merge_threads(self, account_id, threads).await { - Ok(_) => true, + async fn merge_threads(&self, threads: &TaskMergeThreads) -> TaskResult { + match merge_threads(self, threads).await { + Ok(result) => result, Err(err) => { + let result = TaskResult::temporary(err.to_string()); trc::error!( - err.account_id(account_id) + err.account_id(threads.account_id.document_id()) .details("Failed to merge threads") ); - false + result } } } @@ -56,11 +49,19 @@ impl MergeThreadsTask for Server { async fn merge_threads( server: &Server, - account_id: u32, - merge_threads: &MergeThreadIds>, -) -> trc::Result<()> { - let key_len = IndexKeyPrefix::len() + merge_threads.thread_hash.len() + U32_LEN; + task_merge_threads: &TaskMergeThreads, +) -> trc::Result { + let Ok(thread_hash) = CheekyHash::from_str(&task_merge_threads.thread_hash) else { + return Ok(TaskResult::permanent("Invalid thread hash")); + }; + let account_id = task_merge_threads.account_id.document_id(); + let key_len = IndexKeyPrefix::len() + thread_hash.len() + U32_LEN; let document_id_pos = key_len - U32_LEN; + let merge_thread_ids = task_merge_threads + .thread_ids + .iter() + .map(|id| id.document_id()) + .collect::>(); let mut thread_merge = ThreadMerge::new(); let mut thread_index = AHashMap::new(); let mut try_count = 0; @@ -77,7 +78,7 @@ async fn merge_threads( document_id: 0, class: ValueClass::IndexProperty(IndexPropertyClass::Hash { property: EmailField::Threading.into(), - hash: merge_threads.thread_hash, + hash: thread_hash, }), }, ValueKey { @@ -86,7 +87,7 @@ async fn merge_threads( document_id: u32::MAX, class: ValueClass::IndexProperty(IndexPropertyClass::Hash { property: EmailField::Threading.into(), - hash: merge_threads.thread_hash, + hash: thread_hash, }), }, ) @@ -94,7 +95,7 @@ async fn merge_threads( |key, value| { if key.len() == key_len { let thread_id = value.deserialize_be_u32(0)?; - if merge_threads.merge_ids.contains(&thread_id) { + if merge_thread_ids.contains(&thread_id) { let document_id = key.deserialize_be_u32(document_id_pos)?; thread_merge.add(thread_id, document_id); @@ -110,7 +111,7 @@ async fn merge_threads( if thread_merge.num_thread_ids() < 2 { // Another process merged the threads already? - return Ok(()); + return Ok(TaskResult::Success); } let thread_id = thread_merge.merge_thread_id(); @@ -172,7 +173,7 @@ async fn merge_threads( batch.set( ValueClass::IndexProperty(IndexPropertyClass::Hash { property: EmailField::Threading.into(), - hash: merge_threads.thread_hash, + hash: thread_hash, }), thread_index, ); @@ -182,7 +183,7 @@ async fn merge_threads( } match server.commit_batch(batch).await { - Ok(_) => return Ok(()), + Ok(_) => return Ok(TaskResult::Success), Err(err) if err.is_assertion_failure() && try_count < MAX_RETRIES => { let backoff = store::rand::rng().random_range(50..=300); tokio::time::sleep(Duration::from_millis(backoff)).await; diff --git a/crates/services/src/task_manager/mod.rs b/crates/services/src/task_manager/mod.rs index c4ac779f..98fad65b 100644 --- a/crates/services/src/task_manager/mod.rs +++ b/crates/services/src/task_manager/mod.rs @@ -6,7 +6,7 @@ use crate::task_manager::imip::SendImipTask; use crate::task_manager::index::SearchIndexTask; -use crate::task_manager::lock::{TaskLock, TaskLockManager}; +use crate::task_manager::lock::TaskLockManager; use crate::task_manager::merge_threads::MergeThreadsTask; use alarm::SendAlarmTask; use common::config::server::ServerProtocol; @@ -14,25 +14,26 @@ use common::network::limiter::ConcurrencyLimiter; use common::network::{ServerInstance, TcpAcceptor}; use common::{BuildServer, IPC_CHANNEL_BUFFER}; use common::{Inner, KV_LOCK_TASK, Server}; -use email::message::ingest::MergeThreadIds; -use groupware::calendar::alarm::{CalendarAlarm, CalendarAlarmType}; +use registry::pickle::Pickle; +use registry::schema::enums::TaskType; +use registry::schema::structs::{ + Task, TaskManager, TaskRetryStrategy, TaskStatus, TaskStatusFailed, TaskStatusRetry, +}; +use registry::types::EnumImpl; +use registry::types::datetime::UTCDateTime; use std::collections::hash_map::Entry; use std::future::Future; use std::time::Duration; use std::{sync::Arc, time::Instant}; -use store::ahash::AHashSet; -use store::rand; use store::rand::seq::SliceRandom; -use store::write::{SearchIndex, TaskEpoch}; +use store::write::Operation; +use store::write::key::DeserializeBigEndian; use store::{ - IterateParams, U16_LEN, U32_LEN, U64_LEN, ValueKey, + IterateParams, ValueKey, ahash::AHashMap, - write::{ - BatchBuilder, TaskQueueClass, ValueClass, - key::{DeserializeBigEndian, KeySerializer}, - now, - }, + write::{BatchBuilder, TaskQueueClass, ValueClass, now}, }; +use store::{SerializeInfallible, U64_LEN, rand}; use tokio::sync::{mpsc, watch}; use trc::TaskQueueEvent; use utils::snowflake::SnowflakeIdGenerator; @@ -43,57 +44,51 @@ pub mod index; pub mod lock; pub mod merge_threads; -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub struct Task { - pub account_id: u32, - pub document_id: u32, - pub due: TaskEpoch, - pub action: T, -} - -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub enum TaskAction { - UpdateIndex(IndexAction), - SendAlarm(CalendarAlarm), - SendImip, - MergeThreads(MergeThreadIds>), -} - -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub struct IndexAction { - pub index: SearchIndex, - pub is_insert: bool, -} - -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub(crate) struct ImipAction; - -const INDEX_EXPIRY: u64 = 60 * 5; // 5 minutes -const ALARM_EXPIRY: u64 = 60 * 2; // 2 minutes const QUEUE_REFRESH_INTERVAL: u64 = 60 * 5; // 5 minutes +const DEFAULT_LOCK_EXPIRY: u64 = 60 * 5; // 5 minutes pub(crate) struct TaskManagerIpc { - tx_fts: mpsc::Sender>, - tx_alarm: mpsc::Sender>, - tx_imip: mpsc::Sender>, - tx_threads: mpsc::Sender>>>, - locked: AHashMap, Locked>, + txs: [mpsc::Sender; TaskType::COUNT], + locked: AHashMap, revision: u64, } +pub(crate) struct TaskDetails { + task: Task, + info: TaskJob, +} + +pub(crate) struct TaskJob { + id: u64, + due: u64, + typ: TaskType, +} + +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum TaskResult { + Success, + Update([Operation; 2]), + Failure { + typ: TaskFailureType, + message: String, + }, + Ignored, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +#[allow(dead_code)] +pub(crate) enum TaskFailureType { + Retry(u64), + Temporary, + Permanent, +} + struct Locked { expires: Instant, revision: u64, } pub fn spawn_task_manager(inner: Arc) { - // Create mpsc channels for the different task types - let (tx_index_1, mut rx_index_1) = mpsc::channel::>(IPC_CHANNEL_BUFFER); - let (tx_index_2, mut rx_index_2) = mpsc::channel::>(IPC_CHANNEL_BUFFER); - let (tx_index_3, mut rx_index_3) = mpsc::channel::>(IPC_CHANNEL_BUFFER); - let (tx_index_4, mut rx_index_4) = - mpsc::channel::>>>(IPC_CHANNEL_BUFFER); - // Create dummy server instance for alarms let server_instance = Arc::new(ServerInstance { id: "_local".to_string(), @@ -105,201 +100,148 @@ pub fn spawn_task_manager(inner: Arc) { span_id_gen: Arc::new(SnowflakeIdGenerator::new()), }); - // Indexing worker - { + // Spawn workers for each task type + let mut txs = Vec::with_capacity(TaskType::COUNT); + for idx in 0..TaskType::COUNT { + let (tx, mut rx) = mpsc::channel::(IPC_CHANNEL_BUFFER); + txs.push(tx); let inner = inner.clone(); - tokio::spawn(async move { - while let Some(task) = rx_index_1.recv().await { - let server = inner.build_server(); - let batch_size = server.core.email.index_batch_size; - let mut batch = Vec::with_capacity(batch_size); - batch.push(task); + let server_instance = server_instance.clone(); - while batch.len() < batch_size { - match rx_index_1.try_recv() { - Ok(task) => batch.push(task), - Err(_) => break, - } - } + if matches!( + TaskType::from_id(idx as u16).unwrap(), + TaskType::IndexDocument | TaskType::UnindexDocument | TaskType::IndexTrace + ) { + tokio::spawn(async move { + while let Some(job) = rx.recv().await { + let server = inner.build_server(); - if batch.len() > 1 { - batch.shuffle(&mut rand::rng()); - } - - // Lock tasks - let mut locked_batch = Vec::with_capacity(batch.len()); - for task in batch { - if server - .try_lock_task( - task.account_id, - task.document_id, - task.lock_key(), - task.lock_expiry(), - ) + let batch_size = server.core.email.index_batch_size; + let mut batch = Vec::with_capacity(batch_size); + match server + .store() + .get_value::(ValueKey::from(ValueClass::TaskQueue( + TaskQueueClass::Task { id: job.id }, + ))) .await { - locked_batch.push(task); + Ok(Some(task)) => { + batch.push(TaskDetails { task, info: job }); + } + Ok(None) => { + trc::event!( + TaskQueue(TaskQueueEvent::TaskIgnored), + Id = job.id, + Reason = "Task not found in store, likely already processed.", + ); + } + Err(err) => { + trc::error!( + err.id(job.id) + .details("Failed to retrieve task details.") + .caused_by(trc::location!()) + ); + } } - } - // Dispatch - if !locked_batch.is_empty() { - let success = server.index(&locked_batch).await; - - if success.iter().all(|t| t.is_done()) { - delete_tasks(&server, &locked_batch).await; - } else { - trc::event!( - TaskQueue(TaskQueueEvent::TaskFailed), - Total = locked_batch.len(), - Details = "Indexing task failed", - ); - - // Remove successful entries from queue - let mut to_delete = Vec::with_capacity(locked_batch.len()); - for (task, result) in locked_batch.into_iter().zip(success.into_iter()) { - if result.is_done() { - to_delete.push(task); + while batch.len() < batch_size { + match rx.try_recv() { + Ok(job) => { + match server + .store() + .get_value::(ValueKey::from(ValueClass::TaskQueue( + TaskQueueClass::Task { id: job.id }, + ))) + .await + { + Ok(Some(task)) => { + batch.push(TaskDetails { task, info: job }); + } + Ok(None) => { + trc::event!( + TaskQueue(TaskQueueEvent::TaskIgnored), + Id = job.id, + Reason = "Task not found in store, likely already processed.", + ); + } + Err(err) => { + trc::error!( + err.id(job.id) + .details("Failed to retrieve task details.") + .caused_by(trc::location!()) + ); + } + } } + Err(_) => break, } - if !to_delete.is_empty() { - delete_tasks(&server, &to_delete).await; + } + + // Dispatch + let results = server.index(&batch).await.into_iter().map(|r| r.result); + update_tasks(&server, &mut batch, results).await; + } + }); + } else { + let server_instance = server_instance.clone(); + tokio::spawn(async move { + while let Some(job) = rx.recv().await { + let server = inner.build_server(); + + match server + .store() + .get_value::(ValueKey::from(ValueClass::TaskQueue( + TaskQueueClass::Task { id: job.id }, + ))) + .await + { + Ok(Some(task)) => { + let result = match &task { + Task::CalendarAlarmEmail(task) => { + server.send_email_alarm(task, server_instance.clone()).await + } + Task::CalendarAlarmNotification(task) => { + server.send_display_alarm(task).await + } + Task::CalendarItipMessage(task) => { + server.send_imip(task, server_instance.clone()).await + } + Task::MergeThreads(task) => server.merge_threads(task).await, + Task::IndexDocument(_) + | Task::UnindexDocument(_) + | Task::IndexTrace(_) => unreachable!(), + }; + + update_tasks( + &server, + &mut [TaskDetails { task, info: job }], + vec![result], + ) + .await; + } + Ok(None) => { + trc::event!( + TaskQueue(TaskQueueEvent::TaskIgnored), + Id = job.id, + Reason = "Task not found in store, likely already processed.", + ); + } + Err(err) => { + trc::error!( + err.id(job.id) + .details("Failed to retrieve task details.") + .caused_by(trc::location!()) + ); } } } - } - }); - } - - // Send alarm worker - { - let inner = inner.clone(); - let server_instance = server_instance.clone(); - tokio::spawn(async move { - while let Some(task) = rx_index_2.recv().await { - let server = inner.build_server(); - - // Lock task - if server.core.groupware.alarms_enabled - && server - .try_lock_task( - task.account_id, - task.document_id, - task.lock_key(), - task.lock_expiry(), - ) - .await - { - let success = server - .send_alarm( - task.account_id, - task.document_id, - &task.action, - server_instance.clone(), - ) - .await; - - // Remove entry from queue - if success { - delete_tasks(&server, &[task]).await; - } else { - trc::event!( - TaskQueue(TaskQueueEvent::TaskFailed), - AccountId = task.account_id, - DocumentId = task.document_id, - Details = "Sending alarm task failed", - ); - } - } - } - }); - } - - // Send iMIP worker - { - let inner = inner.clone(); - let server_instance = server_instance.clone(); - tokio::spawn(async move { - while let Some(task) = rx_index_3.recv().await { - let server = inner.build_server(); - - // Lock task - if server.core.groupware.itip_enabled - && server - .try_lock_task( - task.account_id, - task.document_id, - task.lock_key(), - task.lock_expiry(), - ) - .await - { - let success = server - .send_imip( - task.account_id, - task.document_id, - task.due, - server_instance.clone(), - ) - .await; - - // Remove entry from queue - if success { - delete_tasks(&server, &[task]).await; - } else { - trc::event!( - TaskQueue(TaskQueueEvent::TaskFailed), - AccountId = task.account_id, - DocumentId = task.document_id, - Details = "Sending iMIP task failed", - ); - } - } - } - }); - } - - // Merge threads worker - { - let inner = inner.clone(); - tokio::spawn(async move { - while let Some(task) = rx_index_4.recv().await { - let server = inner.build_server(); - - // Lock task - if server - .try_lock_task( - task.account_id, - task.document_id, - task.lock_key(), - task.lock_expiry(), - ) - .await - { - let success = server.merge_threads(task.account_id, &task.action).await; - - // Remove entry from queue - if success { - delete_tasks(&server, &[task]).await; - } else { - trc::event!( - TaskQueue(TaskQueueEvent::TaskFailed), - AccountId = task.account_id, - DocumentId = task.document_id, - Details = "Merging threads task failed", - ); - } - } - } - }); + }); + } } tokio::spawn(async move { let mut ipc = TaskManagerIpc { - tx_fts: tx_index_1, - tx_alarm: tx_index_2, - tx_imip: tx_index_3, - tx_threads: tx_index_4, + txs: txs.try_into().expect("Incorrect number of task channels"), locked: Default::default(), revision: 0, }; @@ -325,22 +267,15 @@ impl TaskQueueManager for Server { account_id: 0, collection: 0, document_id: 0, - class: ValueClass::TaskQueue(TaskQueueClass::UpdateIndex { - due: TaskEpoch::from_inner(0), - index: SearchIndex::Email, - is_insert: true, - }), + class: ValueClass::TaskQueue(TaskQueueClass::Due { id: 0, due: 0 }), }; let to_key = ValueKey:: { account_id: u32::MAX, collection: u8::MAX, document_id: u32::MAX, - class: ValueClass::TaskQueue(TaskQueueClass::UpdateIndex { - due: TaskEpoch::new(now_timestamp + QUEUE_REFRESH_INTERVAL) - .with_attempt(u16::MAX) - .with_sequence_id(u16::MAX), - index: SearchIndex::Email, - is_insert: true, + class: ValueClass::TaskQueue(TaskQueueClass::Due { + id: u64::MAX, + due: now_timestamp + QUEUE_REFRESH_INTERVAL, }), }; @@ -354,34 +289,62 @@ impl TaskQueueManager for Server { .iterate( IterateParams::new(from_key, to_key).ascending(), |key, value| { - let task = Task::deserialize(key, value)?; + if key.len() == U64_LEN * 2 { + let task_due = key.deserialize_be_u64(0)?; + let task_id = key.deserialize_be_u64(U64_LEN)?; - let task_due = task.due.due(); - if task_due <= now_timestamp { - match ipc.locked.entry(key.to_vec()) { - Entry::Occupied(mut entry) => { - let locked = entry.get_mut(); - if locked.expires <= now { - locked.expires = Instant::now() - + std::time::Duration::from_secs(task.lock_expiry() + 1); - tasks.push(task); + if task_due <= now_timestamp { + let task_type_idx = value.deserialize_be_u16(0)?; + let task_type = TaskType::from_id(task_type_idx).ok_or_else(|| { + trc::StoreEvent::DataCorruption + .caused_by(trc::location!()) + .ctx(trc::Key::Value, value) + })?; + match ipc.locked.entry(task_id) { + Entry::Occupied(mut entry) => { + let locked = entry.get_mut(); + if locked.expires <= now { + locked.expires = Instant::now() + + std::time::Duration::from_secs( + DEFAULT_LOCK_EXPIRY + 1, + ); + tasks.push(( + TaskJob { + id: task_id, + due: task_due, + typ: task_type, + }, + task_type_idx, + )); + } + locked.revision = ipc.revision; + } + Entry::Vacant(entry) => { + entry.insert(Locked { + expires: Instant::now() + + std::time::Duration::from_secs( + DEFAULT_LOCK_EXPIRY + 1, + ), + revision: ipc.revision, + }); + tasks.push(( + TaskJob { + id: task_id, + due: task_due, + typ: task_type, + }, + task_type_idx, + )); } - locked.revision = ipc.revision; } - Entry::Vacant(entry) => { - entry.insert(Locked { - expires: Instant::now() - + std::time::Duration::from_secs(task.lock_expiry() + 1), - revision: ipc.revision, - }); - tasks.push(task); - } - } - Ok(true) + Ok(true) + } else { + next_event = Some(task_due); + Ok(false) + } } else { - next_event = Some(task_due); - Ok(false) + Ok(true) } }, ) @@ -408,100 +371,42 @@ impl TaskQueueManager for Server { // Dispatch tasks let roles = &self.core.network.roles; - for event in tasks { - match event.action { - TaskAction::UpdateIndex(index) - if roles.fts_indexing.is_enabled_for_hash(&event) => - { - if ipc - .tx_fts - .send(Task { - account_id: event.account_id, - document_id: event.document_id, - due: event.due, - action: index, - }) - .await - .is_err() - { - trc::event!( - Server(trc::ServerEvent::ThreadError), - Details = "Error sending task.", - CausedBy = trc::location!() - ); - } - } - TaskAction::SendAlarm(alarm) - if roles.calendar_alerts.is_enabled_for_hash(&event) => - { - if ipc - .tx_alarm - .send(Task { - account_id: event.account_id, - document_id: event.document_id, - due: event.due, - action: alarm, - }) - .await - .is_err() - { - trc::event!( - Server(trc::ServerEvent::ThreadError), - Details = "Error sending task.", - CausedBy = trc::location!() - ); - } - } - TaskAction::SendImip if roles.imip_processing.is_enabled_for_hash(&event) => { - if ipc - .tx_imip - .send(Task { - account_id: event.account_id, - document_id: event.document_id, - due: event.due, - action: ImipAction, - }) - .await - .is_err() - { - trc::event!( - Server(trc::ServerEvent::ThreadError), - Details = "Error sending task.", - CausedBy = trc::location!() - ); - } - } - TaskAction::MergeThreads(info) - if roles.merge_threads.is_enabled_for_hash(&event) => - { - if ipc - .tx_threads - .send(Task { - account_id: event.account_id, - document_id: event.document_id, - due: event.due, - action: info, - }) - .await - .is_err() - { - trc::event!( - Server(trc::ServerEvent::ThreadError), - Details = "Error sending task.", - CausedBy = trc::location!() - ); - } - } - _ => { - trc::event!( - TaskQueue(TaskQueueEvent::TaskIgnored), - Details = event.action.name(), - AccountId = event.account_id, - DocumentId = event.document_id, - ); + for (task_job, task_type_idx) in tasks { + let enabled = match task_job.typ { + TaskType::IndexDocument | TaskType::UnindexDocument | TaskType::IndexTrace => roles + .fts_indexing + .is_enabled_for_integer(task_job.id as u32), + TaskType::CalendarAlarmEmail | TaskType::CalendarAlarmNotification => roles + .calendar_alerts + .is_enabled_for_integer(task_job.id as u32), + TaskType::CalendarItipMessage => roles + .imip_processing + .is_enabled_for_integer(task_job.id as u32), + TaskType::MergeThreads => roles + .merge_threads + .is_enabled_for_integer(task_job.id as u32), + }; - continue; + if enabled { + if self.try_lock_task(task_job.id).await + && ipc.txs[task_type_idx as usize] + .send(task_job) + .await + .is_err() + { + trc::event!( + Server(trc::ServerEvent::ThreadError), + Details = "Error sending task.", + CausedBy = trc::location!() + ); } + } else { + trc::event!( + TaskQueue(TaskQueueEvent::TaskIgnored), + Id = task_job.id, + Details = task_job.typ.as_str(), + Reason = "Task type is disabled by cluster roles.", + ); } } @@ -515,16 +420,94 @@ impl TaskQueueManager for Server { } } -async fn delete_tasks(server: &Server, tasks: &[T]) { +async fn update_tasks( + server: &Server, + tasks: &mut [TaskDetails], + results: impl IntoIterator, +) { let mut batch = BatchBuilder::new(); - for task in tasks { - batch - .with_account_id(task.account_id()) - .with_document(task.document_id()); + for (task, result) in tasks.iter_mut().zip(results.into_iter()) { + let id = task.info.id; + batch.clear(ValueClass::TaskQueue(TaskQueueClass::Due { + id, + due: task.info.due, + })); + match result { + TaskResult::Success | TaskResult::Ignored => { + batch.clear(ValueClass::TaskQueue(TaskQueueClass::Task { id })); + } + TaskResult::Update(ops) => { + for op in ops { + batch.any_op(op); + } + } + TaskResult::Failure { typ, message } => { + let (attempt_number, created_at) = match task.task.status() { + TaskStatus::Pending(status) => (0, status.created_at), + TaskStatus::Retry(status) => (status.attempt_number, status.created_at), + TaskStatus::Failed(status) => (status.failed_attempt_number, status.failed_at), + }; + let retry_at = match typ { + TaskFailureType::Retry(retry_at) => (attempt_number + < server.core.network.task_manager.max_attempts + && retry_at + < retry_at.saturating_add( + server.core.network.task_manager.total_deadline.as_secs(), + )) + .then_some(retry_at), + TaskFailureType::Temporary => next_retry_time( + &server.core.network.task_manager, + created_at.timestamp() as u64, + attempt_number, + now(), + ), + TaskFailureType::Permanent => None, + }; - for value in task.value_classes() { - batch.clear(value); + let due = if let Some(retry_at) = retry_at { + trc::event!( + TaskQueue(TaskQueueEvent::TaskRetry), + Id = id, + Details = task.task.name(), + Reason = message.to_string(), + NextRetry = trc::Value::Timestamp(retry_at), + ); + + task.task.set_status(TaskStatus::Retry(TaskStatusRetry { + due: UTCDateTime::from_timestamp(retry_at as i64), + attempt_number: attempt_number + 1, + failure_reason: message, + created_at, + })); + + retry_at + } else { + trc::event!( + TaskQueue(TaskQueueEvent::TaskFailed), + Id = id, + Details = task.task.name(), + Reason = message.to_string(), + ); + + task.task.set_status(TaskStatus::Failed(TaskStatusFailed { + failed_at: UTCDateTime::now(), + failed_attempt_number: attempt_number, + failure_reason: message, + created_at, + })); + u64::MAX + }; + batch + .set( + ValueClass::TaskQueue(TaskQueueClass::Due { id, due }), + task.info.typ.to_id().serialize(), + ) + .set( + ValueClass::TaskQueue(TaskQueueClass::Task { id }), + task.task.to_pickled_vec(), + ); + } } } @@ -533,17 +516,75 @@ async fn delete_tasks(server: &Server, tasks: &[T]) { } for task in tasks { - server.remove_index_lock(task.lock_key()).await; + server.remove_index_lock(task.info.id).await; } } -impl TaskAction { - pub fn name(&self) -> &'static str { +pub fn next_retry_time( + manager: &TaskManager, + create_time: u64, + attempt: u64, + now: u64, +) -> Option { + if attempt >= manager.max_attempts { + return None; + } + + let delay_secs: u64 = match &manager.strategy { + TaskRetryStrategy::FixedDelay(fixed) => fixed.delay.as_secs(), + TaskRetryStrategy::ExponentialBackoff(backoff) => { + let delay = (backoff.initial_delay.as_secs() as f64 + * backoff.factor.powi(attempt as i32)) + .min(backoff.max_delay.as_secs() as f64) as u64; + + if backoff.jitter { + let jitter_factor = rand::random::() + 0.5; + ((delay as f64 * jitter_factor) as u64).min(backoff.max_delay.as_secs()) + } else { + delay + } + } + }; + + let next_time = now.saturating_add(delay_secs); + let deadline = create_time.saturating_add(manager.total_deadline.as_secs()); + if next_time > deadline { + return None; + } + + Some(next_time) +} + +pub(crate) trait TaskInfo { + fn name(&self) -> &'static str; +} + +impl TaskInfo for Task { + fn name(&self) -> &'static str { match self { - TaskAction::UpdateIndex(_) => "UpdateIndex", - TaskAction::SendAlarm(_) => "SendAlarm", - TaskAction::SendImip => "SendImip", - TaskAction::MergeThreads(_) => "MergeThreads", + Task::IndexDocument(_) => "IndexDocument", + Task::UnindexDocument(_) => "UnindexDocument", + Task::IndexTrace(_) => "IndexTrace", + Task::CalendarAlarmEmail(_) => "CalendarAlarmEmail", + Task::CalendarAlarmNotification(_) => "CalendarAlarmNotification", + Task::CalendarItipMessage(_) => "CalendarItipMessage", + Task::MergeThreads(_) => "MergeThreads", + } + } +} + +impl TaskResult { + pub fn permanent(message: impl Into) -> Self { + TaskResult::Failure { + typ: TaskFailureType::Permanent, + message: message.into(), + } + } + + pub fn temporary(message: impl Into) -> Self { + TaskResult::Failure { + typ: TaskFailureType::Temporary, + message: message.into(), } } } diff --git a/crates/smtp/src/inbound/data.rs b/crates/smtp/src/inbound/data.rs index ba49e2ee..134f1e4d 100644 --- a/crates/smtp/src/inbound/data.rs +++ b/crates/smtp/src/inbound/data.rs @@ -34,7 +34,7 @@ use mail_auth::{ dmarc::{self, verify::DmarcParameters}, }; use mail_builder::headers::{date::Date, message_id::generate_message_id_header}; -use mail_parser::MessageParser; +use mail_parser::{MessageParser, parsers::fields::thread::thread_name}; use registry::schema::structs::Rate; use sieve::runtime::Variable; use smtp_proto::{ @@ -423,7 +423,12 @@ impl Session { SpamFilterAction::Allow(score) => { // Add headers headers.extend_from_slice(score.headers.as_bytes()); - train_spam = score.train_spam; + train_spam = score.train_spam.map(|is_spam| { + ( + is_spam, + thread_name(parsed_message.subject().unwrap_or_default()).to_string(), + ) + }); // Add scores for local recipients for (is_spam, recipient) in diff --git a/crates/smtp/src/queue/mod.rs b/crates/smtp/src/queue/mod.rs index c444c0b0..2d0eaae3 100644 --- a/crates/smtp/src/queue/mod.rs +++ b/crates/smtp/src/queue/mod.rs @@ -41,12 +41,12 @@ pub struct QueuedMessage { pub queue_name: QueueName, } -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone)] pub enum MessageSource { Authenticated, Unauthenticated { dmarc_pass: bool, - train_spam: Option, + train_spam: Option<(bool, String)>, }, Dsn, Report, diff --git a/crates/smtp/src/queue/spool.rs b/crates/smtp/src/queue/spool.rs index fa42a96e..63698f59 100644 --- a/crates/smtp/src/queue/spool.rs +++ b/crates/smtp/src/queue/spool.rs @@ -16,6 +16,12 @@ use crate::queue::{ use common::config::smtp::queue::QueueName; use common::ipc::QueueEvent; use common::{KV_LOCK_QUEUE_MESSAGE, Server}; +use registry::pickle::Pickle; +use registry::schema::prelude::ObjectType; +use registry::schema::structs::SpamTrainingSample; +use registry::types::EnumImpl; +use registry::types::datetime::UTCDateTime; +use registry::types::id::ObjectId; use std::borrow::Cow; use std::collections::hash_map::Entry; use std::future::Future; @@ -25,12 +31,16 @@ use store::write::key::DeserializeBigEndian; use store::write::serialize::rkyv_deserialize; use store::write::{ AlignedBytes, Archive, Archiver, BatchBuilder, BlobLink, BlobOp, MergeResult, Params, - QueueClass, ValueClass, now, + QueueClass, RegistryClass, ValueClass, now, +}; +use store::{ + Deserialize, IterateParams, Serialize, SerializeInfallible, U64_LEN, ValueKey, xxhash_rust, }; -use store::{Deserialize, IterateParams, Serialize, U64_LEN, ValueKey}; use trc::{AddContext, ServerEvent, SpamEvent}; +use types::blob::BlobId; use types::blob_hash::BlobHash; use utils::DomainPart; +use utils::snowflake::SnowflakeIdGenerator; pub const LOCK_EXPIRY: u64 = 10 * 60; // 10 minutes pub const QUEUE_REFRESH: u64 = 5 * 60; // 5 minutes @@ -443,25 +453,37 @@ impl MessageWrapper { ); } - if let Some(is_spam) = train_spam + if let Some((is_spam, subject)) = train_spam && let Some(config) = &server.core.spam.classifier { let hold_period = now + config.hold_samples_for; + let sample = SpamTrainingSample { + account_id: None, + blob_id: BlobId::new(self.message.blob_hash.clone(), Default::default()), + delete_after_use: false, + expires_at: UTCDateTime::from_timestamp(hold_period as i64), + from: self.message.return_path.to_string(), + is_spam, + subject, + } + .to_pickled_vec(); + let object_id = ObjectType::SpamTrainingSample.to_id(); + let item_id = SnowflakeIdGenerator::from_sequence_id(xxhash_rust::xxh3::xxh3_64( + sample.as_slice(), + )) + .unwrap_or_default(); batch .set( BlobOp::Link { hash: self.message.blob_hash.clone(), to: BlobLink::Temporary { until: hold_period }, }, - vec![BlobLink::SPAM_SAMPLE_LINK], + ObjectId::new(ObjectType::SpamTrainingSample, item_id.into()).serialize(), ) .set( - BlobOp::SpamSample { - hash: self.message.blob_hash.clone(), - until: hold_period, - }, - vec![u8::from(is_spam), 1], + ValueClass::Registry(RegistryClass::Id { object_id, item_id }), + sample, ); trc::event!( diff --git a/crates/spam-filter/src/modules/classifier.rs b/crates/spam-filter/src/modules/classifier.rs index db6ca849..b52d65ed 100644 --- a/crates/spam-filter/src/modules/classifier.rs +++ b/crates/spam-filter/src/modules/classifier.rs @@ -25,7 +25,9 @@ use nlp::classifier::reservoir::SampleReservoir; use nlp::classifier::train::{CcfhTrainer, FhTrainer}; use nlp::tokenizers::types::TypesTokenizer; use nlp::tokenizers::{stream::WordStemTokenizer, types::TokenType}; -use registry::schema::prelude::ObjectType; +use registry::schema::prelude::{ObjectType, Property}; +use registry::schema::structs::SpamTrainingSample; +use registry::types::EnumImpl; use std::time::Instant; use std::{ borrow::Cow, @@ -33,15 +35,17 @@ use std::{ hash::{Hash, RandomState}, sync::Arc, }; +use store::ahash::AHashSet; use store::rand::seq::SliceRandom; -use store::write::{BlobLink, now}; +use store::write::{BlobLink, RegistryClass, now}; use store::{ - Deserialize, IterateParams, Serialize, U32_LEN, U64_LEN, ValueKey, + Deserialize, IterateParams, Serialize, ValueKey, write::{ AlignedBytes, Archive, Archiver, BatchBuilder, BlobOp, ValueClass, key::DeserializeBigEndian, }, }; +use store::{SerializeInfallible, U16_LEN}; use tokio::sync::{mpsc, oneshot}; use trc::{AddContext, SpamEvent}; use types::blob_hash::BlobHash; @@ -63,13 +67,25 @@ pub trait SpamClassifier { ) -> impl Future> + Send; } -#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Clone, PartialEq, Eq, Debug)] +#[derive( + rkyv::Archive, + rkyv::Deserialize, + rkyv::Serialize, + Clone, + PartialEq, + Eq, + Debug, + PartialOrd, + Ord, + Hash, +)] pub struct TrainingSample { hash: BlobHash, account_id: u32, } struct TrainingTask { + id: u64, sample: TrainingSample, is_spam: bool, is_replay: bool, @@ -80,7 +96,7 @@ struct TrainingTask { pub struct SpamTrainer { pub trainer: SpamTrainerClass, pub reservoir: SampleReservoir, - pub last_sample_expiry: u64, + pub last_id: u64, } #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug)] @@ -136,7 +152,7 @@ impl SpamClassifier for Server { )))), }, reservoir: SampleReservoir::default(), - last_sample_expiry: 0, + last_id: 0, } }; @@ -169,77 +185,79 @@ impl SpamClassifier for Server { // Fetch blob hashes for samples let mut samples = Vec::new(); + let mut duplicate_samples = Vec::new(); let mut remove_entries = false; - let from_key = ValueKey { - account_id: 0, - collection: 0, - document_id: 0, - class: ValueClass::Blob(BlobOp::SpamSample { - hash: BlobHash::default(), - until: trainer.last_sample_expiry + 1, - }), - }; - let to_key = ValueKey { - account_id: u32::MAX, - collection: u8::MAX, - document_id: u32::MAX, - class: ValueClass::Blob(BlobOp::SpamSample { - hash: BlobHash::new_max(), - until: u64::MAX, - }), - }; + let object_id = ObjectType::SpamTrainingSample.to_id(); + let from_key = ValueKey::from(ValueClass::Registry(RegistryClass::Id { + object_id, + item_id: trainer.last_id + 1, + })); + let to_key = ValueKey::from(ValueClass::Registry(RegistryClass::Id { + object_id, + item_id: u64::MAX, + })); + let mut seen_samples = AHashSet::new(); let mut spam_count = 0; let mut ham_count = 0; self.store() .iterate( - IterateParams::new(from_key, to_key).ascending(), + IterateParams::new(from_key, to_key).descending(), |key, value| { - let until = key.deserialize_be_u64(1)?; - let account_id = key.deserialize_be_u32(U64_LEN + 1)?; - let hash = BlobHash::try_from_hash_slice( - key.get(U64_LEN + U32_LEN + 1..).ok_or_else(|| { - trc::Error::corrupted_key(key, value.into(), trc::location!()) - })?, - ) - .unwrap(); - let (Some(is_spam), Some(hold)) = (value.first(), value.get(1)) else { - return Err(trc::Error::corrupted_key( - key, - value.into(), - trc::location!(), - )); + let id = key.deserialize_be_u64(U16_LEN + 1)?; + let sample = SpamTrainingSample::deserialize(value)?; + + let until = sample.expires_at.timestamp() as u64; + let do_remove = sample.delete_after_use; + let is_spam = sample.is_spam; + let sample = TrainingSample { + hash: sample.blob_id.hash, + account_id: sample + .account_id + .map(|a| a.document_id()) + .unwrap_or(u32::MAX), }; - let do_remove = *hold == 0; - let is_spam = *is_spam == 1; - let sample = TrainingSample { hash, account_id }; + if seen_samples.insert(sample.clone()) { + // Add to reservoir + if !do_remove { + trainer.reservoir.update_reservoir( + &sample, + is_spam, + config.reservoir_capacity, + ); + } else { + trainer.reservoir.update_counts(is_spam); + } - // Add to reservoir - if !do_remove { - trainer.reservoir.update_reservoir( - &sample, + samples.push(TrainingTask { + id, + sample, is_spam, - config.reservoir_capacity, - ); + is_replay: false, + remove: do_remove.then_some(until), + }); + + remove_entries |= do_remove; + + // Update trainer stats + if is_spam { + spam_count += 1; + } else { + ham_count += 1; + } } else { - trainer.reservoir.update_counts(is_spam); + duplicate_samples.push(TrainingTask { + id, + sample, + is_spam, + is_replay: false, + remove: Some(until), + }); + remove_entries = true; } - samples.push(TrainingTask { - sample, - is_spam, - is_replay: false, - remove: do_remove.then_some(until), - }); - - remove_entries |= do_remove; - - // Update trainer stats - trainer.last_sample_expiry = until; - if is_spam { - spam_count += 1; - } else { - ham_count += 1; + if trainer.last_id == 0 { + trainer.last_id = id; } Ok(true) @@ -284,6 +302,7 @@ impl SpamClassifier for Server { .reservoir .replay_samples((spam_count - ham_count) as usize, false) .map(|sample| TrainingTask { + id: 0, sample: sample.clone(), is_spam: false, is_replay: true, @@ -297,6 +316,7 @@ impl SpamClassifier for Server { .reservoir .replay_samples((ham_count - spam_count) as usize, true) .map(|sample| TrainingTask { + id: 0, sample: sample.clone(), is_spam: true, is_replay: true, @@ -536,18 +556,26 @@ impl SpamClassifier for Server { // Remove samples marked for deletion if remove_entries { let mut batch = BatchBuilder::new(); - for sample in samples { + for sample in samples.into_iter().chain(duplicate_samples.into_iter()) { if let Some(until) = sample.remove { batch .with_account_id(sample.sample.account_id) .clear(BlobOp::Link { - hash: sample.sample.hash.clone(), + hash: sample.sample.hash, to: BlobLink::Temporary { until }, }) - .clear(BlobOp::SpamSample { - hash: sample.sample.hash, - until, - }); + .clear(ValueClass::Registry(RegistryClass::Id { + object_id, + item_id: sample.id, + })); + if sample.sample.account_id != u32::MAX { + batch.clear(ValueClass::Registry(RegistryClass::Index { + index_id: Property::AccountId.to_id(), + object_id, + item_id: sample.id, + key: sample.sample.account_id.serialize(), + })); + } if batch.is_large_batch() { self.store() .write(batch.build_all()) diff --git a/crates/store/src/backend/mysql/main.rs b/crates/store/src/backend/mysql/main.rs index 91afa848..fbb91420 100644 --- a/crates/store/src/backend/mysql/main.rs +++ b/crates/store/src/backend/mysql/main.rs @@ -87,7 +87,8 @@ impl MysqlStore { for table in [ SUBSPACE_ACL, SUBSPACE_TASK_QUEUE, - SUBSPACE_BLOB_EXTRA, + SUBSPACE_DELETED_ITEMS, + SUBSPACE_SPAM_SAMPLES, SUBSPACE_BLOB_LINK, SUBSPACE_IN_MEMORY_VALUE, SUBSPACE_PROPERTY, diff --git a/crates/store/src/backend/postgres/main.rs b/crates/store/src/backend/postgres/main.rs index 5f8292a6..e4033e3d 100644 --- a/crates/store/src/backend/postgres/main.rs +++ b/crates/store/src/backend/postgres/main.rs @@ -89,7 +89,8 @@ impl PostgresStore { for table in [ SUBSPACE_ACL, SUBSPACE_TASK_QUEUE, - SUBSPACE_BLOB_EXTRA, + SUBSPACE_DELETED_ITEMS, + SUBSPACE_SPAM_SAMPLES, SUBSPACE_BLOB_LINK, SUBSPACE_IN_MEMORY_VALUE, SUBSPACE_PROPERTY, diff --git a/crates/store/src/backend/rocksdb/main.rs b/crates/store/src/backend/rocksdb/main.rs index 78d7feae..10a9e0a5 100644 --- a/crates/store/src/backend/rocksdb/main.rs +++ b/crates/store/src/backend/rocksdb/main.rs @@ -46,7 +46,7 @@ impl RocksDbStore { SUBSPACE_INDEXES, SUBSPACE_ACL, SUBSPACE_TASK_QUEUE, - SUBSPACE_BLOB_EXTRA, + SUBSPACE_DELETED_ITEMS, SUBSPACE_BLOB_LINK, SUBSPACE_IN_MEMORY_VALUE, SUBSPACE_PROPERTY, @@ -60,11 +60,11 @@ impl RocksDbStore { SUBSPACE_TELEMETRY_SPAN, SUBSPACE_TELEMETRY_METRIC, SUBSPACE_SEARCH_INDEX, + SUBSPACE_SPAM_SAMPLES, LEGACY_SUBSPACE_BITMAP_ID, LEGACY_SUBSPACE_BITMAP_TAG, LEGACY_SUBSPACE_BITMAP_TEXT, LEGACY_SUBSPACE_FTS_INDEX, - LEGACY_SUBSPACE_TELEMETRY_INDEX, ] { let cf_opts = Options::default(); cfs.push(ColumnFamilyDescriptor::new( diff --git a/crates/store/src/backend/sqlite/main.rs b/crates/store/src/backend/sqlite/main.rs index 5f6fe4de..c40dad0e 100644 --- a/crates/store/src/backend/sqlite/main.rs +++ b/crates/store/src/backend/sqlite/main.rs @@ -64,7 +64,8 @@ impl SqliteStore { for table in [ SUBSPACE_ACL, SUBSPACE_TASK_QUEUE, - SUBSPACE_BLOB_EXTRA, + SUBSPACE_DELETED_ITEMS, + SUBSPACE_SPAM_SAMPLES, SUBSPACE_BLOB_LINK, SUBSPACE_IN_MEMORY_VALUE, SUBSPACE_PROPERTY, diff --git a/crates/store/src/build/registry.rs b/crates/store/src/build/registry.rs index 72f05800..8ef14706 100644 --- a/crates/store/src/build/registry.rs +++ b/crates/store/src/build/registry.rs @@ -76,7 +76,7 @@ impl RegistryStore { "{ERROR_MSG}: \"LocalSettings\" object has invalid nodeId of 0." )); } - inner.id_generator = SnowflakeIdGenerator::with_node_id(inner.node_id); + inner.id_generator = SnowflakeIdGenerator::new(); Ok(Self(inner.into())) } } diff --git a/crates/store/src/dispatch/store.rs b/crates/store/src/dispatch/store.rs index 0310219d..d17b4be8 100644 --- a/crates/store/src/dispatch/store.rs +++ b/crates/store/src/dispatch/store.rs @@ -6,7 +6,7 @@ use super::DocumentSet; use crate::{ - Deserialize, IterateParams, Key, QueryResult, SUBSPACE_BLOB_EXTRA, SUBSPACE_COUNTER, + Deserialize, IterateParams, Key, QueryResult, SUBSPACE_COUNTER, SUBSPACE_DELETED_ITEMS, SUBSPACE_INDEXES, SUBSPACE_LOGS, Store, U32_LEN, Value, ValueKey, write::{ AnyClass, AnyKey, AssignedIds, Batch, BatchBuilder, Operation, ReportClass, ValueClass, @@ -344,12 +344,7 @@ impl Store { } pub async fn danger_destroy_account(&self, account_id: u32) -> trc::Result<()> { - for subspace in [ - SUBSPACE_LOGS, - SUBSPACE_INDEXES, - SUBSPACE_COUNTER, - SUBSPACE_BLOB_EXTRA, - ] { + for subspace in [SUBSPACE_LOGS, SUBSPACE_INDEXES, SUBSPACE_COUNTER] { self.delete_range( AnyKey { subspace, diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index 08acd373..03495e3c 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -97,7 +97,6 @@ pub const U16_LEN: usize = std::mem::size_of::(); pub const SUBSPACE_ACL: u8 = b'a'; pub const SUBSPACE_TASK_QUEUE: u8 = b'f'; pub const SUBSPACE_INDEXES: u8 = b'i'; -pub const SUBSPACE_BLOB_EXTRA: u8 = b'j'; pub const SUBSPACE_BLOB_LINK: u8 = b'k'; pub const SUBSPACE_BLOBS: u8 = b't'; pub const SUBSPACE_LOGS: u8 = b'l'; @@ -106,6 +105,7 @@ pub const SUBSPACE_IN_MEMORY_VALUE: u8 = b'm'; pub const SUBSPACE_IN_MEMORY_COUNTER: u8 = b'y'; pub const SUBSPACE_PROPERTY: u8 = b'p'; pub const SUBSPACE_REGISTRY: u8 = b's'; +pub const SUBSPACE_REGISTRY_DIRECTORY: u8 = b'd'; pub const SUBSPACE_QUEUE_MESSAGE: u8 = b'e'; pub const SUBSPACE_QUEUE_EVENT: u8 = b'q'; pub const SUBSPACE_QUOTA: u8 = b'u'; @@ -114,14 +114,14 @@ pub const SUBSPACE_REPORT_IN: u8 = b'r'; pub const SUBSPACE_TELEMETRY_SPAN: u8 = b'o'; pub const SUBSPACE_TELEMETRY_METRIC: u8 = b'x'; pub const SUBSPACE_SEARCH_INDEX: u8 = b'z'; +pub const SUBSPACE_DELETED_ITEMS: u8 = b'j'; +pub const SUBSPACE_SPAM_SAMPLES: u8 = b'w'; // TODO: Remove in v1.0 pub const LEGACY_SUBSPACE_BITMAP_ID: u8 = b'b'; pub const LEGACY_SUBSPACE_BITMAP_TAG: u8 = b'c'; pub const LEGACY_SUBSPACE_BITMAP_TEXT: u8 = b'v'; pub const LEGACY_SUBSPACE_FTS_INDEX: u8 = b'g'; -pub const LEGACY_SUBSPACE_TELEMETRY_INDEX: u8 = b'w'; -pub const LEGACY_SUBSPACE_DIRECTORY: u8 = b'd'; #[derive(Clone)] pub struct IterateParams { diff --git a/crates/store/src/registry/get.rs b/crates/store/src/registry/get.rs index a0b52a8a..f97dd298 100644 --- a/crates/store/src/registry/get.rs +++ b/crates/store/src/registry/get.rs @@ -5,7 +5,7 @@ */ use crate::{ - Deserialize, IterateParams, RegistryStore, SUBSPACE_REGISTRY, U16_LEN, U64_LEN, ValueKey, + IterateParams, RegistryStore, SUBSPACE_REGISTRY, U16_LEN, U64_LEN, ValueKey, registry::RegistryObject, write::{AnyClass, RegistryClass, ValueClass, key::KeySerializer}, }; @@ -135,14 +135,3 @@ impl RegistryStore { } } } - -impl Deserialize for Object { - fn deserialize(bytes: &[u8]) -> trc::Result { - Object::unpickle(&mut PickledStream::new(bytes)).ok_or_else(|| { - trc::EventType::Registry(trc::RegistryEvent::DeserializationError) - .into_err() - .caused_by(trc::location!()) - .ctx(trc::Key::Value, bytes) - }) - } -} diff --git a/crates/store/src/registry/mod.rs b/crates/store/src/registry/mod.rs index 9279aa27..a44c99ca 100644 --- a/crates/store/src/registry/mod.rs +++ b/crates/store/src/registry/mod.rs @@ -10,10 +10,19 @@ pub mod local; pub mod query; pub mod write; -use registry::{ - schema::prelude::{ObjectType, Property}, - types::{ObjectImpl, id::ObjectId}, +use crate::{ + Deserialize, SerializeInfallible, U16_LEN, U64_LEN, + write::key::{DeserializeBigEndian, KeySerializer}, }; +use registry::{ + pickle::{Pickle, PickledStream}, + schema::{ + prelude::{Object, ObjectType, Property}, + structs::{DeletedItem, SpamTrainingSample, Task}, + }, + types::{EnumImpl, ObjectImpl, id::ObjectId}, +}; +use types::id::Id; pub struct RegistryObject { pub id: ObjectId, @@ -50,3 +59,76 @@ pub enum RegistryFilterValue { U16(u16), Boolean(bool), } + +impl Deserialize for Object { + fn deserialize(bytes: &[u8]) -> trc::Result { + let mut stream = PickledStream::new(bytes); + Object::unpickle(&mut stream).ok_or_else(|| { + trc::EventType::Registry(trc::RegistryEvent::DeserializationError) + .into_err() + .caused_by(trc::location!()) + .ctx(trc::Key::Value, bytes) + }) + } +} + +impl Deserialize for Task { + fn deserialize(bytes: &[u8]) -> trc::Result { + let mut stream = PickledStream::new(bytes); + Task::unpickle(&mut stream).ok_or_else(|| { + trc::EventType::Registry(trc::RegistryEvent::DeserializationError) + .into_err() + .caused_by(trc::location!()) + .ctx(trc::Key::Value, bytes) + }) + } +} + +impl Deserialize for SpamTrainingSample { + fn deserialize(bytes: &[u8]) -> trc::Result { + let mut stream = PickledStream::new(bytes); + SpamTrainingSample::unpickle(&mut stream).ok_or_else(|| { + trc::EventType::Registry(trc::RegistryEvent::DeserializationError) + .into_err() + .caused_by(trc::location!()) + .ctx(trc::Key::Value, bytes) + }) + } +} + +impl Deserialize for DeletedItem { + fn deserialize(bytes: &[u8]) -> trc::Result { + let mut stream = PickledStream::new(bytes); + DeletedItem::unpickle(&mut stream).ok_or_else(|| { + trc::EventType::Registry(trc::RegistryEvent::DeserializationError) + .into_err() + .caused_by(trc::location!()) + .ctx(trc::Key::Value, bytes) + }) + } +} + +impl SerializeInfallible for ObjectId { + fn serialize(&self) -> Vec { + KeySerializer::new(U16_LEN + U64_LEN) + .write(self.object().to_id()) + .write(self.id().id()) + .finalize() + } +} + +impl Deserialize for ObjectId { + fn deserialize(bytes: &[u8]) -> trc::Result { + let object_id = bytes.deserialize_be_u16(0)?; + let item_id = bytes.deserialize_be_u64(U16_LEN)?; + Ok(ObjectId::new( + ObjectType::from_id(object_id).ok_or_else(|| { + trc::EventType::Registry(trc::RegistryEvent::DeserializationError) + .into_err() + .caused_by(trc::location!()) + .ctx(trc::Key::Value, bytes) + })?, + Id::new(item_id), + )) + } +} diff --git a/crates/store/src/write/batch.rs b/crates/store/src/write/batch.rs index a61c8ac1..a989e390 100644 --- a/crates/store/src/write/batch.rs +++ b/crates/store/src/write/batch.rs @@ -10,13 +10,16 @@ use super::{ }; use crate::{ SerializeInfallible, U32_LEN, - write::{LogCollection, MergeFnc, MergeOperation, Params, SetFnc, SetOperation}, + write::{ + LogCollection, MergeFnc, MergeOperation, Params, SetFnc, SetOperation, TaskQueueClass, + }, }; +use registry::{pickle::Pickle, schema::structs::Task, types::EnumImpl}; use types::{ collection::{Collection, SyncCollection, VanishedCollection}, field::FieldType, }; -use utils::map::vec_map::VecMap; +use utils::{map::vec_map::VecMap, snowflake::SnowflakeIdGenerator}; impl BatchBuilder { pub fn new() -> Self { @@ -390,6 +393,13 @@ impl BatchBuilder { } pub fn any_op(&mut self, op: Operation) -> &mut Self { + if let Operation::Value { class, op } = &op { + self.batch_size += class.serialized_size(); + if let ValueOp::Set(value) = op { + self.batch_size += value.len(); + } + } + self.ops.push(op); self.batch_ops += 1; self @@ -459,6 +469,20 @@ impl BatchBuilder { pub fn is_empty(&self) -> bool { self.batch_ops == 0 } + + pub fn schedule_task(&mut self, task: Task) -> &mut Self { + let due = task.due_timestamp(); + let class = task.object_type().to_id(); + let task = task.to_pickled_vec(); + let id = SnowflakeIdGenerator::from_sequence_id(xxhash_rust::xxh3::xxh3_64(&task)) + .unwrap_or_default(); + + self.set(ValueClass::TaskQueue(TaskQueueClass::Task { id }), task) + .set( + ValueClass::TaskQueue(TaskQueueClass::Due { id, due }), + class.serialize(), + ) + } } pub struct CommitPointIterator { diff --git a/crates/store/src/write/blob.rs b/crates/store/src/write/blob.rs index 68c28adf..bdf4ad29 100644 --- a/crates/store/src/write/blob.rs +++ b/crates/store/src/write/blob.rs @@ -4,13 +4,17 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::time::Instant; - use super::{BlobOp, Operation, ValueClass, ValueOp, key::DeserializeBigEndian, now}; use crate::{ - BlobStore, IterateParams, Store, U32_LEN, U64_LEN, ValueKey, - write::{BatchBuilder, BlobLink}, + BlobStore, Deserialize, IterateParams, SerializeInfallible, Store, U16_LEN, U32_LEN, U64_LEN, + ValueKey, + write::{BatchBuilder, BlobLink, RegistryClass}, }; +use registry::{ + schema::prelude::Property, + types::{EnumImpl, id::ObjectId}, +}; +use std::time::Instant; use trc::{AddContext, PurgeEvent}; use types::{ blob::BlobClass, @@ -38,49 +42,6 @@ impl Store { .caused_by(trc::location!()) } - pub async fn blob_quota(&self, account_id: u32) -> trc::Result { - let from_key = ValueKey { - account_id, - collection: 0, - document_id: 0, - class: ValueClass::Blob(BlobOp::Quota { - hash: BlobHash::default(), - until: 0, - }), - }; - let to_key = ValueKey { - account_id: account_id + 1, - collection: 0, - document_id: 0, - class: ValueClass::Blob(BlobOp::Quota { - hash: BlobHash::default(), - until: u64::MAX, - }), - }; - - let now = now(); - let mut quota = BlobQuota { bytes: 0, count: 0 }; - - self.iterate( - IterateParams::new(from_key, to_key).ascending(), - |key, value| { - let until = key.deserialize_be_u64(key.len() - U64_LEN)?; - if until > now { - let bytes = value.deserialize_be_u32(0)?; - if bytes > 0 { - quota.bytes += bytes as usize; - quota.count += 1; - } - } - Ok(true) - }, - ) - .await - .caused_by(trc::location!())?; - - Ok(quota) - } - pub async fn blob_has_access( &self, hash: impl AsRef + Sync + Send, @@ -195,6 +156,31 @@ impl Store { op: ValueOp::Clear, }); } + for (account_id, object_id) in state.delete_registry { + if batch.is_large_batch() { + self.write(batch.build_all()) + .await + .caused_by(trc::location!())?; + batch = BatchBuilder::new(); + } + + let item_id = object_id.id().id(); + let object_id = object_id.object().to_id(); + + if let Some(account_id) = account_id { + batch.clear(ValueClass::Registry(RegistryClass::Index { + index_id: Property::AccountId.to_id(), + object_id, + item_id, + key: account_id.serialize(), + })); + } + + batch.clear(ValueClass::Registry(RegistryClass::Id { + object_id, + item_id, + })); + } if !batch.is_empty() { self.write(batch.build_all()) .await @@ -220,7 +206,7 @@ struct BlobPurgeState { last_hash: BlobHash, last_hash_is_linked: bool, delete_keys: Vec<(Option, BlobOp)>, - spam_train_samples: Vec<(u32, u64)>, + delete_registry: Vec<(Option, ObjectId)>, now: u64, total_deleted: u64, total_active: u64, @@ -232,7 +218,7 @@ impl BlobPurgeState { last_hash: BlobHash::default(), last_hash_is_linked: true, // Avoid deleting non-existing last_hash on first iteration delete_keys: Vec::new(), - spam_train_samples: Vec::new(), + delete_registry: Vec::new(), now: now(), total_deleted: 0, total_active: 0, @@ -257,42 +243,6 @@ impl BlobPurgeState { )); } else { self.total_active += 1; - if !self.spam_train_samples.is_empty() { - if self.spam_train_samples.len() > 1 { - // Sort by account_id ascending, then until descending - self.spam_train_samples - .sort_unstable_by(|(a_id, a_until), (b_id, b_until)| { - a_id.cmp(b_id).then_with(|| b_until.cmp(a_until)) - }); - let mut samples = self.spam_train_samples.iter().peekable(); - while let Some((account_id, _)) = samples.next() { - // Keep only the latest sample per account - while let Some((next_account_id, next_until)) = samples.peek() { - if next_account_id == account_id { - self.delete_keys.push(( - Some(*account_id), - BlobOp::SpamSample { - hash: self.last_hash.clone(), - until: *next_until, - }, - )); - self.delete_keys.push(( - Some(*account_id), - BlobOp::Link { - hash: self.last_hash.clone(), - to: BlobLink::Temporary { until: *next_until }, - }, - )); - samples.next(); - } else { - break; - } - } - } - } - - self.spam_train_samples.clear(); - } self.last_hash = new_hash; } } @@ -319,44 +269,12 @@ impl BlobPurgeState { to: BlobLink::Temporary { until }, }, )); - match value.first().copied() { - Some(BlobLink::QUOTA_LINK) => { - self.delete_keys.push(( - Some(account_id), - BlobOp::Quota { - hash: self.last_hash.clone(), - until, - }, - )); - } - Some(BlobLink::UNDELETE_LINK) => { - self.delete_keys.push(( - Some(account_id), - BlobOp::Undelete { - hash: self.last_hash.clone(), - until, - }, - )); - } - Some(BlobLink::SPAM_SAMPLE_LINK) => { - self.delete_keys.push(( - Some(account_id), - BlobOp::SpamSample { - hash: self.last_hash.clone(), - until, - }, - )); - } - _ => {} + if value.len() == U16_LEN + U64_LEN { + self.delete_registry.push(( + (account_id != u32::MAX).then_some(account_id), + ObjectId::deserialize(value)?, + )); } - } else { - // Delete attempts to train the same message multiple times - if matches!(value.first(), Some(&BlobLink::SPAM_SAMPLE_LINK)) { - let account_id = key.deserialize_be_u32(BLOB_HASH_LEN)?; - self.spam_train_samples.push((account_id, until)); - } - - self.last_hash_is_linked = true; } Ok(()) } diff --git a/crates/store/src/write/key.rs b/crates/store/src/write/key.rs index 73c43669..6d201141 100644 --- a/crates/store/src/write/key.rs +++ b/crates/store/src/write/key.rs @@ -9,16 +9,18 @@ use super::{ TelemetryClass, ValueClass, }; use crate::{ - Deserialize, IndexKey, IndexKeyPrefix, Key, LogKey, SUBSPACE_ACL, SUBSPACE_BLOB_EXTRA, - SUBSPACE_BLOB_LINK, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_IN_MEMORY_VALUE, + Deserialize, IndexKey, IndexKeyPrefix, Key, LogKey, SUBSPACE_ACL, SUBSPACE_BLOB_LINK, + SUBSPACE_COUNTER, SUBSPACE_DELETED_ITEMS, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_IN_MEMORY_VALUE, SUBSPACE_INDEXES, SUBSPACE_LOGS, SUBSPACE_PROPERTY, SUBSPACE_QUEUE_EVENT, - SUBSPACE_QUEUE_MESSAGE, SUBSPACE_QUOTA, SUBSPACE_REGISTRY, SUBSPACE_REPORT_IN, - SUBSPACE_REPORT_OUT, SUBSPACE_SEARCH_INDEX, SUBSPACE_TASK_QUEUE, SUBSPACE_TELEMETRY_METRIC, - SUBSPACE_TELEMETRY_SPAN, U16_LEN, U32_LEN, U64_LEN, ValueKey, WITH_SUBSPACE, + SUBSPACE_QUEUE_MESSAGE, SUBSPACE_QUOTA, SUBSPACE_REGISTRY, SUBSPACE_REGISTRY_DIRECTORY, + SUBSPACE_REPORT_IN, SUBSPACE_REPORT_OUT, SUBSPACE_SEARCH_INDEX, SUBSPACE_SPAM_SAMPLES, + SUBSPACE_TASK_QUEUE, SUBSPACE_TELEMETRY_METRIC, SUBSPACE_TELEMETRY_SPAN, U16_LEN, U32_LEN, + U64_LEN, ValueKey, WITH_SUBSPACE, write::{ BlobLink, IndexPropertyClass, RegistryClass, SearchIndex, SearchIndexId, SearchIndexType, }, }; +use registry::schema::prelude::ObjectType; use std::convert::TryInto; use types::{ blob_hash::BLOB_HASH_LEN, @@ -295,49 +297,8 @@ impl ValueClass { .write(collection) .write(document_id), ValueClass::TaskQueue(task) => match task { - TaskQueueClass::UpdateIndex { - index, - is_insert, - due, - } => serializer - .write(due.inner()) - .write(account_id) - .write(if *is_insert { 7u8 } else { 8u8 }) - .write(document_id) - .write(index.to_u8()), - TaskQueueClass::SendAlarm { - due, - event_id, - alarm_id, - is_email_alert, - } => serializer - .write(due.inner()) - .write(account_id) - .write(if *is_email_alert { 3u8 } else { 6u8 }) - .write(document_id) - .write(*event_id) - .write(*alarm_id), - TaskQueueClass::SendImip { due, is_payload } => { - if !*is_payload { - serializer - .write(due.inner()) - .write(account_id) - .write(4u8) - .write(document_id) - } else { - serializer - .write(u64::MAX) - .write(account_id) - .write(5u8) - .write(document_id) - .write(due.inner()) - } - } - TaskQueueClass::MergeThreads { due } => serializer - .write(due.inner()) - .write(account_id) - .write(9u8) - .write(document_id), + TaskQueueClass::Task { id } => serializer.write(*id), + TaskQueueClass::Due { id, due } => serializer.write(*due).write(*id), }, ValueClass::Blob(op) => match op { BlobOp::Commit { hash } => serializer.write::<&[u8]>(hash.as_ref()), @@ -353,21 +314,6 @@ impl ValueClass { .write(account_id) .write(*until), }, - BlobOp::Quota { hash, until } => serializer - .write(BlobLink::QUOTA_LINK) - .write(account_id) - .write::<&[u8]>(hash.as_ref()) - .write(*until), - BlobOp::Undelete { hash, until } => serializer - .write(BlobLink::UNDELETE_LINK) - .write(account_id) - .write::<&[u8]>(hash.as_ref()) - .write(*until), - BlobOp::SpamSample { hash, until } => serializer - .write(BlobLink::SPAM_SAMPLE_LINK) - .write(*until) - .write(account_id) - .write::<&[u8]>(hash.as_ref()), }, ValueClass::InMemory(lookup) => match lookup { InMemoryClass::Key(key) => serializer.write(key.as_slice()), @@ -545,12 +491,6 @@ impl ValueClass { } } -impl BlobLink { - pub const QUOTA_LINK: u8 = 0; - pub const UNDELETE_LINK: u8 = 1; - pub const SPAM_SAMPLE_LINK: u8 = 2; -} - impl + Sync + Send + Clone> Key for IndexKey { fn subspace(&self) -> u8 { SUBSPACE_INDEXES @@ -592,6 +532,19 @@ impl + Sync + Send + Clone> Key for AnyKey { } } +const MAILBOX_COLLECTION: u8 = Collection::Mailbox as u8; +const MAILBOX_COUNTER_FIELD: u8 = MailboxField::UidCounter as u8; +const REG_DELETED_ITEM: u16 = ObjectType::DeletedItem as u16; +const REG_SPAM_SAMPLE: u16 = ObjectType::SpamTrainingSample as u16; +const REG_ACCOUNT: u16 = ObjectType::Account as u16; +const REG_DOMAIN: u16 = ObjectType::Domain as u16; +const REG_TENANT: u16 = ObjectType::Tenant as u16; +const REG_ROLE: u16 = ObjectType::Role as u16; +const REG_OAUTH_CLIENT: u16 = ObjectType::OAuthClient as u16; +const REG_MAILING_LIST: u16 = ObjectType::MailingList as u16; +const REG_MASKED_EMAIL: u16 = ObjectType::MaskedEmail as u16; +const REG_PUBLIC_KEY: u16 = ObjectType::PublicKey as u16; + impl ValueClass { pub fn serialized_size(&self) -> usize { match self { @@ -621,23 +574,10 @@ impl ValueClass { BlobLink::Temporary { .. } => U32_LEN + U64_LEN, } } - BlobOp::Quota { .. } | BlobOp::Undelete { .. } => { - BLOB_HASH_LEN + U32_LEN + U64_LEN + 1 - } - BlobOp::SpamSample { .. } => BLOB_HASH_LEN + U32_LEN + 2, }, ValueClass::TaskQueue(e) => match e { - TaskQueueClass::UpdateIndex { .. } => (U64_LEN * 2) + 2, - TaskQueueClass::SendAlarm { .. } | TaskQueueClass::MergeThreads { .. } => { - U64_LEN + (U32_LEN * 3) + 1 - } - TaskQueueClass::SendImip { is_payload, .. } => { - if *is_payload { - (U64_LEN * 2) + (U32_LEN * 2) + 1 - } else { - U64_LEN + (U32_LEN * 2) + 1 - } - } + TaskQueueClass::Task { .. } => U64_LEN + 1, + TaskQueueClass::Due { .. } => (U64_LEN * 2) + 1, }, ValueClass::Queue(q) => match q { QueueClass::Message(_) => U64_LEN, @@ -671,9 +611,6 @@ impl ValueClass { } pub fn subspace(&self, collection: u8) -> u8 { - const MAILBOX_COLLECTION: u8 = Collection::Mailbox as u8; - const MAILBOX_COUNTER_FIELD: u8 = MailboxField::UidCounter as u8; - match self { ValueClass::Property(field) => { if collection == MAILBOX_COLLECTION && *field == MAILBOX_COUNTER_FIELD { @@ -687,17 +624,26 @@ impl ValueClass { ValueClass::TaskQueue { .. } => SUBSPACE_TASK_QUEUE, ValueClass::Blob(op) => match op { BlobOp::Commit { .. } | BlobOp::Link { .. } => SUBSPACE_BLOB_LINK, - BlobOp::Quota { .. } | BlobOp::Undelete { .. } | BlobOp::SpamSample { .. } => { - SUBSPACE_BLOB_EXTRA - } }, - ValueClass::Registry(registry) => { - if matches!(registry, RegistryClass::IdCounter { .. }) { - SUBSPACE_COUNTER - } else { - SUBSPACE_REGISTRY - } - } + ValueClass::Registry(registry) => match registry { + RegistryClass::Item { object_id, .. } + | RegistryClass::Id { object_id, .. } + | RegistryClass::Index { object_id, .. } + | RegistryClass::Reference { + to_object_id: object_id, + .. + } => match *object_id { + REG_ACCOUNT | REG_DOMAIN | REG_TENANT | REG_ROLE | REG_OAUTH_CLIENT + | REG_MAILING_LIST | REG_MASKED_EMAIL | REG_PUBLIC_KEY => { + SUBSPACE_REGISTRY_DIRECTORY + } + REG_DELETED_ITEM => SUBSPACE_DELETED_ITEMS, + REG_SPAM_SAMPLE => SUBSPACE_SPAM_SAMPLES, + _ => SUBSPACE_REGISTRY, + }, + RegistryClass::IndexGlobal { .. } => SUBSPACE_REGISTRY, + RegistryClass::IdCounter { .. } => SUBSPACE_COUNTER, + }, ValueClass::InMemory(lookup) => match lookup { InMemoryClass::Key(_) => SUBSPACE_IN_MEMORY_VALUE, InMemoryClass::Counter(_) => SUBSPACE_IN_MEMORY_COUNTER, diff --git a/crates/store/src/write/mod.rs b/crates/store/src/write/mod.rs index 44b5d970..e31c868f 100644 --- a/crates/store/src/write/mod.rs +++ b/crates/store/src/write/mod.rs @@ -222,30 +222,10 @@ pub enum SearchIndexId { #[derive(Debug, PartialEq, Clone, Eq, Hash)] pub enum TaskQueueClass { - UpdateIndex { - due: TaskEpoch, - index: SearchIndex, - is_insert: bool, - }, - SendAlarm { - due: TaskEpoch, - event_id: u16, - alarm_id: u16, - is_email_alert: bool, - }, - SendImip { - due: TaskEpoch, - is_payload: bool, - }, - MergeThreads { - due: TaskEpoch, - }, + Task { id: u64 }, + Due { id: u64, due: u64 }, } -#[derive(Debug, PartialEq, Clone, Copy, Eq, Hash)] -#[repr(transparent)] -pub struct TaskEpoch(pub(crate) u64); - #[derive(Debug, PartialEq, Clone, Copy, Eq, Hash)] pub enum SearchIndex { Email, @@ -396,9 +376,6 @@ pub struct SetOperation { pub enum BlobOp { Commit { hash: BlobHash }, Link { hash: BlobHash, to: BlobLink }, - Quota { hash: BlobHash, until: u64 }, - Undelete { hash: BlobHash, until: u64 }, - SpamSample { hash: BlobHash, until: u64 }, } #[derive(Debug, PartialEq, Clone, Eq, Hash)] @@ -743,56 +720,3 @@ impl AsRef<[Param]> for Params { &self.0 } } - -impl TaskEpoch { - /* - Structure of the 64-bit epoch: - 4 bytes: seconds since custom epoch (1632280000) - 2 bytes: attempt number - 2 bytes: sequence id - */ - - const EPOCH_OFFSET: u64 = 1632280000; - - pub fn now() -> Self { - Self::new(now()) - } - - pub fn new(timestamp: u64) -> Self { - Self(timestamp.saturating_sub(Self::EPOCH_OFFSET) << 32) - } - - pub fn with_attempt(mut self, attempt: u16) -> Self { - self.0 |= (attempt as u64) << 16; - self - } - - pub fn with_sequence_id(mut self, sequence_id: u16) -> Self { - self.0 |= sequence_id as u64; - self - } - - pub fn with_random_sequence_id(self) -> Self { - self.with_sequence_id(rand::random()) - } - - pub fn due(&self) -> u64 { - (self.0 >> 32) + Self::EPOCH_OFFSET - } - - pub fn attempt(&self) -> u16 { - (self.0 >> 16) as u16 - } - - pub fn sequence_id(&self) -> u16 { - self.0 as u16 - } - - pub fn inner(&self) -> u64 { - self.0 - } - - pub fn from_inner(inner: u64) -> Self { - Self(inner) - } -} diff --git a/crates/trc/src/event/enums.rs b/crates/trc/src/event/enums.rs index 06636cbc..31fa8e9a 100644 --- a/crates/trc/src/event/enums.rs +++ b/crates/trc/src/event/enums.rs @@ -608,7 +608,6 @@ pub enum RegistryEvent { NotSupported = 64, ValidationError = 63, Reserved03 = 59, - Reserved04 = 53, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -826,6 +825,7 @@ pub enum TaskQueueEvent { TaskLocked = 144, TaskIgnored = 586, TaskFailed = 587, + TaskRetry = 53, BlobNotFound = 141, MetadataNotFound = 145, } diff --git a/crates/trc/src/event/enums_impl.rs b/crates/trc/src/event/enums_impl.rs index f4ef4937..6e844483 100644 --- a/crates/trc/src/event/enums_impl.rs +++ b/crates/trc/src/event/enums_impl.rs @@ -400,7 +400,6 @@ impl EventType { b"registry.not-supported" => EventType::Registry(RegistryEvent::NotSupported), b"registry.validation-error" => EventType::Registry(RegistryEvent::ValidationError), b"registry.reserved03" => EventType::Registry(RegistryEvent::Reserved03), - b"registry.reserved04" => EventType::Registry(RegistryEvent::Reserved04), b"resource.not-found" => EventType::Resource(ResourceEvent::NotFound), b"resource.bad-parameters" => EventType::Resource(ResourceEvent::BadParameters), b"resource.error" => EventType::Resource(ResourceEvent::Error), @@ -573,6 +572,7 @@ impl EventType { b"task-queue.task-locked" => EventType::TaskQueue(TaskQueueEvent::TaskLocked), b"task-queue.task-ignored" => EventType::TaskQueue(TaskQueueEvent::TaskIgnored), b"task-queue.task-failed" => EventType::TaskQueue(TaskQueueEvent::TaskFailed), + b"task-queue.task-retry" => EventType::TaskQueue(TaskQueueEvent::TaskRetry), b"task-queue.blob-not-found" => EventType::TaskQueue(TaskQueueEvent::BlobNotFound), b"task-queue.metadata-not-found" => EventType::TaskQueue(TaskQueueEvent::MetadataNotFound), b"telemetry.alert" => EventType::Telemetry(TelemetryEvent::Alert), @@ -1106,7 +1106,6 @@ impl EventType { EventType::Registry(RegistryEvent::NotSupported) => "registry.not-supported", EventType::Registry(RegistryEvent::ValidationError) => "registry.validation-error", EventType::Registry(RegistryEvent::Reserved03) => "registry.reserved03", - EventType::Registry(RegistryEvent::Reserved04) => "registry.reserved04", EventType::Resource(ResourceEvent::NotFound) => "resource.not-found", EventType::Resource(ResourceEvent::BadParameters) => "resource.bad-parameters", EventType::Resource(ResourceEvent::Error) => "resource.error", @@ -1283,6 +1282,7 @@ impl EventType { EventType::TaskQueue(TaskQueueEvent::TaskLocked) => "task-queue.task-locked", EventType::TaskQueue(TaskQueueEvent::TaskIgnored) => "task-queue.task-ignored", EventType::TaskQueue(TaskQueueEvent::TaskFailed) => "task-queue.task-failed", + EventType::TaskQueue(TaskQueueEvent::TaskRetry) => "task-queue.task-retry", EventType::TaskQueue(TaskQueueEvent::BlobNotFound) => "task-queue.blob-not-found", EventType::TaskQueue(TaskQueueEvent::MetadataNotFound) => { "task-queue.metadata-not-found" @@ -1721,7 +1721,6 @@ impl EventType { EventType::Registry(RegistryEvent::NotSupported) => 64, EventType::Registry(RegistryEvent::ValidationError) => 63, EventType::Registry(RegistryEvent::Reserved03) => 59, - EventType::Registry(RegistryEvent::Reserved04) => 53, EventType::Resource(ResourceEvent::NotFound) => 389, EventType::Resource(ResourceEvent::BadParameters) => 386, EventType::Resource(ResourceEvent::Error) => 388, @@ -1894,6 +1893,7 @@ impl EventType { EventType::TaskQueue(TaskQueueEvent::TaskLocked) => 144, EventType::TaskQueue(TaskQueueEvent::TaskIgnored) => 586, EventType::TaskQueue(TaskQueueEvent::TaskFailed) => 587, + EventType::TaskQueue(TaskQueueEvent::TaskRetry) => 53, EventType::TaskQueue(TaskQueueEvent::BlobNotFound) => 141, EventType::TaskQueue(TaskQueueEvent::MetadataNotFound) => 145, EventType::Telemetry(TelemetryEvent::Alert) => 548, @@ -2358,7 +2358,6 @@ impl EventType { 64 => Some(EventType::Registry(RegistryEvent::NotSupported)), 63 => Some(EventType::Registry(RegistryEvent::ValidationError)), 59 => Some(EventType::Registry(RegistryEvent::Reserved03)), - 53 => Some(EventType::Registry(RegistryEvent::Reserved04)), 389 => Some(EventType::Resource(ResourceEvent::NotFound)), 386 => Some(EventType::Resource(ResourceEvent::BadParameters)), 388 => Some(EventType::Resource(ResourceEvent::Error)), @@ -2531,6 +2530,7 @@ impl EventType { 144 => Some(EventType::TaskQueue(TaskQueueEvent::TaskLocked)), 586 => Some(EventType::TaskQueue(TaskQueueEvent::TaskIgnored)), 587 => Some(EventType::TaskQueue(TaskQueueEvent::TaskFailed)), + 53 => Some(EventType::TaskQueue(TaskQueueEvent::TaskRetry)), 141 => Some(EventType::TaskQueue(TaskQueueEvent::BlobNotFound)), 145 => Some(EventType::TaskQueue(TaskQueueEvent::MetadataNotFound)), 548 => Some(EventType::Telemetry(TelemetryEvent::Alert)), @@ -3412,7 +3412,6 @@ impl EventType { } EventType::Registry(RegistryEvent::ValidationError) => "Object validation error", EventType::Registry(RegistryEvent::Reserved03) => "Importing external configuration", - EventType::Registry(RegistryEvent::Reserved04) => "Configuration already up to date", EventType::Resource(ResourceEvent::NotFound) => "Resource not found", EventType::Resource(ResourceEvent::BadParameters) => "Bad resource parameters", EventType::Resource(ResourceEvent::Error) => "Resource error", @@ -3595,6 +3594,7 @@ impl EventType { "Task ignored based on current server roles" } EventType::TaskQueue(TaskQueueEvent::TaskFailed) => "Task failed during processing", + EventType::TaskQueue(TaskQueueEvent::TaskRetry) => "Task will be retried", EventType::TaskQueue(TaskQueueEvent::BlobNotFound) => "Blob not found for task", EventType::TaskQueue(TaskQueueEvent::MetadataNotFound) => "Metadata not found for task", EventType::Telemetry(TelemetryEvent::Alert) => "Alert triggered", @@ -4351,9 +4351,6 @@ impl EventType { EventType::Registry(RegistryEvent::Reserved03) => { "An external configuration is being imported" } - EventType::Registry(RegistryEvent::Reserved04) => { - "The configuration is already up to date" - } EventType::Resource(ResourceEvent::NotFound) => "The resource was not found", EventType::Resource(ResourceEvent::BadParameters) => "The resource parameters are bad", EventType::Resource(ResourceEvent::Error) => "An error occurred with the resource", @@ -4668,6 +4665,9 @@ impl EventType { "The task was ignored based on the current server roles" } EventType::TaskQueue(TaskQueueEvent::TaskFailed) => "The task failed during processing", + EventType::TaskQueue(TaskQueueEvent::TaskRetry) => { + "The task will be retried after a failure" + } EventType::TaskQueue(TaskQueueEvent::BlobNotFound) => { "The requested blob was not found for task" } @@ -5404,7 +5404,6 @@ impl EventType { EventType::Registry(RegistryEvent::NotSupported), EventType::Registry(RegistryEvent::ValidationError), EventType::Registry(RegistryEvent::Reserved03), - EventType::Registry(RegistryEvent::Reserved04), EventType::Resource(ResourceEvent::NotFound), EventType::Resource(ResourceEvent::BadParameters), EventType::Resource(ResourceEvent::Error), @@ -5577,6 +5576,7 @@ impl EventType { EventType::TaskQueue(TaskQueueEvent::TaskLocked), EventType::TaskQueue(TaskQueueEvent::TaskIgnored), EventType::TaskQueue(TaskQueueEvent::TaskFailed), + EventType::TaskQueue(TaskQueueEvent::TaskRetry), EventType::TaskQueue(TaskQueueEvent::BlobNotFound), EventType::TaskQueue(TaskQueueEvent::MetadataNotFound), EventType::Telemetry(TelemetryEvent::Alert), diff --git a/crates/types/src/blob.rs b/crates/types/src/blob.rs index 5fbd3f9f..374956a4 100644 --- a/crates/types/src/blob.rs +++ b/crates/types/src/blob.rs @@ -222,6 +222,10 @@ impl BlobId { 0 } } + + pub fn is_empty(&self) -> bool { + self.hash.is_empty() + } } impl serde::Serialize for BlobId { @@ -233,6 +237,16 @@ impl serde::Serialize for BlobId { } } +impl<'de> serde::Deserialize<'de> for BlobId { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + BlobId::from_str(<&str>::deserialize(deserializer)?) + .map_err(|_| serde::de::Error::custom("invalid BlobId")) + } +} + impl std::fmt::Display for BlobId { #[allow(clippy::unused_io_amount)] fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { diff --git a/crates/utils/src/cheeky_hash.rs b/crates/utils/src/cheeky_hash.rs index 22acc4f1..bac96117 100644 --- a/crates/utils/src/cheeky_hash.rs +++ b/crates/utils/src/cheeky_hash.rs @@ -9,6 +9,7 @@ use std::{ collections::{BTreeMap, HashMap, HashSet}, fmt::Debug, hash::Hash, + str::FromStr, }; // A hash that can cheekily store small inputs directly without hashing them. @@ -91,6 +92,10 @@ impl CheekyHash { pub fn payload_len(&self) -> u8 { self.0[0] } + + fn as_u128(&self) -> u128 { + u128::from_be_bytes(self.0) + } } impl AsRef<[u8]> for CheekyHash { @@ -99,6 +104,20 @@ impl AsRef<[u8]> for CheekyHash { } } +impl FromStr for CheekyHash { + type Err = std::num::ParseIntError; + + fn from_str(s: &str) -> Result { + u128::from_str_radix(s, 16).map(|n| CheekyHash(n.to_be_bytes())) + } +} + +impl std::fmt::Display for CheekyHash { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:032x}", self.as_u128()) + } +} + impl Hash for CheekyHash { fn hash(&self, state: &mut H) { let len = self.0[0] as usize; diff --git a/crates/utils/src/snowflake.rs b/crates/utils/src/snowflake.rs index a96f3cec..a9b4f8fb 100644 --- a/crates/utils/src/snowflake.rs +++ b/crates/utils/src/snowflake.rs @@ -12,7 +12,6 @@ use std::{ #[derive(Debug)] pub struct SnowflakeIdGenerator { epoch: SystemTime, - node_id: u64, sequence: AtomicU64, } @@ -23,7 +22,8 @@ const SEQUENCE_MASK: u64 = (1 << SEQUENCE_LEN) - 1; const NODE_ID_MASK: u64 = (1 << NODE_ID_LEN) - 1; const DEFAULT_EPOCH: u64 = 1632280000; // 52 years after UNIX_EPOCH -//const DEFAULT_EPOCH_MS: u128 = (DEFAULT_EPOCH as u128) * 1000; // 52 years after UNIX_EPOCH in milliseconds + +static mut NODE_ID: u64 = 1; /* @@ -35,9 +35,27 @@ ID characteristics: */ +#[inline(always)] +fn node_id() -> u64 { + unsafe { std::ptr::read_volatile(&raw const NODE_ID) } +} + impl SnowflakeIdGenerator { pub fn new() -> Self { - Self::with_node_id(rand::random::()) + Self { + epoch: SystemTime::UNIX_EPOCH + Duration::from_secs(DEFAULT_EPOCH), // 52 years after UNIX_EPOCH + sequence: 0.into(), + } + } + + pub fn set_node_id(set_node_id: u64) { + let set_node_id = set_node_id & NODE_ID_MASK; + + if set_node_id != node_id() { + unsafe { + NODE_ID = set_node_id; + } + } } pub fn from_duration(period: Duration) -> Option { @@ -56,8 +74,7 @@ impl SnowflakeIdGenerator { .and_then(|diff| Self::from_duration(Duration::from_secs(diff))) } - pub fn from_sequence_and_node_id(sequence: u64, node_id: Option) -> Option { - let node_id = node_id.unwrap_or_else(rand::random::); + pub fn from_sequence_id(sequence: u64) -> Option { let sequence = sequence & SEQUENCE_MASK; (SystemTime::UNIX_EPOCH + Duration::from_secs(DEFAULT_EPOCH)) @@ -66,7 +83,7 @@ impl SnowflakeIdGenerator { .map(|elapsed| { ((elapsed.as_millis() as u64) << (SEQUENCE_LEN + NODE_ID_LEN)) | (sequence << NODE_ID_LEN) - | (node_id & NODE_ID_MASK) + | node_id() }) } @@ -74,14 +91,6 @@ impl SnowflakeIdGenerator { (id >> (SEQUENCE_LEN + NODE_ID_LEN)) / 1000 + DEFAULT_EPOCH } - pub fn with_node_id(node_id: u64) -> Self { - Self { - epoch: SystemTime::UNIX_EPOCH + Duration::from_secs(DEFAULT_EPOCH), // 52 years after UNIX_EPOCH - node_id, - sequence: 0.into(), - } - } - #[inline(always)] pub fn past_id(&self, period: Duration) -> Option { self.epoch @@ -104,9 +113,7 @@ impl SnowflakeIdGenerator { .unwrap_or_default() as u64; let sequence = self.sequence.fetch_add(1, Ordering::Relaxed) & SEQUENCE_MASK; - (elapsed << (SEQUENCE_LEN + NODE_ID_LEN)) - | (sequence << NODE_ID_LEN) - | (self.node_id & NODE_ID_MASK) + (elapsed << (SEQUENCE_LEN + NODE_ID_LEN)) | (sequence << NODE_ID_LEN) | node_id() } } @@ -120,7 +127,6 @@ impl Clone for SnowflakeIdGenerator { fn clone(&self) -> Self { Self { epoch: self.epoch, - node_id: self.node_id, sequence: 0.into(), } } diff --git a/tests/src/store/cleanup.rs b/tests/src/store/cleanup.rs index 323b23b9..37ac19d7 100644 --- a/tests/src/store/cleanup.rs +++ b/tests/src/store/cleanup.rs @@ -19,7 +19,8 @@ pub async fn store_destroy(store: &Store) { SUBSPACE_ACL, SUBSPACE_TASK_QUEUE, SUBSPACE_INDEXES, - SUBSPACE_BLOB_EXTRA, + SUBSPACE_DELETED_ITEMS, + SUBSPACE_SPAM_SAMPLES, SUBSPACE_BLOB_LINK, SUBSPACE_LOGS, SUBSPACE_IN_MEMORY_COUNTER, @@ -141,28 +142,6 @@ pub async fn store_blob_expire_all(store: &Store) { .deserialize_be_u64(BLOB_HASH_LEN + U32_LEN) .caused_by(trc::location!())?; - match value.first().copied() { - Some(BlobLink::QUOTA_LINK) => { - batch.clear(ValueClass::Blob(BlobOp::Quota { - hash: hash.clone(), - until, - })); - } - Some(BlobLink::UNDELETE_LINK) => { - batch.clear(ValueClass::Blob(BlobOp::Undelete { - hash: hash.clone(), - until, - })); - } - Some(BlobLink::SPAM_SAMPLE_LINK) => { - batch.clear(ValueClass::Blob(BlobOp::SpamSample { - hash: hash.clone(), - until, - })); - } - _ => {} - } - batch.clear(ValueClass::Blob(BlobOp::Link { hash, to: BlobLink::Temporary { until }, @@ -258,7 +237,8 @@ pub async fn store_assert_is_empty(store: &Store, blob_store: BlobStore, include (SUBSPACE_QUEUE_EVENT, true), (SUBSPACE_REPORT_OUT, true), (SUBSPACE_REPORT_IN, true), - (SUBSPACE_BLOB_EXTRA, true), + (SUBSPACE_DELETED_ITEMS, true), + (SUBSPACE_SPAM_SAMPLES, true), (SUBSPACE_BLOB_LINK, true), (SUBSPACE_BLOBS, true), (SUBSPACE_COUNTER, false), diff --git a/tests/src/store/import_export.rs b/tests/src/store/import_export.rs index b9c5e876..061fb8e7 100644 --- a/tests/src/store/import_export.rs +++ b/tests/src/store/import_export.rs @@ -256,7 +256,8 @@ impl Snapshot { (SUBSPACE_ACL, true), (SUBSPACE_TASK_QUEUE, true), (SUBSPACE_INDEXES, false), - (SUBSPACE_BLOB_EXTRA, true), + (SUBSPACE_DELETED_ITEMS, true), + (SUBSPACE_SPAM_SAMPLES, true), (SUBSPACE_BLOB_LINK, true), (SUBSPACE_BLOBS, true), (SUBSPACE_LOGS, true),