diff --git a/crates/common/src/cache/reload.rs b/crates/common/src/cache/reload.rs index 7706c322..1d54e036 100644 --- a/crates/common/src/cache/reload.rs +++ b/crates/common/src/cache/reload.rs @@ -11,7 +11,7 @@ use crate::{ storage::Storage, telemetry::Telemetry, }, - ipc::RegistryChange, + ipc::{QueueEvent, RegistryChange}, }; use ahash::AHashMap; use directory::Directories; @@ -142,4 +142,16 @@ impl Server { Ok(result) } + + pub async fn reload_core(&self, new_core: Core) { + self.inner.shared_core.store(new_core.into()); + + // Reload queue settings + self.inner + .ipc + .queue_tx + .send(QueueEvent::ReloadSettings) + .await + .ok(); + } } diff --git a/crates/common/src/config/network.rs b/crates/common/src/config/network.rs index 305a3b28..c3004544 100644 --- a/crates/common/src/config/network.rs +++ b/crates/common/src/config/network.rs @@ -57,18 +57,20 @@ pub struct ContactForm { #[derive(Clone, Default)] pub struct ClusterRoles { - pub purge_stores: ClusterRole, - pub purge_accounts: ClusterRole, + pub store_maintenance: ClusterRole, + pub account_maintenance: ClusterRole, pub push_notifications: ClusterRole, - pub fts_indexing: ClusterRole, + pub search_indexing: ClusterRole, pub spam_training: ClusterRole, pub imip_processing: ClusterRole, pub merge_threads: ClusterRole, pub calendar_alerts: ClusterRole, - pub renew_acme: ClusterRole, + pub dns_acme: ClusterRole, pub calculate_metrics: ClusterRole, pub push_metrics: ClusterRole, pub outbound_mta: ClusterRole, + pub task_scheduler: ClusterRole, + pub task_manager: ClusterRole, } #[derive(Clone, Copy, Default)] @@ -171,40 +173,46 @@ impl Network { let is_success = match &range.object { NodeRole::CalculateMetrics(_) | NodeRole::PushMetrics(_) - | NodeRole::TrainSpamClassifier(_) => { + | NodeRole::SpamClassifierTraining(_) + | NodeRole::TaskScheduler(_) => { let (roles, role_obj) = match &range.object { NodeRole::CalculateMetrics(role) => { (&mut network.roles.calculate_metrics, role) } NodeRole::PushMetrics(role) => (&mut network.roles.push_metrics, role), - NodeRole::TrainSpamClassifier(role) => { + NodeRole::SpamClassifierTraining(role) => { (&mut network.roles.spam_training, role) } + NodeRole::TaskScheduler(role) => { + (&mut network.roles.task_scheduler, role) + } _ => unreachable!(), }; roles.set_role(role_obj.node_id == node_id) } - NodeRole::PurgeStores(_) - | NodeRole::PurgeAccounts(_) - | NodeRole::AcmeRenew(_) + NodeRole::StoreMaintenance(_) + | NodeRole::AccountMaintenance(_) | NodeRole::PushNotifications(_) | NodeRole::SearchIndexing(_) | NodeRole::ImipProcessing(_) | NodeRole::CalendarAlerts(_) | NodeRole::MergeThreads(_) - | NodeRole::OutboundMta(_) => { + | NodeRole::DnsAndAcme(_) + | NodeRole::OutboundMta(_) + | NodeRole::TaskQueueProcessing(_) => { let (roles, role_obj) = match &range.object { - NodeRole::PurgeStores(role) => (&mut network.roles.purge_stores, role), - NodeRole::PurgeAccounts(role) => { - (&mut network.roles.purge_accounts, role) + NodeRole::StoreMaintenance(role) => { + (&mut network.roles.store_maintenance, role) + } + NodeRole::AccountMaintenance(role) => { + (&mut network.roles.account_maintenance, role) } - NodeRole::AcmeRenew(role) => (&mut network.roles.renew_acme, role), NodeRole::PushNotifications(role) => { (&mut network.roles.push_notifications, role) } NodeRole::SearchIndexing(role) => { - (&mut network.roles.fts_indexing, role) + (&mut network.roles.search_indexing, role) } NodeRole::ImipProcessing(role) => { (&mut network.roles.imip_processing, role) @@ -216,6 +224,10 @@ impl Network { (&mut network.roles.merge_threads, role) } NodeRole::OutboundMta(role) => (&mut network.roles.outbound_mta, role), + NodeRole::DnsAndAcme(role) => (&mut network.roles.dns_acme, role), + NodeRole::TaskQueueProcessing(role) => { + (&mut network.roles.task_manager, role) + } _ => unreachable!(), }; @@ -258,11 +270,11 @@ impl Network { } let roles = match shard_type { - NodeShardType::PurgeStores => &mut network.roles.purge_stores, - NodeShardType::PurgeAccounts => &mut network.roles.purge_accounts, - NodeShardType::AcmeRenew => &mut network.roles.renew_acme, + NodeShardType::StoreMaintenance => &mut network.roles.store_maintenance, + NodeShardType::AccountMaintenance => &mut network.roles.account_maintenance, + NodeShardType::DnsAndAcme => &mut network.roles.dns_acme, NodeShardType::PushNotifications => &mut network.roles.push_notifications, - NodeShardType::SearchIndexing => &mut network.roles.fts_indexing, + NodeShardType::SearchIndexing => &mut network.roles.search_indexing, NodeShardType::ImipProcessing => &mut network.roles.imip_processing, NodeShardType::CalendarAlerts => &mut network.roles.calendar_alerts, NodeShardType::MergeThreads => &mut network.roles.merge_threads, @@ -419,7 +431,7 @@ impl ClusterRole { matches!(self, ClusterRole::Enabled | ClusterRole::Sharded { .. }) } - pub fn is_enabled_for_integer(&self, value: u32) -> bool { + pub fn is_enabled_for_integer(&self, value: u64) -> bool { debug_assert!(!self.is_uninit() && !self.is_seen_role()); match self { ClusterRole::Enabled => true, @@ -427,7 +439,7 @@ impl ClusterRole { ClusterRole::Sharded { shard_id, total_shards, - } => (value % total_shards) == *shard_id, + } => (value as u32 % total_shards) == *shard_id, } } @@ -440,7 +452,7 @@ impl ClusterRole { shard_id, total_shards, } => { - let mut hasher = Xxh3Builder::new().with_seed(191179).build(); + let mut hasher = Xxh3Builder::new().with_seed(201179).build(); item.hash(&mut hasher); hasher.finish() % (*total_shards as u64) == *shard_id as u64 } @@ -496,17 +508,20 @@ impl ClusterRole { impl ClusterRoles { fn all_mut(&mut self) -> impl Iterator { [ - &mut self.purge_stores, - &mut self.purge_accounts, + &mut self.store_maintenance, + &mut self.account_maintenance, &mut self.push_notifications, - &mut self.fts_indexing, + &mut self.search_indexing, &mut self.spam_training, &mut self.imip_processing, &mut self.merge_threads, &mut self.calendar_alerts, - &mut self.renew_acme, + &mut self.dns_acme, + &mut self.outbound_mta, &mut self.calculate_metrics, &mut self.push_metrics, + &mut self.task_manager, + &mut self.task_scheduler, ] .into_iter() } diff --git a/crates/common/src/enterprise/alerts.rs b/crates/common/src/enterprise/alerts.rs index eecf9589..cbcd3fa0 100644 --- a/crates/common/src/enterprise/alerts.rs +++ b/crates/common/src/enterprise/alerts.rs @@ -60,6 +60,17 @@ impl Server { subject, body, } => { + let subject = subject.build(); + trc::event!( + Telemetry(TelemetryEvent::AlertMessage), + Id = alert.id.id().id(), + To = to + .iter() + .map(|t| trc::Value::from(t.to_string())) + .collect::>(), + Details = subject.clone() + ); + messages.push(AlertMessage { from: from_addr.clone(), to: to.clone(), @@ -82,7 +93,7 @@ impl Server { )), ) .header("Auto-Submitted", HeaderType::Text("auto-generated".into())) - .subject(subject.build()) + .subject(subject) .text_body(body.build()) .write_to_vec() .unwrap_or_default(), @@ -90,14 +101,14 @@ impl Server { } AlertMethod::Event { message } => { trc::event!( - Telemetry(TelemetryEvent::Alert), + Telemetry(TelemetryEvent::AlertEvent), Id = alert.id.id().id(), Details = message.as_ref().map(|m| m.build()) ); #[cfg(feature = "test_mode")] Collector::update_event_counter( - trc::EventType::Telemetry(TelemetryEvent::Alert), + trc::EventType::Telemetry(TelemetryEvent::AlertEvent), 1, ); } diff --git a/crates/common/src/enterprise/config.rs b/crates/common/src/enterprise/config.rs index ed3c4d71..0d950172 100644 --- a/crates/common/src/enterprise/config.rs +++ b/crates/common/src/enterprise/config.rs @@ -170,9 +170,12 @@ impl Enterprise { // Build the enterprise configuration let mut enterprise = Enterprise { license, - undelete_retention: dr + deleted_items_retention: dr .archive_deleted_items_for .map(|retention| retention.into_inner()), + deleted_accounts_retention: dr + .archive_deleted_accounts_for + .map(|retention| retention.into_inner()), logo_url, metrics_alerts: Default::default(), spam_filter_llm: SpamFilterLlmConfig::parse(bp, &ai_apis_ids).await, diff --git a/crates/common/src/enterprise/mod.rs b/crates/common/src/enterprise/mod.rs index 61acefb4..a5f89b94 100644 --- a/crates/common/src/enterprise/mod.rs +++ b/crates/common/src/enterprise/mod.rs @@ -33,7 +33,8 @@ use utils::{HttpLimitResponse, cron::SimpleCron, template::Template}; pub struct Enterprise { pub license: LicenseKey, pub logo_url: Option, - pub undelete_retention: Option, + pub deleted_items_retention: Option, + pub deleted_accounts_retention: Option, pub trace_retention: Option, pub metrics_retention: Option, pub metrics_interval: SimpleCron, diff --git a/crates/common/src/ipc.rs b/crates/common/src/ipc.rs index 1c6f448b..f06075a5 100644 --- a/crates/common/src/ipc.rs +++ b/crates/common/src/ipc.rs @@ -16,39 +16,14 @@ use mail_auth::{ report::{Record, tlsrpt::FailureDetails}, }; use registry::{schema::prelude::ObjectType, types::id::ObjectId}; -use std::{ - sync::{ - Arc, - atomic::{AtomicBool, Ordering}, - }, - time::Instant, +use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, }; use tokio::sync::{Semaphore, SemaphorePermit, mpsc}; use types::type_state::{DataType, StateChange}; use utils::map::bitmap::Bitmap; -pub enum HousekeeperEvent { - AcmeReschedule { - provider_id: String, - renew_at: Instant, - }, - Purge(PurgeType), - ReloadSettings, - Exit, -} - -pub enum PurgeType { - Data, - Blob, - Lookup { - prefix: Option>, - }, - Account { - account_id: Option, - use_roles: bool, - }, -} - #[derive(Debug)] pub enum PushEvent { Subscribe { diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 315b8e51..5a64df23 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -38,7 +38,7 @@ use config::{ storage::Storage, telemetry::Metrics, }; -use ipc::{BroadcastEvent, HousekeeperEvent, PushEvent, QueueEvent, ReportingEvent}; +use ipc::{BroadcastEvent, PushEvent, QueueEvent, ReportingEvent}; use mail_auth::{MX, Txt}; use manager::application::{Resource, WebApplicationManager}; use parking_lot::{Mutex, RwLock}; @@ -122,10 +122,8 @@ pub const KV_RATE_LIMIT_HTTP_ANONYMOUS: u8 = 9; pub const KV_RATE_LIMIT_IMAP: u8 = 10; pub const KV_QUOTA_BLOB: u8 = 11; pub const KV_GREYLIST: u8 = 16; -pub const KV_LOCK_PURGE_ACCOUNT: u8 = 20; pub const KV_LOCK_QUEUE_MESSAGE: u8 = 21; pub const KV_LOCK_TASK: u8 = 23; -pub const KV_LOCK_HOUSEKEEPER: u8 = 24; pub const KV_LOCK_DAV: u8 = 25; pub const KV_SIEVE_ID: u8 = 26; @@ -272,7 +270,6 @@ pub struct HttpAuthCache { pub struct Ipc { pub push_tx: mpsc::Sender, - pub housekeeper_tx: mpsc::Sender, pub task_tx: Arc, pub queue_tx: mpsc::Sender, pub report_tx: mpsc::Sender, diff --git a/crates/common/src/manager/application.rs b/crates/common/src/manager/application.rs index c518ada9..312ad937 100644 --- a/crates/common/src/manager/application.rs +++ b/crates/common/src/manager/application.rs @@ -132,10 +132,11 @@ impl WebApplicationManager { // Update routes self.routes.store(routes.into()); - trc::event!( + let todo = "use new event"; + /*trc::event!( Resource(trc::ResourceEvent::WebadminUnpacked), Path = self.bundle_path.path.to_string_lossy().into_owned(), - ); + );*/ Ok(()) } diff --git a/crates/common/src/manager/boot.rs b/crates/common/src/manager/boot.rs index dd25b72c..3d6a8afb 100644 --- a/crates/common/src/manager/boot.rs +++ b/crates/common/src/manager/boot.rs @@ -10,10 +10,7 @@ use crate::{ config::{ network::AsnGeoLookupConfig, server::Listeners, storage::Storage, telemetry::Telemetry, }, - ipc::{ - BroadcastEvent, HousekeeperEvent, PushEvent, QueueEvent, ReportingEvent, - TrainTaskController, - }, + ipc::{BroadcastEvent, PushEvent, QueueEvent, ReportingEvent, TrainTaskController}, }; use arc_swap::ArcSwap; use pwhash::sha512_crypt; @@ -39,7 +36,6 @@ pub struct BootManager { pub struct IpcReceivers { pub push_rx: Option>, - pub housekeeper_rx: Option>, pub queue_rx: Option>, pub report_rx: Option>, pub broadcast_rx: Option>, @@ -358,14 +354,12 @@ impl BootManager { pub fn build_ipc(has_pubsub: bool) -> (Ipc, IpcReceivers) { // Build ipc receivers let (push_tx, push_rx) = mpsc::channel(IPC_CHANNEL_BUFFER); - let (housekeeper_tx, housekeeper_rx) = mpsc::channel(IPC_CHANNEL_BUFFER); let (queue_tx, queue_rx) = mpsc::channel(IPC_CHANNEL_BUFFER); let (report_tx, report_rx) = mpsc::channel(IPC_CHANNEL_BUFFER); let (broadcast_tx, broadcast_rx) = mpsc::channel(IPC_CHANNEL_BUFFER); ( Ipc { push_tx, - housekeeper_tx, queue_tx, report_tx, broadcast_tx: has_pubsub.then_some(broadcast_tx), @@ -374,7 +368,6 @@ pub fn build_ipc(has_pubsub: bool) -> (Ipc, IpcReceivers) { }, IpcReceivers { push_rx: Some(push_rx), - housekeeper_rx: Some(housekeeper_rx), queue_rx: Some(queue_rx), report_rx: Some(report_rx), broadcast_rx: has_pubsub.then_some(broadcast_rx), diff --git a/crates/common/src/telemetry/metrics/store.rs b/crates/common/src/telemetry/metrics/store.rs index 61710e87..f09b400d 100644 --- a/crates/common/src/telemetry/metrics/store.rs +++ b/crates/common/src/telemetry/metrics/store.rs @@ -61,10 +61,10 @@ impl MetricsStore for Store { MetricType::ManageSieveConnectionStart, MetricType::HttpConnectionStart, MetricType::DeliveryAttemptStart, - MetricType::QueueQueueMessage, - MetricType::QueueQueueMessageAuthenticated, - MetricType::QueueQueueDsn, - MetricType::QueueQueueReport, + MetricType::QueueMessageQueued, + MetricType::QueueAuthenticatedMessageQueued, + MetricType::QueueDsnQueued, + MetricType::QueueReportQueued, MetricType::MessageIngestHam, MetricType::MessageIngestSpam, MetricType::AuthFailed, diff --git a/crates/common/src/telemetry/tracers/store.rs b/crates/common/src/telemetry/tracers/store.rs index 117910fb..46682ea7 100644 --- a/crates/common/src/telemetry/tracers/store.rs +++ b/crates/common/src/telemetry/tracers/store.rs @@ -222,11 +222,11 @@ impl StoreTracer { | EventType::Dkim(_) | EventType::MailAuth(_) | EventType::Queue( - QueueEvent::QueueMessage - | QueueEvent::QueueMessageAuthenticated - | QueueEvent::QueueReport - | QueueEvent::QueueDsn - | QueueEvent::QueueAutogenerated + QueueEvent::MessageQueued + | QueueEvent::AuthenticatedMessageQueued + | QueueEvent::ReportQueued + | QueueEvent::DsnQueued + | QueueEvent::AutogeneratedQueued | QueueEvent::Rescheduled | QueueEvent::RateLimitExceeded | QueueEvent::ConcurrencyLimitExceeded diff --git a/crates/email/src/message/delete.rs b/crates/email/src/message/delete.rs index 89467db5..6180ef95 100644 --- a/crates/email/src/message/delete.rs +++ b/crates/email/src/message/delete.rs @@ -5,14 +5,11 @@ */ use super::metadata::MessageData; -use common::{KV_LOCK_PURGE_ACCOUNT, Server, storage::index::ObjectIndexBuilder}; +use common::{Server, storage::index::ObjectIndexBuilder}; use groupware::calendar::storage::ItipAutoExpunge; use registry::schema::enums::IndexDocumentType; -use registry::schema::prelude::ObjectType; use registry::schema::structs::{Task, TaskIndexDocument, TaskStatus}; use std::future::Future; -use store::ahash::AHashSet; -use store::registry::RegistryQuery; use store::write::key::DeserializeBigEndian; use store::write::{IndexPropertyClass, now}; use store::{IterateParams, U32_LEN, U64_LEN, ValueKey}; @@ -33,9 +30,7 @@ pub trait EmailDeletion: Sync + Send { document_ids: RoaringBitmap, ) -> impl Future> + Send; - fn purge_accounts(&self, use_roles: bool) -> impl Future + Send; - - fn purge_account(&self, account_id: u32) -> impl Future + Send; + fn purge_account(&self, account_id: u32) -> impl Future> + Send; fn purge_email_submissions( &self, @@ -110,107 +105,38 @@ impl EmailDeletion for Server { Ok(not_destroyed) } - async fn purge_accounts(&self, use_roles: bool) { - match self - .registry() - .query::>(RegistryQuery::new(ObjectType::Account)) - .await - { - Ok(account_ids) => { - for account_id in account_ids { - let account_id = account_id as u32; - if !use_roles - || self - .core - .network - .roles - .purge_accounts - .is_enabled_for_integer(account_id) - { - self.purge_account(account_id).await; - } - } - } - Err(err) => { - trc::error!(err.caused_by(trc::location!())); - } - } - } - - async fn purge_account(&self, account_id: u32) { - // Lock account - match self - .in_memory_store() - .try_lock(KV_LOCK_PURGE_ACCOUNT, &account_id.to_be_bytes(), 3600) - .await - { - Ok(true) => (), - Ok(false) => { - trc::event!(Purge(trc::PurgeEvent::InProgress), AccountId = account_id,); - return; - } - Err(err) => { - trc::error!( - err.details("Failed to lock account.") - .account_id(account_id) - ); - return; - } - } - + async fn purge_account(&self, account_id: u32) -> trc::Result<()> { // Auto-expunge deleted and junk messages - if let Some(hold_period) = self.core.email.mail_autoexpunge_after - && let Err(err) = self.emails_auto_expunge(account_id, hold_period).await - { - trc::error!( - err.details("Failed to auto-expunge e-mail messages.") - .account_id(account_id) - ); + if let Some(hold_period) = self.core.email.mail_autoexpunge_after { + self.emails_auto_expunge(account_id, hold_period) + .await + .caused_by(trc::location!())?; } // Auto-expunge iMIP messages - if let Some(hold_period) = self.core.groupware.itip_inbox_auto_expunge - && let Err(err) = self.itip_auto_expunge(account_id, hold_period).await - { - trc::error!( - err.details("Failed to auto-expunge iTIP messages.") - .account_id(account_id) - ); + if let Some(hold_period) = self.core.groupware.itip_inbox_auto_expunge { + self.itip_auto_expunge(account_id, hold_period) + .await + .caused_by(trc::location!())?; } // Delete old e-mail submissions - if let Some(hold_period) = self.core.email.email_submission_autoexpunge_after - && let Err(err) = self.purge_email_submissions(account_id, hold_period).await - { - trc::error!( - err.details("Failed to auto-expunge e-mail submissions.") - .account_id(account_id) - ); + if let Some(hold_period) = self.core.email.email_submission_autoexpunge_after { + self.purge_email_submissions(account_id, hold_period) + .await + .caused_by(trc::location!())?; } // Purge changelogs - if let Err(err) = self - .delete_changes( - account_id, - self.core.email.changes_max_history, - self.core.email.share_notification_max_history, - ) - .await - { - trc::error!( - err.details("Failed to purge changes.") - .account_id(account_id) - ); - } + self.delete_changes( + account_id, + self.core.email.changes_max_history, + self.core.email.share_notification_max_history, + ) + .await + .caused_by(trc::location!())?; - // Delete lock - if let Err(err) = self - .in_memory_store() - .remove_lock(KV_LOCK_PURGE_ACCOUNT, &account_id.to_be_bytes()) - .await - { - trc::error!(err.details("Failed to delete lock.").account_id(account_id)); - } + Ok(()) } async fn emails_auto_expunge(&self, account_id: u32, hold_period: u64) -> trc::Result<()> { @@ -251,7 +177,7 @@ impl EmailDeletion for Server { } trc::event!( - Purge(trc::PurgeEvent::AutoExpunge), + Store(trc::StoreEvent::AutoExpunge), Collection = Collection::Email.as_str(), AccountId = account_id, Total = destroy_ids.len(), @@ -316,7 +242,7 @@ impl EmailDeletion for Server { } trc::event!( - Purge(trc::PurgeEvent::AutoExpunge), + Store(trc::StoreEvent::AutoExpunge), Collection = Collection::EmailSubmission.as_str(), AccountId = account_id, Total = destroy_ids.len(), diff --git a/crates/groupware/src/calendar/storage.rs b/crates/groupware/src/calendar/storage.rs index 63822dd2..1620a470 100644 --- a/crates/groupware/src/calendar/storage.rs +++ b/crates/groupware/src/calendar/storage.rs @@ -129,7 +129,7 @@ impl ItipAutoExpunge for Server { } trc::event!( - Purge(trc::PurgeEvent::AutoExpunge), + Store(trc::StoreEvent::AutoExpunge), AccountId = account_id, Collection = Collection::CalendarEventNotification.as_str(), Total = destroy_ids.len(), diff --git a/crates/http/src/auth/oauth/auth.rs b/crates/http/src/auth/oauth/auth.rs index 23074a6d..7d665342 100644 --- a/crates/http/src/auth/oauth/auth.rs +++ b/crates/http/src/auth/oauth/auth.rs @@ -85,14 +85,14 @@ impl OAuthApiHandler for Server { } => { // Validate clientId if client_id.len() > CLIENT_ID_MAX_LEN { - return Err(trc::ManageEvent::Error + return Err(trc::AuthEvent::Error .into_err() .details("Client ID is invalid.")); } else if redirect_uri .as_ref() .is_some_and(|uri| uri.starts_with("http://")) { - return Err(trc::ManageEvent::Error + return Err(trc::AuthEvent::Error .into_err() .details("Redirect URI must be HTTPS.")); } diff --git a/crates/http/src/management/mod.rs b/crates/http/src/management/mod.rs index 03884b0f..a390d8a2 100644 --- a/crates/http/src/management/mod.rs +++ b/crates/http/src/management/mod.rs @@ -14,38 +14,11 @@ pub mod diagnose; use crate::management::diagnose::TroubleshootApi; use common::{Server, auth::AccessToken}; -use http_proto::{ - HttpRequest, HttpResponse, HttpSessionData, JsonResponse, ToHttpResponse, request::fetch_body, -}; +use http_proto::{HttpRequest, HttpResponse, HttpSessionData, request::fetch_body}; use hyper::{StatusCode, header}; use jmap::api::{ToJmapHttpResponse, ToRequestError}; use jmap_proto::error::request::RequestError; use registry::schema::enums::Permission; -use serde::Serialize; - -#[derive(Serialize)] -#[serde(tag = "error")] -#[serde(rename_all = "camelCase")] -pub enum ManagementApiError<'x> { - FieldAlreadyExists { - field: &'x str, - value: &'x str, - }, - FieldMissing { - field: &'x str, - }, - NotFound { - item: &'x str, - }, - Unsupported { - details: &'x str, - }, - AssertFailed, - Other { - details: &'x str, - reason: Option<&'x str>, - }, -} pub trait ManagementApi: Sync + Send { fn handle_api_manage_request( @@ -95,7 +68,7 @@ impl ManagementApi for Server { self.handle_telemetry_api_request(req, path, access_token) .await } else { - Err(trc::ManageEvent::NotSupported.ctx(trc::Key::Details, "Enterprise feature")) + Err(trc::ResourceEvent::NotFound.ctx(trc::Key::Details, "Enterprise feature")) } } // SPDX-SnippetEnd @@ -111,35 +84,6 @@ pub trait ToManageHttpResponse { impl ToManageHttpResponse for &trc::Error { fn into_http_response(self) -> HttpResponse { match self.as_ref() { - trc::EventType::Manage(cause) => { - match cause { - trc::ManageEvent::MissingParameter => ManagementApiError::FieldMissing { - field: self.value_as_str(trc::Key::Key).unwrap_or_default(), - }, - trc::ManageEvent::AlreadyExists => ManagementApiError::FieldAlreadyExists { - field: self.value_as_str(trc::Key::Key).unwrap_or_default(), - value: self.value_as_str(trc::Key::Value).unwrap_or_default(), - }, - trc::ManageEvent::NotFound => ManagementApiError::NotFound { - item: self.value_as_str(trc::Key::Key).unwrap_or_default(), - }, - trc::ManageEvent::NotSupported => ManagementApiError::Unsupported { - details: self - .value(trc::Key::Details) - .or_else(|| self.value(trc::Key::Reason)) - .and_then(|v| v.as_str()) - .unwrap_or("Requested action is unsupported"), - }, - trc::ManageEvent::AssertFailed => ManagementApiError::AssertFailed, - trc::ManageEvent::Error => ManagementApiError::Other { - reason: self.value_as_str(trc::Key::Reason), - details: self - .value_as_str(trc::Key::Details) - .unwrap_or("Unknown error"), - }, - } - } - .into_http_response(), trc::EventType::Auth( trc::AuthEvent::Failed | trc::AuthEvent::Error | trc::AuthEvent::TokenExpired, ) => HttpResponse::unauthorized(true), @@ -165,9 +109,3 @@ impl UnauthorizedResponse for HttpResponse { .with_text_body(serde_json::to_string(&RequestError::unauthorized()).unwrap_or_default()) } } - -impl ManagementApiError<'_> { - fn into_http_response(self) -> HttpResponse { - JsonResponse::new(self).into_http_response() - } -} diff --git a/crates/jmap/src/api/request.rs b/crates/jmap/src/api/request.rs index 60494353..eb91f534 100644 --- a/crates/jmap/src/api/request.rs +++ b/crates/jmap/src/api/request.rs @@ -536,9 +536,14 @@ impl RequestHandler for Server { set_account_id_if_missing(&mut req.account_id, access_token); access_token.assert_is_member(req.account_id)?; - self.registry_set(method_name.obj.unwrap_registry(), req, access_token) - .await? - .into() + self.registry_set( + method_name.obj.unwrap_registry(), + req, + access_token, + session, + ) + .await? + .into() } }, RequestMethod::Changes(mut req) => { diff --git a/crates/jmap/src/registry/get.rs b/crates/jmap/src/registry/get.rs index 8ee64a2e..9c878f6f 100644 --- a/crates/jmap/src/registry/get.rs +++ b/crates/jmap/src/registry/get.rs @@ -234,6 +234,7 @@ impl RegistryGet for Server { continue; }; + let todo = "include account quota if requested"; match &object.inner { ObjectInner::DkimSignature(obj) if get.properties.is_empty() @@ -278,6 +279,10 @@ impl RegistryGet for Server { ObjectType::AccountSettings | ObjectType::Credential => { account_get(get).await.map(|get| get.into_response()) } + ObjectType::Action => { + let todo = "actions"; + todo!() + } } } } diff --git a/crates/jmap/src/registry/mapping/account.rs b/crates/jmap/src/registry/mapping/account.rs index 2be5b7b1..7b457c8a 100644 --- a/crates/jmap/src/registry/mapping/account.rs +++ b/crates/jmap/src/registry/mapping/account.rs @@ -5,17 +5,29 @@ */ use crate::registry::{ - mapping::{RegistryGetResponse, RegistrySetResponse}, + mapping::{RegistryGetResponse, RegistrySetResponse, principal::build_set_error}, set::map_write_error, }; +use common::{ + auth::{ + AccessToken, Permissions, PermissionsGroup, + credential::{ApiKey, AppPassword}, + permissions::BuildPermissions, + }, + cache::invalidate::CacheInvalidationBuilder, + ipc::CacheInvalidation, +}; use directory::core::secret::{hash_secret, verify_otp_auth, verify_secret_hash}; use jmap_proto::error::set::SetError; -use jmap_tools::{JsonPointer, JsonPointerItem, Key}; +use jmap_tools::{JsonPointer, JsonPointerItem, Key, Map, Value}; use registry::{ - jmap::{IntoValue, JsonPointerPatch, MaybeUnpatched, RegistryJsonPatch}, + jmap::{IntoValue, JsonPointerPatch, MaybeUnpatched, RegistryJsonPatch, RegistryValue}, schema::{ + enums::StorageQuota, prelude::{MASKED_PASSWORD, Object, ObjectInner, ObjectType, Property}, - structs::{Account, AccountSettings, Credential}, + structs::{ + Account, AccountSettings, Credential, CredentialPermissions, SecondaryCredential, + }, }, types::id::ObjectId, }; @@ -77,12 +89,185 @@ pub(crate) async fn account_set( } } } - - if account.encryption_at_rest != old_account.encryption_at_rest { - let todo = "validate pk"; - } } ObjectType::Credential => { + // Process creations + if !set.create.is_empty() { + let account_cache = set.server.account(set.account_id).await?; + let app_pass_quota = set + .server + .object_quota(account_cache.object_quotas(), StorageQuota::MaxAppPasswords); + let api_key_quota = set + .server + .object_quota(account_cache.object_quotas(), StorageQuota::MaxApiKeys); + let mut last_credential_id = 0; + let mut app_pass_total = 0; + let mut api_key_total = 0; + + for credential in account.credentials.values() { + match credential { + Credential::Password(c) => { + let credential_id = c.credential_id.id(); + if credential_id > last_credential_id { + last_credential_id = credential_id; + } + } + Credential::AppPassword(c) => { + let credential_id = c.credential_id.id(); + if credential_id > last_credential_id { + last_credential_id = credential_id; + } + app_pass_total += 1; + } + Credential::ApiKey(c) => { + let credential_id = c.credential_id.id(); + if credential_id > last_credential_id { + last_credential_id = credential_id; + } + api_key_total += 1; + } + } + } + + 'outer: for (id, value) in set.create.drain() { + let mut credential = Credential::default(); + + for (key, value) in value.into_expanded_object() { + let Key::Property(prop) = key else { + set.response.not_created.append( + id, + SetError::invalid_properties().with_property(key.into_owned()), + ); + continue 'outer; + }; + let ptr = JsonPointer::new(vec![JsonPointerItem::Key(Key::Property(prop))]); + + // Patch object + match credential.patch(JsonPointerPatch::new(&ptr).with_create(true), value) + { + Ok(MaybeUnpatched::Patched) => {} + Ok( + MaybeUnpatched::Unpatched { .. } + | MaybeUnpatched::UnpatchedMany { .. }, + ) => { + set.response.not_created.append( + id, + SetError::invalid_properties() + .with_property(prop) + .with_description("Cannot set property during creation."), + ); + continue 'outer; + } + Err(err) => { + set.response.not_created.append(id, err.into()); + continue 'outer; + } + } + } + + // Validate credential + match &mut credential { + Credential::AppPassword(credential) => { + if app_pass_total >= app_pass_quota { + set.response.not_created.append( + id, + SetError::over_quota().with_description(format!( + "You have exceeded your quota of {} app passwords.", + app_pass_quota + )), + ); + continue 'outer; + } + if let Err(err) = + validate_credential_permissions(set.access_token, credential) + { + set.response.not_created.append(id, err); + continue 'outer; + } + + // Assign id + last_credential_id += 1; + app_pass_total += 1; + credential.credential_id = last_credential_id.into(); + + // Generate App password and hash secret + let app_ass = AppPassword::new(last_credential_id as u32).build(); + credential.secret = hash_secret( + set.server.core.network.security.password_hash_algorithm, + app_ass.clone(), + ) + .await + .caused_by(trc::location!())?; + set.response.created.insert( + id, + Value::Object(Map::from(vec![ + ( + Key::Property(Property::Secret), + Value::Element(RegistryValue::Id( + last_credential_id.into(), + )), + ), + (Key::Property(Property::Secret), Value::Str(app_ass.into())), + ])), + ); + } + Credential::ApiKey(credential) => { + if api_key_total >= api_key_quota { + set.response.not_created.append( + id, + SetError::over_quota().with_description(format!( + "You have exceeded your quota of {} API keys.", + api_key_quota + )), + ); + continue 'outer; + } + if let Err(err) = + validate_credential_permissions(set.access_token, credential) + { + set.response.not_created.append(id, err); + continue 'outer; + } + + // Assign id + last_credential_id += 1; + api_key_total += 1; + credential.credential_id = last_credential_id.into(); + + // Generate API key and hash secret + let api_key = + ApiKey::new(set.account_id, last_credential_id as u32).build(); + credential.secret = hash_secret( + set.server.core.network.security.password_hash_algorithm, + api_key.clone(), + ) + .await + .caused_by(trc::location!())?; + set.response.created.insert( + id, + Value::Object(Map::from(vec![ + ( + Key::Property(Property::Secret), + Value::Element(RegistryValue::Id( + last_credential_id.into(), + )), + ), + (Key::Property(Property::Secret), Value::Str(api_key.into())), + ])), + ); + } + Credential::Password(_) => { + set.response.not_created.append( + id, + SetError::forbidden() + .with_description("Cannot create a password credential."), + ); + } + } + } + } + + // Process updates 'outer: for (id, value) in set.update.drain(..) { if let Some(credential) = account .credentials @@ -145,6 +330,20 @@ pub(crate) async fn account_set( credential.otp_auth = old_credential.otp_auth.clone(); } + // Users cannot modify their allowedIps or expiration + if credential.allowed_ips != old_credential.allowed_ips + || credential.expires_at != old_credential.expires_at + { + set.response.not_updated.append( + id, + SetError::forbidden().with_description( + "Modifying allowed IPs or expiration is not allowed.", + ), + ); + continue 'outer; + } + + // Password changes are not supported when using external directories if (credential.secret != old_credential.secret || credential.otp_auth != old_credential.otp_auth) && set @@ -289,6 +488,7 @@ pub(crate) async fn account_set( } } + // Process deletions for id in set.destroy.drain(..) { if let Some(idx) = account.credentials.0.inner.iter_mut().position(|c| { c.value.credential_id() == id && !matches!(c.value, Credential::Password(_)) @@ -304,6 +504,17 @@ pub(crate) async fn account_set( } if account != old_account { + let mut cache_invalidator = CacheInvalidationBuilder::default(); + if account.encryption_at_rest != old_account.encryption_at_rest + || account.description != old_account.description + || account.locale != old_account.locale + { + cache_invalidator.invalidate(CacheInvalidation::Account(set.account_id)); + } + if account.credentials != old_account.credentials { + cache_invalidator.invalidate(CacheInvalidation::AccessToken(set.account_id)); + } + let object = Object::new(ObjectInner::Account(Account::User(account))); let old_object = Object::with_revision( ObjectInner::Account(Account::User(old_account.clone())), @@ -320,7 +531,10 @@ pub(crate) async fn account_set( }) .await? { - RegistryWriteResult::Success(_) => {} + RegistryWriteResult::Success(_) => { + // Invalidate caches + set.server.invalidate_caches(cache_invalidator).await?; + } err => { let err = map_write_error(err); let failed_create = set @@ -405,8 +619,19 @@ pub(crate) async fn account_get( .collect::>() }; - for credential in account.credentials { - let id = credential.credential_id(); + for mut credential in account.credentials { + let id = match &mut credential { + Credential::Password(credential) => { + credential.allowed_ips.clear(); + credential.credential_id + } + Credential::AppPassword(credential_properties) => { + credential_properties.credential_id + } + Credential::ApiKey(credential_properties) => { + credential_properties.credential_id + } + }; if ids.contains(&id) { get.insert(id, credential.into_value()); ids.retain(|i| i != &id); @@ -422,3 +647,22 @@ pub(crate) async fn account_get( Ok(get) } + +fn validate_credential_permissions( + access_token: &AccessToken, + credential: &SecondaryCredential, +) -> Result<(), SetError> { + match &credential.permissions { + CredentialPermissions::Inherit | CredentialPermissions::Disable(_) => Ok(()), + CredentialPermissions::Replace(permissions) => access_token + .can_grant_permissions( + PermissionsGroup { + enabled: Permissions::from_permission(permissions.permissions.as_slice()), + disabled: Default::default(), + merge: false, + } + .finalize(), + ) + .map_err(build_set_error), + } +} diff --git a/crates/jmap/src/registry/mapping/log.rs b/crates/jmap/src/registry/mapping/log.rs index 6d38242c..8f9dfd4c 100644 --- a/crates/jmap/src/registry/mapping/log.rs +++ b/crates/jmap/src/registry/mapping/log.rs @@ -51,7 +51,7 @@ pub(crate) async fn log_get( .caused_by(trc::location!()) })? .map_err(|err| { - trc::ManageEvent::Error + trc::EventType::Telemetry(trc::TelemetryEvent::LogError) .reason(err) .details("Failed to read log files") .caused_by(trc::location!()) diff --git a/crates/jmap/src/registry/mapping/principal.rs b/crates/jmap/src/registry/mapping/principal.rs index 18470dd3..300ba697 100644 --- a/crates/jmap/src/registry/mapping/principal.rs +++ b/crates/jmap/src/registry/mapping/principal.rs @@ -11,15 +11,16 @@ use common::{ }; use directory::core::secret::hash_secret; use jmap_proto::error::set::SetError; +use registry::schema::structs::TaskStatus; use registry::{ schema::{ enums::{AccountType, Permission, TenantStorageQuota}, prelude::{MASKED_PASSWORD, ObjectType, Property}, - structs::{Account, Credential, Role}, + structs::{Account, Credential, Role, Task, TaskDestroyAccount}, }, types::EnumImpl, }; -use store::registry::RegistryQuery; +use store::{registry::RegistryQuery, write::BatchBuilder}; use trc::AddContext; use types::id::Id; @@ -340,7 +341,60 @@ pub(crate) async fn validate_tenant_quota( Ok(Ok(ObjectResponse::default())) } -fn build_set_error(permissions: Vec) -> SetError { +pub(crate) async fn schedule_account_destruction( + server: &Server, + account_id: Id, + account: &Account, +) -> trc::Result<()> { + // SPDX-SnippetBegin + // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + // SPDX-License-Identifier: LicenseRef-SEL + #[cfg(feature = "enterprise")] + let status = server + .core + .enterprise + .as_ref() + .and_then(|e| e.deleted_accounts_retention.as_ref()) + .map(|retention| TaskStatus::at(store::write::now() as i64 + retention.as_secs() as i64)) + .unwrap_or_else(TaskStatus::now); + // SPDX-SnippetEnd + + #[cfg(not(feature = "enterprise"))] + let status = TaskStatus::now(); + + let account_domain_id; + let account_name; + let account_type; + + match account { + Account::User(account) => { + account_domain_id = account.domain_id; + account_name = account.name.clone(); + account_type = AccountType::User; + } + Account::Group(account) => { + account_domain_id = account.domain_id; + account_name = account.name.clone(); + account_type = AccountType::Group; + } + } + + let mut batch = BatchBuilder::new(); + batch.schedule_task(Task::DestroyAccount(TaskDestroyAccount { + account_domain_id, + account_id, + account_name, + account_type, + status, + })); + + server.store().write(batch.build_all()).await?; + server.notify_task_queue(); + + Ok(()) +} + +pub(crate) fn build_set_error(permissions: Vec) -> SetError { let mut missing_permissions = String::with_capacity(16); let mut total_missing = permissions.len(); for permission in permissions.into_iter().take(5) { diff --git a/crates/jmap/src/registry/mapping/queued_message.rs b/crates/jmap/src/registry/mapping/queued_message.rs index c18c6f18..77191242 100644 --- a/crates/jmap/src/registry/mapping/queued_message.rs +++ b/crates/jmap/src/registry/mapping/queued_message.rs @@ -4,12 +4,19 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::registry::mapping::RegistryGetResponse; -use common::{Server, config::smtp::queue::ArchivedQueueExpiry}; +use crate::registry::mapping::{RegistryGetResponse, RegistrySetResponse}; +use common::{ + Server, + config::smtp::queue::{ArchivedQueueExpiry, QueueName}, + ipc::QueueEvent, +}; +use jmap_proto::error::set::SetError; +use jmap_tools::{JsonPointer, JsonPointerItem, Key}; use registry::{ - jmap::IntoValue, + jmap::{IntoValue, JsonPointerPatch, RegistryJsonPatch}, schema::{ enums::{DeliveryErrorType, MessageFlag, RecipientFlag}, + prelude::ObjectType, structs::{ DeliveryError, QueueExpiry, QueueExpiryAttempts, QueueExpiryTtl, QueuedMessage, QueuedRecipient, RecipientStatus, ServerResponse, @@ -17,19 +24,201 @@ use registry::{ }, types::{datetime::UTCDateTime, ipaddr::IpAddr, list::List, map::Map}, }; -use smtp::queue::{spool::SmtpSpool, *}; +use smtp::queue::{ + self, ArchivedError, ArchivedErrorDetails, ArchivedMessage, ArchivedStatus, ErrorDetails, + FROM_AUTHENTICATED, FROM_AUTOGENERATED, FROM_DSN, FROM_REPORT, FROM_UNAUTHENTICATED, + FROM_UNAUTHENTICATED_DMARC, Message, MessageWrapper, RCPT_DSN_SENT, RCPT_SPAM_PAYLOAD, + Schedule, Status, spool::SmtpSpool, +}; use store::{ IterateParams, U64_LEN, ValueKey, ahash::AHashSet, + registry::RegistryQuery, write::{QueueClass, ValueClass, key::DeserializeBigEndian}, }; use trc::AddContext; use types::{blob::BlobId, blob_hash::BlobHash, id::Id}; use utils::DomainPart; +pub(crate) async fn queued_message_set( + mut set: RegistrySetResponse<'_>, +) -> trc::Result> { + // Fail all create operations + set.fail_all_create("Queued messages cannot be created"); + + // Obtain tenant domains + let tenant_domains = if let Some(tenant_id) = set.access_token.tenant_id() { + Some(tenant_domains(set.server, tenant_id).await?) + } else { + None + }; + + // Process update operations + 'outer: for (id, value) in set.update.drain(..) { + let queue_id = id.id(); + let Some(archive) = set.server.read_message_archive(queue_id).await? else { + set.response.not_updated.append(id, SetError::not_found()); + continue; + }; + let archived_message = archive.to_unarchived::()?; + if !tenant_domains.as_ref().is_none_or(|domains| { + archived_message + .inner + .return_path + .try_domain_part() + .is_some_and(|domain| domains.contains(domain)) + }) { + set.response.not_updated.append(id, SetError::not_found()); + continue; + } + + // Process patches + let mut message = map_message(archived_message.inner); + for (key, value) in value.into_expanded_object() { + let ptr = match key { + Key::Property(prop) => { + JsonPointer::new(vec![JsonPointerItem::Key(Key::Property(prop))]) + } + Key::Borrowed(other) => JsonPointer::parse(other), + Key::Owned(other) => JsonPointer::parse(&other), + }; + if let Err(err) = message.patch(JsonPointerPatch::new(&ptr).with_create(false), value) { + set.response.not_updated.append(id, err.into()); + continue 'outer; + } + } + + // Process changes + let mut has_changes = false; + let mut queued_message = archived_message.deserialize()?; + if queued_message.env_id.as_deref() != message.env_id.as_deref() { + queued_message.env_id = message.env_id.as_deref().map(|v| v.into()); + has_changes = true; + } + if queued_message.priority as i64 != message.priority { + queued_message.priority = message.priority as i16; + has_changes = true; + } + for rcpt in queued_message.recipients.iter_mut() { + if !message + .recipients + .iter() + .any(|r| r.address.as_str() == rcpt.address.as_ref()) + { + rcpt.status = Status::PermanentFailure(ErrorDetails { + entity: "localhost".into(), + details: queue::Error::Io("Delivery canceled.".into()), + }); + has_changes = true; + } + } + for rcpt in message.recipients.into_iter() { + let Some(queued_rcpt) = queued_message + .recipients + .iter_mut() + .find(|r| r.address.as_ref() == rcpt.address.as_str()) + else { + continue; + }; + if rcpt.orcpt.as_deref() != queued_rcpt.orcpt.as_deref() { + queued_rcpt.orcpt = rcpt.orcpt.as_deref().map(|v| v.into()); + has_changes = true; + } + let expiry = match rcpt.expires { + QueueExpiry::Ttl(ttl) => common::config::smtp::queue::QueueExpiry::Ttl( + (ttl.expires_at.timestamp() as u64).saturating_sub(queued_message.created), + ), + QueueExpiry::Attempts(attempts) => { + common::config::smtp::queue::QueueExpiry::Attempts( + attempts.expires_attempts as u32, + ) + } + }; + if expiry != queued_rcpt.expires { + queued_rcpt.expires = expiry; + has_changes = true; + } + + for (due, count, field) in [ + (rcpt.retry_due, rcpt.retry_count, &mut queued_rcpt.retry), + (rcpt.notify_due, rcpt.notify_count, &mut queued_rcpt.notify), + ] { + let schedule = Schedule { + due: due.timestamp() as u64, + inner: count as u32, + }; + if schedule != *field { + *field = schedule; + has_changes = true; + } + } + + if matches!(rcpt.status, RecipientStatus::Scheduled) + && !matches!(queued_rcpt.status, Status::Scheduled) + { + queued_rcpt.status = Status::Scheduled; + has_changes = true; + } + } + + if has_changes { + if !MessageWrapper::new(queued_message, queue_id, QueueName::default()) + .save_changes(set.server, None) + .await + { + set.response.not_updated.append( + id, + SetError::forbidden().with_description("Queue update operation failed"), + ); + continue; + } + + let _ = set + .server + .inner + .ipc + .queue_tx + .send(QueueEvent::Refresh) + .await; + } + + set.response.updated.append(id, None); + } + + // Process destroy operations + for id in set.destroy.drain(..) { + let Some(message) = set.server.read_message(id.id(), QueueName::default()).await else { + set.response.not_destroyed.append(id, SetError::not_found()); + continue; + }; + + if tenant_domains.as_ref().is_none_or(|domains| { + message + .message + .return_path + .try_domain_part() + .is_some_and(|domain| domains.contains(domain)) + }) { + if message.remove(set.server, None).await { + set.response.destroyed.push(id); + } else { + set.response.not_destroyed.append( + id, + SetError::forbidden().with_description("Queue delete operation failed"), + ); + } + } else { + set.response.not_destroyed.append(id, SetError::not_found()); + } + } + + Ok(set) +} + pub(crate) async fn queued_message_get( mut get: RegistryGetResponse<'_>, ) -> trc::Result> { + let client_ids = get.ids.is_some(); let ids = if let Some(ids) = get.ids.take() { ids } else { @@ -40,29 +229,55 @@ pub(crate) async fn queued_message_get( .collect() }; + // Obtain tenant domains + let tenant_domains = if let Some(tenant_id) = get.access_token.tenant_id() { + Some(tenant_domains(get.server, tenant_id).await?) + } else { + None + }; + for id in ids { let Some(message_archive) = get.server.read_message_archive(id.id()).await? else { - get.not_found(id); + if client_ids { + get.not_found(id); + } continue; }; let message_in = message_archive.unarchive::()?; - if get.access_token.tenant_id().is_some() { - if let Some(domain) = message_in.return_path.try_domain_part() - && let Some(domain) = get.server.domain(domain).await? - && domain.id_tenant == get.access_token.tenant_id() - { - get.insert(id, map_message(message_in).into_value()); - } else { - get.not_found(id); - } - } else { + if tenant_domains.as_ref().is_none_or(|domains| { + message_in + .return_path + .try_domain_part() + .is_some_and(|domain| domains.contains(domain)) + }) { get.insert(id, map_message(message_in).into_value()); + } else if client_ids { + get.not_found(id); } } Ok(get) } +async fn tenant_domains(server: &Server, tenant_id: u32) -> trc::Result> { + let domain_ids = server + .registry() + .query::>( + RegistryQuery::new(ObjectType::Domain).with_tenant(tenant_id.into()), + ) + .await?; + + let mut domains = AHashSet::with_capacity(domain_ids.len()); + + for domain_id in domain_ids { + if let Some(domain) = server.domain_by_id(domain_id as u32).await? { + domains.extend(domain.names.iter().map(|name| name.to_string())); + } + } + + Ok(domains) +} + fn map_message(message_in: &ArchivedMessage) -> QueuedMessage { let mut message_out = QueuedMessage { blob_id: BlobId::new(BlobHash::from(&message_in.blob_hash), Default::default()), @@ -101,7 +316,9 @@ fn map_message(message_in: &ArchivedMessage) -> QueuedMessage { address: rcpt_in.address.to_string(), expires: match &rcpt_in.expires { ArchivedQueueExpiry::Ttl(ttl) => QueueExpiry::Ttl(QueueExpiryTtl { - expires_at: UTCDateTime::from_timestamp(ttl.to_native() as i64), + expires_at: UTCDateTime::from_timestamp( + message_in.created.to_native() as i64 + ttl.to_native() as i64, + ), }), ArchivedQueueExpiry::Attempts(attempts) => { QueueExpiry::Attempts(QueueExpiryAttempts { diff --git a/crates/jmap/src/registry/mapping/task.rs b/crates/jmap/src/registry/mapping/task.rs index eb9519bf..6aebf412 100644 --- a/crates/jmap/src/registry/mapping/task.rs +++ b/crates/jmap/src/registry/mapping/task.rs @@ -4,16 +4,310 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::registry::mapping::RegistryGetResponse; +use crate::registry::mapping::{RegistryGetResponse, RegistrySetResponse}; use common::Server; -use registry::{jmap::IntoValue, schema::prelude::Object, types::EnumImpl}; +use jmap_proto::error::set::{SetError, SetErrorType}; +use jmap_tools::{JsonPointer, JsonPointerItem, Key}; +use registry::{ + jmap::{IntoValue, JsonPointerPatch, RegistryJsonPatch}, + pickle::Pickle, + schema::{enums::TaskType, prelude::Object, structs::Task}, + types::{ + EnumImpl, ObjectImpl, + index::{IndexBuilder, IndexKey}, + }, +}; +use services::task_manager::lock::TaskLockManager; use store::{ - IterateParams, U64_LEN, ValueKey, - write::{RegistryClass, TaskQueueClass, ValueClass, key::DeserializeBigEndian}, + IterateParams, SerializeInfallible, U64_LEN, ValueKey, + write::{BatchBuilder, RegistryClass, TaskQueueClass, ValueClass, key::DeserializeBigEndian}, }; use trc::AddContext; use types::id::Id; +pub(crate) async fn task_set( + mut set: RegistrySetResponse<'_>, +) -> trc::Result> { + let mut batch = BatchBuilder::new(); + let mut locked_tasks = Vec::new(); + + // Process creations + 'outer: for (id, value) in set.create.drain() { + let mut task = Task::default(); + for (key, value) in value.into_expanded_object() { + let Key::Property(prop) = key else { + set.response.not_created.append( + id, + SetError::invalid_properties().with_property(key.into_owned()), + ); + continue 'outer; + }; + let ptr = JsonPointer::new(vec![JsonPointerItem::Key(Key::Property(prop))]); + if let Err(err) = task.patch(JsonPointerPatch::new(&ptr).with_create(false), value) { + set.response.not_created.append(id, err.into()); + continue 'outer; + } + } + + let mut validation_errors = Vec::new(); + if !task.validate(&mut validation_errors) { + set.response.not_created.append( + id, + SetError::new(SetErrorType::ValidationFailed) + .with_validation_errors(validation_errors), + ); + continue 'outer; + } + + let task_type = task.object_type(); + match task_type { + TaskType::IndexDocument + | TaskType::UnindexDocument + | TaskType::IndexTrace + | TaskType::AccountMaintenance + | TaskType::StoreMaintenance + | TaskType::SpamFilterMaintenance => { + let mut index = IndexBuilder::default(); + task.index(&mut index); + + // Validate foreign keys + for key in index.keys { + if let IndexKey::ForeignKey { + object_id: foreign_id, + .. + } = key + && set + .server + .store() + .get_value::<()>(ValueKey::from(ValueClass::Registry( + RegistryClass::IndexId { + object_id: foreign_id.object().to_id(), + item_id: foreign_id.id().id(), + }, + ))) + .await + .caused_by(trc::location!())? + .is_none() + { + set.response.not_created.append( + id, + SetError::new(SetErrorType::InvalidForeignKey) + .with_object_id(foreign_id), + ); + continue 'outer; + } + } + + let task_id = set.server.registry().assign_id(); + batch.schedule_task_with_id(task_id, task).commit_point(); + set.response.created(id, task_id); + } + TaskType::CalendarAlarmEmail + | TaskType::CalendarAlarmNotification + | TaskType::CalendarItipMessage + | TaskType::MergeThreads + | TaskType::DmarcReport + | TaskType::TlsReport + | TaskType::DestroyAccount + | TaskType::RestoreArchivedItem => { + set.response.not_created.append( + id, + SetError::forbidden().with_description(format!( + "{} is an internal task type that cannot be created by clients", + task_type.as_str() + )), + ); + } + } + } + + // Process updates + 'outer: for (id, value) in set.update.drain(..) { + let task_id = id.id(); + let Some(mut task) = set + .server + .store() + .get_value::(ValueKey::from(ValueClass::TaskQueue( + TaskQueueClass::Task { id: task_id }, + ))) + .await? + else { + set.response.not_updated.append(id, SetError::not_found()); + continue; + }; + + if !set.server.try_lock_task(task_id).await { + set.response.not_updated.append( + id, + SetError::forbidden().with_description( + "Task is currently being processed and cannot be updated".to_string(), + ), + ); + continue; + } + locked_tasks.push(task_id); + + let old_timestamp = task.due_timestamp(); + let old_status = task.status().clone(); + for (key, value) in value.into_expanded_object() { + let ptr = match key { + Key::Property(prop) => { + JsonPointer::new(vec![JsonPointerItem::Key(Key::Property(prop))]) + } + Key::Borrowed(other) => JsonPointer::parse(other), + Key::Owned(other) => JsonPointer::parse(&other), + }; + + if let Err(err) = task.patch(JsonPointerPatch::new(&ptr).with_create(false), value) { + set.response.not_updated.append(id, err.into()); + continue 'outer; + } + } + + if task.status() != &old_status { + let timestamp = task.due_timestamp(); + if timestamp != old_timestamp { + batch + .clear(ValueClass::TaskQueue(TaskQueueClass::Due { + id: task_id, + due: old_timestamp, + })) + .set( + ValueClass::TaskQueue(TaskQueueClass::Due { + id: task_id, + due: timestamp, + }), + task.object_type().to_id().serialize(), + ); + } + + batch + .set( + ValueClass::TaskQueue(TaskQueueClass::Task { id: task_id }), + task.to_pickled_vec(), + ) + .commit_point(); + } + } + + // Process destructions + for id in set.destroy.drain(..) { + let task_id = id.id(); + let Some(task) = set + .server + .store() + .get_value::(ValueKey::from(ValueClass::TaskQueue( + TaskQueueClass::Task { id: task_id }, + ))) + .await? + else { + set.response.not_destroyed.append(id, SetError::not_found()); + continue; + }; + + if !set.server.try_lock_task(task_id).await { + set.response.not_destroyed.append( + id, + SetError::forbidden().with_description( + "Task is currently being processed and cannot be destroyed".to_string(), + ), + ); + continue; + } + locked_tasks.push(task_id); + + let due = task.due_timestamp(); + + // SPDX-SnippetBegin + // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + // SPDX-License-Identifier: LicenseRef-SEL + #[cfg(feature = "enterprise")] + if let Task::DestroyAccount(task) = task { + use crate::registry::set::map_write_error; + use registry::schema::{ + enums::AccountType, + structs::{Account, GroupAccount, UserAccount}, + }; + use store::registry::write::{RegistryWrite, RegistryWriteResult}; + + if !set.server.is_enterprise_edition() { + set.response.not_destroyed.append( + id, + SetError::forbidden().with_description( + "Account recovery is not supported in this deployment".to_string(), + ), + ); + continue; + } + + let object = match task.account_type { + AccountType::User => Account::User(UserAccount { + name: task.account_name, + domain_id: task.account_domain_id, + ..Default::default() + }), + AccountType::Group => Account::Group(GroupAccount { + name: task.account_name, + domain_id: task.account_domain_id, + ..Default::default() + }), + } + .into(); + + match set + .server + .registry() + .write(RegistryWrite::insert_with_id(task.account_id, &object)) + .await? + { + RegistryWriteResult::Success(_) => {} + err => { + set.response.not_destroyed.append( + id, + map_write_error(err).with_description("Account recovery failed"), + ); + continue; + } + } + } + // SPDX-SnippetEnd + + #[cfg(not(feature = "enterprise"))] + { + set.response.not_destroyed.append( + id, + SetError::forbidden().with_description( + "Account recovery is not supported in this deployment".to_string(), + ), + ); + continue; + } + + batch + .clear(ValueClass::TaskQueue(TaskQueueClass::Task { id: task_id })) + .clear(ValueClass::TaskQueue(TaskQueueClass::Due { + id: task_id, + due, + })) + .commit_point(); + } + + if !batch.is_empty() { + set.server + .store() + .write(batch.build_all()) + .await + .caused_by(trc::location!())?; + set.server.notify_task_queue(); + } + + for task_id in locked_tasks { + set.server.remove_index_lock(task_id).await; + } + + Ok(set) +} + pub(crate) async fn task_get( mut get: RegistryGetResponse<'_>, ) -> trc::Result> { @@ -22,16 +316,14 @@ pub(crate) async fn task_get( } else { task_ids(get.server, get.server.core.jmap.get_max_objects).await? }; - let object_id = get.object_type.to_id(); for id in ids { if let Some(task) = get .server .store() - .get_value::(ValueKey::from(ValueClass::Registry(RegistryClass::Item { - object_id, - item_id: id.id(), - }))) + .get_value::(ValueKey::from(ValueClass::TaskQueue( + TaskQueueClass::Task { id: id.id() }, + ))) .await? { get.insert(id, task.into_value()); diff --git a/crates/jmap/src/registry/set.rs b/crates/jmap/src/registry/set.rs index 003e31fa..b8fcfc63 100644 --- a/crates/jmap/src/registry/set.rs +++ b/crates/jmap/src/registry/set.rs @@ -8,14 +8,20 @@ use std::borrow::Cow; use crate::registry::mapping::{ ObjectResponse, RegistrySetResponse, + account::account_set, archived_item::archived_item_set, masked_email::validate_masked_email, - principal::{validate_account, validate_role, validate_tenant_quota}, + principal::{ + schedule_account_destruction, validate_account, validate_role, validate_tenant_quota, + }, public_key::validate_public_key, + queued_message::queued_message_set, report::report_set, spam_sample::spam_sample_set, + task::task_set, }; use common::{Server, auth::AccessToken, cache::invalidate::CacheInvalidationBuilder}; +use http_proto::HttpSessionData; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::set::{SetRequest, SetResponse}, @@ -46,6 +52,7 @@ pub trait RegistrySet: Sync + Send { object_type: ObjectType, request: SetRequest<'_, Registry>, access_token: &AccessToken, + session: &HttpSessionData, ) -> impl Future>> + Send; } @@ -61,6 +68,7 @@ impl RegistrySet for Server { object_type: ObjectType, mut request: SetRequest<'_, Registry>, access_token: &AccessToken, + session: &HttpSessionData, ) -> trc::Result> { let object_flags = object_type.flags(); let is_singleton = (object_flags & OBJ_SINGLETON) != 0; @@ -113,6 +121,7 @@ impl RegistrySet for Server { let mut set = RegistrySetResponse { access_token, server: self, + remote_ip: session.remote_ip, account_id: request.account_id.document_id(), object_type, response, @@ -481,10 +490,16 @@ impl RegistrySet for Server { .write(RegistryWrite::Delete { object_id, object: Some(&object), + force: object_type == ObjectType::Account, // Force delete accounts to allow recovery, but not other objects }) .await? { RegistryWriteResult::Success(_) => { + // Schedule account deletion + if let ObjectInner::Account(account) = &object.inner { + schedule_account_destruction(set.server, id, account).await?; + } + cache_invalidator.process_delete(id, &object); set.response.destroyed.push(id); } @@ -514,19 +529,21 @@ impl RegistrySet for Server { spam_sample_set(set).await.map(|set| set.into_response()) } - ObjectType::AccountSettings => { - todo!() - } - ObjectType::Credential => { - todo!() + ObjectType::AccountSettings | ObjectType::Credential => { + account_set(set).await.map(|set| set.into_response()) } ObjectType::QueuedMessage => { + queued_message_set(set).await.map(|set| set.into_response()) + } + + ObjectType::Task => task_set(set).await.map(|set| set.into_response()), + + ObjectType::Action => { + let todo = "actions"; todo!() } - ObjectType::Task => { - todo!() - } + ObjectType::Log | ObjectType::Metric | ObjectType::Trace => { set.fail_all_create("Telemetry objects cannot be created"); set.fail_all_update("Telemetry objects cannot be modified"); @@ -535,8 +552,7 @@ impl RegistrySet for Server { } } - // Schedule account and tenant deletions - + // locks for expensive tasks should be longer or renewed // management objects for actions (reload, etc)"; // DkimSignature = Generate keys + Enforce count? // PublicKey = Validate PK? Store decoded? Enforce count? Update ingest diff --git a/crates/registry/src/types/index.rs b/crates/registry/src/types/index.rs index cc2c8813..3858ff9d 100644 --- a/crates/registry/src/types/index.rs +++ b/crates/registry/src/types/index.rs @@ -129,6 +129,10 @@ impl<'x> IndexBuilder<'x> { }); } } + + pub fn is_empty(&self) -> bool { + self.keys.is_empty() + } } impl From for IndexValue<'_> { diff --git a/crates/registry/src/types/map.rs b/crates/registry/src/types/map.rs index 1bb8de4a..19fbf5f7 100644 --- a/crates/registry/src/types/map.rs +++ b/crates/registry/src/types/map.rs @@ -67,6 +67,10 @@ impl Map { pub fn push_unchecked(&mut self, item: T) { self.0.push(item); } + + pub fn clear(&mut self) { + self.0.clear(); + } } impl Pickle for Map diff --git a/crates/registry/src/utils/task.rs b/crates/registry/src/utils/task.rs index d6f809b8..1957d8f3 100644 --- a/crates/registry/src/utils/task.rs +++ b/crates/registry/src/utils/task.rs @@ -19,6 +19,10 @@ impl Task { Task::DmarcReport(task) => task.status = status, Task::TlsReport(task) => task.status = status, Task::RestoreArchivedItem(task) => task.status = status, + Task::DestroyAccount(task) => task.status = status, + Task::AccountMaintenance(task) => task.status = status, + Task::StoreMaintenance(task) => task.status = status, + Task::SpamFilterMaintenance(task) => task.status = status, } } @@ -34,6 +38,10 @@ impl Task { Task::DmarcReport(task) => &task.status, Task::TlsReport(task) => &task.status, Task::RestoreArchivedItem(task) => &task.status, + Task::DestroyAccount(task) => &task.status, + Task::AccountMaintenance(task) => &task.status, + Task::StoreMaintenance(task) => &task.status, + Task::SpamFilterMaintenance(task) => &task.status, } } diff --git a/crates/services/src/broadcast/subscriber.rs b/crates/services/src/broadcast/subscriber.rs index f78c2db6..b6438a40 100644 --- a/crates/services/src/broadcast/subscriber.rs +++ b/crates/services/src/broadcast/subscriber.rs @@ -7,7 +7,7 @@ use crate::broadcast::{BROADCAST_TOPIC, BroadcastBatch}; use common::{ BuildServer, Inner, - ipc::{BroadcastEvent, HousekeeperEvent, PushEvent, PushNotification, RegistryChange}, + ipc::{BroadcastEvent, PushEvent, PushNotification, RegistryChange}, }; use registry::types::EnumImpl; use std::{sync::Arc, time::Duration}; @@ -146,20 +146,6 @@ pub fn spawn_broadcast_subscriber(inner: Arc, mut shutdown_rx: watch::Rec if let Some(new_core) = result.new_core { // Update core inner.shared_core.store(new_core.into()); - - if inner - .ipc - .housekeeper_tx - .send(HousekeeperEvent::ReloadSettings) - .await - .is_err() - { - trc::event!( - Server(trc::ServerEvent::ThreadError), - Details = "Failed to send setting reload event to housekeeper", - CausedBy = trc::location!(), - ); - } } } Err(err) => { diff --git a/crates/services/src/housekeeper/mod.rs b/crates/services/src/housekeeper/mod.rs deleted file mode 100644 index 2748e707..00000000 --- a/crates/services/src/housekeeper/mod.rs +++ /dev/null @@ -1,897 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use common::{ - BuildServer, Inner, KV_LOCK_HOUSEKEEPER, LONG_1D_SLUMBER, Server, - config::{mailstore::spamfilter, telemetry::OtelMetrics}, - ipc::{BroadcastEvent, HousekeeperEvent, PurgeType}, -}; -use email::message::delete::EmailDeletion; -use smtp::reporting::send::MtaReportSend; -use spam_filter::modules::classifier::SpamClassifier; -use std::{ - collections::BinaryHeap, - future::Future, - sync::Arc, - time::{Duration, Instant, SystemTime}, -}; -use store::write::now; -use tokio::sync::mpsc; -use trc::{Collector, MetricType, PurgeEvent}; - -// SPDX-SnippetBegin -// SPDX-FileCopyrightText: 2020 Stalwart Labs LLC -// SPDX-License-Identifier: LicenseRef-SEL -#[cfg(feature = "enterprise")] -use common::telemetry::{ - metrics::store::{MetricsStore, SharedMetricHistory}, - tracers::store::TracingStore, -}; -// SPDX-SnippetEnd - -#[derive(PartialEq, Eq)] -struct Action { - due: Instant, - event: ActionClass, -} - -#[derive(PartialEq, Eq, Debug)] -enum ActionClass { - PurgeAccount, - PurgeDataStore, - PurgeBlobStore, - Acme(String), - OtelMetrics, - CalculateMetrics, - // SPDX-SnippetBegin - // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - // SPDX-License-Identifier: LicenseRef-SEL - #[cfg(feature = "enterprise")] - InternalMetrics, - #[cfg(feature = "enterprise")] - AlertMetrics, - #[cfg(feature = "enterprise")] - RenewLicense, - // SPDX-SnippetEnd - TrainSpamClassifier, -} - -#[derive(Default)] -struct Queue { - heap: BinaryHeap, -} - -// SPDX-SnippetBegin -// SPDX-FileCopyrightText: 2020 Stalwart Labs LLC -// SPDX-License-Identifier: LicenseRef-SEL -#[cfg(feature = "enterprise")] -const METRIC_ALERTS_INTERVAL: Duration = Duration::from_secs(5 * 60); -// SPDX-SnippetEnd - -pub fn spawn_housekeeper(inner: Arc, mut rx: mpsc::Receiver) { - tokio::spawn(async move { - trc::event!(Housekeeper(trc::HousekeeperEvent::Start)); - let start_time = SystemTime::now(); - - // Add all events to queue - let mut queue = Queue::default(); - { - let server = inner.build_server(); - let roles = &server.core.network.roles; - - // Account purge - if roles.purge_accounts.is_enabled_or_sharded() { - queue.schedule( - Instant::now() + server.core.email.account_purge_frequency.time_to_next(), - ActionClass::PurgeAccount, - ); - } - - // Store purges - if roles.purge_stores.is_enabled_or_sharded() { - queue.schedule( - Instant::now() + server.core.email.data_purge_frequency.time_to_next(), - ActionClass::PurgeDataStore, - ); - queue.schedule( - Instant::now() + server.core.email.blob_purge_frequency.time_to_next(), - ActionClass::PurgeBlobStore, - ); - } - - // Spam classifier training - if roles.spam_training.is_enabled_or_sharded() - && let Some(train_frequency) = server - .core - .spam - .classifier - .as_ref() - .and_then(|c| c.train_frequency) - { - let next_train = match server.inner.data.spam_classifier.load().as_ref() { - spamfilter::SpamClassifier::FhClassifier { - last_trained_at, .. - } - | spamfilter::SpamClassifier::CcfhClassifier { - last_trained_at, .. - } => now().saturating_sub(*last_trained_at).min(train_frequency), - spamfilter::SpamClassifier::Disabled => train_frequency, - }; - - queue.schedule( - Instant::now() + Duration::from_secs(next_train), - ActionClass::TrainSpamClassifier, - ); - } - - // OTEL Push Metrics - if roles.push_metrics.is_enabled_or_sharded() - && let Some(otel) = &server.core.metrics.otel - { - OtelMetrics::enable_errors(); - queue.schedule(Instant::now() + otel.interval, ActionClass::OtelMetrics); - } - - // Calculate expensive metrics - queue.schedule(Instant::now(), ActionClass::CalculateMetrics); - - // Add all ACME renewals to heap - let todo = "fix"; - /*for provider in server.core.acme.providers.values() { - if roles.renew_acme.is_enabled_for_hash(&provider.id) { - match server.init_acme(provider).await { - Ok(renew_at) => { - queue.schedule( - Instant::now() + renew_at, - ActionClass::Acme(provider.id.clone()), - ); - } - Err(err) => { - trc::error!( - err.details("Failed to initialize ACME certificate manager.") - ); - } - }; - } - }*/ - - // SPDX-SnippetBegin - // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - // SPDX-License-Identifier: LicenseRef-SEL - - // Enterprise Edition license management - #[cfg(feature = "enterprise")] - if let Some(enterprise) = &server.core.enterprise { - queue.schedule( - Instant::now() + enterprise.license.renew_in(), - ActionClass::RenewLicense, - ); - - if server.core.storage.metrics.is_active() { - queue.schedule( - Instant::now() + enterprise.metrics_interval.time_to_next(), - ActionClass::InternalMetrics, - ); - } - - if !enterprise.metrics_alerts.is_empty() { - queue.schedule( - Instant::now() + METRIC_ALERTS_INTERVAL, - ActionClass::AlertMetrics, - ); - } - } - - // SPDX-SnippetEnd - } - - // SPDX-SnippetBegin - // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - // SPDX-License-Identifier: LicenseRef-SEL - // Metrics history - #[cfg(feature = "enterprise")] - let metrics_history = SharedMetricHistory::default(); - // SPDX-SnippetEnd - - let mut next_metric_update = Instant::now(); - - loop { - match tokio::time::timeout(queue.wake_up_time(), rx.recv()).await { - Ok(Some(event)) => { - match event { - HousekeeperEvent::ReloadSettings => { - let server = inner.build_server(); - - // Reload OTEL push metrics - match &server.core.metrics.otel { - Some(otel) if !queue.has_action(&ActionClass::OtelMetrics) => { - OtelMetrics::enable_errors(); - - queue.schedule( - Instant::now() + otel.interval, - ActionClass::OtelMetrics, - ); - } - _ => {} - } - - // SPDX-SnippetBegin - // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - // SPDX-License-Identifier: LicenseRef-SEL - #[cfg(feature = "enterprise")] - if let Some(enterprise) = &server.core.enterprise { - if !queue.has_action(&ActionClass::RenewLicense) { - queue.schedule( - Instant::now() + enterprise.license.renew_in(), - ActionClass::RenewLicense, - ); - } - - if server.core.storage.metrics.is_active() - && !queue.has_action(&ActionClass::InternalMetrics) - { - queue.schedule( - Instant::now() + enterprise.metrics_interval.time_to_next(), - ActionClass::InternalMetrics, - ); - } - - if !enterprise.metrics_alerts.is_empty() - && !queue.has_action(&ActionClass::AlertMetrics) - { - queue.schedule(Instant::now(), ActionClass::AlertMetrics); - } - } - // SPDX-SnippetEnd - - // Reload queue settings - server - .inner - .ipc - .queue_tx - .send(common::ipc::QueueEvent::ReloadSettings) - .await - .ok(); - - // Reload ACME certificates - let todo = "fix"; - /*tokio::spawn(async move { - for provider in server.core.acme.providers.values() { - match server.init_acme(provider).await { - Ok(renew_at) => { - server - .inner - .ipc - .housekeeper_tx - .send(HousekeeperEvent::AcmeReschedule { - provider_id: provider.id.clone(), - renew_at: Instant::now() + renew_at, - }) - .await - .ok(); - } - Err(err) => { - trc::error!(err.details( - "Failed to reload ACME certificate manager." - )); - } - }; - } - });*/ - } - HousekeeperEvent::AcmeReschedule { - provider_id, - renew_at, - } => { - let action = ActionClass::Acme(provider_id); - queue.remove_action(&action); - queue.schedule(renew_at, action); - } - HousekeeperEvent::Purge(purge) => { - let server = inner.build_server(); - tokio::spawn(async move { - server.purge(purge).await; - }); - } - HousekeeperEvent::Exit => { - trc::event!( - Housekeeper(trc::HousekeeperEvent::Stop), - Reason = "Shutdown" - ); - - return; - } - } - } - Ok(None) => { - trc::event!( - Housekeeper(trc::HousekeeperEvent::Stop), - Reason = "Channel closed" - ); - return; - } - Err(_) => { - let server = inner.build_server(); - while let Some(event) = queue.pop() { - match event.event { - ActionClass::Acme(provider_id) => { - trc::event!(Housekeeper(trc::HousekeeperEvent::Run), Type = "acme"); - - let todo = "fix"; - /*let server = server.clone(); - tokio::spawn(async move { - if let Some(provider) = - server.core.acme.providers.get(&provider_id) - { - trc::event!( - Acme(trc::AcmeEvent::OrderStart), - Hostname = provider.domains.as_slice() - ); - - let renew_at = match server.renew(provider).await { - Ok(renew_at) => { - trc::event!( - Acme(trc::AcmeEvent::OrderCompleted), - Domain = provider.domains.as_slice(), - Expires = trc::Value::Timestamp( - now() + renew_at.as_secs() - ) - ); - - renew_at - } - Err(err) => { - trc::error!( - err.details("Failed to renew certificates.") - ); - - Duration::from_secs(3600) - } - }; - - server - .cluster_broadcast(BroadcastEvent::ReloadSettings) - .await; - - server - .inner - .ipc - .housekeeper_tx - .send(HousekeeperEvent::AcmeReschedule { - provider_id: provider_id.clone(), - renew_at: Instant::now() + renew_at, - }) - .await - .ok(); - } - });*/ - } - ActionClass::PurgeAccount => { - trc::event!( - Housekeeper(trc::HousekeeperEvent::Run), - Type = "purge_account" - ); - - let server = server.clone(); - queue.schedule( - Instant::now() - + server.core.email.account_purge_frequency.time_to_next(), - ActionClass::PurgeAccount, - ); - tokio::spawn(async move { - server - .purge(PurgeType::Account { - account_id: None, - use_roles: true, - }) - .await; - }); - } - ActionClass::PurgeDataStore => { - trc::event!( - Housekeeper(trc::HousekeeperEvent::Run), - Type = "purge_data_store" - ); - let todo = "make sure all store types are purged, in memory, metrics, tracing, etc"; - let todo = "make sure spam samples with their indexes and undelete items are purged as well"; - - queue.schedule( - Instant::now() - + server.core.email.data_purge_frequency.time_to_next(), - ActionClass::PurgeDataStore, - ); - let server_ = server.clone(); - tokio::spawn(async move { - server_.purge(PurgeType::Data).await; - }); - - let server = server.clone(); - tokio::spawn(async move { - server.purge(PurgeType::Lookup { prefix: None }).await; - }); - } - ActionClass::PurgeBlobStore => { - trc::event!( - Housekeeper(trc::HousekeeperEvent::Run), - Type = "purge_blob_store" - ); - - queue.schedule( - Instant::now() - + server.core.email.blob_purge_frequency.time_to_next(), - ActionClass::PurgeBlobStore, - ); - let server = server.clone(); - tokio::spawn(async move { - server.purge(PurgeType::Blob).await; - }); - } - ActionClass::OtelMetrics => { - if let Some(otel) = &server.core.metrics.otel { - trc::event!( - Housekeeper(trc::HousekeeperEvent::Run), - Type = "metrics_report" - ); - - queue.schedule( - Instant::now() + otel.interval, - ActionClass::OtelMetrics, - ); - - let otel = otel.clone(); - - // SPDX-SnippetBegin - // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - // SPDX-License-Identifier: LicenseRef-SEL - #[cfg(feature = "enterprise")] - let is_enterprise = server.is_enterprise_edition(); - // SPDX-SnippetEnd - - #[cfg(not(feature = "enterprise"))] - let is_enterprise = false; - - tokio::spawn(async move { - otel.push_metrics(is_enterprise, start_time).await; - }); - } - } - ActionClass::CalculateMetrics => { - trc::event!( - Housekeeper(trc::HousekeeperEvent::Run), - Type = "metrics_calculate" - ); - - // Calculate expensive metrics every 5 minutes - queue.schedule( - Instant::now() + Duration::from_secs(5 * 60), - ActionClass::CalculateMetrics, - ); - - let update_other_metrics = if Instant::now() >= next_metric_update { - next_metric_update = - Instant::now() + Duration::from_secs(86400); - true - } else { - false - }; - - let server = server.clone(); - tokio::spawn(async move { - if server - .core - .network - .roles - .calculate_metrics - .is_enabled_or_sharded() - { - // SPDX-SnippetBegin - // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - // SPDX-License-Identifier: LicenseRef-SEL - #[cfg(feature = "enterprise")] - if server.is_enterprise_edition() { - // Obtain queue size - match server.total_queued_messages().await { - Ok(total) => { - Collector::update_gauge( - MetricType::QueueCount, - total, - ); - } - Err(err) => { - trc::error!( - err.details("Failed to obtain queue size") - ); - } - } - } - // SPDX-SnippetEnd - - if update_other_metrics { - match server.total_accounts().await { - Ok(total) => { - Collector::update_gauge( - MetricType::UserCount, - total as u64, - ); - } - Err(err) => { - trc::error!( - err.details( - "Failed to obtain account count" - ) - ); - } - } - - match server.total_domains().await { - Ok(total) => { - Collector::update_gauge( - MetricType::DomainCount, - total as u64, - ); - } - Err(err) => { - trc::error!( - err.details( - "Failed to obtain domain count" - ) - ); - } - } - } - } - - match tokio::task::spawn_blocking(memory_stats::memory_stats) - .await - { - Ok(Some(stats)) => { - Collector::update_gauge( - MetricType::ServerMemory, - stats.physical_mem as u64, - ); - } - Ok(None) => {} - Err(err) => { - trc::error!( - trc::EventType::Server( - trc::ServerEvent::ThreadError, - ) - .reason(err) - .caused_by(trc::location!()) - .details("Join Error") - ); - } - } - }); - } - ActionClass::TrainSpamClassifier => { - if server - .core - .network - .roles - .spam_training - .is_enabled_or_sharded() - && let Some(train_frequency) = server - .core - .spam - .classifier - .as_ref() - .and_then(|c| c.train_frequency) - { - trc::event!( - Housekeeper(trc::HousekeeperEvent::Run), - Type = "spam_classifier_train" - ); - - // Schedule next training - queue.schedule( - Instant::now() + Duration::from_secs(train_frequency), - ActionClass::TrainSpamClassifier, - ); - - let server = server.clone(); - tokio::spawn(async move { - if let Err(err) = server.spam_train(false).await { - trc::error!( - err.details("Failed to train spam classifier") - ); - } - }); - } - } - - // SPDX-SnippetBegin - // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - // SPDX-License-Identifier: LicenseRef-SEL - #[cfg(feature = "enterprise")] - ActionClass::InternalMetrics => { - if server.core.storage.metrics.is_active() { - trc::event!( - Housekeeper(trc::HousekeeperEvent::Run), - Type = "metrics_internal" - ); - - queue.schedule( - Instant::now() - + server - .core - .enterprise - .as_ref() - .unwrap() - .metrics_interval - .time_to_next(), - ActionClass::InternalMetrics, - ); - - let metrics_store = server.core.storage.metrics.clone(); - let metrics_history = metrics_history.clone(); - tokio::spawn(async move { - if let Err(err) = - metrics_store.write_metrics(None, metrics_history).await - { - trc::error!(err.details("Failed to write metrics")); - } - }); - } - } - - #[cfg(feature = "enterprise")] - ActionClass::AlertMetrics => { - trc::event!( - Housekeeper(trc::HousekeeperEvent::Run), - Type = "metrics_alert" - ); - - let server = server.clone(); - - tokio::spawn(async move { - if let Some(messages) = server.process_alerts().await { - for message in messages { - server - .send_autogenerated( - message.from, - message.to.into_iter(), - message.body, - None, - 0, - ) - .await; - } - } - }); - } - - #[cfg(feature = "enterprise")] - ActionClass::RenewLicense => { - use common::ipc::RegistryChange; - use registry::schema::prelude::ObjectType; - - trc::event!( - Housekeeper(trc::HousekeeperEvent::Run), - Type = "renew_license" - ); - - match server - .reload_registry(RegistryChange::Reload(ObjectType::Enterprise)) - .await - { - Ok(result) => { - if let Some(new_core) = result.new_core { - if let Some(enterprise) = &new_core.enterprise { - let renew_in = - if enterprise.license.is_near_expiration() { - // Something went wrong during renewal, try again in 1 day or 1 hour, - // depending on the time left on the license - if enterprise.license.expires_in() - < Duration::from_secs(86400) - { - Duration::from_secs(3600) - } else { - Duration::from_secs(86400) - } - } else { - enterprise.license.renew_in() - }; - - queue.schedule( - Instant::now() + renew_in, - ActionClass::RenewLicense, - ); - } - - // Update core - server.inner.shared_core.store(new_core.into()); - - server - .cluster_broadcast(BroadcastEvent::reload( - ObjectType::Enterprise, - )) - .await; - } - } - Err(err) => { - trc::error!(err.details("Failed to reload configuration.")); - } - } - } // SPDX-SnippetEnd - } - } - } - } - } - }); -} - -pub trait Purge: Sync + Send { - fn purge(&self, purge: PurgeType) -> impl Future + Send; -} - -impl Purge for Server { - async fn purge(&self, purge: PurgeType) { - // Lock task - let (lock_type, lock_name) = match &purge { - PurgeType::Data => ("data", [0u8].into()), - PurgeType::Blob => ("blob", [1u8].into()), - PurgeType::Lookup { prefix: None } => ("in-memory", [2u8].into()), - PurgeType::Lookup { .. } => ("in-memory-prefix", None), - PurgeType::Account { .. } => ("account", None), - }; - if let Some(lock_name) = &lock_name { - match self - .in_memory_store() - .try_lock(KV_LOCK_HOUSEKEEPER, lock_name, 3600) - .await - { - Ok(true) => (), - Ok(false) => { - trc::event!(Purge(PurgeEvent::InProgress), Details = lock_type); - return; - } - Err(err) => { - trc::error!(err.details("Failed to lock task.").details(lock_type)); - return; - } - } - } - - trc::event!(Purge(PurgeEvent::Started), Type = lock_type); - let time = Instant::now(); - - match purge { - PurgeType::Data => { - if let Err(err) = self.store().purge_store().await { - trc::error!(err.details("Failed to purge data store")); - } - - // SPDX-SnippetBegin - // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - // SPDX-License-Identifier: LicenseRef-SEL - #[cfg(feature = "enterprise")] - if let Some(trace_retention) = self - .core - .enterprise - .as_ref() - .and_then(|e| e.trace_retention) - && self.tracing_store().is_active() - && let Err(err) = self - .tracing_store() - .purge_spans(trace_retention, self.search_store().into()) - .await - { - trc::error!(err.details("Failed to purge tracing spans")); - } - - #[cfg(feature = "enterprise")] - if let Some(metrics_retention) = self - .core - .enterprise - .as_ref() - .and_then(|e| e.metrics_retention) - && self.metrics_store().is_active() - && let Err(err) = self.metrics_store().purge_metrics(metrics_retention).await - { - trc::error!(err.details("Failed to purge metrics")); - } - // SPDX-SnippetEnd - } - PurgeType::Blob => { - if let Err(err) = self.store().purge_blobs(self.blob_store().clone()).await { - trc::error!(err.details("Failed to purge blob store")); - } - } - PurgeType::Lookup { prefix } => { - if let Some(prefix) = prefix { - if let Err(err) = self.in_memory_store().key_delete_prefix(&prefix).await { - trc::error!( - err.details("Failed to delete key prefix") - .ctx(trc::Key::Key, prefix) - ); - } - } else if let Err(err) = self.in_memory_store().purge_in_memory_store().await { - trc::error!(err.details("Failed to purge in-memory store")); - } - } - PurgeType::Account { - account_id, - use_roles, - } => { - if let Some(account_id) = account_id { - self.purge_account(account_id).await; - } else { - self.purge_accounts(use_roles).await; - } - } - } - - trc::event!( - Purge(PurgeEvent::Finished), - Type = lock_type, - Elapsed = time.elapsed() - ); - - // Remove lock - if let Some(lock_name) = &lock_name - && let Err(err) = self - .in_memory_store() - .remove_lock(KV_LOCK_HOUSEKEEPER, lock_name) - .await - { - trc::error!( - err.details("Failed to delete task lock.") - .details(lock_type) - ); - } - } -} - -impl Queue { - pub fn schedule(&mut self, due: Instant, event: ActionClass) { - trc::event!( - Housekeeper(trc::HousekeeperEvent::Schedule), - Due = trc::Value::Timestamp( - now() + due.saturating_duration_since(Instant::now()).as_secs() - ), - Id = format!("{:?}", event) - ); - - self.heap.push(Action { due, event }); - } - - pub fn remove_action(&mut self, event: &ActionClass) { - self.heap.retain(|e| &e.event != event); - } - - pub fn wake_up_time(&self) -> Duration { - self.heap - .peek() - .map(|e| e.due.saturating_duration_since(Instant::now())) - .unwrap_or(LONG_1D_SLUMBER) - } - - pub fn pop(&mut self) -> Option { - if self.heap.peek()?.due <= Instant::now() { - self.heap.pop() - } else { - None - } - } - - pub fn has_action(&self, event: &ActionClass) -> bool { - self.heap.iter().any(|e| &e.event == event) - } -} - -impl Ord for Action { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - self.due.cmp(&other.due).reverse() - } -} - -impl PartialOrd for Action { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} diff --git a/crates/services/src/lib.rs b/crates/services/src/lib.rs index 7b37dc40..683bc946 100644 --- a/crates/services/src/lib.rs +++ b/crates/services/src/lib.rs @@ -9,13 +9,12 @@ use common::{ Inner, manager::boot::{BootManager, IpcReceivers}, }; -use housekeeper::spawn_housekeeper; use state_manager::manager::spawn_push_router; use std::sync::Arc; -use task_manager::spawn_task_manager; + +use crate::task_manager::{manager::spawn_task_manager, scheduler::spawn_task_scheduler}; pub mod broadcast; -pub mod housekeeper; pub mod state_manager; pub mod task_manager; @@ -53,9 +52,6 @@ impl SpawnServices for IpcReceivers { // Spawn push manager spawn_push_router(inner.clone(), self.push_rx.take().unwrap()); - // Spawn housekeeper - spawn_housekeeper(inner.clone(), self.housekeeper_rx.take().unwrap()); - // Spawn broadcast publisher if let Some(event_rx) = self.broadcast_rx.take() { // Spawn broadcast publisher @@ -63,6 +59,9 @@ impl SpawnServices for IpcReceivers { } // Spawn task manager - spawn_task_manager(inner); + spawn_task_manager(inner.clone()); + + // Spawn task scheduler + spawn_task_scheduler(inner); } } diff --git a/crates/services/src/state_manager/push.rs b/crates/services/src/state_manager/push.rs index 2c354258..c3021f7f 100644 --- a/crates/services/src/state_manager/push.rs +++ b/crates/services/src/state_manager/push.rs @@ -59,7 +59,7 @@ pub fn spawn_push_manager(inner: Arc) -> mpsc::Sender { .network .roles .push_notifications - .is_enabled_for_integer(account_id) + .is_enabled_for_integer(account_id as u64) { // Load push subscriptions for account let (subscriptions, member_account_ids) = @@ -147,7 +147,7 @@ pub fn spawn_push_manager(inner: Arc) -> mpsc::Sender { .network .roles .push_notifications - .is_enabled_for_integer(account_id) + .is_enabled_for_integer(account_id as u64) { continue; } diff --git a/crates/services/src/task_manager/alarm.rs b/crates/services/src/task_manager/alarm.rs index 56c71d43..3573d60a 100644 --- a/crates/services/src/task_manager/alarm.rs +++ b/crates/services/src/task_manager/alarm.rs @@ -38,7 +38,7 @@ use store::{ ValueKey, write::{AlignedBytes, Archive, now}, }; -use trc::{AddContext, TaskQueueEvent}; +use trc::{AddContext, TaskManagerEvent}; use types::collection::Collection; use utils::{sanitize_email, template::Variables}; @@ -145,7 +145,7 @@ async fn send_email_alarm( .caused_by(trc::location!())? else { trc::event!( - TaskQueue(TaskQueueEvent::MetadataNotFound), + TaskManager(TaskManagerEvent::MetadataNotFound), Details = "Calendar Event metadata not found", AccountId = account_id, DocumentId = document_id, @@ -329,7 +329,7 @@ async fn send_display_alarm( .caused_by(trc::location!())? else { trc::event!( - TaskQueue(TaskQueueEvent::MetadataNotFound), + TaskManager(TaskManagerEvent::MetadataNotFound), Details = "Calendar Event metadata not found", AccountId = account_id, DocumentId = document_id, @@ -435,7 +435,7 @@ async fn build_template( event.data.event.components.get(alarm.alarm_id as usize), ) else { trc::event!( - TaskQueue(TaskQueueEvent::MetadataNotFound), + TaskManager(TaskManagerEvent::MetadataNotFound), Details = "Calendar Alarm component not found", AccountId = account_id, DocumentId = document_id, diff --git a/crates/services/src/task_manager/destroy_account.rs b/crates/services/src/task_manager/destroy_account.rs new file mode 100644 index 00000000..4f3a2b15 --- /dev/null +++ b/crates/services/src/task_manager/destroy_account.rs @@ -0,0 +1,147 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::task_manager::TaskResult; +use common::Server; +use email::{message::metadata::MessageMetadata, sieve::SieveScript}; +use groupware::file::FileNode; +use registry::schema::structs::TaskDestroyAccount; +use store::{ + search::SearchQuery, + write::{BatchBuilder, BlobLink, BlobOp, SearchIndex, ValueClass}, +}; +use trc::AddContext; +use types::{ + blob_hash::BlobHash, + collection::Collection, + field::{EmailField, Field}, +}; + +pub(crate) trait DestroyAccountTask: Sync + Send { + fn destroy_account(&self, task: &TaskDestroyAccount) + -> impl Future + Send; +} + +impl DestroyAccountTask for Server { + async fn destroy_account(&self, task: &TaskDestroyAccount) -> TaskResult { + match destroy_account(self, task).await { + Ok(result) => result, + Err(err) => { + let result = TaskResult::temporary(err.to_string()); + trc::error!( + err.account_id(task.account_id.document_id()) + .details("Failed to destroy account") + ); + result + } + } + } +} + +async fn destroy_account(server: &Server, task: &TaskDestroyAccount) -> trc::Result { + let account_id = task.account_id.document_id(); + let todo = "destroy spam samples, undelete, registry objects, etc."; + + // Remove search index + for index in [ + SearchIndex::Email, + SearchIndex::Contacts, + SearchIndex::Calendar, + ] { + server + .search_store() + .unindex(SearchQuery::new(index).with_account_id(account_id)) + .await?; + } + + // Unlink all accounts's blobs + destroy_account_blobs(server, account_id).await?; + + // Destroy account data + server + .store() + .danger_destroy_account(account_id) + .await + .caused_by(trc::location!())?; + + Ok(TaskResult::Success) +} + +pub async fn destroy_account_blobs(server: &Server, account_id: u32) -> trc::Result<()> { + let mut delete_keys = Vec::new(); + for (collection, field) in [ + (Collection::Email, u8::from(EmailField::Metadata)), + (Collection::FileNode, u8::from(Field::ARCHIVE)), + (Collection::SieveScript, u8::from(Field::ARCHIVE)), + ] { + server + .all_archives(account_id, collection, field, |document_id, archive| { + match collection { + Collection::Email => { + let message = archive.unarchive::()?; + delete_keys.push(( + collection, + document_id, + BlobHash::from(&message.blob_hash), + )); + } + Collection::FileNode => { + if let Some(file) = archive.unarchive::()?.file.as_ref() { + delete_keys.push(( + collection, + document_id, + BlobHash::from(&file.blob_hash), + )); + } + } + Collection::SieveScript => { + let sieve = archive.unarchive::()?; + delete_keys.push(( + collection, + document_id, + BlobHash::from(&sieve.blob_hash), + )); + } + _ => {} + } + Ok(()) + }) + .await + .caused_by(trc::location!())?; + } + + let mut batch = BatchBuilder::new(); + batch.with_account_id(account_id); + + for (collection, document_id, hash) in delete_keys { + if batch.is_large_batch() { + server + .store() + .write(batch.build_all()) + .await + .caused_by(trc::location!())?; + batch = BatchBuilder::new(); + batch.with_account_id(account_id); + } + batch + .with_collection(collection) + .with_document(document_id) + .clear(ValueClass::Blob(BlobOp::Link { + hash, + to: BlobLink::Document, + })); + } + + if !batch.is_empty() { + server + .store() + .write(batch.build_all()) + .await + .caused_by(trc::location!())?; + } + + Ok(()) +} diff --git a/crates/services/src/task_manager/index.rs b/crates/services/src/task_manager/index.rs index e2938c1c..63e4b449 100644 --- a/crates/services/src/task_manager/index.rs +++ b/crates/services/src/task_manager/index.rs @@ -21,15 +21,13 @@ use store::{ IterateParams, ValueKey, ahash::AHashMap, rand::{self, Rng}, - registry::RegistryQuery, - roaring::RoaringBitmap, search::{IndexDocument, SearchField, SearchFilter, SearchQuery}, write::{ AlignedBytes, Archive, BatchBuilder, SearchIndex, TelemetryClass, ValueClass, key::DeserializeBigEndian, now, }, }; -use trc::{AddContext, TaskQueueEvent}; +use trc::{AddContext, TaskManagerEvent}; use types::{ blob_hash::BlobHash, collection::{Collection, SyncCollection}, @@ -40,15 +38,6 @@ pub(crate) trait SearchIndexTask: Sync + Send { fn index(&self, tasks: &[TaskDetails]) -> impl Future> + Send; } -pub trait ReindexIndexTask: Sync + Send { - fn reindex( - &self, - index: SearchIndex, - account_id: Option, - tenant_id: Option, - ) -> impl Future> + Send; -} - const NUM_INDEXES: usize = 5; #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -112,7 +101,7 @@ impl SearchIndexTask for Server { } _ => { trc::event!( - TaskQueue(TaskQueueEvent::TaskIgnored), + TaskManager(TaskManagerEvent::TaskIgnored), Collection = task.document_type.as_str(), Reason = "Nothing to index", AccountId = account_id, @@ -145,7 +134,7 @@ impl SearchIndexTask for Server { } _ => { trc::event!( - TaskQueue(TaskQueueEvent::TaskIgnored), + TaskManager(TaskManagerEvent::TaskIgnored), Reason = "Nothing to index", Id = task.trace_id.id(), ); @@ -301,135 +290,108 @@ impl SearchIndexTask for Server { } } -impl ReindexIndexTask for Server { - async fn reindex( - &self, - index: SearchIndex, - account_id: Option, - tenant_id: Option, - ) -> trc::Result<()> { - let accounts = if let Some(account_id) = account_id { - RoaringBitmap::from_sorted_iter([account_id]).unwrap() - } else { - self.registry() - .query( - RegistryQuery::new(ObjectType::Account) - .equal_opt(Property::MemberTenantId, tenant_id), - ) - .await - .caused_by(trc::location!())? - }; +pub(crate) async fn reindex_telemetry(server: &Server) -> trc::Result<()> { + let mut spans = Vec::new(); + server + .store() + .iterate( + IterateParams::new( + ValueKey::from(ValueClass::Telemetry(TelemetryClass::Span(0))), + ValueKey::from(ValueClass::Telemetry(TelemetryClass::Span(u64::MAX))), + ) + .no_values(), + |key, _| { + spans.push(key.deserialize_be_u64(0)?); + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; - let now = now() as i64; - match index { - SearchIndex::Email => { - for account_id in accounts { - let mut batch = BatchBuilder::new(); - - for document_id in self - .get_cached_messages(account_id) - .await - .caused_by(trc::location!())? - .emails - .items - .iter() - .map(|v| v.document_id) - { - batch.schedule_task(Task::IndexDocument(TaskIndexDocument { - account_id: account_id.into(), - document_id: document_id.into(), - document_type: IndexDocumentType::Email, - status: TaskStatus::at(now + rand::rng().random_range(0..=300)), - })); - - if batch.len() >= 2000 { - self.core.storage.data.write(batch.build_all()).await?; - batch = BatchBuilder::new(); - } - } - - if !batch.is_empty() { - self.core.storage.data.write(batch.build_all()).await?; - } - } - } - SearchIndex::Calendar | SearchIndex::Contacts => { - for account_id in accounts { - let cache = self - .fetch_dav_resources( - account_id, - account_id, - if index == SearchIndex::Calendar { - SyncCollection::Calendar - } else { - SyncCollection::AddressBook - }, - ) - .await - .caused_by(trc::location!())?; - let mut batch = BatchBuilder::new(); - - for document_id in cache.document_ids(false) { - batch.schedule_task(Task::IndexDocument(TaskIndexDocument { - account_id: account_id.into(), - document_id: document_id.into(), - document_type: if index == SearchIndex::Calendar { - IndexDocumentType::Calendar - } else { - IndexDocumentType::Contacts - }, - status: TaskStatus::at(now + rand::rng().random_range(0..=300)), - })); - - if batch.len() >= 2000 { - self.core.storage.data.write(batch.build_all()).await?; - batch = BatchBuilder::new(); - } - } - - if !batch.is_empty() { - self.core.storage.data.write(batch.build_all()).await?; - } - } - } - SearchIndex::Tracing => { - let mut spans = Vec::new(); - self.store() - .iterate( - IterateParams::new( - ValueKey::from(ValueClass::Telemetry(TelemetryClass::Span(0))), - ValueKey::from(ValueClass::Telemetry(TelemetryClass::Span(u64::MAX))), - ) - .no_values(), - |key, _| { - spans.push(key.deserialize_be_u64(0)?); - Ok(true) - }, - ) - .await - .caused_by(trc::location!())?; - - let mut batch = BatchBuilder::new(); - for span_id in spans { - batch.schedule_task(Task::IndexTrace(TaskIndexTrace { - trace_id: span_id.into(), - status: TaskStatus::at(now + rand::rng().random_range(0..=300)), - })); - if batch.len() >= 2000 { - self.core.storage.data.write(batch.build_all()).await?; - } - } - - // SPDX-SnippetEnd - } - SearchIndex::File | SearchIndex::InMemory => (), + let mut batch = BatchBuilder::new(); + let now = now() as i64; + for span_id in spans { + batch.schedule_task(Task::IndexTrace(TaskIndexTrace { + trace_id: span_id.into(), + status: TaskStatus::at(now + rand::rng().random_range(0..=300)), + })); + if batch.is_large_batch() { + server.core.storage.data.write(batch.build_all()).await?; + batch = BatchBuilder::new(); } - - // Request indexing - self.notify_task_queue(); - - Ok(()) } + + if !batch.is_empty() { + server.core.storage.data.write(batch.build_all()).await?; + } + Ok(()) +} + +pub(crate) async fn reindex_account(server: &Server, account_id: u32) -> trc::Result<()> { + let now = now() as i64; + + let mut batch = BatchBuilder::new(); + + for document_id in server + .get_cached_messages(account_id) + .await + .caused_by(trc::location!())? + .emails + .items + .iter() + .map(|v| v.document_id) + { + batch.schedule_task(Task::IndexDocument(TaskIndexDocument { + account_id: account_id.into(), + document_id: document_id.into(), + document_type: IndexDocumentType::Email, + status: TaskStatus::at(now + rand::rng().random_range(0..=300)), + })); + + if batch.is_large_batch() { + server.core.storage.data.write(batch.build_all()).await?; + batch = BatchBuilder::new(); + } + } + + for document_type in [IndexDocumentType::Calendar, IndexDocumentType::Contacts] { + let cache = server + .fetch_dav_resources( + account_id, + account_id, + if document_type == IndexDocumentType::Calendar { + SyncCollection::Calendar + } else { + SyncCollection::AddressBook + }, + ) + .await + .caused_by(trc::location!())?; + let mut batch = BatchBuilder::new(); + + for document_id in cache.document_ids(false) { + batch.schedule_task(Task::IndexDocument(TaskIndexDocument { + account_id: account_id.into(), + document_id: document_id.into(), + document_type, + status: TaskStatus::at(now + rand::rng().random_range(0..=300)), + })); + + if batch.len() >= 2000 { + server.core.storage.data.write(batch.build_all()).await?; + batch = BatchBuilder::new(); + } + } + } + + if !batch.is_empty() { + server.core.storage.data.write(batch.build_all()).await?; + } + + // Request indexing + server.notify_task_queue(); + + Ok(()) } async fn build_email_document( @@ -616,7 +578,7 @@ async fn delete_email_metadata( .core .enterprise .as_ref() - .and_then(|e| e.undelete_retention.as_ref()) + .and_then(|e| e.deleted_items_retention.as_ref()) { use email::message::metadata::MESSAGE_RECEIVED_MASK; use registry::{ @@ -707,7 +669,7 @@ async fn delete_email_metadata( } None => { trc::event!( - TaskQueue(TaskQueueEvent::MetadataNotFound), + TaskManager(TaskManagerEvent::MetadataNotFound), Details = "E-mail metadata not found", AccountId = account_id, DocumentId = document_id, diff --git a/crates/services/src/task_manager/lock.rs b/crates/services/src/task_manager/lock.rs index 78b75468..33b9984f 100644 --- a/crates/services/src/task_manager/lock.rs +++ b/crates/services/src/task_manager/lock.rs @@ -6,7 +6,7 @@ use crate::task_manager::*; -pub(crate) trait TaskLockManager: Sync + Send { +pub trait TaskLockManager: Sync + Send { fn try_lock_task(&self, task: u64) -> impl Future + Send; fn remove_index_lock(&self, id: u64) -> impl Future + Send; } @@ -21,7 +21,7 @@ impl TaskLockManager for Server { Ok(result) => { if !result { trc::event!( - TaskQueue(TaskQueueEvent::TaskLocked), + TaskManager(TaskManagerEvent::TaskLocked), Id = id, Details = "Task details not available", Expires = trc::Value::Timestamp(now() + DEFAULT_LOCK_EXPIRY), diff --git a/crates/services/src/task_manager/maintenance.rs b/crates/services/src/task_manager/maintenance.rs new file mode 100644 index 00000000..2f293f62 --- /dev/null +++ b/crates/services/src/task_manager/maintenance.rs @@ -0,0 +1,417 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::time::Instant; + +use crate::task_manager::{ + TaskResult, + index::{reindex_account, reindex_telemetry}, +}; +use common::{ + KV_ACME, KV_GREYLIST, KV_LOCK_DAV, KV_LOCK_QUEUE_MESSAGE, KV_LOCK_TASK, KV_OAUTH, + KV_QUOTA_BLOB, KV_RATE_LIMIT_AUTH, KV_RATE_LIMIT_CONTACT, KV_RATE_LIMIT_HTTP_ANONYMOUS, + KV_RATE_LIMIT_HTTP_AUTHENTICATED, KV_RATE_LIMIT_IMAP, KV_RATE_LIMIT_LOITER, KV_RATE_LIMIT_RCPT, + KV_RATE_LIMIT_SCAN, KV_RATE_LIMIT_SMTP, KV_SIEVE_ID, Server, + storage::index::ObjectIndexBuilder, +}; +use email::{ + cache::MessageCacheFetch, + message::{delete::EmailDeletion, ingest::EmailIngest, metadata::MessageData}, +}; +use groupware::{ + calendar::{Calendar, CalendarEvent, CalendarEventNotification}, + contact::{AddressBook, ContactCard}, + file::FileNode, +}; +use registry::schema::{ + enums::{TaskAccountMaintenanceType, TaskStoreMaintenanceType}, + prelude::ObjectType, + structs::{Task, TaskAccountMaintenance, TaskStatus, TaskStoreMaintenance}, +}; +use store::{ + Serialize, ValueKey, + rand::{self, Rng}, + registry::RegistryQuery, + roaring::RoaringBitmap, + write::{AlignedBytes, Archive, Archiver, BatchBuilder, ValueClass, now}, +}; +use trc::{AddContext, StoreEvent}; +use types::{ + collection::Collection, + field::{EmailField, MailboxField}, +}; + +pub(crate) trait MaintenanceTask: Sync + Send { + fn store_maintenance( + &self, + task: &TaskStoreMaintenance, + ) -> impl Future + Send; + fn account_maintenance( + &self, + task: &TaskAccountMaintenance, + ) -> impl Future + Send; +} + +impl MaintenanceTask for Server { + async fn store_maintenance(&self, task: &TaskStoreMaintenance) -> TaskResult { + match store_maintenance(self, task).await { + Ok(result) => result, + Err(err) => { + let result = TaskResult::temporary(err.to_string()); + trc::error!(err.details("Failed to perform store maintenance task")); + result + } + } + } + + async fn account_maintenance(&self, task: &TaskAccountMaintenance) -> TaskResult { + match account_maintenance(self, task).await { + Ok(result) => result, + Err(err) => { + let result = TaskResult::temporary(err.to_string()); + trc::error!( + err.account_id(task.account_id.document_id()) + .details("Failed to perform account maintenance task") + ); + result + } + } + } +} + +async fn store_maintenance( + server: &Server, + task: &TaskStoreMaintenance, +) -> trc::Result { + match task.maintenance_type { + TaskStoreMaintenanceType::ReindexAccounts | TaskStoreMaintenanceType::PurgeAccounts => { + let mut batch = BatchBuilder::new(); + let now = now() as i64; + let maintenance_type = + if task.maintenance_type == TaskStoreMaintenanceType::ReindexAccounts { + TaskAccountMaintenanceType::Reindex + } else { + TaskAccountMaintenanceType::Purge + }; + for account_id in server + .registry() + .query::(RegistryQuery::new(ObjectType::Account)) + .await? + { + batch.schedule_task(Task::AccountMaintenance(TaskAccountMaintenance { + account_id: account_id.into(), + maintenance_type, + status: TaskStatus::at(now + rand::rng().random_range(0..=300)), + })); + + if batch.is_large_batch() { + server.core.storage.data.write(batch.build_all()).await?; + server.notify_task_queue(); + batch = BatchBuilder::new(); + } + } + + if !batch.is_empty() { + server.core.storage.data.write(batch.build_all()).await?; + server.notify_task_queue(); + } + } + TaskStoreMaintenanceType::ReindexTelemetry => { + reindex_telemetry(server).await?; + } + TaskStoreMaintenanceType::PurgeData => { + let todo = "make sure all store types are purged, in memory, metrics, tracing, etc"; + let todo = + "make sure spam samples with their indexes and undelete items are purged as well"; + + let mut started = Instant::now(); + + server + .store() + .purge_store() + .await + .caused_by(trc::location!())?; + + server + .in_memory_store() + .purge_in_memory_store() + .await + .caused_by(trc::location!())?; + + // SPDX-SnippetBegin + // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + // SPDX-License-Identifier: LicenseRef-SEL + #[cfg(feature = "enterprise")] + { + use common::telemetry::metrics::store::MetricsStore; + use common::telemetry::tracers::store::TracingStore; + + if let Some(trace_retention) = server + .core + .enterprise + .as_ref() + .and_then(|e| e.trace_retention) + && server.tracing_store().is_active() + { + server + .tracing_store() + .purge_spans(trace_retention, server.search_store().into()) + .await + .caused_by(trc::location!())?; + } + + if let Some(metrics_retention) = server + .core + .enterprise + .as_ref() + .and_then(|e| e.metrics_retention) + && server.metrics_store().is_active() + { + server + .metrics_store() + .purge_metrics(metrics_retention) + .await + .caused_by(trc::location!())?; + } + } + // SPDX-SnippetEnd + + trc::event!( + Store(StoreEvent::DataStorePurged), + Elapsed = started.elapsed() + ); + } + TaskStoreMaintenanceType::PurgeBlob => { + server + .store() + .purge_blobs(server.blob_store().clone()) + .await + .caused_by(trc::location!())?; + } + TaskStoreMaintenanceType::RemoveGreylist + | TaskStoreMaintenanceType::RemoveLockQueueMessage + | TaskStoreMaintenanceType::RemoveLockTask + | TaskStoreMaintenanceType::RemoveLockDav + | TaskStoreMaintenanceType::RemoveSieveId + | TaskStoreMaintenanceType::ResetRateLimiters + | TaskStoreMaintenanceType::ResetBlobQuotas + | TaskStoreMaintenanceType::RemoveAuthTokens => { + let prefixes = match task.maintenance_type { + TaskStoreMaintenanceType::RemoveGreylist => &[KV_GREYLIST][..], + TaskStoreMaintenanceType::RemoveLockQueueMessage => &[KV_LOCK_QUEUE_MESSAGE][..], + TaskStoreMaintenanceType::RemoveLockTask => &[KV_LOCK_TASK][..], + TaskStoreMaintenanceType::RemoveLockDav => &[KV_LOCK_DAV][..], + TaskStoreMaintenanceType::RemoveSieveId => &[KV_SIEVE_ID][..], + TaskStoreMaintenanceType::ResetRateLimiters => &[ + KV_RATE_LIMIT_RCPT, + KV_RATE_LIMIT_SCAN, + KV_RATE_LIMIT_LOITER, + KV_RATE_LIMIT_AUTH, + KV_RATE_LIMIT_SMTP, + KV_RATE_LIMIT_CONTACT, + KV_RATE_LIMIT_HTTP_AUTHENTICATED, + KV_RATE_LIMIT_HTTP_ANONYMOUS, + KV_RATE_LIMIT_IMAP, + ][..], + TaskStoreMaintenanceType::ResetBlobQuotas => &[KV_QUOTA_BLOB][..], + TaskStoreMaintenanceType::RemoveAuthTokens => &[KV_ACME, KV_OAUTH][..], + _ => unreachable!(), + }; + + for &prefix in prefixes { + server + .in_memory_store() + .key_delete_prefix(&[prefix]) + .await?; + } + } + } + + Ok(TaskResult::Success) +} + +async fn account_maintenance( + server: &Server, + task: &TaskAccountMaintenance, +) -> trc::Result { + match task.maintenance_type { + TaskAccountMaintenanceType::Purge => { + server.purge_account(task.account_id.document_id()).await?; + } + TaskAccountMaintenanceType::Reindex => { + reindex_account(server, task.account_id.document_id()).await?; + } + TaskAccountMaintenanceType::RecalculateImapUid => { + reset_imap_uids(server, task.account_id.document_id()).await?; + } + TaskAccountMaintenanceType::RecalculateQuota => { + recalculate_quota(server, task.account_id.document_id()).await?; + } + } + + Ok(TaskResult::Success) +} + +async fn recalculate_quota(server: &Server, account_id: u32) -> trc::Result<()> { + let mut quota = 0; + + for collection in [ + Collection::Email, + Collection::Calendar, + Collection::CalendarEvent, + Collection::CalendarEventNotification, + Collection::AddressBook, + Collection::ContactCard, + Collection::FileNode, + ] { + server + .archives(account_id, collection, &(), |_, archive| { + match collection { + Collection::Email => { + quota += archive.unarchive::()?.size.to_native() as i64; + } + Collection::Calendar => { + quota += archive.unarchive::()?.size() as i64; + } + Collection::CalendarEvent => { + quota += archive.unarchive::()?.size() as i64; + } + Collection::CalendarEventNotification => { + quota += archive.unarchive::()?.size() as i64; + } + Collection::AddressBook => { + quota += archive.unarchive::()?.size() as i64; + } + Collection::ContactCard => { + quota += archive.unarchive::()?.size() as i64; + } + Collection::FileNode => { + quota += archive.unarchive::()?.size() as i64; + } + _ => {} + } + Ok(true) + }) + .await + .caused_by(trc::location!())?; + } + + let mut batch = BatchBuilder::new(); + batch + .with_account_id(account_id) + .clear(ValueClass::Quota) + .add(ValueClass::Quota, quota); + server + .store() + .write(batch.build_all()) + .await + .caused_by(trc::location!()) + .map(|_| ()) +} + +async fn reset_imap_uids(server: &Server, account_id: u32) -> trc::Result<(u32, u32)> { + let mut mailbox_count = 0; + let mut email_count = 0; + + let cache = server + .get_cached_messages(account_id) + .await + .caused_by(trc::location!())?; + + for &mailbox_id in cache.mailboxes.index.keys() { + let mailbox = server + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::Mailbox, + mailbox_id, + )) + .await + .caused_by(trc::location!())? + .ok_or_else(|| trc::ImapEvent::Error.into_err().caused_by(trc::location!()))? + .into_deserialized::() + .caused_by(trc::location!())?; + let mut new_mailbox = mailbox.inner.clone(); + new_mailbox.uid_validity = rand::random::(); + let mut batch = BatchBuilder::new(); + batch + .with_account_id(account_id) + .with_collection(Collection::Mailbox) + .with_document(mailbox_id) + .custom( + ObjectIndexBuilder::new() + .with_current(mailbox) + .with_changes(new_mailbox), + ) + .caused_by(trc::location!())? + .clear(MailboxField::UidCounter); + server + .store() + .write(batch.build_all()) + .await + .caused_by(trc::location!())?; + mailbox_count += 1; + } + + // Reset all UIDs + for message_id in cache.emails.items.iter().map(|i| i.document_id) { + let data = server + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::Email, + message_id, + )) + .await + .caused_by(trc::location!())?; + let data_ = if let Some(data) = data { + data + } else { + continue; + }; + let data = data_ + .to_unarchived::() + .caused_by(trc::location!())?; + let mut new_data = data + .deserialize::() + .caused_by(trc::location!())?; + + let ids = server + .assign_email_ids( + account_id, + new_data.mailboxes.iter().map(|m| m.mailbox_id), + false, + ) + .await + .caused_by(trc::location!())?; + + for (uid_mailbox, uid) in new_data.mailboxes.iter_mut().zip(ids) { + uid_mailbox.uid = uid; + } + + // Prepare write batch + let mut batch = BatchBuilder::new(); + batch + .with_account_id(account_id) + .with_collection(Collection::Email) + .with_document(message_id) + .assert_value(ValueClass::Property(EmailField::Archive.into()), &data) + .set( + EmailField::Archive, + Archiver::new(new_data) + .serialize() + .caused_by(trc::location!())?, + ); + server + .store() + .write(batch.build_all()) + .await + .caused_by(trc::location!())?; + email_count += 1; + } + + Ok((mailbox_count, email_count)) +} diff --git a/crates/services/src/task_manager/manager.rs b/crates/services/src/task_manager/manager.rs new file mode 100644 index 00000000..99fbedea --- /dev/null +++ b/crates/services/src/task_manager/manager.rs @@ -0,0 +1,557 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::task_manager::alarm::SendAlarmTask; +use crate::task_manager::destroy_account::DestroyAccountTask; +use crate::task_manager::imip::SendImipTask; +use crate::task_manager::index::SearchIndexTask; +use crate::task_manager::lock::TaskLockManager; +use crate::task_manager::maintenance::MaintenanceTask; +use crate::task_manager::merge_threads::MergeThreadsTask; +use crate::task_manager::report::{self, SubmitReportTask}; +use crate::task_manager::restore_item::RestoreItemTask; +use crate::task_manager::spam_classifier::SpamFilterMaintenanceTask; +use crate::task_manager::{ + DEFAULT_LOCK_EXPIRY, Locked, QUEUE_REFRESH_INTERVAL, TaskDetails, TaskFailureType, TaskInfo, + TaskJob, TaskManagerIpc, TaskResult, +}; +use common::config::server::ServerProtocol; +use common::network::limiter::ConcurrencyLimiter; +use common::network::{ServerInstance, TcpAcceptor}; +use common::{BuildServer, IPC_CHANNEL_BUFFER}; +use common::{Inner, Server}; +use registry::pickle::Pickle; +use registry::schema::enums::TaskType; +use registry::schema::structs::{ + Task, TaskManager, TaskRetryStrategy, TaskStatus, TaskStatusFailed, TaskStatusRetry, +}; +use registry::types::EnumImpl; +use registry::types::datetime::UTCDateTime; +use std::collections::hash_map::Entry; +use std::future::Future; +use std::time::Duration; +use std::{sync::Arc, time::Instant}; +use store::rand::seq::SliceRandom; +use store::write::key::DeserializeBigEndian; +use store::{ + IterateParams, ValueKey, + write::{BatchBuilder, TaskQueueClass, ValueClass, now}, +}; +use store::{SerializeInfallible, U64_LEN, rand}; +use tokio::sync::{mpsc, watch}; +use trc::TaskManagerEvent; +use utils::snowflake::SnowflakeIdGenerator; + +pub fn spawn_task_manager(inner: Arc) { + if !inner + .build_server() + .core + .network + .roles + .task_manager + .is_enabled_or_sharded() + { + return; + } + + trc::event!(TaskManager(TaskManagerEvent::ManagerStarted)); + + // Create dummy server instance for alarms + let server_instance = Arc::new(ServerInstance { + id: "_local".to_string(), + protocol: ServerProtocol::Smtp, + acceptor: TcpAcceptor::Plain, + limiter: ConcurrencyLimiter::new(100), + shutdown_rx: watch::channel(false).1, + proxy_networks: vec![], + span_id_gen: Arc::new(SnowflakeIdGenerator::new()), + }); + + // Spawn workers for each task type + let mut txs = Vec::with_capacity(TaskType::COUNT); + for idx in 0..TaskType::COUNT { + let (tx, mut rx) = mpsc::channel::(IPC_CHANNEL_BUFFER); + txs.push(tx); + let inner = inner.clone(); + let server_instance = server_instance.clone(); + + if matches!( + TaskType::from_id(idx as u16).unwrap(), + TaskType::IndexDocument | TaskType::UnindexDocument | TaskType::IndexTrace + ) { + tokio::spawn(async move { + while let Some(job) = rx.recv().await { + let server = inner.build_server(); + + let batch_size = server.core.email.index_batch_size; + let mut batch = Vec::with_capacity(batch_size); + match server + .store() + .get_value::(ValueKey::from(ValueClass::TaskQueue( + TaskQueueClass::Task { id: job.id }, + ))) + .await + { + Ok(Some(task)) => { + batch.push(TaskDetails { task, info: job }); + } + Ok(None) => { + trc::event!( + TaskManager(TaskManagerEvent::TaskIgnored), + Id = job.id, + Reason = "Task not found in store, likely already processed.", + ); + } + Err(err) => { + trc::error!( + err.id(job.id) + .details("Failed to retrieve task details.") + .caused_by(trc::location!()) + ); + } + } + + while batch.len() < batch_size { + match rx.try_recv() { + Ok(job) => { + match server + .store() + .get_value::(ValueKey::from(ValueClass::TaskQueue( + TaskQueueClass::Task { id: job.id }, + ))) + .await + { + Ok(Some(task)) => { + batch.push(TaskDetails { task, info: job }); + } + Ok(None) => { + trc::event!( + TaskManager(TaskManagerEvent::TaskIgnored), + Id = job.id, + Reason = "Task not found in store, likely already processed.", + ); + } + Err(err) => { + trc::error!( + err.id(job.id) + .details("Failed to retrieve task details.") + .caused_by(trc::location!()) + ); + } + } + } + Err(_) => break, + } + } + + // Dispatch + let results = server.index(&batch).await.into_iter().map(|r| r.result); + update_tasks(&server, &mut batch, results).await; + } + }); + } else { + let server_instance = server_instance.clone(); + tokio::spawn(async move { + while let Some(job) = rx.recv().await { + let server = inner.build_server(); + + match server + .store() + .get_value::(ValueKey::from(ValueClass::TaskQueue( + TaskQueueClass::Task { id: job.id }, + ))) + .await + { + Ok(Some(task)) => { + let result = match &task { + Task::CalendarAlarmEmail(task) => { + server.send_email_alarm(task, server_instance.clone()).await + } + Task::CalendarAlarmNotification(task) => { + server.send_display_alarm(task).await + } + Task::CalendarItipMessage(task) => { + server.send_imip(task, server_instance.clone()).await + } + Task::MergeThreads(task) => server.merge_threads(task).await, + Task::DmarcReport(task) => { + server + .submit_report(report::ReportId::Dmarc(task.report_id.id())) + .await + } + Task::TlsReport(task) => { + server + .submit_report(report::ReportId::Tls(task.report_id.id())) + .await + } + Task::RestoreArchivedItem(task) => server.restore_item(task).await, + Task::DestroyAccount(task) => server.destroy_account(task).await, + Task::AccountMaintenance(task) => { + server.account_maintenance(task).await + } + Task::StoreMaintenance(task) => { + server.store_maintenance(task).await + } + Task::SpamFilterMaintenance(task) => { + server.spam_filter_maintenance(task).await + } + Task::IndexDocument(_) + | Task::UnindexDocument(_) + | Task::IndexTrace(_) => unreachable!(), + }; + + update_tasks( + &server, + &mut [TaskDetails { task, info: job }], + vec![result], + ) + .await; + } + Ok(None) => { + trc::event!( + TaskManager(TaskManagerEvent::TaskIgnored), + Id = job.id, + Reason = "Task not found in store, likely already processed.", + ); + } + Err(err) => { + trc::error!( + err.id(job.id) + .details("Failed to retrieve task details.") + .caused_by(trc::location!()) + ); + } + } + } + }); + } + } + + tokio::spawn(async move { + let mut ipc = TaskManagerIpc { + txs: txs.try_into().expect("Incorrect number of task channels"), + locked: Default::default(), + revision: 0, + }; + let rx = inner.ipc.task_tx.clone(); + loop { + // Index any queued tasks + let sleep_for = inner.build_server().process_tasks(&mut ipc).await; + + // Wait for a signal or sleep until the next task is due + let _ = tokio::time::timeout(sleep_for, rx.notified()).await; + } + }); +} + +pub(crate) trait TaskQueueManager: Sync + Send { + fn process_tasks(&self, ipc: &mut TaskManagerIpc) -> impl Future + Send; +} + +impl TaskQueueManager for Server { + async fn process_tasks(&self, ipc: &mut TaskManagerIpc) -> Duration { + let now_timestamp = now(); + let from_key = ValueKey:: { + account_id: 0, + collection: 0, + document_id: 0, + class: ValueClass::TaskQueue(TaskQueueClass::Due { id: 0, due: 0 }), + }; + let to_key = ValueKey:: { + account_id: u32::MAX, + collection: u8::MAX, + document_id: u32::MAX, + class: ValueClass::TaskQueue(TaskQueueClass::Due { + id: u64::MAX, + due: now_timestamp + QUEUE_REFRESH_INTERVAL, + }), + }; + + // Retrieve tasks pending to be processed + let mut tasks = Vec::new(); + let now = Instant::now(); + let mut next_event = None; + ipc.revision += 1; + let _ = self + .store() + .iterate( + IterateParams::new(from_key, to_key).ascending(), + |key, value| { + if key.len() == U64_LEN * 2 { + let task_due = key.deserialize_be_u64(0)?; + let task_id = key.deserialize_be_u64(U64_LEN)?; + + if task_due <= now_timestamp { + let task_type_idx = value.deserialize_be_u16(0)?; + let task_type = TaskType::from_id(task_type_idx).ok_or_else(|| { + trc::StoreEvent::DataCorruption + .caused_by(trc::location!()) + .ctx(trc::Key::Value, value) + })?; + match ipc.locked.entry(task_id) { + Entry::Occupied(mut entry) => { + let locked = entry.get_mut(); + if locked.expires <= now { + locked.expires = Instant::now() + + std::time::Duration::from_secs( + DEFAULT_LOCK_EXPIRY + 1, + ); + tasks.push(( + TaskJob { + id: task_id, + due: task_due, + typ: task_type, + }, + task_type_idx, + )); + } + locked.revision = ipc.revision; + } + Entry::Vacant(entry) => { + entry.insert(Locked { + expires: Instant::now() + + std::time::Duration::from_secs( + DEFAULT_LOCK_EXPIRY + 1, + ), + revision: ipc.revision, + }); + tasks.push(( + TaskJob { + id: task_id, + due: task_due, + typ: task_type, + }, + task_type_idx, + )); + } + } + + Ok(true) + } else { + next_event = Some(task_due); + Ok(false) + } + } else { + Ok(true) + } + }, + ) + .await + .map_err(|err| { + trc::error!( + err.caused_by(trc::location!()) + .details("Failed to iterate over task queue.") + ); + }); + + if !tasks.is_empty() || !ipc.locked.is_empty() { + trc::event!( + TaskManager(TaskManagerEvent::TaskAcquired), + Total = tasks.len(), + Details = ipc.locked.len(), + ); + } + + // Shuffle tasks + if tasks.len() > 1 { + tasks.shuffle(&mut rand::rng()); + } + + // Dispatch tasks + let roles = &self.core.network.roles; + for (task_job, task_type_idx) in tasks { + let enabled = match task_job.typ { + TaskType::IndexDocument | TaskType::UnindexDocument | TaskType::IndexTrace => roles + .search_indexing + .is_enabled_for_integer(task_job.id_hash()), + TaskType::CalendarAlarmEmail | TaskType::CalendarAlarmNotification => roles + .calendar_alerts + .is_enabled_for_integer(task_job.id_hash()), + TaskType::CalendarItipMessage => roles + .imip_processing + .is_enabled_for_integer(task_job.id_hash()), + TaskType::MergeThreads => roles + .merge_threads + .is_enabled_for_integer(task_job.id_hash()), + TaskType::AccountMaintenance | TaskType::DestroyAccount => roles + .account_maintenance + .is_enabled_for_integer(task_job.id_hash()), + TaskType::StoreMaintenance => roles + .store_maintenance + .is_enabled_for_integer(task_job.id_hash()), + TaskType::SpamFilterMaintenance => roles + .spam_training + .is_enabled_for_integer(task_job.id_hash()), + TaskType::DmarcReport | TaskType::TlsReport | TaskType::RestoreArchivedItem => true, + }; + + if enabled { + if self.try_lock_task(task_job.id).await + && ipc.txs[task_type_idx as usize] + .send(task_job) + .await + .is_err() + { + trc::event!( + Server(trc::ServerEvent::ThreadError), + Details = "Error sending task.", + CausedBy = trc::location!() + ); + } + } else { + trc::event!( + TaskManager(TaskManagerEvent::TaskIgnored), + Id = task_job.id, + Details = task_job.typ.as_str(), + Reason = "Task type is disabled by cluster roles.", + ); + } + } + + // Delete expired locks + let now = Instant::now(); + ipc.locked + .retain(|_, locked| locked.expires > now && locked.revision == ipc.revision); + Duration::from_secs(next_event.map_or(QUEUE_REFRESH_INTERVAL, |timestamp| { + timestamp.saturating_sub(store::write::now()) + })) + } +} + +async fn update_tasks( + server: &Server, + tasks: &mut [TaskDetails], + results: impl IntoIterator, +) { + let mut batch = BatchBuilder::new(); + + for (task, result) in tasks.iter_mut().zip(results.into_iter()) { + let id = task.info.id; + batch.clear(ValueClass::TaskQueue(TaskQueueClass::Due { + id, + due: task.info.due, + })); + match result { + TaskResult::Success | TaskResult::Ignored => { + batch.clear(ValueClass::TaskQueue(TaskQueueClass::Task { id })); + } + TaskResult::Update(ops) => { + for op in ops { + batch.any_op(op); + } + } + TaskResult::Failure { typ, message } => { + let (attempt_number, created_at) = match task.task.status() { + TaskStatus::Pending(status) => (0, status.created_at), + TaskStatus::Retry(status) => (status.attempt_number, status.created_at), + TaskStatus::Failed(status) => (status.failed_attempt_number, status.failed_at), + }; + let retry_at = match typ { + TaskFailureType::Retry(retry_at) => (attempt_number + < server.core.network.task_manager.max_attempts + && retry_at + < retry_at.saturating_add( + server.core.network.task_manager.total_deadline.as_secs(), + )) + .then_some(retry_at), + TaskFailureType::Temporary => next_retry_time( + &server.core.network.task_manager, + created_at.timestamp() as u64, + attempt_number, + now(), + ), + TaskFailureType::Permanent => None, + }; + + let due = if let Some(retry_at) = retry_at { + trc::event!( + TaskManager(TaskManagerEvent::TaskRetry), + Id = id, + Details = task.task.name(), + Reason = message.to_string(), + NextRetry = trc::Value::Timestamp(retry_at), + ); + + task.task.set_status(TaskStatus::Retry(TaskStatusRetry { + due: UTCDateTime::from_timestamp(retry_at as i64), + attempt_number: attempt_number + 1, + failure_reason: message, + created_at, + })); + + retry_at + } else { + trc::event!( + TaskManager(TaskManagerEvent::TaskFailed), + Id = id, + Details = task.task.name(), + Reason = message.to_string(), + ); + + task.task.set_status(TaskStatus::Failed(TaskStatusFailed { + failed_at: UTCDateTime::now(), + failed_attempt_number: attempt_number, + failure_reason: message, + created_at, + })); + u64::MAX + }; + batch + .set( + ValueClass::TaskQueue(TaskQueueClass::Due { id, due }), + task.info.typ.to_id().serialize(), + ) + .set( + ValueClass::TaskQueue(TaskQueueClass::Task { id }), + task.task.to_pickled_vec(), + ); + } + } + } + + if let Err(err) = server.store().write(batch.build_all()).await { + trc::error!(err.details("Failed to remove task(s) from queue.")); + } + + for task in tasks { + server.remove_index_lock(task.info.id).await; + } +} + +pub fn next_retry_time( + manager: &TaskManager, + create_time: u64, + attempt: u64, + now: u64, +) -> Option { + if attempt >= manager.max_attempts { + return None; + } + + let delay_secs: u64 = match &manager.strategy { + TaskRetryStrategy::FixedDelay(fixed) => fixed.delay.as_secs(), + TaskRetryStrategy::ExponentialBackoff(backoff) => { + let delay = (backoff.initial_delay.as_secs() as f64 + * backoff.factor.into_inner().powi(attempt as i32)) + .min(backoff.max_delay.as_secs() as f64) as u64; + + if backoff.jitter { + let jitter_factor = rand::random::() + 0.5; + ((delay as f64 * jitter_factor) as u64).min(backoff.max_delay.as_secs()) + } else { + delay + } + } + }; + + let next_time = now.saturating_add(delay_secs); + let deadline = create_time.saturating_add(manager.total_deadline.as_secs()); + if next_time > deadline { + return None; + } + + Some(next_time) +} diff --git a/crates/services/src/task_manager/mod.rs b/crates/services/src/task_manager/mod.rs index ebecf638..db5e3f48 100644 --- a/crates/services/src/task_manager/mod.rs +++ b/crates/services/src/task_manager/mod.rs @@ -4,49 +4,30 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::task_manager::imip::SendImipTask; -use crate::task_manager::index::SearchIndexTask; -use crate::task_manager::lock::TaskLockManager; -use crate::task_manager::merge_threads::MergeThreadsTask; -use crate::task_manager::report::SubmitReportTask; -use crate::task_manager::restore_item::RestoreItemTask; -use alarm::SendAlarmTask; -use common::config::server::ServerProtocol; -use common::network::limiter::ConcurrencyLimiter; -use common::network::{ServerInstance, TcpAcceptor}; -use common::{BuildServer, IPC_CHANNEL_BUFFER}; -use common::{Inner, KV_LOCK_TASK, Server}; -use registry::pickle::Pickle; +use common::{KV_LOCK_TASK, Server}; use registry::schema::enums::TaskType; -use registry::schema::structs::{ - Task, TaskManager, TaskRetryStrategy, TaskStatus, TaskStatusFailed, TaskStatusRetry, -}; +use registry::schema::structs::Task; use registry::types::EnumImpl; -use registry::types::datetime::UTCDateTime; -use std::collections::hash_map::Entry; use std::future::Future; -use std::time::Duration; -use std::{sync::Arc, time::Instant}; -use store::rand::seq::SliceRandom; +use std::time::Instant; use store::write::Operation; -use store::write::key::DeserializeBigEndian; -use store::{ - IterateParams, ValueKey, - ahash::AHashMap, - write::{BatchBuilder, TaskQueueClass, ValueClass, now}, -}; -use store::{SerializeInfallible, U64_LEN, rand}; -use tokio::sync::{mpsc, watch}; -use trc::TaskQueueEvent; -use utils::snowflake::SnowflakeIdGenerator; +use store::xxhash_rust::xxh3::xxh3_64; +use store::{ahash::AHashMap, write::now}; +use tokio::sync::mpsc; +use trc::TaskManagerEvent; pub mod alarm; +pub mod destroy_account; pub mod imip; pub mod index; pub mod lock; +pub mod maintenance; +pub mod manager; pub mod merge_threads; pub mod report; pub mod restore_item; +pub mod scheduler; +pub mod spam_classifier; const QUEUE_REFRESH_INTERVAL: u64 = 60 * 5; // 5 minutes const DEFAULT_LOCK_EXPIRY: u64 = 60 * 5; // 5 minutes @@ -57,6 +38,11 @@ pub(crate) struct TaskManagerIpc { revision: u64, } +pub(crate) struct Locked { + expires: Instant, + revision: u64, +} + pub(crate) struct TaskDetails { task: Task, info: TaskJob, @@ -87,490 +73,6 @@ pub(crate) enum TaskFailureType { Permanent, } -struct Locked { - expires: Instant, - revision: u64, -} - -pub fn spawn_task_manager(inner: Arc) { - // Create dummy server instance for alarms - let server_instance = Arc::new(ServerInstance { - id: "_local".to_string(), - protocol: ServerProtocol::Smtp, - acceptor: TcpAcceptor::Plain, - limiter: ConcurrencyLimiter::new(100), - shutdown_rx: watch::channel(false).1, - proxy_networks: vec![], - span_id_gen: Arc::new(SnowflakeIdGenerator::new()), - }); - - // Spawn workers for each task type - let mut txs = Vec::with_capacity(TaskType::COUNT); - for idx in 0..TaskType::COUNT { - let (tx, mut rx) = mpsc::channel::(IPC_CHANNEL_BUFFER); - txs.push(tx); - let inner = inner.clone(); - let server_instance = server_instance.clone(); - - if matches!( - TaskType::from_id(idx as u16).unwrap(), - TaskType::IndexDocument | TaskType::UnindexDocument | TaskType::IndexTrace - ) { - tokio::spawn(async move { - while let Some(job) = rx.recv().await { - let server = inner.build_server(); - - let batch_size = server.core.email.index_batch_size; - let mut batch = Vec::with_capacity(batch_size); - match server - .store() - .get_value::(ValueKey::from(ValueClass::TaskQueue( - TaskQueueClass::Task { id: job.id }, - ))) - .await - { - Ok(Some(task)) => { - batch.push(TaskDetails { task, info: job }); - } - Ok(None) => { - trc::event!( - TaskQueue(TaskQueueEvent::TaskIgnored), - Id = job.id, - Reason = "Task not found in store, likely already processed.", - ); - } - Err(err) => { - trc::error!( - err.id(job.id) - .details("Failed to retrieve task details.") - .caused_by(trc::location!()) - ); - } - } - - while batch.len() < batch_size { - match rx.try_recv() { - Ok(job) => { - match server - .store() - .get_value::(ValueKey::from(ValueClass::TaskQueue( - TaskQueueClass::Task { id: job.id }, - ))) - .await - { - Ok(Some(task)) => { - batch.push(TaskDetails { task, info: job }); - } - Ok(None) => { - trc::event!( - TaskQueue(TaskQueueEvent::TaskIgnored), - Id = job.id, - Reason = "Task not found in store, likely already processed.", - ); - } - Err(err) => { - trc::error!( - err.id(job.id) - .details("Failed to retrieve task details.") - .caused_by(trc::location!()) - ); - } - } - } - Err(_) => break, - } - } - - // Dispatch - let results = server.index(&batch).await.into_iter().map(|r| r.result); - update_tasks(&server, &mut batch, results).await; - } - }); - } else { - let server_instance = server_instance.clone(); - tokio::spawn(async move { - while let Some(job) = rx.recv().await { - let server = inner.build_server(); - - match server - .store() - .get_value::(ValueKey::from(ValueClass::TaskQueue( - TaskQueueClass::Task { id: job.id }, - ))) - .await - { - Ok(Some(task)) => { - let result = match &task { - Task::CalendarAlarmEmail(task) => { - server.send_email_alarm(task, server_instance.clone()).await - } - Task::CalendarAlarmNotification(task) => { - server.send_display_alarm(task).await - } - Task::CalendarItipMessage(task) => { - server.send_imip(task, server_instance.clone()).await - } - Task::MergeThreads(task) => server.merge_threads(task).await, - Task::DmarcReport(task) => { - server - .submit_report(report::ReportId::Dmarc(task.report_id.id())) - .await - } - Task::TlsReport(task) => { - server - .submit_report(report::ReportId::Tls(task.report_id.id())) - .await - } - Task::RestoreArchivedItem(task) => server.restore_item(task).await, - Task::IndexDocument(_) - | Task::UnindexDocument(_) - | Task::IndexTrace(_) => unreachable!(), - }; - - update_tasks( - &server, - &mut [TaskDetails { task, info: job }], - vec![result], - ) - .await; - } - Ok(None) => { - trc::event!( - TaskQueue(TaskQueueEvent::TaskIgnored), - Id = job.id, - Reason = "Task not found in store, likely already processed.", - ); - } - Err(err) => { - trc::error!( - err.id(job.id) - .details("Failed to retrieve task details.") - .caused_by(trc::location!()) - ); - } - } - } - }); - } - } - - tokio::spawn(async move { - let mut ipc = TaskManagerIpc { - txs: txs.try_into().expect("Incorrect number of task channels"), - locked: Default::default(), - revision: 0, - }; - let rx = inner.ipc.task_tx.clone(); - loop { - // Index any queued tasks - let sleep_for = inner.build_server().process_tasks(&mut ipc).await; - - // Wait for a signal or sleep until the next task is due - let _ = tokio::time::timeout(sleep_for, rx.notified()).await; - } - }); -} - -pub(crate) trait TaskQueueManager: Sync + Send { - fn process_tasks(&self, ipc: &mut TaskManagerIpc) -> impl Future + Send; -} - -impl TaskQueueManager for Server { - async fn process_tasks(&self, ipc: &mut TaskManagerIpc) -> Duration { - let now_timestamp = now(); - let from_key = ValueKey:: { - account_id: 0, - collection: 0, - document_id: 0, - class: ValueClass::TaskQueue(TaskQueueClass::Due { id: 0, due: 0 }), - }; - let to_key = ValueKey:: { - account_id: u32::MAX, - collection: u8::MAX, - document_id: u32::MAX, - class: ValueClass::TaskQueue(TaskQueueClass::Due { - id: u64::MAX, - due: now_timestamp + QUEUE_REFRESH_INTERVAL, - }), - }; - - // Retrieve tasks pending to be processed - let mut tasks = Vec::new(); - let now = Instant::now(); - let mut next_event = None; - ipc.revision += 1; - let _ = self - .store() - .iterate( - IterateParams::new(from_key, to_key).ascending(), - |key, value| { - if key.len() == U64_LEN * 2 { - let task_due = key.deserialize_be_u64(0)?; - let task_id = key.deserialize_be_u64(U64_LEN)?; - - if task_due <= now_timestamp { - let task_type_idx = value.deserialize_be_u16(0)?; - let task_type = TaskType::from_id(task_type_idx).ok_or_else(|| { - trc::StoreEvent::DataCorruption - .caused_by(trc::location!()) - .ctx(trc::Key::Value, value) - })?; - match ipc.locked.entry(task_id) { - Entry::Occupied(mut entry) => { - let locked = entry.get_mut(); - if locked.expires <= now { - locked.expires = Instant::now() - + std::time::Duration::from_secs( - DEFAULT_LOCK_EXPIRY + 1, - ); - tasks.push(( - TaskJob { - id: task_id, - due: task_due, - typ: task_type, - }, - task_type_idx, - )); - } - locked.revision = ipc.revision; - } - Entry::Vacant(entry) => { - entry.insert(Locked { - expires: Instant::now() - + std::time::Duration::from_secs( - DEFAULT_LOCK_EXPIRY + 1, - ), - revision: ipc.revision, - }); - tasks.push(( - TaskJob { - id: task_id, - due: task_due, - typ: task_type, - }, - task_type_idx, - )); - } - } - - Ok(true) - } else { - next_event = Some(task_due); - Ok(false) - } - } else { - Ok(true) - } - }, - ) - .await - .map_err(|err| { - trc::error!( - err.caused_by(trc::location!()) - .details("Failed to iterate over task queue.") - ); - }); - - if !tasks.is_empty() || !ipc.locked.is_empty() { - trc::event!( - TaskQueue(TaskQueueEvent::TaskAcquired), - Total = tasks.len(), - Details = ipc.locked.len(), - ); - } - - // Shuffle tasks - if tasks.len() > 1 { - tasks.shuffle(&mut rand::rng()); - } - - // Dispatch tasks - let roles = &self.core.network.roles; - for (task_job, task_type_idx) in tasks { - let enabled = match task_job.typ { - TaskType::IndexDocument | TaskType::UnindexDocument | TaskType::IndexTrace => roles - .fts_indexing - .is_enabled_for_integer(task_job.id as u32), - TaskType::CalendarAlarmEmail | TaskType::CalendarAlarmNotification => roles - .calendar_alerts - .is_enabled_for_integer(task_job.id as u32), - TaskType::CalendarItipMessage => roles - .imip_processing - .is_enabled_for_integer(task_job.id as u32), - TaskType::MergeThreads => roles - .merge_threads - .is_enabled_for_integer(task_job.id as u32), - TaskType::DmarcReport | TaskType::TlsReport | TaskType::RestoreArchivedItem => true, - }; - - if enabled { - if self.try_lock_task(task_job.id).await - && ipc.txs[task_type_idx as usize] - .send(task_job) - .await - .is_err() - { - trc::event!( - Server(trc::ServerEvent::ThreadError), - Details = "Error sending task.", - CausedBy = trc::location!() - ); - } - } else { - trc::event!( - TaskQueue(TaskQueueEvent::TaskIgnored), - Id = task_job.id, - Details = task_job.typ.as_str(), - Reason = "Task type is disabled by cluster roles.", - ); - } - } - - // Delete expired locks - let now = Instant::now(); - ipc.locked - .retain(|_, locked| locked.expires > now && locked.revision == ipc.revision); - Duration::from_secs(next_event.map_or(QUEUE_REFRESH_INTERVAL, |timestamp| { - timestamp.saturating_sub(store::write::now()) - })) - } -} - -async fn update_tasks( - server: &Server, - tasks: &mut [TaskDetails], - results: impl IntoIterator, -) { - let mut batch = BatchBuilder::new(); - - for (task, result) in tasks.iter_mut().zip(results.into_iter()) { - let id = task.info.id; - batch.clear(ValueClass::TaskQueue(TaskQueueClass::Due { - id, - due: task.info.due, - })); - match result { - TaskResult::Success | TaskResult::Ignored => { - batch.clear(ValueClass::TaskQueue(TaskQueueClass::Task { id })); - } - TaskResult::Update(ops) => { - for op in ops { - batch.any_op(op); - } - } - TaskResult::Failure { typ, message } => { - let (attempt_number, created_at) = match task.task.status() { - TaskStatus::Pending(status) => (0, status.created_at), - TaskStatus::Retry(status) => (status.attempt_number, status.created_at), - TaskStatus::Failed(status) => (status.failed_attempt_number, status.failed_at), - }; - let retry_at = match typ { - TaskFailureType::Retry(retry_at) => (attempt_number - < server.core.network.task_manager.max_attempts - && retry_at - < retry_at.saturating_add( - server.core.network.task_manager.total_deadline.as_secs(), - )) - .then_some(retry_at), - TaskFailureType::Temporary => next_retry_time( - &server.core.network.task_manager, - created_at.timestamp() as u64, - attempt_number, - now(), - ), - TaskFailureType::Permanent => None, - }; - - let due = if let Some(retry_at) = retry_at { - trc::event!( - TaskQueue(TaskQueueEvent::TaskRetry), - Id = id, - Details = task.task.name(), - Reason = message.to_string(), - NextRetry = trc::Value::Timestamp(retry_at), - ); - - task.task.set_status(TaskStatus::Retry(TaskStatusRetry { - due: UTCDateTime::from_timestamp(retry_at as i64), - attempt_number: attempt_number + 1, - failure_reason: message, - created_at, - })); - - retry_at - } else { - trc::event!( - TaskQueue(TaskQueueEvent::TaskFailed), - Id = id, - Details = task.task.name(), - Reason = message.to_string(), - ); - - task.task.set_status(TaskStatus::Failed(TaskStatusFailed { - failed_at: UTCDateTime::now(), - failed_attempt_number: attempt_number, - failure_reason: message, - created_at, - })); - u64::MAX - }; - batch - .set( - ValueClass::TaskQueue(TaskQueueClass::Due { id, due }), - task.info.typ.to_id().serialize(), - ) - .set( - ValueClass::TaskQueue(TaskQueueClass::Task { id }), - task.task.to_pickled_vec(), - ); - } - } - } - - if let Err(err) = server.store().write(batch.build_all()).await { - trc::error!(err.details("Failed to remove task(s) from queue.")); - } - - for task in tasks { - server.remove_index_lock(task.info.id).await; - } -} - -pub fn next_retry_time( - manager: &TaskManager, - create_time: u64, - attempt: u64, - now: u64, -) -> Option { - if attempt >= manager.max_attempts { - return None; - } - - let delay_secs: u64 = match &manager.strategy { - TaskRetryStrategy::FixedDelay(fixed) => fixed.delay.as_secs(), - TaskRetryStrategy::ExponentialBackoff(backoff) => { - let delay = (backoff.initial_delay.as_secs() as f64 - * backoff.factor.into_inner().powi(attempt as i32)) - .min(backoff.max_delay.as_secs() as f64) as u64; - - if backoff.jitter { - let jitter_factor = rand::random::() + 0.5; - ((delay as f64 * jitter_factor) as u64).min(backoff.max_delay.as_secs()) - } else { - delay - } - } - }; - - let next_time = now.saturating_add(delay_secs); - let deadline = create_time.saturating_add(manager.total_deadline.as_secs()); - if next_time > deadline { - return None; - } - - Some(next_time) -} - pub(crate) trait TaskInfo { fn name(&self) -> &'static str; } @@ -588,6 +90,10 @@ impl TaskInfo for Task { Task::DmarcReport(_) => "DmarcReport", Task::TlsReport(_) => "TlsReport", Task::RestoreArchivedItem(_) => "RestoreArchivedItem", + Task::DestroyAccount(_) => "DestroyAccount", + Task::AccountMaintenance(_) => "AccountMaintenance", + Task::StoreMaintenance(_) => "StoreMaintenance", + Task::SpamFilterMaintenance(_) => "SpamFilterMaintenance", } } } @@ -607,3 +113,10 @@ impl TaskResult { } } } + +impl TaskJob { + #[inline(always)] + pub fn id_hash(&self) -> u64 { + xxh3_64(&self.id.to_le_bytes()) + } +} diff --git a/crates/services/src/task_manager/scheduler.rs b/crates/services/src/task_manager/scheduler.rs new file mode 100644 index 00000000..92bdcce3 --- /dev/null +++ b/crates/services/src/task_manager/scheduler.rs @@ -0,0 +1,561 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +#[cfg(feature = "enterprise")] +use std::time::Duration; +use std::{ + collections::BinaryHeap, + sync::Arc, + time::{Instant, SystemTime}, +}; + +use common::{ + BuildServer, Inner, LONG_1D_SLUMBER, + config::{mailstore::spamfilter, telemetry::OtelMetrics}, +}; +use registry::{ + schema::{ + enums::{TaskSpamFilterMaintenanceType, TaskStoreMaintenanceType, TaskType}, + structs::{Task, TaskSpamFilterMaintenance, TaskStatus, TaskStoreMaintenance}, + }, + types::EnumImpl, +}; +use store::write::{BatchBuilder, now}; +use trc::{Collector, MetricType, TaskManagerEvent, TelemetryEvent}; + +#[derive(PartialEq, Eq)] +struct Action { + due: Instant, + event: Event, +} + +#[derive(PartialEq, Eq, Debug)] +enum Event { + PurgeAccount, + PurgeDataStore, + PurgeBlobStore, + OtelMetrics, + CalculateMetrics, + TrainSpamClassifier, + // SPDX-SnippetBegin + // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + // SPDX-License-Identifier: LicenseRef-SEL + #[cfg(feature = "enterprise")] + InternalMetrics, + #[cfg(feature = "enterprise")] + AlertMetrics, + #[cfg(feature = "enterprise")] + RenewLicense, + // SPDX-SnippetEnd +} + +#[derive(Default)] +struct Queue { + heap: BinaryHeap, +} + +// SPDX-SnippetBegin +// SPDX-FileCopyrightText: 2020 Stalwart Labs LLC +// SPDX-License-Identifier: LicenseRef-SEL +#[cfg(feature = "enterprise")] +const METRIC_ALERTS_INTERVAL: Duration = Duration::from_secs(5 * 60); +// SPDX-SnippetEnd + +pub fn spawn_task_scheduler(inner: Arc) { + tokio::spawn(async move { + trc::event!(TaskManager(TaskManagerEvent::SchedulerStarted)); + let start_time = SystemTime::now(); + + // Add all events to queue + let mut queue = Queue::default(); + { + let server = inner.build_server(); + + // Account purge + queue.schedule( + Instant::now() + server.core.email.account_purge_frequency.time_to_next(), + Event::PurgeAccount, + ); + queue.schedule( + Instant::now() + server.core.email.data_purge_frequency.time_to_next(), + Event::PurgeDataStore, + ); + queue.schedule( + Instant::now() + server.core.email.blob_purge_frequency.time_to_next(), + Event::PurgeBlobStore, + ); + + // Spam classifier training + if let Some(train_frequency) = server + .core + .spam + .classifier + .as_ref() + .and_then(|c| c.train_frequency) + { + let next_train = match server.inner.data.spam_classifier.load().as_ref() { + spamfilter::SpamClassifier::FhClassifier { + last_trained_at, .. + } + | spamfilter::SpamClassifier::CcfhClassifier { + last_trained_at, .. + } => now().saturating_sub(*last_trained_at).min(train_frequency), + spamfilter::SpamClassifier::Disabled => train_frequency, + }; + + queue.schedule( + Instant::now() + Duration::from_secs(next_train), + Event::TrainSpamClassifier, + ); + } + + // OTEL Push Metrics + if let Some(otel) = &server.core.metrics.otel { + OtelMetrics::enable_errors(); + queue.schedule(Instant::now() + otel.interval, Event::OtelMetrics); + } + + // Calculate expensive metrics + queue.schedule(Instant::now(), Event::CalculateMetrics); + + // SPDX-SnippetBegin + // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + // SPDX-License-Identifier: LicenseRef-SEL + + // Enterprise Edition license management + #[cfg(feature = "enterprise")] + if let Some(enterprise) = &server.core.enterprise { + queue.schedule( + Instant::now() + enterprise.license.renew_in(), + Event::RenewLicense, + ); + + queue.schedule( + Instant::now() + enterprise.metrics_interval.time_to_next(), + Event::InternalMetrics, + ); + + queue.schedule(Instant::now() + METRIC_ALERTS_INTERVAL, Event::AlertMetrics); + } + + // SPDX-SnippetEnd + } + + // SPDX-SnippetBegin + // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + // SPDX-License-Identifier: LicenseRef-SEL + // Metrics history + #[cfg(feature = "enterprise")] + let metrics_history = common::telemetry::metrics::store::SharedMetricHistory::default(); + // SPDX-SnippetEnd + + let mut next_metric_update = Instant::now(); + + loop { + tokio::time::sleep(queue.wake_up_time()).await; + + let server = inner.build_server(); + let roles = &server.core.network.roles; + let mut batch = (roles.task_scheduler.is_enabled_or_sharded()).then(BatchBuilder::new); + + while let Some(event) = queue.pop() { + match event.event { + Event::PurgeAccount => { + queue.schedule( + Instant::now() + + server.core.email.account_purge_frequency.time_to_next(), + Event::PurgeAccount, + ); + + if let Some(batch) = batch.as_mut() { + trc::event!( + TaskManager(TaskManagerEvent::TaskQueued), + Type = TaskStoreMaintenanceType::PurgeAccounts.as_str() + ); + + batch.schedule_task(Task::StoreMaintenance(TaskStoreMaintenance { + maintenance_type: TaskStoreMaintenanceType::PurgeAccounts, + status: TaskStatus::now(), + })); + } + } + Event::PurgeDataStore => { + queue.schedule( + Instant::now() + server.core.email.data_purge_frequency.time_to_next(), + Event::PurgeDataStore, + ); + + if let Some(batch) = batch.as_mut() { + trc::event!( + TaskManager(TaskManagerEvent::TaskQueued), + Type = TaskStoreMaintenanceType::PurgeData.as_str() + ); + + batch.schedule_task(Task::StoreMaintenance(TaskStoreMaintenance { + maintenance_type: TaskStoreMaintenanceType::PurgeData, + status: TaskStatus::now(), + })); + } + } + Event::PurgeBlobStore => { + queue.schedule( + Instant::now() + server.core.email.blob_purge_frequency.time_to_next(), + Event::PurgeBlobStore, + ); + + if let Some(batch) = batch.as_mut() { + trc::event!( + TaskManager(TaskManagerEvent::TaskQueued), + Type = TaskStoreMaintenanceType::PurgeBlob.as_str() + ); + + batch.schedule_task(Task::StoreMaintenance(TaskStoreMaintenance { + maintenance_type: TaskStoreMaintenanceType::PurgeBlob, + status: TaskStatus::now(), + })); + } + } + Event::OtelMetrics => { + if let Some(otel) = &server.core.metrics.otel { + queue.schedule(Instant::now() + otel.interval, Event::OtelMetrics); + + if roles.push_metrics.is_enabled_or_sharded() { + let otel = otel.clone(); + + // SPDX-SnippetBegin + // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + // SPDX-License-Identifier: LicenseRef-SEL + #[cfg(feature = "enterprise")] + let is_enterprise = server.is_enterprise_edition(); + // SPDX-SnippetEnd + + #[cfg(not(feature = "enterprise"))] + let is_enterprise = false; + + tokio::spawn(async move { + let elapsed = Instant::now(); + otel.push_metrics(is_enterprise, start_time).await; + + trc::event!( + Telemetry(TelemetryEvent::MetricsPushed), + Elapsed = elapsed.elapsed() + ); + }); + } + } + } + Event::CalculateMetrics => { + // Calculate expensive metrics every 5 minutes + queue.schedule( + Instant::now() + Duration::from_secs(5 * 60), + Event::CalculateMetrics, + ); + + let update_other_metrics = if Instant::now() >= next_metric_update { + next_metric_update = Instant::now() + Duration::from_secs(86400); + true + } else { + false + }; + + let server = server.clone(); + tokio::spawn(async move { + let elapsed = Instant::now(); + if server + .core + .network + .roles + .calculate_metrics + .is_enabled_or_sharded() + { + // SPDX-SnippetBegin + // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + // SPDX-License-Identifier: LicenseRef-SEL + #[cfg(feature = "enterprise")] + if server.is_enterprise_edition() { + // Obtain queue size + match server.total_queued_messages().await { + Ok(total) => { + Collector::update_gauge(MetricType::QueueCount, total); + } + Err(err) => { + trc::error!(err.details("Failed to obtain queue size")); + } + } + } + // SPDX-SnippetEnd + + if update_other_metrics { + match server.total_accounts().await { + Ok(total) => { + Collector::update_gauge( + MetricType::UserCount, + total as u64, + ); + } + Err(err) => { + trc::error!( + err.details("Failed to obtain account count") + ); + } + } + + match server.total_domains().await { + Ok(total) => { + Collector::update_gauge( + MetricType::DomainCount, + total as u64, + ); + } + Err(err) => { + trc::error!( + err.details("Failed to obtain domain count") + ); + } + } + } + } + + match tokio::task::spawn_blocking(memory_stats::memory_stats).await { + Ok(Some(stats)) => { + Collector::update_gauge( + MetricType::ServerMemory, + stats.physical_mem as u64, + ); + } + Ok(None) => {} + Err(err) => { + trc::error!( + trc::EventType::Server(trc::ServerEvent::ThreadError,) + .reason(err) + .caused_by(trc::location!()) + .details("Join Error") + ); + } + } + + trc::event!( + Telemetry(TelemetryEvent::MetricsCollected), + Elapsed = elapsed.elapsed() + ); + }); + } + Event::TrainSpamClassifier => { + if let Some(train_frequency) = server + .core + .spam + .classifier + .as_ref() + .and_then(|c| c.train_frequency) + { + // Schedule next training + queue.schedule( + Instant::now() + Duration::from_secs(train_frequency), + Event::TrainSpamClassifier, + ); + + if let Some(batch) = batch.as_mut() { + trc::event!( + TaskManager(TaskManagerEvent::TaskQueued), + Type = TaskType::SpamFilterMaintenance.as_str() + ); + + batch.schedule_task(Task::SpamFilterMaintenance( + TaskSpamFilterMaintenance { + maintenance_type: TaskSpamFilterMaintenanceType::Train, + status: TaskStatus::now(), + }, + )); + } + } + } + + // SPDX-SnippetBegin + // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + // SPDX-License-Identifier: LicenseRef-SEL + #[cfg(feature = "enterprise")] + Event::InternalMetrics => { + if let Some(enterprise) = &server.core.enterprise { + queue.schedule( + Instant::now() + enterprise.metrics_interval.time_to_next(), + Event::InternalMetrics, + ); + } + + if server.core.storage.metrics.is_active() { + use common::telemetry::metrics::store::MetricsStore; + + let metrics_store = server.core.storage.metrics.clone(); + let metrics_history = metrics_history.clone(); + tokio::spawn(async move { + let elapsed = Instant::now(); + if let Err(err) = + metrics_store.write_metrics(None, metrics_history).await + { + trc::error!(err.details("Failed to write metrics")); + } + trc::event!( + Telemetry(TelemetryEvent::MetricsStored), + Elapsed = elapsed.elapsed() + ); + }); + } + } + + #[cfg(feature = "enterprise")] + Event::AlertMetrics => { + queue + .schedule(Instant::now() + METRIC_ALERTS_INTERVAL, Event::AlertMetrics); + + let server = server.clone(); + tokio::spawn(async move { + if let Some(messages) = server.process_alerts().await { + use smtp::reporting::send::MtaReportSend; + + for message in messages { + server + .send_autogenerated( + message.from, + message.to.into_iter(), + message.body, + None, + 0, + ) + .await; + } + } + }); + } + + #[cfg(feature = "enterprise")] + Event::RenewLicense => { + use common::ipc::RegistryChange; + use registry::schema::prelude::ObjectType; + + trc::event!( + TaskManager(TaskManagerEvent::TaskQueued), + Type = "validateLicense" + ); + + match server + .reload_registry(RegistryChange::Reload(ObjectType::Enterprise)) + .await + { + Ok(result) => { + if let Some(new_core) = result.new_core { + if let Some(enterprise) = &new_core.enterprise { + let renew_in = if enterprise.license.is_near_expiration() { + // Something went wrong during renewal, try again in 1 day or 1 hour, + // depending on the time left on the license + if enterprise.license.expires_in() + < Duration::from_secs(86400) + { + Duration::from_secs(3600) + } else { + Duration::from_secs(86400) + } + } else { + enterprise.license.renew_in() + }; + + queue.schedule( + Instant::now() + renew_in, + Event::RenewLicense, + ); + } + + // Update core + server.inner.shared_core.store(new_core.into()); + + server + .cluster_broadcast(common::ipc::BroadcastEvent::reload( + ObjectType::Enterprise, + )) + .await; + } + } + Err(err) => { + trc::error!(err.details("Failed to reload configuration.")); + } + } + } // SPDX-SnippetEnd + } + } + + if let Some(mut batch) = batch + && !batch.is_empty() + && let Err(err) = server.store().write(batch.build_all()).await + { + trc::error!(err.details("Failed to write scheduled tasks")); + } + } + }); +} + +impl Queue { + pub fn schedule(&mut self, due: Instant, event: Event) { + trc::event!( + TaskManager(TaskManagerEvent::TaskScheduled), + Due = trc::Value::Timestamp( + now() + due.saturating_duration_since(Instant::now()).as_secs() + ), + Id = event.name() + ); + + self.heap.push(Action { due, event }); + } + + pub fn wake_up_time(&self) -> Duration { + self.heap + .peek() + .map(|e| e.due.saturating_duration_since(Instant::now())) + .unwrap_or(LONG_1D_SLUMBER) + } + + pub fn pop(&mut self) -> Option { + if self.heap.peek()?.due <= Instant::now() { + self.heap.pop() + } else { + None + } + } +} + +impl Ord for Action { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.due.cmp(&other.due).reverse() + } +} + +impl PartialOrd for Action { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Event { + fn name(&self) -> &'static str { + match self { + Event::PurgeAccount => "purgeAccount", + Event::PurgeDataStore => "purgeDataStore", + Event::PurgeBlobStore => "purgeBlobStore", + Event::OtelMetrics => "otelMetrics", + Event::CalculateMetrics => "calculateMetrics", + Event::TrainSpamClassifier => "trainSpamClassifier", + + // SPDX-SnippetBegin + // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + // SPDX-License-Identifier: LicenseRef-SEL + #[cfg(feature = "enterprise")] + Event::InternalMetrics => "internalMetrics", + #[cfg(feature = "enterprise")] + Event::AlertMetrics => "alertMetrics", + #[cfg(feature = "enterprise")] + Event::RenewLicense => "renewLicense", + // SPDX-SnippetEnd + } + } +} diff --git a/crates/services/src/task_manager/spam_classifier.rs b/crates/services/src/task_manager/spam_classifier.rs new file mode 100644 index 00000000..45ee0f92 --- /dev/null +++ b/crates/services/src/task_manager/spam_classifier.rs @@ -0,0 +1,63 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::task_manager::TaskResult; +use common::{ + Server, + manager::{SPAM_CLASSIFIER_KEY, SPAM_TRAINER_KEY}, +}; +use registry::schema::{enums::TaskSpamFilterMaintenanceType, structs::TaskSpamFilterMaintenance}; +use spam_filter::modules::classifier::SpamClassifier; + +pub(crate) trait SpamFilterMaintenanceTask: Sync + Send { + fn spam_filter_maintenance( + &self, + task: &TaskSpamFilterMaintenance, + ) -> impl Future + Send; +} + +impl SpamFilterMaintenanceTask for Server { + async fn spam_filter_maintenance(&self, task: &TaskSpamFilterMaintenance) -> TaskResult { + match spam_filter_maintenance(self, task).await { + Ok(result) => result, + Err(err) => { + let result = TaskResult::temporary(err.to_string()); + trc::error!(err.details("Failed to perform spam filter maintenance task")); + result + } + } + } +} + +async fn spam_filter_maintenance( + server: &Server, + task: &TaskSpamFilterMaintenance, +) -> trc::Result { + match task.maintenance_type { + TaskSpamFilterMaintenanceType::Train => { + if !server.inner.ipc.train_task_controller.is_running() { + server.spam_train(false).await?; + } + } + TaskSpamFilterMaintenanceType::Retrain => { + if !server.inner.ipc.train_task_controller.is_running() { + server.spam_train(true).await?; + } + } + TaskSpamFilterMaintenanceType::Reset => { + for key in [SPAM_CLASSIFIER_KEY, SPAM_TRAINER_KEY] { + server.blob_store().delete_blob(key).await?; + } + } + TaskSpamFilterMaintenanceType::Abort => { + if server.inner.ipc.train_task_controller.is_running() { + server.inner.ipc.train_task_controller.stop(); + } + } + } + + Ok(TaskResult::Success) +} diff --git a/crates/smtp/src/lib.rs b/crates/smtp/src/lib.rs index afc3f44b..f0bd3d1b 100644 --- a/crates/smtp/src/lib.rs +++ b/crates/smtp/src/lib.rs @@ -7,7 +7,7 @@ #![warn(clippy::large_futures)] use common::{ - Inner, + BuildServer, Inner, manager::boot::{BootManager, IpcReceivers}, }; use queue::manager::SpawnQueue; @@ -37,10 +37,19 @@ impl StartQueueManager for BootManager { impl SpawnQueueManager for IpcReceivers { fn spawn_queue_manager(&mut self, inner: Arc) { - // Spawn queue manager - self.queue_rx.take().unwrap().spawn(inner.clone()); + if inner + .build_server() + .core + .network + .roles + .outbound_mta + .is_enabled_or_sharded() + { + // Spawn queue manager + self.queue_rx.take().unwrap().spawn(inner.clone()); - // Spawn report manager - self.report_rx.take().unwrap().spawn(inner); + // Spawn report manager + self.report_rx.take().unwrap().spawn(inner); + } } } diff --git a/crates/smtp/src/queue/manager.rs b/crates/smtp/src/queue/manager.rs index 26184055..91885292 100644 --- a/crates/smtp/src/queue/manager.rs +++ b/crates/smtp/src/queue/manager.rs @@ -68,6 +68,8 @@ impl Queue { } pub async fn start(&mut self) { + trc::event!(Queue(trc::QueueEvent::Started)); + loop { let mut refresh_queue; diff --git a/crates/smtp/src/queue/spool.rs b/crates/smtp/src/queue/spool.rs index fc982c3b..1ea033b9 100644 --- a/crates/smtp/src/queue/spool.rs +++ b/crates/smtp/src/queue/spool.rs @@ -261,16 +261,7 @@ impl SmtpSpool for Server { Some(a) => a.deserialize::().map(Some), None => Ok(None), }) { - Ok(Some(message)) => Some(MessageWrapper { - is_multi_queue: message.recipients.iter().any(|rcpt| { - matches!(rcpt.status, Status::Scheduled | Status::TemporaryFailure(_)) - && rcpt.queue != queue_name - }), - queue_id, - queue_name, - span_id: 0, - message, - }), + Ok(Some(message)) => Some(MessageWrapper::new(message, queue_id, queue_name)), Ok(None) => None, Err(err) => { trc::error!( @@ -295,6 +286,21 @@ impl SmtpSpool for Server { } } +impl MessageWrapper { + pub fn new(message: Message, queue_id: QueueId, queue_name: QueueName) -> Self { + MessageWrapper { + is_multi_queue: message.recipients.iter().any(|rcpt| { + matches!(rcpt.status, Status::Scheduled | Status::TemporaryFailure(_)) + && rcpt.queue != queue_name + }), + queue_id, + queue_name, + span_id: 0, + message, + } + } +} + fn lock_id(queue_id: QueueId, queue_name: QueueName) -> [u8; 16] { let mut id = [0; 16]; id[..8].copy_from_slice(&queue_id.to_be_bytes()); @@ -315,7 +321,7 @@ impl MessageWrapper { let (flags, event, train_spam) = match source { MessageSource::Authenticated => ( FROM_AUTHENTICATED, - trc::QueueEvent::QueueMessageAuthenticated, + trc::QueueEvent::AuthenticatedMessageQueued, None, ), MessageSource::Unauthenticated { @@ -323,7 +329,7 @@ impl MessageWrapper { train_spam, } => ( FROM_UNAUTHENTICATED_DMARC, - trc::QueueEvent::QueueMessage, + trc::QueueEvent::MessageQueued, train_spam, ), MessageSource::Unauthenticated { @@ -331,14 +337,14 @@ impl MessageWrapper { train_spam, } => ( FROM_UNAUTHENTICATED, - trc::QueueEvent::QueueMessage, + trc::QueueEvent::MessageQueued, train_spam, ), - MessageSource::Dsn => (FROM_DSN, trc::QueueEvent::QueueDsn, None), - MessageSource::Report => (FROM_REPORT, trc::QueueEvent::QueueReport, None), + MessageSource::Dsn => (FROM_DSN, trc::QueueEvent::DsnQueued, None), + MessageSource::Report => (FROM_REPORT, trc::QueueEvent::ReportQueued, None), MessageSource::Autogenerated => ( FROM_AUTOGENERATED, - trc::QueueEvent::QueueAutogenerated, + trc::QueueEvent::AutogeneratedQueued, None, ), }; diff --git a/crates/store/src/registry/write.rs b/crates/store/src/registry/write.rs index 2c2c9d52..aa0fe827 100644 --- a/crates/store/src/registry/write.rs +++ b/crates/store/src/registry/write.rs @@ -65,6 +65,7 @@ pub enum RegistryWrite<'x> { Delete { object_id: ObjectId, object: Option<&'x Object>, + force: bool, }, } @@ -137,9 +138,13 @@ impl RegistryStore { AssertValue::Hash(old_object.revision), ); } - RegistryWrite::Delete { object_id, object } => { + RegistryWrite::Delete { + object_id, + object, + force, + } => { return if object_id.object().flags() & OBJ_SINGLETON == 0 { - self.delete(object_id, object).await + self.delete(object_id, object, force).await } else { Ok(RegistryWriteResult::CannotDeleteSingleton) }; @@ -171,7 +176,11 @@ impl RegistryStore { // Validate foreign keys let tenant_id = object.inner.member_tenant_id().map(|id| id.id()); - let account_id = object.inner.account_id().map(|id| id.id()); + let account_id = object + .inner + .account_id() + .map(|id| id.id()) + .or_else(|| (object_type == ObjectType::Account).then_some(item_id)); for key in &set_index.keys { match key { IndexKey::ForeignKey { @@ -222,8 +231,8 @@ impl RegistryStore { return Ok(RegistryWriteResult::InvalidForeignKey { object_id: *foreign_id, }); - } else if let Some(account_id) = account_id - && (object_flags & OBJ_FILTER_ACCOUNT) != 0 + } else if (object_flags & OBJ_FILTER_ACCOUNT) != 0 + && let Some(account_id) = account_id && self .0 .store @@ -314,6 +323,7 @@ impl RegistryStore { &self, object_id: ObjectId, object: Option<&Object>, + force: bool, ) -> trc::Result { let object_type = object_id.object(); let object_type_id = object_type.to_id(); @@ -347,12 +357,14 @@ impl RegistryStore { object.index(&mut clear_index); // Validate relationships - let linked = self.linked_objects(object_id).await?; - if !linked.is_empty() { - return Ok(RegistryWriteResult::CannotDeleteLinked { - object_id: ObjectId::new(object_type, id), - linked_objects: linked, - }); + if !force { + let linked = self.linked_objects(object_id).await?; + if !linked.is_empty() { + return Ok(RegistryWriteResult::CannotDeleteLinked { + object_id: ObjectId::new(object_type, id), + linked_objects: linked, + }); + } } // Build deletion batch @@ -545,6 +557,7 @@ impl<'x> RegistryWrite<'x> { RegistryWrite::Delete { object_id, object: None, + force: false, } } @@ -552,6 +565,7 @@ impl<'x> RegistryWrite<'x> { RegistryWrite::Delete { object_id, object: Some(object), + force: false, } } } diff --git a/crates/store/src/write/blob.rs b/crates/store/src/write/blob.rs index f03adde9..05af93b8 100644 --- a/crates/store/src/write/blob.rs +++ b/crates/store/src/write/blob.rs @@ -15,7 +15,7 @@ use registry::{ types::{EnumImpl, id::ObjectId}, }; use std::time::Instant; -use trc::{AddContext, PurgeEvent}; +use trc::{AddContext, StoreEvent}; use types::{ blob::BlobClass, blob_hash::{BLOB_HASH_LEN, BlobHash}, @@ -190,7 +190,7 @@ impl Store { } trc::event!( - Purge(PurgeEvent::BlobCleanup), + Store(StoreEvent::BlobStorePurged), Expires = total_deleted, Total = total_active, Elapsed = started.elapsed() diff --git a/crates/trc/src/event/enums.rs b/crates/trc/src/event/enums.rs index 2c118302..7b56fc9f 100644 --- a/crates/trc/src/event/enums.rs +++ b/crates/trc/src/event/enums.rs @@ -23,7 +23,6 @@ pub enum EventType { Dmarc(DmarcEvent), Dns(DnsEvent), Eval(EvalEvent), - Housekeeper(HousekeeperEvent), Http(HttpEvent), Imap(ImapEvent), IncomingReport(IncomingReportEvent), @@ -40,7 +39,6 @@ pub enum EventType { Network(NetworkEvent), OutgoingReport(OutgoingReportEvent), Pop3(Pop3Event), - Purge(PurgeEvent), PushSubscription(PushSubscriptionEvent), Queue(QueueEvent), Registry(RegistryEvent), @@ -52,7 +50,7 @@ pub enum EventType { Spam(SpamEvent), Spf(SpfEvent), Store(StoreEvent), - TaskQueue(TaskQueueEvent), + TaskManager(TaskManagerEvent), Telemetry(TelemetryEvent), Tls(TlsEvent), TlsRpt(TlsRptEvent), @@ -257,15 +255,6 @@ pub enum EvalEvent { StoreNotFound = 140, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[repr(u16)] -pub enum HousekeeperEvent { - Start = 150, - Stop = 151, - Schedule = 149, - Run = 146, -} - #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum HttpEvent { @@ -410,12 +399,12 @@ pub enum MailAuthEvent { #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum ManageEvent { - MissingParameter = 278, - AlreadyExists = 275, - AssertFailed = 276, - NotFound = 279, - NotSupported = 280, - Error = 277, + Reserved1 = 278, + Reserved2 = 275, + Reserved3 = 276, + Reserved4 = 279, + Reserved5 = 280, + Reserved6 = 277, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -559,18 +548,6 @@ pub enum Pop3Event { RawOutput = 357, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[repr(u16)] -pub enum PurgeEvent { - Started = 369, - Finished = 366, - Running = 368, - Error = 365, - InProgress = 367, - AutoExpunge = 364, - BlobCleanup = 370, -} - #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum PushSubscriptionEvent { @@ -582,11 +559,12 @@ pub enum PushSubscriptionEvent { #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum QueueEvent { - QueueMessage = 380, - QueueMessageAuthenticated = 381, - QueueReport = 382, - QueueDsn = 379, - QueueAutogenerated = 378, + Started = 390, + MessageQueued = 380, + AuthenticatedMessageQueued = 381, + ReportQueued = 382, + DsnQueued = 379, + AutogeneratedQueued = 378, Rescheduled = 385, Locked = 377, BlobNotFound = 374, @@ -618,7 +596,6 @@ pub enum ResourceEvent { BadParameters = 386, Error = 388, DownloadExternal = 387, - WebadminUnpacked = 390, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -817,30 +794,41 @@ pub enum StoreEvent { LdapQuery = 521, LdapWarning = 519, HttpStoreFetch = 492, + AutoExpunge = 364, + BlobStorePurged = 369, + DataStorePurged = 368, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] -pub enum TaskQueueEvent { +pub enum TaskManagerEvent { TaskAcquired = 578, + TaskQueued = 149, + TaskScheduled = 370, TaskLocked = 144, TaskIgnored = 586, TaskFailed = 587, TaskRetry = 53, BlobNotFound = 141, MetadataNotFound = 145, + SchedulerStarted = 150, + ManagerStarted = 367, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] #[repr(u16)] pub enum TelemetryEvent { - Alert = 548, + AlertEvent = 548, + AlertMessage = 365, LogError = 535, WebhookError = 539, OtelExporterError = 536, OtelMetricsExporterError = 537, PrometheusExporterError = 538, JournalError = 534, + MetricsCollected = 151, + MetricsStored = 366, + MetricsPushed = 146, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -1090,16 +1078,15 @@ pub enum MetricType { Pop3ActiveConnections = 19, Pop3ConnectionStart = 210, Pop3ConnectionEnd = 211, - PurgeError = 212, PushSubscriptionSuccess = 213, PushSubscriptionError = 214, PushSubscriptionNotFound = 215, QueueCount = 24, - QueueQueueMessage = 216, - QueueQueueMessageAuthenticated = 217, - QueueQueueReport = 218, - QueueQueueDsn = 219, - QueueQueueAutogenerated = 220, + QueueMessageQueued = 216, + QueueAuthenticatedMessageQueued = 217, + QueueReportQueued = 218, + QueueDsnQueued = 219, + QueueAutogeneratedQueued = 220, QueueRescheduled = 221, QueueBlobNotFound = 222, QueueRateLimitExceeded = 223, @@ -1216,8 +1203,9 @@ pub enum MetricType { StoreBlobRead = 325, StoreBlobWrite = 326, StoreBlobDelete = 327, - TaskQueueBlobNotFound = 328, - TaskQueueMetadataNotFound = 329, + TaskManagerBlobNotFound = 328, + TaskManagerMetadataNotFound = 329, + TelemetryAlertMessage = 212, TelemetryLogError = 330, TelemetryWebhookError = 331, TelemetryOtelExporterError = 332, diff --git a/crates/trc/src/event/enums_impl.rs b/crates/trc/src/event/enums_impl.rs index 711ed5e5..82cfdae2 100644 --- a/crates/trc/src/event/enums_impl.rs +++ b/crates/trc/src/event/enums_impl.rs @@ -151,10 +151,6 @@ impl EventType { b"eval.error" => EventType::Eval(EvalEvent::Error), b"eval.directory-not-found" => EventType::Eval(EvalEvent::DirectoryNotFound), b"eval.store-not-found" => EventType::Eval(EvalEvent::StoreNotFound), - b"housekeeper.start" => EventType::Housekeeper(HousekeeperEvent::Start), - b"housekeeper.stop" => EventType::Housekeeper(HousekeeperEvent::Stop), - b"housekeeper.schedule" => EventType::Housekeeper(HousekeeperEvent::Schedule), - b"housekeeper.run" => EventType::Housekeeper(HousekeeperEvent::Run), b"http.connection-start" => EventType::Http(HttpEvent::ConnectionStart), b"http.connection-end" => EventType::Http(HttpEvent::ConnectionEnd), b"http.error" => EventType::Http(HttpEvent::Error), @@ -261,12 +257,12 @@ impl EventType { b"mail-auth.dns-record-not-found" => EventType::MailAuth(MailAuthEvent::DnsRecordNotFound), b"mail-auth.dns-invalid-record-type" => EventType::MailAuth(MailAuthEvent::DnsInvalidRecordType), b"mail-auth.policy-not-aligned" => EventType::MailAuth(MailAuthEvent::PolicyNotAligned), - b"manage.missing-parameter" => EventType::Manage(ManageEvent::MissingParameter), - b"manage.already-exists" => EventType::Manage(ManageEvent::AlreadyExists), - b"manage.assert-failed" => EventType::Manage(ManageEvent::AssertFailed), - b"manage.not-found" => EventType::Manage(ManageEvent::NotFound), - b"manage.not-supported" => EventType::Manage(ManageEvent::NotSupported), - b"manage.error" => EventType::Manage(ManageEvent::Error), + b"manage.reserved1" => EventType::Manage(ManageEvent::Reserved1), + b"manage.reserved2" => EventType::Manage(ManageEvent::Reserved2), + b"manage.reserved3" => EventType::Manage(ManageEvent::Reserved3), + b"manage.reserved4" => EventType::Manage(ManageEvent::Reserved4), + b"manage.reserved5" => EventType::Manage(ManageEvent::Reserved5), + b"manage.reserved6" => EventType::Manage(ManageEvent::Reserved6), b"manage-sieve.connection-start" => EventType::ManageSieve(ManageSieveEvent::ConnectionStart), b"manage-sieve.connection-end" => EventType::ManageSieve(ManageSieveEvent::ConnectionEnd), b"manage-sieve.create-script" => EventType::ManageSieve(ManageSieveEvent::CreateScript), @@ -368,21 +364,15 @@ impl EventType { b"pop3.error" => EventType::Pop3(Pop3Event::Error), b"pop3.raw-input" => EventType::Pop3(Pop3Event::RawInput), b"pop3.raw-output" => EventType::Pop3(Pop3Event::RawOutput), - b"purge.started" => EventType::Purge(PurgeEvent::Started), - b"purge.finished" => EventType::Purge(PurgeEvent::Finished), - b"purge.running" => EventType::Purge(PurgeEvent::Running), - b"purge.error" => EventType::Purge(PurgeEvent::Error), - b"purge.in-progress" => EventType::Purge(PurgeEvent::InProgress), - b"purge.auto-expunge" => EventType::Purge(PurgeEvent::AutoExpunge), - b"purge.blob-cleanup" => EventType::Purge(PurgeEvent::BlobCleanup), b"push-subscription.success" => EventType::PushSubscription(PushSubscriptionEvent::Success), b"push-subscription.error" => EventType::PushSubscription(PushSubscriptionEvent::Error), b"push-subscription.not-found" => EventType::PushSubscription(PushSubscriptionEvent::NotFound), - b"queue.queue-message" => EventType::Queue(QueueEvent::QueueMessage), - b"queue.queue-message-authenticated" => EventType::Queue(QueueEvent::QueueMessageAuthenticated), - b"queue.queue-report" => EventType::Queue(QueueEvent::QueueReport), - b"queue.queue-dsn" => EventType::Queue(QueueEvent::QueueDsn), - b"queue.queue-autogenerated" => EventType::Queue(QueueEvent::QueueAutogenerated), + b"queue.started" => EventType::Queue(QueueEvent::Started), + b"queue.message-queued" => EventType::Queue(QueueEvent::MessageQueued), + b"queue.authenticated-message-queued" => EventType::Queue(QueueEvent::AuthenticatedMessageQueued), + b"queue.report-queued" => EventType::Queue(QueueEvent::ReportQueued), + b"queue.dsn-queued" => EventType::Queue(QueueEvent::DsnQueued), + b"queue.autogenerated-queued" => EventType::Queue(QueueEvent::AutogeneratedQueued), b"queue.rescheduled" => EventType::Queue(QueueEvent::Rescheduled), b"queue.locked" => EventType::Queue(QueueEvent::Locked), b"queue.blob-not-found" => EventType::Queue(QueueEvent::BlobNotFound), @@ -404,7 +394,6 @@ impl EventType { b"resource.bad-parameters" => EventType::Resource(ResourceEvent::BadParameters), b"resource.error" => EventType::Resource(ResourceEvent::Error), b"resource.download-external" => EventType::Resource(ResourceEvent::DownloadExternal), - b"resource.webadmin-unpacked" => EventType::Resource(ResourceEvent::WebadminUnpacked), b"security.authentication-ban" => EventType::Security(SecurityEvent::AuthenticationBan), b"security.abuse-ban" => EventType::Security(SecurityEvent::AbuseBan), b"security.scan-ban" => EventType::Security(SecurityEvent::ScanBan), @@ -568,20 +557,31 @@ impl EventType { b"store.ldap-query" => EventType::Store(StoreEvent::LdapQuery), b"store.ldap-warning" => EventType::Store(StoreEvent::LdapWarning), b"store.http-store-fetch" => EventType::Store(StoreEvent::HttpStoreFetch), - b"task-queue.task-acquired" => EventType::TaskQueue(TaskQueueEvent::TaskAcquired), - b"task-queue.task-locked" => EventType::TaskQueue(TaskQueueEvent::TaskLocked), - b"task-queue.task-ignored" => EventType::TaskQueue(TaskQueueEvent::TaskIgnored), - b"task-queue.task-failed" => EventType::TaskQueue(TaskQueueEvent::TaskFailed), - b"task-queue.task-retry" => EventType::TaskQueue(TaskQueueEvent::TaskRetry), - b"task-queue.blob-not-found" => EventType::TaskQueue(TaskQueueEvent::BlobNotFound), - b"task-queue.metadata-not-found" => EventType::TaskQueue(TaskQueueEvent::MetadataNotFound), - b"telemetry.alert" => EventType::Telemetry(TelemetryEvent::Alert), + b"store.auto-expunge" => EventType::Store(StoreEvent::AutoExpunge), + b"store.blob-store-purged" => EventType::Store(StoreEvent::BlobStorePurged), + b"store.data-store-purged" => EventType::Store(StoreEvent::DataStorePurged), + b"task-manager.task-acquired" => EventType::TaskManager(TaskManagerEvent::TaskAcquired), + b"task-manager.task-queued" => EventType::TaskManager(TaskManagerEvent::TaskQueued), + b"task-manager.task-scheduled" => EventType::TaskManager(TaskManagerEvent::TaskScheduled), + b"task-manager.task-locked" => EventType::TaskManager(TaskManagerEvent::TaskLocked), + b"task-manager.task-ignored" => EventType::TaskManager(TaskManagerEvent::TaskIgnored), + b"task-manager.task-failed" => EventType::TaskManager(TaskManagerEvent::TaskFailed), + b"task-manager.task-retry" => EventType::TaskManager(TaskManagerEvent::TaskRetry), + b"task-manager.blob-not-found" => EventType::TaskManager(TaskManagerEvent::BlobNotFound), + b"task-manager.metadata-not-found" => EventType::TaskManager(TaskManagerEvent::MetadataNotFound), + b"task-manager.scheduler-started" => EventType::TaskManager(TaskManagerEvent::SchedulerStarted), + b"task-manager.manager-started" => EventType::TaskManager(TaskManagerEvent::ManagerStarted), + b"telemetry.alert-event" => EventType::Telemetry(TelemetryEvent::AlertEvent), + b"telemetry.alert-message" => EventType::Telemetry(TelemetryEvent::AlertMessage), b"telemetry.log-error" => EventType::Telemetry(TelemetryEvent::LogError), b"telemetry.webhook-error" => EventType::Telemetry(TelemetryEvent::WebhookError), b"telemetry.otel-exporter-error" => EventType::Telemetry(TelemetryEvent::OtelExporterError), b"telemetry.otel-metrics-exporter-error" => EventType::Telemetry(TelemetryEvent::OtelMetricsExporterError), b"telemetry.prometheus-exporter-error" => EventType::Telemetry(TelemetryEvent::PrometheusExporterError), b"telemetry.journal-error" => EventType::Telemetry(TelemetryEvent::JournalError), + b"telemetry.metrics-collected" => EventType::Telemetry(TelemetryEvent::MetricsCollected), + b"telemetry.metrics-stored" => EventType::Telemetry(TelemetryEvent::MetricsStored), + b"telemetry.metrics-pushed" => EventType::Telemetry(TelemetryEvent::MetricsPushed), b"tls.handshake" => EventType::Tls(TlsEvent::Handshake), b"tls.handshake-error" => EventType::Tls(TlsEvent::HandshakeError), b"tls.not-configured" => EventType::Tls(TlsEvent::NotConfigured), @@ -773,10 +773,6 @@ impl EventType { EventType::Eval(EvalEvent::Error) => "eval.error", EventType::Eval(EvalEvent::DirectoryNotFound) => "eval.directory-not-found", EventType::Eval(EvalEvent::StoreNotFound) => "eval.store-not-found", - EventType::Housekeeper(HousekeeperEvent::Start) => "housekeeper.start", - EventType::Housekeeper(HousekeeperEvent::Stop) => "housekeeper.stop", - EventType::Housekeeper(HousekeeperEvent::Schedule) => "housekeeper.schedule", - EventType::Housekeeper(HousekeeperEvent::Run) => "housekeeper.run", EventType::Http(HttpEvent::ConnectionStart) => "http.connection-start", EventType::Http(HttpEvent::ConnectionEnd) => "http.connection-end", EventType::Http(HttpEvent::Error) => "http.error", @@ -919,12 +915,12 @@ impl EventType { "mail-auth.dns-invalid-record-type" } EventType::MailAuth(MailAuthEvent::PolicyNotAligned) => "mail-auth.policy-not-aligned", - EventType::Manage(ManageEvent::MissingParameter) => "manage.missing-parameter", - EventType::Manage(ManageEvent::AlreadyExists) => "manage.already-exists", - EventType::Manage(ManageEvent::AssertFailed) => "manage.assert-failed", - EventType::Manage(ManageEvent::NotFound) => "manage.not-found", - EventType::Manage(ManageEvent::NotSupported) => "manage.not-supported", - EventType::Manage(ManageEvent::Error) => "manage.error", + EventType::Manage(ManageEvent::Reserved1) => "manage.reserved1", + EventType::Manage(ManageEvent::Reserved2) => "manage.reserved2", + EventType::Manage(ManageEvent::Reserved3) => "manage.reserved3", + EventType::Manage(ManageEvent::Reserved4) => "manage.reserved4", + EventType::Manage(ManageEvent::Reserved5) => "manage.reserved5", + EventType::Manage(ManageEvent::Reserved6) => "manage.reserved6", EventType::ManageSieve(ManageSieveEvent::ConnectionStart) => { "manage-sieve.connection-start" } @@ -1066,13 +1062,6 @@ impl EventType { EventType::Pop3(Pop3Event::Error) => "pop3.error", EventType::Pop3(Pop3Event::RawInput) => "pop3.raw-input", EventType::Pop3(Pop3Event::RawOutput) => "pop3.raw-output", - EventType::Purge(PurgeEvent::Started) => "purge.started", - EventType::Purge(PurgeEvent::Finished) => "purge.finished", - EventType::Purge(PurgeEvent::Running) => "purge.running", - EventType::Purge(PurgeEvent::Error) => "purge.error", - EventType::Purge(PurgeEvent::InProgress) => "purge.in-progress", - EventType::Purge(PurgeEvent::AutoExpunge) => "purge.auto-expunge", - EventType::Purge(PurgeEvent::BlobCleanup) => "purge.blob-cleanup", EventType::PushSubscription(PushSubscriptionEvent::Success) => { "push-subscription.success" } @@ -1080,13 +1069,14 @@ impl EventType { EventType::PushSubscription(PushSubscriptionEvent::NotFound) => { "push-subscription.not-found" } - EventType::Queue(QueueEvent::QueueMessage) => "queue.queue-message", - EventType::Queue(QueueEvent::QueueMessageAuthenticated) => { - "queue.queue-message-authenticated" + EventType::Queue(QueueEvent::Started) => "queue.started", + EventType::Queue(QueueEvent::MessageQueued) => "queue.message-queued", + EventType::Queue(QueueEvent::AuthenticatedMessageQueued) => { + "queue.authenticated-message-queued" } - EventType::Queue(QueueEvent::QueueReport) => "queue.queue-report", - EventType::Queue(QueueEvent::QueueDsn) => "queue.queue-dsn", - EventType::Queue(QueueEvent::QueueAutogenerated) => "queue.queue-autogenerated", + EventType::Queue(QueueEvent::ReportQueued) => "queue.report-queued", + EventType::Queue(QueueEvent::DsnQueued) => "queue.dsn-queued", + EventType::Queue(QueueEvent::AutogeneratedQueued) => "queue.autogenerated-queued", EventType::Queue(QueueEvent::Rescheduled) => "queue.rescheduled", EventType::Queue(QueueEvent::Locked) => "queue.locked", EventType::Queue(QueueEvent::BlobNotFound) => "queue.blob-not-found", @@ -1112,7 +1102,6 @@ impl EventType { EventType::Resource(ResourceEvent::BadParameters) => "resource.bad-parameters", EventType::Resource(ResourceEvent::Error) => "resource.error", EventType::Resource(ResourceEvent::DownloadExternal) => "resource.download-external", - EventType::Resource(ResourceEvent::WebadminUnpacked) => "resource.webadmin-unpacked", EventType::Security(SecurityEvent::AuthenticationBan) => "security.authentication-ban", EventType::Security(SecurityEvent::AbuseBan) => "security.abuse-ban", EventType::Security(SecurityEvent::ScanBan) => "security.scan-ban", @@ -1280,16 +1269,30 @@ impl EventType { EventType::Store(StoreEvent::LdapQuery) => "store.ldap-query", EventType::Store(StoreEvent::LdapWarning) => "store.ldap-warning", EventType::Store(StoreEvent::HttpStoreFetch) => "store.http-store-fetch", - EventType::TaskQueue(TaskQueueEvent::TaskAcquired) => "task-queue.task-acquired", - EventType::TaskQueue(TaskQueueEvent::TaskLocked) => "task-queue.task-locked", - EventType::TaskQueue(TaskQueueEvent::TaskIgnored) => "task-queue.task-ignored", - EventType::TaskQueue(TaskQueueEvent::TaskFailed) => "task-queue.task-failed", - EventType::TaskQueue(TaskQueueEvent::TaskRetry) => "task-queue.task-retry", - EventType::TaskQueue(TaskQueueEvent::BlobNotFound) => "task-queue.blob-not-found", - EventType::TaskQueue(TaskQueueEvent::MetadataNotFound) => { - "task-queue.metadata-not-found" + EventType::Store(StoreEvent::AutoExpunge) => "store.auto-expunge", + EventType::Store(StoreEvent::BlobStorePurged) => "store.blob-store-purged", + EventType::Store(StoreEvent::DataStorePurged) => "store.data-store-purged", + EventType::TaskManager(TaskManagerEvent::TaskAcquired) => "task-manager.task-acquired", + EventType::TaskManager(TaskManagerEvent::TaskQueued) => "task-manager.task-queued", + EventType::TaskManager(TaskManagerEvent::TaskScheduled) => { + "task-manager.task-scheduled" } - EventType::Telemetry(TelemetryEvent::Alert) => "telemetry.alert", + EventType::TaskManager(TaskManagerEvent::TaskLocked) => "task-manager.task-locked", + EventType::TaskManager(TaskManagerEvent::TaskIgnored) => "task-manager.task-ignored", + EventType::TaskManager(TaskManagerEvent::TaskFailed) => "task-manager.task-failed", + EventType::TaskManager(TaskManagerEvent::TaskRetry) => "task-manager.task-retry", + EventType::TaskManager(TaskManagerEvent::BlobNotFound) => "task-manager.blob-not-found", + EventType::TaskManager(TaskManagerEvent::MetadataNotFound) => { + "task-manager.metadata-not-found" + } + EventType::TaskManager(TaskManagerEvent::SchedulerStarted) => { + "task-manager.scheduler-started" + } + EventType::TaskManager(TaskManagerEvent::ManagerStarted) => { + "task-manager.manager-started" + } + EventType::Telemetry(TelemetryEvent::AlertEvent) => "telemetry.alert-event", + EventType::Telemetry(TelemetryEvent::AlertMessage) => "telemetry.alert-message", EventType::Telemetry(TelemetryEvent::LogError) => "telemetry.log-error", EventType::Telemetry(TelemetryEvent::WebhookError) => "telemetry.webhook-error", EventType::Telemetry(TelemetryEvent::OtelExporterError) => { @@ -1302,6 +1305,9 @@ impl EventType { "telemetry.prometheus-exporter-error" } EventType::Telemetry(TelemetryEvent::JournalError) => "telemetry.journal-error", + EventType::Telemetry(TelemetryEvent::MetricsCollected) => "telemetry.metrics-collected", + EventType::Telemetry(TelemetryEvent::MetricsStored) => "telemetry.metrics-stored", + EventType::Telemetry(TelemetryEvent::MetricsPushed) => "telemetry.metrics-pushed", EventType::Tls(TlsEvent::Handshake) => "tls.handshake", EventType::Tls(TlsEvent::HandshakeError) => "tls.handshake-error", EventType::Tls(TlsEvent::NotConfigured) => "tls.not-configured", @@ -1474,10 +1480,6 @@ impl EventType { EventType::Eval(EvalEvent::Error) => 138, EventType::Eval(EvalEvent::DirectoryNotFound) => 137, EventType::Eval(EvalEvent::StoreNotFound) => 140, - EventType::Housekeeper(HousekeeperEvent::Start) => 150, - EventType::Housekeeper(HousekeeperEvent::Stop) => 151, - EventType::Housekeeper(HousekeeperEvent::Schedule) => 149, - EventType::Housekeeper(HousekeeperEvent::Run) => 146, EventType::Http(HttpEvent::ConnectionStart) => 153, EventType::Http(HttpEvent::ConnectionEnd) => 152, EventType::Http(HttpEvent::Error) => 154, @@ -1584,12 +1586,12 @@ impl EventType { EventType::MailAuth(MailAuthEvent::DnsRecordNotFound) => 250, EventType::MailAuth(MailAuthEvent::DnsInvalidRecordType) => 249, EventType::MailAuth(MailAuthEvent::PolicyNotAligned) => 255, - EventType::Manage(ManageEvent::MissingParameter) => 278, - EventType::Manage(ManageEvent::AlreadyExists) => 275, - EventType::Manage(ManageEvent::AssertFailed) => 276, - EventType::Manage(ManageEvent::NotFound) => 279, - EventType::Manage(ManageEvent::NotSupported) => 280, - EventType::Manage(ManageEvent::Error) => 277, + EventType::Manage(ManageEvent::Reserved1) => 278, + EventType::Manage(ManageEvent::Reserved2) => 275, + EventType::Manage(ManageEvent::Reserved3) => 276, + EventType::Manage(ManageEvent::Reserved4) => 279, + EventType::Manage(ManageEvent::Reserved5) => 280, + EventType::Manage(ManageEvent::Reserved6) => 277, EventType::ManageSieve(ManageSieveEvent::ConnectionStart) => 259, EventType::ManageSieve(ManageSieveEvent::ConnectionEnd) => 258, EventType::ManageSieve(ManageSieveEvent::CreateScript) => 260, @@ -1691,21 +1693,15 @@ impl EventType { EventType::Pop3(Pop3Event::Error) => 350, EventType::Pop3(Pop3Event::RawInput) => 356, EventType::Pop3(Pop3Event::RawOutput) => 357, - EventType::Purge(PurgeEvent::Started) => 369, - EventType::Purge(PurgeEvent::Finished) => 366, - EventType::Purge(PurgeEvent::Running) => 368, - EventType::Purge(PurgeEvent::Error) => 365, - EventType::Purge(PurgeEvent::InProgress) => 367, - EventType::Purge(PurgeEvent::AutoExpunge) => 364, - EventType::Purge(PurgeEvent::BlobCleanup) => 370, EventType::PushSubscription(PushSubscriptionEvent::Success) => 373, EventType::PushSubscription(PushSubscriptionEvent::Error) => 371, EventType::PushSubscription(PushSubscriptionEvent::NotFound) => 372, - EventType::Queue(QueueEvent::QueueMessage) => 380, - EventType::Queue(QueueEvent::QueueMessageAuthenticated) => 381, - EventType::Queue(QueueEvent::QueueReport) => 382, - EventType::Queue(QueueEvent::QueueDsn) => 379, - EventType::Queue(QueueEvent::QueueAutogenerated) => 378, + EventType::Queue(QueueEvent::Started) => 390, + EventType::Queue(QueueEvent::MessageQueued) => 380, + EventType::Queue(QueueEvent::AuthenticatedMessageQueued) => 381, + EventType::Queue(QueueEvent::ReportQueued) => 382, + EventType::Queue(QueueEvent::DsnQueued) => 379, + EventType::Queue(QueueEvent::AutogeneratedQueued) => 378, EventType::Queue(QueueEvent::Rescheduled) => 385, EventType::Queue(QueueEvent::Locked) => 377, EventType::Queue(QueueEvent::BlobNotFound) => 374, @@ -1727,7 +1723,6 @@ impl EventType { EventType::Resource(ResourceEvent::BadParameters) => 386, EventType::Resource(ResourceEvent::Error) => 388, EventType::Resource(ResourceEvent::DownloadExternal) => 387, - EventType::Resource(ResourceEvent::WebadminUnpacked) => 390, EventType::Security(SecurityEvent::AuthenticationBan) => 33, EventType::Security(SecurityEvent::AbuseBan) => 549, EventType::Security(SecurityEvent::ScanBan) => 558, @@ -1891,20 +1886,31 @@ impl EventType { EventType::Store(StoreEvent::LdapQuery) => 521, EventType::Store(StoreEvent::LdapWarning) => 519, EventType::Store(StoreEvent::HttpStoreFetch) => 492, - EventType::TaskQueue(TaskQueueEvent::TaskAcquired) => 578, - EventType::TaskQueue(TaskQueueEvent::TaskLocked) => 144, - EventType::TaskQueue(TaskQueueEvent::TaskIgnored) => 586, - EventType::TaskQueue(TaskQueueEvent::TaskFailed) => 587, - EventType::TaskQueue(TaskQueueEvent::TaskRetry) => 53, - EventType::TaskQueue(TaskQueueEvent::BlobNotFound) => 141, - EventType::TaskQueue(TaskQueueEvent::MetadataNotFound) => 145, - EventType::Telemetry(TelemetryEvent::Alert) => 548, + EventType::Store(StoreEvent::AutoExpunge) => 364, + EventType::Store(StoreEvent::BlobStorePurged) => 369, + EventType::Store(StoreEvent::DataStorePurged) => 368, + EventType::TaskManager(TaskManagerEvent::TaskAcquired) => 578, + EventType::TaskManager(TaskManagerEvent::TaskQueued) => 149, + EventType::TaskManager(TaskManagerEvent::TaskScheduled) => 370, + EventType::TaskManager(TaskManagerEvent::TaskLocked) => 144, + EventType::TaskManager(TaskManagerEvent::TaskIgnored) => 586, + EventType::TaskManager(TaskManagerEvent::TaskFailed) => 587, + EventType::TaskManager(TaskManagerEvent::TaskRetry) => 53, + EventType::TaskManager(TaskManagerEvent::BlobNotFound) => 141, + EventType::TaskManager(TaskManagerEvent::MetadataNotFound) => 145, + EventType::TaskManager(TaskManagerEvent::SchedulerStarted) => 150, + EventType::TaskManager(TaskManagerEvent::ManagerStarted) => 367, + EventType::Telemetry(TelemetryEvent::AlertEvent) => 548, + EventType::Telemetry(TelemetryEvent::AlertMessage) => 365, EventType::Telemetry(TelemetryEvent::LogError) => 535, EventType::Telemetry(TelemetryEvent::WebhookError) => 539, EventType::Telemetry(TelemetryEvent::OtelExporterError) => 536, EventType::Telemetry(TelemetryEvent::OtelMetricsExporterError) => 537, EventType::Telemetry(TelemetryEvent::PrometheusExporterError) => 538, EventType::Telemetry(TelemetryEvent::JournalError) => 534, + EventType::Telemetry(TelemetryEvent::MetricsCollected) => 151, + EventType::Telemetry(TelemetryEvent::MetricsStored) => 366, + EventType::Telemetry(TelemetryEvent::MetricsPushed) => 146, EventType::Tls(TlsEvent::Handshake) => 543, EventType::Tls(TlsEvent::HandshakeError) => 544, EventType::Tls(TlsEvent::NotConfigured) => 547, @@ -2075,10 +2081,6 @@ impl EventType { 138 => Some(EventType::Eval(EvalEvent::Error)), 137 => Some(EventType::Eval(EvalEvent::DirectoryNotFound)), 140 => Some(EventType::Eval(EvalEvent::StoreNotFound)), - 150 => Some(EventType::Housekeeper(HousekeeperEvent::Start)), - 151 => Some(EventType::Housekeeper(HousekeeperEvent::Stop)), - 149 => Some(EventType::Housekeeper(HousekeeperEvent::Schedule)), - 146 => Some(EventType::Housekeeper(HousekeeperEvent::Run)), 153 => Some(EventType::Http(HttpEvent::ConnectionStart)), 152 => Some(EventType::Http(HttpEvent::ConnectionEnd)), 154 => Some(EventType::Http(HttpEvent::Error)), @@ -2203,12 +2205,12 @@ impl EventType { 250 => Some(EventType::MailAuth(MailAuthEvent::DnsRecordNotFound)), 249 => Some(EventType::MailAuth(MailAuthEvent::DnsInvalidRecordType)), 255 => Some(EventType::MailAuth(MailAuthEvent::PolicyNotAligned)), - 278 => Some(EventType::Manage(ManageEvent::MissingParameter)), - 275 => Some(EventType::Manage(ManageEvent::AlreadyExists)), - 276 => Some(EventType::Manage(ManageEvent::AssertFailed)), - 279 => Some(EventType::Manage(ManageEvent::NotFound)), - 280 => Some(EventType::Manage(ManageEvent::NotSupported)), - 277 => Some(EventType::Manage(ManageEvent::Error)), + 278 => Some(EventType::Manage(ManageEvent::Reserved1)), + 275 => Some(EventType::Manage(ManageEvent::Reserved2)), + 276 => Some(EventType::Manage(ManageEvent::Reserved3)), + 279 => Some(EventType::Manage(ManageEvent::Reserved4)), + 280 => Some(EventType::Manage(ManageEvent::Reserved5)), + 277 => Some(EventType::Manage(ManageEvent::Reserved6)), 259 => Some(EventType::ManageSieve(ManageSieveEvent::ConnectionStart)), 258 => Some(EventType::ManageSieve(ManageSieveEvent::ConnectionEnd)), 260 => Some(EventType::ManageSieve(ManageSieveEvent::CreateScript)), @@ -2330,21 +2332,15 @@ impl EventType { 350 => Some(EventType::Pop3(Pop3Event::Error)), 356 => Some(EventType::Pop3(Pop3Event::RawInput)), 357 => Some(EventType::Pop3(Pop3Event::RawOutput)), - 369 => Some(EventType::Purge(PurgeEvent::Started)), - 366 => Some(EventType::Purge(PurgeEvent::Finished)), - 368 => Some(EventType::Purge(PurgeEvent::Running)), - 365 => Some(EventType::Purge(PurgeEvent::Error)), - 367 => Some(EventType::Purge(PurgeEvent::InProgress)), - 364 => Some(EventType::Purge(PurgeEvent::AutoExpunge)), - 370 => Some(EventType::Purge(PurgeEvent::BlobCleanup)), 373 => Some(EventType::PushSubscription(PushSubscriptionEvent::Success)), 371 => Some(EventType::PushSubscription(PushSubscriptionEvent::Error)), 372 => Some(EventType::PushSubscription(PushSubscriptionEvent::NotFound)), - 380 => Some(EventType::Queue(QueueEvent::QueueMessage)), - 381 => Some(EventType::Queue(QueueEvent::QueueMessageAuthenticated)), - 382 => Some(EventType::Queue(QueueEvent::QueueReport)), - 379 => Some(EventType::Queue(QueueEvent::QueueDsn)), - 378 => Some(EventType::Queue(QueueEvent::QueueAutogenerated)), + 390 => Some(EventType::Queue(QueueEvent::Started)), + 380 => Some(EventType::Queue(QueueEvent::MessageQueued)), + 381 => Some(EventType::Queue(QueueEvent::AuthenticatedMessageQueued)), + 382 => Some(EventType::Queue(QueueEvent::ReportQueued)), + 379 => Some(EventType::Queue(QueueEvent::DsnQueued)), + 378 => Some(EventType::Queue(QueueEvent::AutogeneratedQueued)), 385 => Some(EventType::Queue(QueueEvent::Rescheduled)), 377 => Some(EventType::Queue(QueueEvent::Locked)), 374 => Some(EventType::Queue(QueueEvent::BlobNotFound)), @@ -2366,7 +2362,6 @@ impl EventType { 386 => Some(EventType::Resource(ResourceEvent::BadParameters)), 388 => Some(EventType::Resource(ResourceEvent::Error)), 387 => Some(EventType::Resource(ResourceEvent::DownloadExternal)), - 390 => Some(EventType::Resource(ResourceEvent::WebadminUnpacked)), 33 => Some(EventType::Security(SecurityEvent::AuthenticationBan)), 549 => Some(EventType::Security(SecurityEvent::AbuseBan)), 558 => Some(EventType::Security(SecurityEvent::ScanBan)), @@ -2530,14 +2525,22 @@ impl EventType { 521 => Some(EventType::Store(StoreEvent::LdapQuery)), 519 => Some(EventType::Store(StoreEvent::LdapWarning)), 492 => Some(EventType::Store(StoreEvent::HttpStoreFetch)), - 578 => Some(EventType::TaskQueue(TaskQueueEvent::TaskAcquired)), - 144 => Some(EventType::TaskQueue(TaskQueueEvent::TaskLocked)), - 586 => Some(EventType::TaskQueue(TaskQueueEvent::TaskIgnored)), - 587 => Some(EventType::TaskQueue(TaskQueueEvent::TaskFailed)), - 53 => Some(EventType::TaskQueue(TaskQueueEvent::TaskRetry)), - 141 => Some(EventType::TaskQueue(TaskQueueEvent::BlobNotFound)), - 145 => Some(EventType::TaskQueue(TaskQueueEvent::MetadataNotFound)), - 548 => Some(EventType::Telemetry(TelemetryEvent::Alert)), + 364 => Some(EventType::Store(StoreEvent::AutoExpunge)), + 369 => Some(EventType::Store(StoreEvent::BlobStorePurged)), + 368 => Some(EventType::Store(StoreEvent::DataStorePurged)), + 578 => Some(EventType::TaskManager(TaskManagerEvent::TaskAcquired)), + 149 => Some(EventType::TaskManager(TaskManagerEvent::TaskQueued)), + 370 => Some(EventType::TaskManager(TaskManagerEvent::TaskScheduled)), + 144 => Some(EventType::TaskManager(TaskManagerEvent::TaskLocked)), + 586 => Some(EventType::TaskManager(TaskManagerEvent::TaskIgnored)), + 587 => Some(EventType::TaskManager(TaskManagerEvent::TaskFailed)), + 53 => Some(EventType::TaskManager(TaskManagerEvent::TaskRetry)), + 141 => Some(EventType::TaskManager(TaskManagerEvent::BlobNotFound)), + 145 => Some(EventType::TaskManager(TaskManagerEvent::MetadataNotFound)), + 150 => Some(EventType::TaskManager(TaskManagerEvent::SchedulerStarted)), + 367 => Some(EventType::TaskManager(TaskManagerEvent::ManagerStarted)), + 548 => Some(EventType::Telemetry(TelemetryEvent::AlertEvent)), + 365 => Some(EventType::Telemetry(TelemetryEvent::AlertMessage)), 535 => Some(EventType::Telemetry(TelemetryEvent::LogError)), 539 => Some(EventType::Telemetry(TelemetryEvent::WebhookError)), 536 => Some(EventType::Telemetry(TelemetryEvent::OtelExporterError)), @@ -2548,6 +2551,9 @@ impl EventType { TelemetryEvent::PrometheusExporterError, )), 534 => Some(EventType::Telemetry(TelemetryEvent::JournalError)), + 151 => Some(EventType::Telemetry(TelemetryEvent::MetricsCollected)), + 366 => Some(EventType::Telemetry(TelemetryEvent::MetricsStored)), + 146 => Some(EventType::Telemetry(TelemetryEvent::MetricsPushed)), 543 => Some(EventType::Tls(TlsEvent::Handshake)), 544 => Some(EventType::Tls(TlsEvent::HandshakeError)), 547 => Some(EventType::Tls(TlsEvent::NotConfigured)), @@ -2593,7 +2599,6 @@ impl EventType { EventType::Network(NetworkEvent::BindError) => Level::Error, EventType::Network(NetworkEvent::SplitError) => Level::Error, EventType::Network(NetworkEvent::SetOptError) => Level::Error, - EventType::Purge(PurgeEvent::Error) => Level::Error, EventType::Registry(RegistryEvent::LocalReadError) => Level::Error, EventType::Registry(RegistryEvent::LocalWriteError) => Level::Error, EventType::Registry(RegistryEvent::LocalParseError) => Level::Error, @@ -2688,8 +2693,6 @@ impl EventType { EventType::Delivery(DeliveryEvent::DsnPermFail) => Level::Info, EventType::Dns(DnsEvent::RecordCreated) => Level::Info, EventType::Dns(DnsEvent::RecordPropagated) => Level::Info, - EventType::Housekeeper(HousekeeperEvent::Start) => Level::Info, - EventType::Housekeeper(HousekeeperEvent::Stop) => Level::Info, EventType::IncomingReport(IncomingReportEvent::DmarcReport) => Level::Info, EventType::IncomingReport(IncomingReportEvent::TlsReport) => Level::Info, EventType::IncomingReport(IncomingReportEvent::AbuseReport) => Level::Info, @@ -2748,19 +2751,17 @@ impl EventType { EventType::OutgoingReport(OutgoingReportEvent::SubmissionError) => Level::Info, EventType::OutgoingReport(OutgoingReportEvent::NoRecipientsFound) => Level::Info, EventType::OutgoingReport(OutgoingReportEvent::Locked) => Level::Info, - EventType::Purge(PurgeEvent::Running) => Level::Info, - EventType::Purge(PurgeEvent::BlobCleanup) => Level::Info, - EventType::Queue(QueueEvent::QueueMessage) => Level::Info, - EventType::Queue(QueueEvent::QueueMessageAuthenticated) => Level::Info, - EventType::Queue(QueueEvent::QueueReport) => Level::Info, - EventType::Queue(QueueEvent::QueueDsn) => Level::Info, - EventType::Queue(QueueEvent::QueueAutogenerated) => Level::Info, + EventType::Queue(QueueEvent::Started) => Level::Info, + EventType::Queue(QueueEvent::MessageQueued) => Level::Info, + EventType::Queue(QueueEvent::AuthenticatedMessageQueued) => Level::Info, + EventType::Queue(QueueEvent::ReportQueued) => Level::Info, + EventType::Queue(QueueEvent::DsnQueued) => Level::Info, + EventType::Queue(QueueEvent::AutogeneratedQueued) => Level::Info, EventType::Queue(QueueEvent::Rescheduled) => Level::Info, EventType::Queue(QueueEvent::RateLimitExceeded) => Level::Info, EventType::Queue(QueueEvent::ConcurrencyLimitExceeded) => Level::Info, EventType::Queue(QueueEvent::QuotaExceeded) => Level::Info, EventType::Resource(ResourceEvent::DownloadExternal) => Level::Info, - EventType::Resource(ResourceEvent::WebadminUnpacked) => Level::Info, EventType::Security(SecurityEvent::AuthenticationBan) => Level::Info, EventType::Security(SecurityEvent::AbuseBan) => Level::Info, EventType::Security(SecurityEvent::ScanBan) => Level::Info, @@ -2817,6 +2818,14 @@ impl EventType { EventType::Spam(SpamEvent::ModelLoaded) => Level::Info, EventType::Spam(SpamEvent::ModelNotReady) => Level::Info, EventType::Spam(SpamEvent::ModelNotFound) => Level::Info, + EventType::Store(StoreEvent::BlobStorePurged) => Level::Info, + EventType::Store(StoreEvent::DataStorePurged) => Level::Info, + EventType::TaskManager(TaskManagerEvent::TaskQueued) => Level::Info, + EventType::TaskManager(TaskManagerEvent::TaskScheduled) => Level::Info, + EventType::TaskManager(TaskManagerEvent::SchedulerStarted) => Level::Info, + EventType::TaskManager(TaskManagerEvent::ManagerStarted) => Level::Info, + EventType::Telemetry(TelemetryEvent::AlertMessage) => Level::Info, + EventType::Telemetry(TelemetryEvent::MetricsCollected) => Level::Info, EventType::Tls(TlsEvent::Handshake) => Level::Info, EventType::TlsRpt(TlsRptEvent::RecordFetch) => Level::Info, EventType::TlsRpt(TlsRptEvent::RecordFetchError) => Level::Info, @@ -2896,8 +2905,8 @@ impl EventType { EventType::Spam(SpamEvent::TrainSampleNotFound) => Level::Warn, EventType::Store(StoreEvent::HttpStoreError) => Level::Warn, EventType::Store(StoreEvent::BlobMissingMarker) => Level::Warn, - EventType::TaskQueue(TaskQueueEvent::TaskFailed) => Level::Warn, - EventType::Telemetry(TelemetryEvent::Alert) => Level::Warn, + EventType::TaskManager(TaskManagerEvent::TaskFailed) => Level::Warn, + EventType::Telemetry(TelemetryEvent::AlertEvent) => Level::Warn, EventType::Telemetry(TelemetryEvent::LogError) => Level::Warn, EventType::Telemetry(TelemetryEvent::WebhookError) => Level::Warn, EventType::Telemetry(TelemetryEvent::OtelExporterError) => Level::Warn, @@ -3080,10 +3089,6 @@ impl EventType { EventType::Eval(EvalEvent::StoreNotFound) => { "Store not found while evaluating expression" } - EventType::Housekeeper(HousekeeperEvent::Start) => "Housekeeper process started", - EventType::Housekeeper(HousekeeperEvent::Stop) => "Housekeeper process stopped", - EventType::Housekeeper(HousekeeperEvent::Schedule) => "Housekeeper task scheduled", - EventType::Housekeeper(HousekeeperEvent::Run) => "Housekeeper task run", EventType::Http(HttpEvent::ConnectionStart) => "HTTP connection started", EventType::Http(HttpEvent::ConnectionEnd) => "HTTP connection ended", EventType::Http(HttpEvent::Error) => "HTTP error occurred", @@ -3216,12 +3221,12 @@ impl EventType { EventType::MailAuth(MailAuthEvent::DnsRecordNotFound) => "DNS record not found", EventType::MailAuth(MailAuthEvent::DnsInvalidRecordType) => "Invalid DNS record type", EventType::MailAuth(MailAuthEvent::PolicyNotAligned) => "Policy not aligned", - EventType::Manage(ManageEvent::MissingParameter) => "Missing parameter", - EventType::Manage(ManageEvent::AlreadyExists) => "Record already exists", - EventType::Manage(ManageEvent::AssertFailed) => "Assertion failed", - EventType::Manage(ManageEvent::NotFound) => "Resource not found", - EventType::Manage(ManageEvent::NotSupported) => "Management operation not supported", - EventType::Manage(ManageEvent::Error) => "Management error", + EventType::Manage(ManageEvent::Reserved1) => "Missing parameter", + EventType::Manage(ManageEvent::Reserved2) => "Record already exists", + EventType::Manage(ManageEvent::Reserved3) => "Assertion failed", + EventType::Manage(ManageEvent::Reserved4) => "Resource not found", + EventType::Manage(ManageEvent::Reserved5) => "Management operation not supported", + EventType::Manage(ManageEvent::Reserved6) => "Management error", EventType::ManageSieve(ManageSieveEvent::ConnectionStart) => { "ManageSieve connection started" } @@ -3373,13 +3378,6 @@ impl EventType { EventType::Pop3(Pop3Event::Error) => "POP3 error occurred", EventType::Pop3(Pop3Event::RawInput) => "Raw POP3 input received", EventType::Pop3(Pop3Event::RawOutput) => "Raw POP3 output sent", - EventType::Purge(PurgeEvent::Started) => "Purge started", - EventType::Purge(PurgeEvent::Finished) => "Purge finished", - EventType::Purge(PurgeEvent::Running) => "Purge running", - EventType::Purge(PurgeEvent::Error) => "Purge error", - EventType::Purge(PurgeEvent::InProgress) => "Active purge in progress", - EventType::Purge(PurgeEvent::AutoExpunge) => "Auto-expunge executed", - EventType::Purge(PurgeEvent::BlobCleanup) => "Blob storage cleanup completed", EventType::PushSubscription(PushSubscriptionEvent::Success) => { "Push subscription successful" } @@ -3387,13 +3385,14 @@ impl EventType { EventType::PushSubscription(PushSubscriptionEvent::NotFound) => { "Push subscription not found" } - EventType::Queue(QueueEvent::QueueMessage) => "Queued message for delivery", - EventType::Queue(QueueEvent::QueueMessageAuthenticated) => { + EventType::Queue(QueueEvent::Started) => "MTA queue started", + EventType::Queue(QueueEvent::MessageQueued) => "Queued message for delivery", + EventType::Queue(QueueEvent::AuthenticatedMessageQueued) => { "Queued message submission for delivery" } - EventType::Queue(QueueEvent::QueueReport) => "Queued report for delivery", - EventType::Queue(QueueEvent::QueueDsn) => "Queued DSN for delivery", - EventType::Queue(QueueEvent::QueueAutogenerated) => { + EventType::Queue(QueueEvent::ReportQueued) => "Queued report for delivery", + EventType::Queue(QueueEvent::DsnQueued) => "Queued DSN for delivery", + EventType::Queue(QueueEvent::AutogeneratedQueued) => { "Queued autogenerated message for delivery" } EventType::Queue(QueueEvent::Rescheduled) => "Message rescheduled for delivery", @@ -3421,7 +3420,6 @@ impl EventType { EventType::Resource(ResourceEvent::BadParameters) => "Bad resource parameters", EventType::Resource(ResourceEvent::Error) => "Resource error", EventType::Resource(ResourceEvent::DownloadExternal) => "Downloading external resource", - EventType::Resource(ResourceEvent::WebadminUnpacked) => "Webadmin resource unpacked", EventType::Security(SecurityEvent::AuthenticationBan) => { "Banned due to authentication errors" } @@ -3593,16 +3591,30 @@ impl EventType { EventType::Store(StoreEvent::LdapQuery) => "LDAP query executed", EventType::Store(StoreEvent::LdapWarning) => "LDAP authentication warning", EventType::Store(StoreEvent::HttpStoreFetch) => "HTTP store updated", - EventType::TaskQueue(TaskQueueEvent::TaskAcquired) => "Task acquired from queue", - EventType::TaskQueue(TaskQueueEvent::TaskLocked) => "Task is locked by another process", - EventType::TaskQueue(TaskQueueEvent::TaskIgnored) => { + EventType::Store(StoreEvent::AutoExpunge) => "Auto-expunge executed", + EventType::Store(StoreEvent::BlobStorePurged) => "Blob store purge completed", + EventType::Store(StoreEvent::DataStorePurged) => "Data store purge completed", + EventType::TaskManager(TaskManagerEvent::TaskAcquired) => "Task acquired from queue", + EventType::TaskManager(TaskManagerEvent::TaskQueued) => "Task queued for processing", + EventType::TaskManager(TaskManagerEvent::TaskScheduled) => { + "Task scheduled for future execution" + } + EventType::TaskManager(TaskManagerEvent::TaskLocked) => { + "Task is locked by another process" + } + EventType::TaskManager(TaskManagerEvent::TaskIgnored) => { "Task ignored based on current server roles" } - EventType::TaskQueue(TaskQueueEvent::TaskFailed) => "Task failed during processing", - EventType::TaskQueue(TaskQueueEvent::TaskRetry) => "Task will be retried", - EventType::TaskQueue(TaskQueueEvent::BlobNotFound) => "Blob not found for task", - EventType::TaskQueue(TaskQueueEvent::MetadataNotFound) => "Metadata not found for task", - EventType::Telemetry(TelemetryEvent::Alert) => "Alert triggered", + EventType::TaskManager(TaskManagerEvent::TaskFailed) => "Task failed during processing", + EventType::TaskManager(TaskManagerEvent::TaskRetry) => "Task will be retried", + EventType::TaskManager(TaskManagerEvent::BlobNotFound) => "Blob not found for task", + EventType::TaskManager(TaskManagerEvent::MetadataNotFound) => { + "Metadata not found for task" + } + EventType::TaskManager(TaskManagerEvent::SchedulerStarted) => "Task scheduler started", + EventType::TaskManager(TaskManagerEvent::ManagerStarted) => "Task manager started", + EventType::Telemetry(TelemetryEvent::AlertEvent) => "Alert event triggered", + EventType::Telemetry(TelemetryEvent::AlertMessage) => "Alert message sent", EventType::Telemetry(TelemetryEvent::LogError) => "Log collector error", EventType::Telemetry(TelemetryEvent::WebhookError) => "Webhook collector error", EventType::Telemetry(TelemetryEvent::OtelExporterError) => { @@ -3615,6 +3627,9 @@ impl EventType { "Prometheus exporter error" } EventType::Telemetry(TelemetryEvent::JournalError) => "Journal collector error", + EventType::Telemetry(TelemetryEvent::MetricsCollected) => "Housekeeper process stopped", + EventType::Telemetry(TelemetryEvent::MetricsStored) => "Purge finished", + EventType::Telemetry(TelemetryEvent::MetricsPushed) => "Housekeeper task run", EventType::Tls(TlsEvent::Handshake) => "TLS handshake", EventType::Tls(TlsEvent::HandshakeError) => "TLS handshake error", EventType::Tls(TlsEvent::NotConfigured) => "TLS not configured", @@ -3901,14 +3916,6 @@ impl EventType { EventType::Eval(EvalEvent::StoreNotFound) => { "The store was not found while evaluating the expression" } - EventType::Housekeeper(HousekeeperEvent::Start) => { - "The housekeeper process has started" - } - EventType::Housekeeper(HousekeeperEvent::Stop) => "The housekeeper process has stopped", - EventType::Housekeeper(HousekeeperEvent::Schedule) => { - "A housekeeper task has been scheduled" - } - EventType::Housekeeper(HousekeeperEvent::Run) => "A housekeeper task is running", EventType::Http(HttpEvent::ConnectionStart) => "An HTTP connection was started", EventType::Http(HttpEvent::ConnectionEnd) => "An HTTP connection was ended", EventType::Http(HttpEvent::Error) => "An error occurred during an HTTP request", @@ -4079,16 +4086,16 @@ impl EventType { "The DNS record type is invalid" } EventType::MailAuth(MailAuthEvent::PolicyNotAligned) => "The policy is not aligned", - EventType::Manage(ManageEvent::MissingParameter) => "A parameter is missing", - EventType::Manage(ManageEvent::AlreadyExists) => { + EventType::Manage(ManageEvent::Reserved1) => "A parameter is missing", + EventType::Manage(ManageEvent::Reserved2) => { "A record with the same name already exists" } - EventType::Manage(ManageEvent::AssertFailed) => "A management assertion has failed", - EventType::Manage(ManageEvent::NotFound) => "The managed resource was not found", - EventType::Manage(ManageEvent::NotSupported) => { + EventType::Manage(ManageEvent::Reserved3) => "A management assertion has failed", + EventType::Manage(ManageEvent::Reserved4) => "The managed resource was not found", + EventType::Manage(ManageEvent::Reserved5) => { "The management operation is not supported" } - EventType::Manage(ManageEvent::Error) => "A management error occurred", + EventType::Manage(ManageEvent::Reserved6) => "A management error occurred", EventType::ManageSieve(ManageSieveEvent::ConnectionStart) => { "ManageSieve connection started" } @@ -4288,13 +4295,6 @@ impl EventType { EventType::Pop3(Pop3Event::Error) => "An error occurred during a POP3 command", EventType::Pop3(Pop3Event::RawInput) => "Raw POP3 input received", EventType::Pop3(Pop3Event::RawOutput) => "Raw POP3 output sent", - EventType::Purge(PurgeEvent::Started) => "The purge has started", - EventType::Purge(PurgeEvent::Finished) => "The purge has finished", - EventType::Purge(PurgeEvent::Running) => "The purge is running", - EventType::Purge(PurgeEvent::Error) => "An error occurred with the purge", - EventType::Purge(PurgeEvent::InProgress) => "An active purge is in progress", - EventType::Purge(PurgeEvent::AutoExpunge) => "Auto-expunge has been executed", - EventType::Purge(PurgeEvent::BlobCleanup) => "Blob storage cleanup has completed", EventType::PushSubscription(PushSubscriptionEvent::Success) => { "The push subscription was successful" } @@ -4304,15 +4304,16 @@ impl EventType { EventType::PushSubscription(PushSubscriptionEvent::NotFound) => { "The push subscription was not found" } - EventType::Queue(QueueEvent::QueueMessage) => "A new message was queued for delivery", - EventType::Queue(QueueEvent::QueueMessageAuthenticated) => { + EventType::Queue(QueueEvent::Started) => "The MTA queue has started", + EventType::Queue(QueueEvent::MessageQueued) => "A new message was queued for delivery", + EventType::Queue(QueueEvent::AuthenticatedMessageQueued) => { "A new message was queued for delivery from an authenticated client" } - EventType::Queue(QueueEvent::QueueReport) => "A new report was queued for delivery", - EventType::Queue(QueueEvent::QueueDsn) => { + EventType::Queue(QueueEvent::ReportQueued) => "A new report was queued for delivery", + EventType::Queue(QueueEvent::DsnQueued) => { "A delivery status notification was queued for delivery" } - EventType::Queue(QueueEvent::QueueAutogenerated) => { + EventType::Queue(QueueEvent::AutogeneratedQueued) => { "A system generated message was queued for delivery" } EventType::Queue(QueueEvent::Rescheduled) => "The message was rescheduled for delivery", @@ -4362,9 +4363,6 @@ impl EventType { EventType::Resource(ResourceEvent::DownloadExternal) => { "The external resource is being downloaded" } - EventType::Resource(ResourceEvent::WebadminUnpacked) => { - "The webadmin resource has been unpacked" - } EventType::Security(SecurityEvent::AuthenticationBan) => { "IP address was banned due to multiple authentication errors" } @@ -4660,26 +4658,50 @@ impl EventType { EventType::Store(StoreEvent::LdapQuery) => "An LDAP query was executed", EventType::Store(StoreEvent::LdapWarning) => "An LDAP authentication warning occurred", EventType::Store(StoreEvent::HttpStoreFetch) => "The HTTP store was updated", - EventType::TaskQueue(TaskQueueEvent::TaskAcquired) => { + EventType::Store(StoreEvent::AutoExpunge) => "Auto-expunge has been executed", + EventType::Store(StoreEvent::BlobStorePurged) => { + "The blob store purge process has completed" + } + EventType::Store(StoreEvent::DataStorePurged) => { + "The data store purge process has completed" + } + EventType::TaskManager(TaskManagerEvent::TaskAcquired) => { "A task has been acquired from the queue" } - EventType::TaskQueue(TaskQueueEvent::TaskLocked) => { + EventType::TaskManager(TaskManagerEvent::TaskQueued) => { + "A task has been queued for processing" + } + EventType::TaskManager(TaskManagerEvent::TaskScheduled) => { + "A task has been scheduled for future execution" + } + EventType::TaskManager(TaskManagerEvent::TaskLocked) => { "The task id is locked by another process" } - EventType::TaskQueue(TaskQueueEvent::TaskIgnored) => { + EventType::TaskManager(TaskManagerEvent::TaskIgnored) => { "The task was ignored based on the current server roles" } - EventType::TaskQueue(TaskQueueEvent::TaskFailed) => "The task failed during processing", - EventType::TaskQueue(TaskQueueEvent::TaskRetry) => { + EventType::TaskManager(TaskManagerEvent::TaskFailed) => { + "The task failed during processing" + } + EventType::TaskManager(TaskManagerEvent::TaskRetry) => { "The task will be retried after a failure" } - EventType::TaskQueue(TaskQueueEvent::BlobNotFound) => { + EventType::TaskManager(TaskManagerEvent::BlobNotFound) => { "The requested blob was not found for task" } - EventType::TaskQueue(TaskQueueEvent::MetadataNotFound) => { + EventType::TaskManager(TaskManagerEvent::MetadataNotFound) => { "The metadata was not found for task" } - EventType::Telemetry(TelemetryEvent::Alert) => "An alert was triggered", + EventType::TaskManager(TaskManagerEvent::SchedulerStarted) => { + "The task scheduler has started" + } + EventType::TaskManager(TaskManagerEvent::ManagerStarted) => { + "The task manager has started" + } + EventType::Telemetry(TelemetryEvent::AlertEvent) => "An alert event was triggered", + EventType::Telemetry(TelemetryEvent::AlertMessage) => { + "An alert message was sent via email" + } EventType::Telemetry(TelemetryEvent::LogError) => { "An error occurred with the log collector" } @@ -4698,6 +4720,11 @@ impl EventType { EventType::Telemetry(TelemetryEvent::JournalError) => { "An error occurred with the journal collector" } + EventType::Telemetry(TelemetryEvent::MetricsCollected) => { + "The housekeeper process has stopped" + } + EventType::Telemetry(TelemetryEvent::MetricsStored) => "The purge has finished", + EventType::Telemetry(TelemetryEvent::MetricsPushed) => "A housekeeper task is running", EventType::Tls(TlsEvent::Handshake) => "Successful TLS handshake", EventType::Tls(TlsEvent::HandshakeError) => { "An error occurred during the TLS handshake" @@ -4828,12 +4855,12 @@ impl EventType { EventType::Limit(LimitEvent::BlobQuota) => "Blob quota exceeded", EventType::Limit(LimitEvent::TenantQuota) => "Tenant quota exceeded", EventType::Limit(LimitEvent::TooManyRequests) => "Too many requests", - EventType::Manage(ManageEvent::MissingParameter) => "Missing parameter", - EventType::Manage(ManageEvent::AlreadyExists) => "Already exists", - EventType::Manage(ManageEvent::AssertFailed) => "Assertion failed", - EventType::Manage(ManageEvent::NotFound) => "Not found", - EventType::Manage(ManageEvent::NotSupported) => "Operation not supported", - EventType::Manage(ManageEvent::Error) => "Management API Error", + EventType::Manage(ManageEvent::Reserved1) => "Missing parameter", + EventType::Manage(ManageEvent::Reserved2) => "Already exists", + EventType::Manage(ManageEvent::Reserved3) => "Assertion failed", + EventType::Manage(ManageEvent::Reserved4) => "Not found", + EventType::Manage(ManageEvent::Reserved5) => "Operation not supported", + EventType::Manage(ManageEvent::Reserved6) => "Management API Error", EventType::ManageSieve(ManageSieveEvent::ConnectionStart) => "ManageSieve error", EventType::ManageSieve(ManageSieveEvent::ConnectionEnd) => "ManageSieve error", EventType::ManageSieve(ManageSieveEvent::CreateScript) => "ManageSieve error", @@ -4888,7 +4915,6 @@ impl EventType { EventType::Resource(ResourceEvent::BadParameters) => "Bad parameters", EventType::Resource(ResourceEvent::Error) => "Resource error", EventType::Resource(ResourceEvent::DownloadExternal) => "Other status", - EventType::Resource(ResourceEvent::WebadminUnpacked) => "Other status", EventType::Security(SecurityEvent::AuthenticationBan) => "Insufficient permissions", EventType::Security(SecurityEvent::AbuseBan) => "Insufficient permissions", EventType::Security(SecurityEvent::ScanBan) => "Insufficient permissions", @@ -5160,10 +5186,6 @@ impl EventType { EventType::Eval(EvalEvent::Error), EventType::Eval(EvalEvent::DirectoryNotFound), EventType::Eval(EvalEvent::StoreNotFound), - EventType::Housekeeper(HousekeeperEvent::Start), - EventType::Housekeeper(HousekeeperEvent::Stop), - EventType::Housekeeper(HousekeeperEvent::Schedule), - EventType::Housekeeper(HousekeeperEvent::Run), EventType::Http(HttpEvent::ConnectionStart), EventType::Http(HttpEvent::ConnectionEnd), EventType::Http(HttpEvent::Error), @@ -5270,12 +5292,12 @@ impl EventType { EventType::MailAuth(MailAuthEvent::DnsRecordNotFound), EventType::MailAuth(MailAuthEvent::DnsInvalidRecordType), EventType::MailAuth(MailAuthEvent::PolicyNotAligned), - EventType::Manage(ManageEvent::MissingParameter), - EventType::Manage(ManageEvent::AlreadyExists), - EventType::Manage(ManageEvent::AssertFailed), - EventType::Manage(ManageEvent::NotFound), - EventType::Manage(ManageEvent::NotSupported), - EventType::Manage(ManageEvent::Error), + EventType::Manage(ManageEvent::Reserved1), + EventType::Manage(ManageEvent::Reserved2), + EventType::Manage(ManageEvent::Reserved3), + EventType::Manage(ManageEvent::Reserved4), + EventType::Manage(ManageEvent::Reserved5), + EventType::Manage(ManageEvent::Reserved6), EventType::ManageSieve(ManageSieveEvent::ConnectionStart), EventType::ManageSieve(ManageSieveEvent::ConnectionEnd), EventType::ManageSieve(ManageSieveEvent::CreateScript), @@ -5377,21 +5399,15 @@ impl EventType { EventType::Pop3(Pop3Event::Error), EventType::Pop3(Pop3Event::RawInput), EventType::Pop3(Pop3Event::RawOutput), - EventType::Purge(PurgeEvent::Started), - EventType::Purge(PurgeEvent::Finished), - EventType::Purge(PurgeEvent::Running), - EventType::Purge(PurgeEvent::Error), - EventType::Purge(PurgeEvent::InProgress), - EventType::Purge(PurgeEvent::AutoExpunge), - EventType::Purge(PurgeEvent::BlobCleanup), EventType::PushSubscription(PushSubscriptionEvent::Success), EventType::PushSubscription(PushSubscriptionEvent::Error), EventType::PushSubscription(PushSubscriptionEvent::NotFound), - EventType::Queue(QueueEvent::QueueMessage), - EventType::Queue(QueueEvent::QueueMessageAuthenticated), - EventType::Queue(QueueEvent::QueueReport), - EventType::Queue(QueueEvent::QueueDsn), - EventType::Queue(QueueEvent::QueueAutogenerated), + EventType::Queue(QueueEvent::Started), + EventType::Queue(QueueEvent::MessageQueued), + EventType::Queue(QueueEvent::AuthenticatedMessageQueued), + EventType::Queue(QueueEvent::ReportQueued), + EventType::Queue(QueueEvent::DsnQueued), + EventType::Queue(QueueEvent::AutogeneratedQueued), EventType::Queue(QueueEvent::Rescheduled), EventType::Queue(QueueEvent::Locked), EventType::Queue(QueueEvent::BlobNotFound), @@ -5413,7 +5429,6 @@ impl EventType { EventType::Resource(ResourceEvent::BadParameters), EventType::Resource(ResourceEvent::Error), EventType::Resource(ResourceEvent::DownloadExternal), - EventType::Resource(ResourceEvent::WebadminUnpacked), EventType::Security(SecurityEvent::AuthenticationBan), EventType::Security(SecurityEvent::AbuseBan), EventType::Security(SecurityEvent::ScanBan), @@ -5577,20 +5592,31 @@ impl EventType { EventType::Store(StoreEvent::LdapQuery), EventType::Store(StoreEvent::LdapWarning), EventType::Store(StoreEvent::HttpStoreFetch), - EventType::TaskQueue(TaskQueueEvent::TaskAcquired), - EventType::TaskQueue(TaskQueueEvent::TaskLocked), - EventType::TaskQueue(TaskQueueEvent::TaskIgnored), - EventType::TaskQueue(TaskQueueEvent::TaskFailed), - EventType::TaskQueue(TaskQueueEvent::TaskRetry), - EventType::TaskQueue(TaskQueueEvent::BlobNotFound), - EventType::TaskQueue(TaskQueueEvent::MetadataNotFound), - EventType::Telemetry(TelemetryEvent::Alert), + EventType::Store(StoreEvent::AutoExpunge), + EventType::Store(StoreEvent::BlobStorePurged), + EventType::Store(StoreEvent::DataStorePurged), + EventType::TaskManager(TaskManagerEvent::TaskAcquired), + EventType::TaskManager(TaskManagerEvent::TaskQueued), + EventType::TaskManager(TaskManagerEvent::TaskScheduled), + EventType::TaskManager(TaskManagerEvent::TaskLocked), + EventType::TaskManager(TaskManagerEvent::TaskIgnored), + EventType::TaskManager(TaskManagerEvent::TaskFailed), + EventType::TaskManager(TaskManagerEvent::TaskRetry), + EventType::TaskManager(TaskManagerEvent::BlobNotFound), + EventType::TaskManager(TaskManagerEvent::MetadataNotFound), + EventType::TaskManager(TaskManagerEvent::SchedulerStarted), + EventType::TaskManager(TaskManagerEvent::ManagerStarted), + EventType::Telemetry(TelemetryEvent::AlertEvent), + EventType::Telemetry(TelemetryEvent::AlertMessage), EventType::Telemetry(TelemetryEvent::LogError), EventType::Telemetry(TelemetryEvent::WebhookError), EventType::Telemetry(TelemetryEvent::OtelExporterError), EventType::Telemetry(TelemetryEvent::OtelMetricsExporterError), EventType::Telemetry(TelemetryEvent::PrometheusExporterError), EventType::Telemetry(TelemetryEvent::JournalError), + EventType::Telemetry(TelemetryEvent::MetricsCollected), + EventType::Telemetry(TelemetryEvent::MetricsStored), + EventType::Telemetry(TelemetryEvent::MetricsPushed), EventType::Tls(TlsEvent::Handshake), EventType::Tls(TlsEvent::HandshakeError), EventType::Tls(TlsEvent::NotConfigured), @@ -5829,16 +5855,15 @@ impl MetricType { b"pop3.active-connections" => MetricType::Pop3ActiveConnections, b"pop3.connection-start" => MetricType::Pop3ConnectionStart, b"pop3.connection-end" => MetricType::Pop3ConnectionEnd, - b"purge.error" => MetricType::PurgeError, b"push-subscription.success" => MetricType::PushSubscriptionSuccess, b"push-subscription.error" => MetricType::PushSubscriptionError, b"push-subscription.not-found" => MetricType::PushSubscriptionNotFound, b"queue.count" => MetricType::QueueCount, - b"queue.queue-message" => MetricType::QueueQueueMessage, - b"queue.queue-message-authenticated" => MetricType::QueueQueueMessageAuthenticated, - b"queue.queue-report" => MetricType::QueueQueueReport, - b"queue.queue-dsn" => MetricType::QueueQueueDsn, - b"queue.queue-autogenerated" => MetricType::QueueQueueAutogenerated, + b"queue.message-queued" => MetricType::QueueMessageQueued, + b"queue.authenticated-message-queued" => MetricType::QueueAuthenticatedMessageQueued, + b"queue.report-queued" => MetricType::QueueReportQueued, + b"queue.dsn-queued" => MetricType::QueueDsnQueued, + b"queue.autogenerated-queued" => MetricType::QueueAutogeneratedQueued, b"queue.rescheduled" => MetricType::QueueRescheduled, b"queue.blob-not-found" => MetricType::QueueBlobNotFound, b"queue.rate-limit-exceeded" => MetricType::QueueRateLimitExceeded, @@ -5955,8 +5980,9 @@ impl MetricType { b"store.blob-read" => MetricType::StoreBlobRead, b"store.blob-write" => MetricType::StoreBlobWrite, b"store.blob-delete" => MetricType::StoreBlobDelete, - b"task-queue.blob-not-found" => MetricType::TaskQueueBlobNotFound, - b"task-queue.metadata-not-found" => MetricType::TaskQueueMetadataNotFound, + b"task-manager.blob-not-found" => MetricType::TaskManagerBlobNotFound, + b"task-manager.metadata-not-found" => MetricType::TaskManagerMetadataNotFound, + b"telemetry.alert-message" => MetricType::TelemetryAlertMessage, b"telemetry.log-error" => MetricType::TelemetryLogError, b"telemetry.webhook-error" => MetricType::TelemetryWebhookError, b"telemetry.otel-exporter-error" => MetricType::TelemetryOtelExporterError, @@ -6183,16 +6209,15 @@ impl MetricType { MetricType::Pop3ActiveConnections => "pop3.active-connections", MetricType::Pop3ConnectionStart => "pop3.connection-start", MetricType::Pop3ConnectionEnd => "pop3.connection-end", - MetricType::PurgeError => "purge.error", MetricType::PushSubscriptionSuccess => "push-subscription.success", MetricType::PushSubscriptionError => "push-subscription.error", MetricType::PushSubscriptionNotFound => "push-subscription.not-found", MetricType::QueueCount => "queue.count", - MetricType::QueueQueueMessage => "queue.queue-message", - MetricType::QueueQueueMessageAuthenticated => "queue.queue-message-authenticated", - MetricType::QueueQueueReport => "queue.queue-report", - MetricType::QueueQueueDsn => "queue.queue-dsn", - MetricType::QueueQueueAutogenerated => "queue.queue-autogenerated", + MetricType::QueueMessageQueued => "queue.message-queued", + MetricType::QueueAuthenticatedMessageQueued => "queue.authenticated-message-queued", + MetricType::QueueReportQueued => "queue.report-queued", + MetricType::QueueDsnQueued => "queue.dsn-queued", + MetricType::QueueAutogeneratedQueued => "queue.autogenerated-queued", MetricType::QueueRescheduled => "queue.rescheduled", MetricType::QueueBlobNotFound => "queue.blob-not-found", MetricType::QueueRateLimitExceeded => "queue.rate-limit-exceeded", @@ -6309,8 +6334,9 @@ impl MetricType { MetricType::StoreBlobRead => "store.blob-read", MetricType::StoreBlobWrite => "store.blob-write", MetricType::StoreBlobDelete => "store.blob-delete", - MetricType::TaskQueueBlobNotFound => "task-queue.blob-not-found", - MetricType::TaskQueueMetadataNotFound => "task-queue.metadata-not-found", + MetricType::TaskManagerBlobNotFound => "task-manager.blob-not-found", + MetricType::TaskManagerMetadataNotFound => "task-manager.metadata-not-found", + MetricType::TelemetryAlertMessage => "telemetry.alert-message", MetricType::TelemetryLogError => "telemetry.log-error", MetricType::TelemetryWebhookError => "telemetry.webhook-error", MetricType::TelemetryOtelExporterError => "telemetry.otel-exporter-error", @@ -6528,16 +6554,15 @@ impl MetricType { MetricType::Pop3ActiveConnections => 19, MetricType::Pop3ConnectionStart => 210, MetricType::Pop3ConnectionEnd => 211, - MetricType::PurgeError => 212, MetricType::PushSubscriptionSuccess => 213, MetricType::PushSubscriptionError => 214, MetricType::PushSubscriptionNotFound => 215, MetricType::QueueCount => 24, - MetricType::QueueQueueMessage => 216, - MetricType::QueueQueueMessageAuthenticated => 217, - MetricType::QueueQueueReport => 218, - MetricType::QueueQueueDsn => 219, - MetricType::QueueQueueAutogenerated => 220, + MetricType::QueueMessageQueued => 216, + MetricType::QueueAuthenticatedMessageQueued => 217, + MetricType::QueueReportQueued => 218, + MetricType::QueueDsnQueued => 219, + MetricType::QueueAutogeneratedQueued => 220, MetricType::QueueRescheduled => 221, MetricType::QueueBlobNotFound => 222, MetricType::QueueRateLimitExceeded => 223, @@ -6654,8 +6679,9 @@ impl MetricType { MetricType::StoreBlobRead => 325, MetricType::StoreBlobWrite => 326, MetricType::StoreBlobDelete => 327, - MetricType::TaskQueueBlobNotFound => 328, - MetricType::TaskQueueMetadataNotFound => 329, + MetricType::TaskManagerBlobNotFound => 328, + MetricType::TaskManagerMetadataNotFound => 329, + MetricType::TelemetryAlertMessage => 212, MetricType::TelemetryLogError => 330, MetricType::TelemetryWebhookError => 331, MetricType::TelemetryOtelExporterError => 332, @@ -6871,16 +6897,15 @@ impl MetricType { 19 => Some(MetricType::Pop3ActiveConnections), 210 => Some(MetricType::Pop3ConnectionStart), 211 => Some(MetricType::Pop3ConnectionEnd), - 212 => Some(MetricType::PurgeError), 213 => Some(MetricType::PushSubscriptionSuccess), 214 => Some(MetricType::PushSubscriptionError), 215 => Some(MetricType::PushSubscriptionNotFound), 24 => Some(MetricType::QueueCount), - 216 => Some(MetricType::QueueQueueMessage), - 217 => Some(MetricType::QueueQueueMessageAuthenticated), - 218 => Some(MetricType::QueueQueueReport), - 219 => Some(MetricType::QueueQueueDsn), - 220 => Some(MetricType::QueueQueueAutogenerated), + 216 => Some(MetricType::QueueMessageQueued), + 217 => Some(MetricType::QueueAuthenticatedMessageQueued), + 218 => Some(MetricType::QueueReportQueued), + 219 => Some(MetricType::QueueDsnQueued), + 220 => Some(MetricType::QueueAutogeneratedQueued), 221 => Some(MetricType::QueueRescheduled), 222 => Some(MetricType::QueueBlobNotFound), 223 => Some(MetricType::QueueRateLimitExceeded), @@ -6997,8 +7022,9 @@ impl MetricType { 325 => Some(MetricType::StoreBlobRead), 326 => Some(MetricType::StoreBlobWrite), 327 => Some(MetricType::StoreBlobDelete), - 328 => Some(MetricType::TaskQueueBlobNotFound), - 329 => Some(MetricType::TaskQueueMetadataNotFound), + 328 => Some(MetricType::TaskManagerBlobNotFound), + 329 => Some(MetricType::TaskManagerMetadataNotFound), + 212 => Some(MetricType::TelemetryAlertMessage), 330 => Some(MetricType::TelemetryLogError), 331 => Some(MetricType::TelemetryWebhookError), 332 => Some(MetricType::TelemetryOtelExporterError), @@ -7199,15 +7225,14 @@ impl MetricType { MetricType::OutgoingReportNoRecipientsFound => 338, MetricType::Pop3ConnectionStart => 348, MetricType::Pop3ConnectionEnd => 347, - MetricType::PurgeError => 365, MetricType::PushSubscriptionSuccess => 373, MetricType::PushSubscriptionError => 371, MetricType::PushSubscriptionNotFound => 372, - MetricType::QueueQueueMessage => 380, - MetricType::QueueQueueMessageAuthenticated => 381, - MetricType::QueueQueueReport => 382, - MetricType::QueueQueueDsn => 379, - MetricType::QueueQueueAutogenerated => 378, + MetricType::QueueMessageQueued => 380, + MetricType::QueueAuthenticatedMessageQueued => 381, + MetricType::QueueReportQueued => 382, + MetricType::QueueDsnQueued => 379, + MetricType::QueueAutogeneratedQueued => 378, MetricType::QueueRescheduled => 385, MetricType::QueueBlobNotFound => 374, MetricType::QueueRateLimitExceeded => 384, @@ -7315,8 +7340,9 @@ impl MetricType { MetricType::StoreBlobRead => 508, MetricType::StoreBlobWrite => 509, MetricType::StoreBlobDelete => 506, - MetricType::TaskQueueBlobNotFound => 141, - MetricType::TaskQueueMetadataNotFound => 145, + MetricType::TaskManagerBlobNotFound => 141, + MetricType::TaskManagerMetadataNotFound => 145, + MetricType::TelemetryAlertMessage => 365, MetricType::TelemetryLogError => 535, MetricType::TelemetryWebhookError => 539, MetricType::TelemetryOtelExporterError => 536, @@ -7544,16 +7570,15 @@ impl MetricType { MetricType::Pop3ActiveConnections => "Active POP3 connections", MetricType::Pop3ConnectionStart => "POP3 connection started", MetricType::Pop3ConnectionEnd => "POP3 connection ended", - MetricType::PurgeError => "Purge error", MetricType::PushSubscriptionSuccess => "Push subscription successful", MetricType::PushSubscriptionError => "Push subscription error", MetricType::PushSubscriptionNotFound => "Push subscription not found", MetricType::QueueCount => "Total number of messages in the queue", - MetricType::QueueQueueMessage => "Queued message for delivery", - MetricType::QueueQueueMessageAuthenticated => "Queued message submission for delivery", - MetricType::QueueQueueReport => "Queued report for delivery", - MetricType::QueueQueueDsn => "Queued DSN for delivery", - MetricType::QueueQueueAutogenerated => "Queued autogenerated message for delivery", + MetricType::QueueMessageQueued => "Queued message for delivery", + MetricType::QueueAuthenticatedMessageQueued => "Queued message submission for delivery", + MetricType::QueueReportQueued => "Queued report for delivery", + MetricType::QueueDsnQueued => "Queued DSN for delivery", + MetricType::QueueAutogeneratedQueued => "Queued autogenerated message for delivery", MetricType::QueueRescheduled => "Message rescheduled for delivery", MetricType::QueueBlobNotFound => "Message blob not found", MetricType::QueueRateLimitExceeded => "Rate limit exceeded", @@ -7670,8 +7695,9 @@ impl MetricType { MetricType::StoreBlobRead => "Blob read operation", MetricType::StoreBlobWrite => "Blob write operation", MetricType::StoreBlobDelete => "Blob delete operation", - MetricType::TaskQueueBlobNotFound => "Blob not found for task", - MetricType::TaskQueueMetadataNotFound => "Metadata not found for task", + MetricType::TaskManagerBlobNotFound => "Blob not found for task", + MetricType::TaskManagerMetadataNotFound => "Metadata not found for task", + MetricType::TelemetryAlertMessage => "Alert message sent", MetricType::TelemetryLogError => "Log collector error", MetricType::TelemetryWebhookError => "Webhook collector error", MetricType::TelemetryOtelExporterError => "OpenTelemetry exporter error", @@ -7881,15 +7907,14 @@ impl MetricType { | MetricType::OutgoingReportNoRecipientsFound | MetricType::Pop3ConnectionStart | MetricType::Pop3ConnectionEnd - | MetricType::PurgeError | MetricType::PushSubscriptionSuccess | MetricType::PushSubscriptionError | MetricType::PushSubscriptionNotFound - | MetricType::QueueQueueMessage - | MetricType::QueueQueueMessageAuthenticated - | MetricType::QueueQueueReport - | MetricType::QueueQueueDsn - | MetricType::QueueQueueAutogenerated + | MetricType::QueueMessageQueued + | MetricType::QueueAuthenticatedMessageQueued + | MetricType::QueueReportQueued + | MetricType::QueueDsnQueued + | MetricType::QueueAutogeneratedQueued | MetricType::QueueRescheduled | MetricType::QueueBlobNotFound | MetricType::QueueRateLimitExceeded @@ -7997,8 +8022,9 @@ impl MetricType { | MetricType::StoreBlobRead | MetricType::StoreBlobWrite | MetricType::StoreBlobDelete - | MetricType::TaskQueueBlobNotFound - | MetricType::TaskQueueMetadataNotFound + | MetricType::TaskManagerBlobNotFound + | MetricType::TaskManagerMetadataNotFound + | MetricType::TelemetryAlertMessage | MetricType::TelemetryLogError | MetricType::TelemetryWebhookError | MetricType::TelemetryOtelExporterError @@ -8230,16 +8256,15 @@ impl MetricType { MetricType::Pop3ActiveConnections, MetricType::Pop3ConnectionStart, MetricType::Pop3ConnectionEnd, - MetricType::PurgeError, MetricType::PushSubscriptionSuccess, MetricType::PushSubscriptionError, MetricType::PushSubscriptionNotFound, MetricType::QueueCount, - MetricType::QueueQueueMessage, - MetricType::QueueQueueMessageAuthenticated, - MetricType::QueueQueueReport, - MetricType::QueueQueueDsn, - MetricType::QueueQueueAutogenerated, + MetricType::QueueMessageQueued, + MetricType::QueueAuthenticatedMessageQueued, + MetricType::QueueReportQueued, + MetricType::QueueDsnQueued, + MetricType::QueueAutogeneratedQueued, MetricType::QueueRescheduled, MetricType::QueueBlobNotFound, MetricType::QueueRateLimitExceeded, @@ -8356,8 +8381,9 @@ impl MetricType { MetricType::StoreBlobRead, MetricType::StoreBlobWrite, MetricType::StoreBlobDelete, - MetricType::TaskQueueBlobNotFound, - MetricType::TaskQueueMetadataNotFound, + MetricType::TaskManagerBlobNotFound, + MetricType::TaskManagerMetadataNotFound, + MetricType::TelemetryAlertMessage, MetricType::TelemetryLogError, MetricType::TelemetryWebhookError, MetricType::TelemetryOtelExporterError, diff --git a/crates/trc/src/ipc/metrics.rs b/crates/trc/src/ipc/metrics.rs index 3c64d4ac..a0057f42 100644 --- a/crates/trc/src/ipc/metrics.rs +++ b/crates/trc/src/ipc/metrics.rs @@ -163,19 +163,19 @@ impl Collector { ) => { MESSAGE_INGESTION_TIME.observe(elapsed); } - EventType::Queue(QueueEvent::QueueMessage) => { + EventType::Queue(QueueEvent::MessageQueued) => { MESSAGE_INCOMING_SIZE.observe(size); QUEUE_COUNT.increment(); } - EventType::Queue(QueueEvent::QueueMessageAuthenticated) => { + EventType::Queue(QueueEvent::AuthenticatedMessageQueued) => { MESSAGE_SUBMISSION_SIZE.observe(size); QUEUE_COUNT.increment(); } - EventType::Queue(QueueEvent::QueueReport) => { + EventType::Queue(QueueEvent::ReportQueued) => { MESSAGE_OUT_REPORT_SIZE.observe(size); QUEUE_COUNT.increment(); } - EventType::Queue(QueueEvent::QueueAutogenerated | QueueEvent::QueueDsn) => { + EventType::Queue(QueueEvent::AutogeneratedQueued | QueueEvent::DsnQueued) => { QUEUE_COUNT.increment(); } EventType::MessageIngest(MessageIngestEvent::FtsIndex) => {