diff --git a/Cargo.lock b/Cargo.lock index a3813a9f..912d9b69 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3939,6 +3939,28 @@ dependencies = [ "tokio-tungstenite", ] +[[package]] +name = "jmap-client" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0a269c4668bc7c12c61bf142f4f30c8d568d5afa8472f9dd881d83d2551323a" +dependencies = [ + "ahash", + "async-stream", + "base64 0.13.1", + "chrono", + "futures-util", + "maybe-async", + "parking_lot", + "reqwest 0.12.23", + "rustls 0.23.32", + "rustls-pki-types", + "serde", + "serde_json", + "tokio", + "tokio-tungstenite", +] + [[package]] name = "jmap-tools" version = "0.1.3" @@ -7797,7 +7819,7 @@ dependencies = [ "futures", "human-size", "indicatif", - "jmap-client", + "jmap-client 0.3.3", "mail-auth", "mail-parser", "num_cpus", @@ -8058,7 +8080,7 @@ dependencies = [ "imap_proto", "jemallocator", "jmap", - "jmap-client", + "jmap-client 0.4.0", "jmap_proto", "mail-auth", "mail-parser", diff --git a/crates/common/src/auth/access_token.rs b/crates/common/src/auth/access_token.rs index b8819612..7fbd5018 100644 --- a/crates/common/src/auth/access_token.rs +++ b/crates/common/src/auth/access_token.rs @@ -352,7 +352,7 @@ impl Server { // Invalidate DAV caches if !changed_names.is_empty() { - self.cluster_broadcast(BroadcastEvent::InvalidateDavCache(changed_names)) + self.cluster_broadcast(BroadcastEvent::InvalidateGroupwareCache(changed_names)) .await; } } @@ -420,6 +420,12 @@ impl AccessToken { .chain(self.access_to.iter().map(|(id, _)| id)) } + pub fn member_ids(&self) -> impl Iterator { + [self.primary_id] + .into_iter() + .chain(self.member_of.iter().copied()) + } + pub fn all_ids(&self) -> impl Iterator { [self.primary_id] .into_iter() diff --git a/crates/common/src/config/network.rs b/crates/common/src/config/network.rs index ea84669e..d8c9921d 100644 --- a/crates/common/src/config/network.rs +++ b/crates/common/src/config/network.rs @@ -9,7 +9,7 @@ use std::time::Duration; use crate::expr::{if_block::IfBlock, tokenizer::TokenMap}; use ahash::AHashSet; -use utils::config::{Config, Rate}; +use utils::config::{Config, Rate, utils::ParseValue}; use super::*; @@ -38,13 +38,29 @@ pub struct ContactForm { pub field_honey_pot: Option, } -#[derive(Clone)] +#[derive(Clone, Default)] pub struct ClusterRoles { - pub purge_stores: bool, - pub purge_accounts: bool, - pub renew_acme: bool, - pub calculate_metrics: bool, - pub push_metrics: bool, + pub purge_stores: ClusterRole, + pub purge_accounts: ClusterRole, + pub push_notifications: ClusterRole, + pub fts_indexing: ClusterRole, + pub bayes_training: ClusterRole, + pub imip_processing: ClusterRole, + pub calendar_alerts: ClusterRole, + pub renew_acme: ClusterRole, + pub calculate_metrics: ClusterRole, + pub push_metrics: ClusterRole, +} + +#[derive(Clone, Copy, Default)] +pub enum ClusterRole { + #[default] + Enabled, + Disabled, + Sharded { + shard_id: u32, + total_shards: u32, + }, } #[derive(Clone, Default)] @@ -104,13 +120,7 @@ impl Default for Network { asn_geo_lookup: AsnGeoLookupConfig::Disabled, server_name: Default::default(), report_domain: Default::default(), - roles: ClusterRoles { - purge_stores: true, - purge_accounts: true, - renew_acme: true, - calculate_metrics: true, - push_metrics: true, - }, + roles: ClusterRoles::default(), } } } @@ -228,14 +238,49 @@ impl Network { &mut network.roles.push_metrics, "cluster.roles.metrics.push", ), + ( + &mut network.roles.push_notifications, + "cluster.roles.push-notifications", + ), + ( + &mut network.roles.fts_indexing, + "cluster.roles.fts-indexing", + ), + ( + &mut network.roles.bayes_training, + "cluster.roles.bayes-training", + ), + ( + &mut network.roles.imip_processing, + "cluster.roles.imip-processing", + ), + ( + &mut network.roles.calendar_alerts, + "cluster.roles.calendar-alerts", + ), ] { - let node_ids = config - .properties::(key) + let shards = config + .properties::(key) .into_iter() .map(|(_, v)| v) - .collect::>(); - if !node_ids.is_empty() && !node_ids.contains(&network.node_id) { - *value = false; + .collect::>(); + let shard_size = shards.len() as u32; + let mut found_node = false; + for (shard_id, shard) in shards.iter().enumerate() { + if shard.0.contains(&network.node_id) { + if shard_size > 1 { + *value = ClusterRole::Sharded { + shard_id: shard_id as u32, + total_shards: shard_size, + }; + } + found_node = true; + break; + } + } + + if !shards.is_empty() && !found_node { + *value = ClusterRole::Disabled; } } @@ -252,6 +297,18 @@ impl Network { } } +struct NodeList(AHashSet); + +impl ParseValue for NodeList { + fn parse_value(value: &str) -> utils::config::Result { + value + .split(',') + .map(|s| s.trim().parse::().map_err(|e| e.to_string())) + .collect::, String>>() + .map(NodeList) + } +} + impl AsnGeoLookupConfig { pub fn parse(config: &mut Config) -> Option { match config.value("asn.type")? { @@ -297,3 +354,20 @@ impl AsnGeoLookupConfig { } } } + +impl ClusterRole { + pub fn is_enabled(&self) -> bool { + matches!(self, ClusterRole::Enabled) + } + + pub fn is_enabled_for_account(&self, account_id: u32) -> bool { + match self { + ClusterRole::Enabled => true, + ClusterRole::Disabled => false, + ClusterRole::Sharded { + shard_id, + total_shards, + } => (account_id % total_shards) == *shard_id, + } + } +} diff --git a/crates/common/src/core.rs b/crates/common/src/core.rs index 84e36ee4..58e91fe1 100644 --- a/crates/common/src/core.rs +++ b/crates/common/src/core.rs @@ -14,7 +14,7 @@ use crate::{ QueueStrategy, RequireOptional, RoutingStrategy, TlsStrategy, VirtualQueue, }, }, - ipc::{BroadcastEvent, StateEvent}, + ipc::{BroadcastEvent, PushEvent, PushNotification}, }; use directory::{Directory, QueryParams, Type, backend::internal::manage::ManageDirectory}; use mail_auth::IpLookupStrategy; @@ -41,7 +41,7 @@ use types::{ field::{EmailField, Field}, type_state::{DataType, StateChange}, }; -use utils::snowflake::SnowflakeIdGenerator; +use utils::{map::bitmap::Bitmap, snowflake::SnowflakeIdGenerator}; impl Server { #[inline(always)] @@ -345,7 +345,6 @@ impl Server { pub async fn recalculate_quota(&self, account_id: u32) -> trc::Result<()> { let mut quota = 0i64; - let todo = "include sieve scripts and calendars, contacts, files in quota"; self.store() .iterate( @@ -651,8 +650,7 @@ impl Server { if let Some(changes) = builder.changes() { for (account_id, changed_collections) in changes { - let mut state_change = - StateChange::new(account_id, assigned_ids.last_change_id(account_id)?); + let mut state_change = StateChange::new(account_id); for changed_collection in changed_collections.changed_containers { if let Some(data_type) = DataType::try_from_sync(changed_collection, true) { state_change.set_change(data_type); @@ -664,7 +662,18 @@ impl Server { } } if state_change.has_changes() { - self.broadcast_state_change(state_change).await; + self.broadcast_push_notification(PushNotification::StateChange( + state_change.with_change_id(assigned_ids.last_change_id(account_id)?), + )) + .await; + } + if let Some(change_id) = changed_collections.share_notification_id { + self.broadcast_push_notification(PushNotification::StateChange(StateChange { + account_id, + change_id, + types: Bitmap::from_iter([DataType::ShareNotification]), + })) + .await; } } } @@ -800,14 +809,14 @@ impl Server { Ok(()) } - pub async fn broadcast_state_change(&self, state_change: StateChange) -> bool { + pub async fn broadcast_push_notification(&self, notification: PushNotification) -> bool { match self .inner .ipc - .state_tx + .push_tx .clone() - .send(StateEvent::Publish { - state_change, + .send(PushEvent::Publish { + notification, broadcast: true, }) .await diff --git a/crates/common/src/ipc.rs b/crates/common/src/ipc.rs index 18d56047..eedd8c61 100644 --- a/crates/common/src/ipc.rs +++ b/crates/common/src/ipc.rs @@ -45,59 +45,58 @@ pub enum PurgeType { } #[derive(Debug)] -pub enum StateEvent { +pub enum PushEvent { Subscribe { - account_id: u32, + account_ids: Vec, types: Bitmap, - tx: mpsc::Sender, + tx: mpsc::Sender, }, Publish { - state_change: StateChange, + notification: PushNotification, broadcast: bool, }, - UpdateSharedAccounts { - account_id: u32, + PushServerRegister { + activate: Vec, + expired: Vec, }, - UpdateSubscriptions { + PushServerUpdate { account_id: u32, - subscriptions: Vec, + broadcast: bool, }, Stop, } -#[derive(Debug)] -pub enum BroadcastEvent { +#[derive(Debug, Clone)] +pub enum PushNotification { StateChange(StateChange), - InvalidateAccessTokens(Vec), - InvalidateDavCache(Vec), - ReloadSettings, - ReloadBlockedIps, -} - -#[derive(Debug)] -pub enum UpdateSubscription { - Unverified { - id: u32, - url: String, - code: String, - keys: Option, - }, - Verified(PushSubscription), -} - -#[derive(Debug)] -pub struct PushSubscription { - pub id: u32, - pub url: String, - pub expires: u64, - pub types: Bitmap, - pub keys: Option, + CalendarAlert(CalendarAlert), + EmailPush(EmailPush), } #[derive(Debug, Clone)] -pub struct EncryptionKeys { - pub p256dh: Vec, - pub auth: Vec, +pub struct EmailPush { + pub account_id: u32, + pub email_id: u32, + pub change_id: u64, +} + +#[derive(Debug, Clone)] +pub struct CalendarAlert { + pub account_id: u32, + pub event_id: u32, + pub recurrence_id: Option, + pub uid: String, + pub alert_id: String, +} + +#[derive(Debug)] +pub enum BroadcastEvent { + PushNotification(PushNotification), + InvalidateAccessTokens(Vec), + InvalidateGroupwareCache(Vec), + ReloadPushServers(u32), + ReloadSettings, + ReloadBlockedIps, } #[derive(Debug)] @@ -212,3 +211,68 @@ impl From<(&Option>, &Option>)> for PolicyType { } } } + +impl PushNotification { + pub fn account_id(&self) -> u32 { + match self { + PushNotification::StateChange(state_change) => state_change.account_id, + PushNotification::CalendarAlert(calendar_alert) => calendar_alert.account_id, + PushNotification::EmailPush(email_push) => email_push.account_id, + } + } + + pub fn filter_types(&self, types: &Bitmap) -> Option { + match self { + PushNotification::StateChange(state_change) => { + let mut filtered_types = state_change.types; + filtered_types.intersection(types); + if !filtered_types.is_empty() { + Some(PushNotification::StateChange(StateChange { + account_id: state_change.account_id, + change_id: state_change.change_id, + types: filtered_types, + })) + } else { + None + } + } + PushNotification::CalendarAlert(_) => { + if types.contains(DataType::CalendarAlert) { + Some(self.clone()) + } else { + None + } + } + PushNotification::EmailPush(_) => { + if types.contains_any( + [ + DataType::EmailDelivery, + DataType::Email, + DataType::Mailbox, + DataType::Thread, + ] + .into_iter(), + ) { + Some(self.clone()) + } else { + None + } + } + } + } +} + +impl EmailPush { + pub fn to_state_change(&self) -> StateChange { + StateChange { + account_id: self.account_id, + change_id: self.change_id, + types: Bitmap::from_iter([ + DataType::EmailDelivery, + DataType::Email, + DataType::Mailbox, + DataType::Thread, + ]), + } + } +} diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 27fd0a86..3f5ff347 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -24,7 +24,7 @@ use config::{ storage::Storage, telemetry::Metrics, }; -use ipc::{BroadcastEvent, HousekeeperEvent, QueueEvent, ReportingEvent, StateEvent}; +use ipc::{BroadcastEvent, HousekeeperEvent, PushEvent, QueueEvent, ReportingEvent}; use listener::{asn::AsnGeoLookupData, blocked::Security, tls::AcmeProviders}; use mail_auth::{MX, Txt}; use manager::webadmin::{Resource, WebAdminManager}; @@ -244,7 +244,7 @@ pub struct HttpAuthCache { } pub struct Ipc { - pub state_tx: mpsc::Sender, + pub push_tx: mpsc::Sender, pub housekeeper_tx: mpsc::Sender, pub task_tx: Arc, pub queue_tx: mpsc::Sender, @@ -503,7 +503,7 @@ impl Default for Caches { impl Default for Ipc { fn default() -> Self { Self { - state_tx: mpsc::channel(IPC_CHANNEL_BUFFER).0, + push_tx: mpsc::channel(IPC_CHANNEL_BUFFER).0, housekeeper_tx: mpsc::channel(IPC_CHANNEL_BUFFER).0, task_tx: Default::default(), queue_tx: mpsc::channel(IPC_CHANNEL_BUFFER).0, diff --git a/crates/common/src/manager/boot.rs b/crates/common/src/manager/boot.rs index c26edd08..24be0e4c 100644 --- a/crates/common/src/manager/boot.rs +++ b/crates/common/src/manager/boot.rs @@ -14,7 +14,7 @@ use crate::{ Caches, Core, Data, IPC_CHANNEL_BUFFER, Inner, Ipc, config::{network::AsnGeoLookupConfig, server::Listeners, telemetry::Telemetry}, core::BuildServer, - ipc::{BroadcastEvent, HousekeeperEvent, QueueEvent, ReportingEvent, StateEvent}, + ipc::{BroadcastEvent, HousekeeperEvent, PushEvent, QueueEvent, ReportingEvent}, }; use arc_swap::ArcSwap; use pwhash::sha512_crypt; @@ -42,7 +42,7 @@ pub struct BootManager { } pub struct IpcReceivers { - pub state_rx: Option>, + pub push_rx: Option>, pub housekeeper_rx: Option>, pub queue_rx: Option>, pub report_rx: Option>, @@ -541,14 +541,14 @@ impl BootManager { pub fn build_ipc(has_pubsub: bool) -> (Ipc, IpcReceivers) { // Build ipc receivers - let (state_tx, state_rx) = mpsc::channel(IPC_CHANNEL_BUFFER); + let (push_tx, push_rx) = mpsc::channel(IPC_CHANNEL_BUFFER); let (housekeeper_tx, housekeeper_rx) = mpsc::channel(IPC_CHANNEL_BUFFER); let (queue_tx, queue_rx) = mpsc::channel(IPC_CHANNEL_BUFFER); let (report_tx, report_rx) = mpsc::channel(IPC_CHANNEL_BUFFER); let (broadcast_tx, broadcast_rx) = mpsc::channel(IPC_CHANNEL_BUFFER); ( Ipc { - state_tx, + push_tx, housekeeper_tx, queue_tx, report_tx, @@ -556,7 +556,7 @@ pub fn build_ipc(has_pubsub: bool) -> (Ipc, IpcReceivers) { task_tx: Arc::new(Notify::new()), }, IpcReceivers { - state_rx: Some(state_rx), + push_rx: Some(push_rx), housekeeper_rx: Some(housekeeper_rx), queue_rx: Some(queue_rx), report_rx: Some(report_rx), diff --git a/crates/common/src/storage/index.rs b/crates/common/src/storage/index.rs index dc29dc3e..6b495949 100644 --- a/crates/common/src/storage/index.rs +++ b/crates/common/src/storage/index.rs @@ -14,10 +14,7 @@ use rkyv::{ use std::{borrow::Cow, fmt::Debug}; use store::{ Serialize, SerializeInfallible, - write::{ - Archive, Archiver, BatchBuilder, BlobOp, DirectoryClass, IntoOperations, TagValue, - ValueClass, - }, + write::{Archive, Archiver, BatchBuilder, BlobOp, DirectoryClass, IntoOperations, TagValue}, }; use types::{ acl::AclGrant, @@ -411,11 +408,9 @@ fn build_index( for item in value.as_ref() { if set { batch.acl_grant(item.account_id, item.grants.bitmap.serialize()); - batch.set( - ValueClass::ShareNotification { - notification_id, - notify_account_id: item.account_id, - }, + batch.log_share_notification( + notification_id, + item.account_id, ShareNotification { object_account_id, object_id, @@ -424,16 +419,13 @@ fn build_index( old_rights: Default::default(), new_rights: item.grants, name: Default::default(), - } - .serialize(), + }, ); } else { batch.acl_revoke(item.account_id); - batch.set( - ValueClass::ShareNotification { - notification_id, - notify_account_id: item.account_id, - }, + batch.log_share_notification( + notification_id, + item.account_id, ShareNotification { object_account_id, object_id, @@ -442,8 +434,7 @@ fn build_index( old_rights: item.grants, new_rights: Default::default(), name: Default::default(), - } - .serialize(), + }, ); } } @@ -584,11 +575,9 @@ fn merge_index( .any(|item| item.account_id == current_item.account_id) { batch.acl_revoke(current_item.account_id); - batch.set( - ValueClass::ShareNotification { - notification_id, - notify_account_id: current_item.account_id, - }, + batch.log_share_notification( + notification_id, + current_item.account_id, ShareNotification { object_account_id, object_id, @@ -597,8 +586,7 @@ fn merge_index( old_rights: current_item.grants, new_rights: Default::default(), name: Default::default(), - } - .serialize(), + }, ); } } @@ -619,11 +607,9 @@ fn merge_index( } if add_item { batch.acl_grant(item.account_id, item.grants.bitmap.serialize()); - batch.set( - ValueClass::ShareNotification { - notification_id, - notify_account_id: item.account_id, - }, + batch.log_share_notification( + notification_id, + item.account_id, ShareNotification { object_account_id, object_id, @@ -632,8 +618,7 @@ fn merge_index( old_rights, new_rights: item.grants, name: Default::default(), - } - .serialize(), + }, ); } } @@ -642,11 +627,9 @@ fn merge_index( // Add all ACLs for item in new_acl.as_ref() { batch.acl_grant(item.account_id, item.grants.bitmap.serialize()); - batch.set( - ValueClass::ShareNotification { - notification_id, - notify_account_id: item.account_id, - }, + batch.log_share_notification( + notification_id, + item.account_id, ShareNotification { object_account_id, object_id, @@ -655,8 +638,7 @@ fn merge_index( old_rights: Default::default(), new_rights: item.grants, name: Default::default(), - } - .serialize(), + }, ); } } @@ -664,11 +646,9 @@ fn merge_index( // Remove all ACLs for item in old_acl.as_ref() { batch.acl_revoke(item.account_id); - batch.set( - ValueClass::ShareNotification { - notification_id, - notify_account_id: item.account_id, - }, + batch.log_share_notification( + notification_id, + item.account_id, ShareNotification { object_account_id, object_id, @@ -677,8 +657,7 @@ fn merge_index( old_rights: item.grants, new_rights: Default::default(), name: Default::default(), - } - .serialize(), + }, ); } } diff --git a/crates/common/src/storage/state.rs b/crates/common/src/storage/state.rs index 39a996f7..e9d2f30d 100644 --- a/crates/common/src/storage/state.rs +++ b/crates/common/src/storage/state.rs @@ -4,35 +4,37 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{IPC_CHANNEL_BUFFER, Server, ipc::StateEvent}; +use crate::{ + IPC_CHANNEL_BUFFER, Server, + auth::AccessToken, + ipc::{PushEvent, PushNotification}, +}; use tokio::sync::mpsc; -use types::type_state::{DataType, StateChange}; +use types::type_state::DataType; use utils::map::bitmap::Bitmap; impl Server { - pub async fn subscribe_state_manager( + pub async fn subscribe_push_manager( &self, - account_id: u32, + access_token: &AccessToken, types: Bitmap, - ) -> trc::Result> { - let (change_tx, change_rx) = mpsc::channel::(IPC_CHANNEL_BUFFER); - let state_tx = self.inner.ipc.state_tx.clone(); + ) -> trc::Result> { + let (tx, rx) = mpsc::channel::(IPC_CHANNEL_BUFFER); + let push_tx = self.inner.ipc.push_tx.clone(); - for event in [ - StateEvent::UpdateSharedAccounts { account_id }, - StateEvent::Subscribe { - account_id, + push_tx + .send(PushEvent::Subscribe { + account_ids: access_token.member_ids().collect(), types, - tx: change_tx, - }, - ] { - state_tx.send(event).await.map_err(|err| { + tx, + }) + .await + .map_err(|err| { trc::EventType::Server(trc::ServerEvent::ThreadError) .reason(err) .caused_by(trc::location!()) })?; - } - Ok(change_rx) + Ok(rx) } } diff --git a/crates/directory/src/backend/internal/manage.rs b/crates/directory/src/backend/internal/manage.rs index f63dbfd4..4545611b 100644 --- a/crates/directory/src/backend/internal/manage.rs +++ b/crates/directory/src/backend/internal/manage.rs @@ -875,6 +875,13 @@ impl ManageDirectory for Store { }); } + // Delete push subscriptions + if matches!(typ, Type::Individual) { + batch + .with_collection(Collection::PushSubscription) + .delete_document(principal_id); + } + self.write(batch.build_all()) .await .caused_by(trc::location!())?; diff --git a/crates/email/src/message/delivery.rs b/crates/email/src/message/delivery.rs index 75b53221..2d48b874 100644 --- a/crates/email/src/message/delivery.rs +++ b/crates/email/src/message/delivery.rs @@ -6,15 +6,15 @@ use super::ingest::{EmailIngest, IngestEmail, IngestSource}; use crate::{mailbox::INBOX_ID, sieve::ingest::SieveScriptIngest}; -use common::Server; +use common::{ + Server, + ipc::{EmailPush, PushNotification}, +}; use directory::Permission; use mail_parser::MessageParser; use std::{borrow::Cow, future::Future}; use store::ahash::AHashMap; -use types::{ - blob_hash::BlobHash, - type_state::{DataType, StateChange}, -}; +use types::blob_hash::BlobHash; #[derive(Debug)] pub struct IngestMessage { @@ -102,19 +102,20 @@ impl MailDelivery for Server { } }; - // Obtain the UIDs for each recipient - let mut uids: AHashMap = AHashMap::with_capacity(message.recipients.len()); + // Obtain the account IDs for each recipient + let mut account_ids: AHashMap = + AHashMap::with_capacity(message.recipients.len()); let mut result = LocalDeliveryResult { status: Vec::with_capacity(message.recipients.len()), autogenerated: Vec::new(), }; for rcpt in message.recipients { - let uid = match self + let account_id = match self .email_to_id(&self.core.storage.directory, &rcpt, message.session_id) .await { - Ok(Some(uid)) => uid, + Ok(Some(account_id)) => account_id, Ok(None) => { // Something went wrong result.status.push(LocalDeliveryStatus::PermanentFailure { @@ -136,20 +137,23 @@ impl MailDelivery for Server { continue; } }; - if let Some(status) = uids.get(&uid).and_then(|pos| result.status.get(*pos)) { + if let Some(status) = account_ids + .get(&account_id) + .and_then(|pos| result.status.get(*pos)) + { result.status.push(status.clone()); continue; } // Obtain access token - let status = match self.get_access_token(uid).await.and_then(|token| { + let status = match self.get_access_token(account_id).await.and_then(|token| { token .assert_has_permission(Permission::EmailReceive) .map(|_| token) }) { Ok(access_token) => { // Check if there is an active sieve script - match self.sieve_script_get_active(uid).await { + match self.sieve_script_get_active(account_id).await { Ok(None) => { // Ingest message self.email_ingest(IngestEmail { @@ -194,13 +198,11 @@ impl MailDelivery for Server { Ok(ingested_message) => { // Notify state change if ingested_message.change_id != u64::MAX { - self.broadcast_state_change( - StateChange::new(uid, ingested_message.change_id) - .with_change(DataType::EmailDelivery) - .with_change(DataType::Email) - .with_change(DataType::Mailbox) - .with_change(DataType::Thread), - ) + self.broadcast_push_notification(PushNotification::EmailPush(EmailPush { + account_id, + email_id: ingested_message.document_id, + change_id: ingested_message.change_id, + })) .await; } @@ -255,7 +257,7 @@ impl MailDelivery for Server { }; // Cache response for UID to avoid duplicate deliveries - uids.insert(uid, result.status.len()); + account_ids.insert(account_id, result.status.len()); result.status.push(status); } diff --git a/crates/email/src/push/mod.rs b/crates/email/src/push/mod.rs index 568a3072..8b13dd18 100644 --- a/crates/email/src/push/mod.rs +++ b/crates/email/src/push/mod.rs @@ -11,6 +11,7 @@ use utils::map::bitmap::Bitmap; rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Default, Debug, Clone, PartialEq, Eq, )] pub struct PushSubscription { + pub id: u32, pub url: String, pub device_client_id: String, pub expires: u64, @@ -18,6 +19,7 @@ pub struct PushSubscription { pub verified: bool, pub types: Bitmap, pub keys: Option, + pub email_push: Vec, } #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)] @@ -25,3 +27,37 @@ pub struct Keys { pub p256dh: Vec, pub auth: Vec, } + +#[derive( + rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Default, Debug, Clone, PartialEq, Eq, +)] +pub struct PushSubscriptions { + pub subscriptions: Vec, +} + +#[derive( + rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Default, Debug, Clone, PartialEq, Eq, +)] +pub struct EmailPush { + pub account_id: u32, + pub properties: u64, + pub filters: Vec, + pub flags: u16, +} + +#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)] +pub enum EmailPushFilter { + Condition { field: u8, value: EmailPushValue }, + And, + Or, + Not, + End, +} + +#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)] +pub enum EmailPushValue { + Text(String), + Number(u64), + TextList(Vec), + NumberList(Vec), +} diff --git a/crates/groupware/src/calendar/alarm.rs b/crates/groupware/src/calendar/alarm.rs index 162ef72d..86e22874 100644 --- a/crates/groupware/src/calendar/alarm.rs +++ b/crates/groupware/src/calendar/alarm.rs @@ -17,15 +17,25 @@ use std::str::FromStr; use store::write::bitpack::BitpackIterator; use utils::codec::leb128::Leb128Reader; -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct CalendarAlarm { pub alarm_id: u16, pub event_id: u16, pub alarm_time: i64, - pub event_start: i64, - pub event_start_tz: u16, - pub event_end: i64, - pub event_end_tz: u16, + pub typ: CalendarAlarmType, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum CalendarAlarmType { + Email { + event_start: i64, + event_start_tz: u16, + event_end: i64, + event_end_tz: u16, + }, + Display { + recurrence_id: Option, + }, } impl ArchivedCalendarEventData { @@ -39,11 +49,7 @@ impl ArchivedCalendarEventData { 'outer: for range in self.time_ranges.iter() { let comp_id = range.id.to_native(); - let Some(alarm) = self - .alarms - .iter() - .find(|a| a.is_email_alert && a.parent_id == comp_id) - else { + let Some(alarm) = self.alarms.iter().find(|a| a.parent_id == comp_id) else { continue; }; @@ -83,30 +89,34 @@ impl ArchivedCalendarEventData { if let Some(alarm_time) = alarm.delta.to_timestamp(start, end, default_tz) && alarm_time > start_time + && next_alarm + .as_ref() + .is_none_or(|next| alarm_time < next.alarm_time) { - if let Some(next) = next_alarm { - if alarm_time < next.alarm_time { - next_alarm = Some(CalendarAlarm { - alarm_id: alarm.id.to_native(), - event_id: alarm.parent_id.to_native(), - alarm_time, + next_alarm = Some(CalendarAlarm { + alarm_id: alarm.id.to_native(), + event_id: alarm.parent_id.to_native(), + alarm_time, + typ: if alarm.is_email_alert { + CalendarAlarmType::Email { event_start: start_date_naive, event_start_tz: start_tz.as_id(), event_end: end_date_naive, event_end_tz: end_tz.as_id(), - }); - } - } else { - next_alarm = Some(CalendarAlarm { - alarm_id: alarm.id.to_native(), - event_id: alarm.parent_id.to_native(), - alarm_time, - event_start: start_date_naive, - event_start_tz: start_tz.as_id(), - event_end: end_date_naive, - event_end_tz: end_tz.as_id(), - }); - } + } + } else { + let comp = + &self.event.components[alarm.parent_id.to_native() as usize]; + + CalendarAlarmType::Display { + recurrence_id: if comp.is_recurrent_or_override() { + start_date_naive.into() + } else { + None + }, + } + }, + }); continue 'outer; } } @@ -129,30 +139,33 @@ impl ArchivedCalendarEventData { if let Some(alarm_time) = alarm.delta.to_timestamp(start, end, default_tz) && alarm_time > start_time + && next_alarm + .as_ref() + .is_none_or(|next| alarm_time < next.alarm_time) { - if let Some(next) = next_alarm { - if alarm_time < next.alarm_time { - next_alarm = Some(CalendarAlarm { - alarm_id: alarm.id.to_native(), - event_id: alarm.parent_id.to_native(), - alarm_time, + next_alarm = Some(CalendarAlarm { + alarm_id: alarm.id.to_native(), + event_id: alarm.parent_id.to_native(), + alarm_time, + typ: if alarm.is_email_alert { + CalendarAlarmType::Email { event_start: start_date_naive, event_start_tz: start_tz.as_id(), event_end: end_date_naive, event_end_tz: end_tz.as_id(), - }); - } - } else { - next_alarm = Some(CalendarAlarm { - alarm_id: alarm.id.to_native(), - event_id: alarm.parent_id.to_native(), - alarm_time, - event_start: start_date_naive, - event_start_tz: start_tz.as_id(), - event_end: end_date_naive, - event_end_tz: end_tz.as_id(), - }); - } + } + } else { + let comp = &self.event.components[alarm.parent_id.to_native() as usize]; + + CalendarAlarmType::Display { + recurrence_id: if comp.is_recurrent_or_override() { + start_date_naive.into() + } else { + None + }, + } + }, + }); } } } diff --git a/crates/groupware/src/calendar/dates.rs b/crates/groupware/src/calendar/dates.rs index 50154f13..136b83ce 100644 --- a/crates/groupware/src/calendar/dates.rs +++ b/crates/groupware/src/calendar/dates.rs @@ -8,7 +8,7 @@ use super::{ ArchivedCalendarEventData, ArchivedTimezone, CalendarEventData, Timezone, alarm::{CalendarAlarm, ExpandAlarm}, }; -use crate::calendar::ComponentTimeRange; +use crate::calendar::{ComponentTimeRange, alarm::CalendarAlarmType}; use calcard::{ common::timezone::Tz, icalendar::{ICalendar, ICalendarComponentType, dates::TimeOrDelta}, @@ -88,30 +88,34 @@ impl CalendarEventData { if alarm_time > max { max = alarm_time; } - if alarm.is_email_alert && alarm_time > now { - if let Some(next) = next_email_alarm { - if alarm_time < next.alarm_time { - *next = CalendarAlarm { - alarm_id: alarm.id, - event_id: alarm.parent_id, - alarm_time, + if alarm_time > now + && next_email_alarm + .as_ref() + .is_none_or(|next| alarm_time < next.alarm_time) + { + *next_email_alarm = Some(CalendarAlarm { + alarm_id: alarm.id, + event_id: alarm.parent_id, + alarm_time, + typ: if alarm.is_email_alert { + CalendarAlarmType::Email { event_start: start_timestamp_naive, event_end: end_timestamp_naive, event_start_tz: start_tz, event_end_tz: end_tz, - }; - } - } else { - *next_email_alarm = Some(CalendarAlarm { - alarm_id: alarm.id, - event_id: alarm.parent_id, - alarm_time, - event_start: start_timestamp_naive, - event_end: end_timestamp_naive, - event_start_tz: start_tz, - event_end_tz: end_tz, - }); - } + } + } else { + CalendarAlarmType::Display { + recurrence_id: if ical.components[alarm.parent_id as usize] + .is_recurrent_or_override() + { + start_timestamp_naive.into() + } else { + None + }, + } + }, + }); } } } diff --git a/crates/groupware/src/calendar/storage.rs b/crates/groupware/src/calendar/storage.rs index 50f27ca9..333b1e51 100644 --- a/crates/groupware/src/calendar/storage.rs +++ b/crates/groupware/src/calendar/storage.rs @@ -10,7 +10,9 @@ use super::{ }; use crate::{ DavResourceName, DestroyArchive, RFC_3986, - calendar::{ArchivedCalendarEventNotification, CalendarEventNotification}, + calendar::{ + ArchivedCalendarEventNotification, CalendarEventNotification, alarm::CalendarAlarmType, + }, scheduling::{ItipMessages, event_cancel::itip_cancel}, }; use calcard::common::timezone::Tz; @@ -470,19 +472,42 @@ impl DestroyArchive> { impl CalendarAlarm { pub fn write_task(&self, batch: &mut BatchBuilder) { - batch.set( - ValueClass::TaskQueue(TaskQueueClass::SendAlarm { - due: self.alarm_time as u64, - event_id: self.event_id, - alarm_id: self.alarm_id, - }), - KeySerializer::new((U64_LEN * 2) + (U16_LEN * 2)) - .write(self.event_start as u64) - .write(self.event_end as u64) - .write(self.event_start_tz) - .write(self.event_end_tz) - .finalize(), - ); + match &self.typ { + CalendarAlarmType::Email { + event_start, + event_start_tz, + event_end, + event_end_tz, + } => { + batch.set( + ValueClass::TaskQueue(TaskQueueClass::SendAlarm { + due: self.alarm_time as u64, + event_id: self.event_id, + alarm_id: self.alarm_id, + is_email_alert: true, + }), + KeySerializer::new((U64_LEN * 2) + (U16_LEN * 2)) + .write(*event_start as u64) + .write(*event_end as u64) + .write(*event_start_tz) + .write(*event_end_tz) + .finalize(), + ); + } + CalendarAlarmType::Display { recurrence_id } => { + batch.set( + ValueClass::TaskQueue(TaskQueueClass::SendAlarm { + due: self.alarm_time as u64, + event_id: self.event_id, + alarm_id: self.alarm_id, + is_email_alert: false, + }), + KeySerializer::new(U64_LEN) + .write(recurrence_id.unwrap_or_default() as u64) + .finalize(), + ); + } + } } pub fn delete_task(&self, batch: &mut BatchBuilder) { @@ -490,6 +515,7 @@ impl CalendarAlarm { due: self.alarm_time as u64, event_id: self.event_id, alarm_id: self.alarm_id, + is_email_alert: matches!(self.typ, CalendarAlarmType::Email { .. }), })); } } diff --git a/crates/http/src/request.rs b/crates/http/src/request.rs index 06e4aadc..e9071e21 100644 --- a/crates/http/src/request.rs +++ b/crates/http/src/request.rs @@ -23,7 +23,7 @@ use common::{ Inner, KV_ACME, Server, auth::{AccessToken, oauth::GrantType}, core::BuildServer, - ipc::StateEvent, + ipc::PushEvent, listener::{SessionData, SessionManager, SessionStream}, manager::webadmin::Resource, }; @@ -850,7 +850,7 @@ impl SessionManager for HttpSessionManager { #[allow(clippy::manual_async_fn)] fn shutdown(&self) -> impl std::future::Future + Send { async { - let _ = self.inner.ipc.state_tx.send(StateEvent::Stop).await; + let _ = self.inner.ipc.push_tx.send(PushEvent::Stop).await; } } } diff --git a/crates/imap/src/op/append.rs b/crates/imap/src/op/append.rs index acadf04e..643265af 100644 --- a/crates/imap/src/op/append.rs +++ b/crates/imap/src/op/append.rs @@ -9,7 +9,7 @@ use crate::{ core::{ImapUidToId, MailboxId, SelectedMailbox, Session, SessionData}, spawn_op, }; -use common::listener::SessionStream; +use common::{ipc::PushNotification, listener::SessionStream}; use directory::Permission; use email::message::ingest::{EmailIngest, IngestEmail, IngestSource}; use imap_proto::{ @@ -144,12 +144,13 @@ impl SessionData { // Broadcast changes if let Some(change_id) = last_change_id { self.server - .broadcast_state_change( - StateChange::new(account_id, change_id) + .broadcast_push_notification(PushNotification::StateChange( + StateChange::new(account_id) + .with_change_id(change_id) .with_change(DataType::Email) .with_change(DataType::Mailbox) .with_change(DataType::Thread), - ) + )) .await; } diff --git a/crates/imap/src/op/copy_move.rs b/crates/imap/src/op/copy_move.rs index 2e3ff1c9..50f34b2e 100644 --- a/crates/imap/src/op/copy_move.rs +++ b/crates/imap/src/op/copy_move.rs @@ -9,7 +9,7 @@ use crate::{ core::{MailboxId, SelectedMailbox, Session, SessionData}, spawn_op, }; -use common::{listener::SessionStream, storage::index::ObjectIndexBuilder}; +use common::{ipc::PushNotification, listener::SessionStream, storage::index::ObjectIndexBuilder}; use directory::Permission; use email::{ cache::{MessageCacheFetch, email::MessageCacheAccess}, @@ -437,12 +437,13 @@ impl SessionData { // Broadcast changes on destination account if let Some(change_id) = dest_change_id { self.server - .broadcast_state_change( - StateChange::new(dest_account_id, change_id) + .broadcast_push_notification(PushNotification::StateChange( + StateChange::new(dest_account_id) + .with_change_id(change_id) .with_change(DataType::Email) .with_change(DataType::Thread) .with_change(DataType::Mailbox), - ) + )) .await; } } diff --git a/crates/imap/src/op/idle.rs b/crates/imap/src/op/idle.rs index bc0058c0..450258bf 100644 --- a/crates/imap/src/op/idle.rs +++ b/crates/imap/src/op/idle.rs @@ -9,7 +9,7 @@ use crate::{ op::ImapContext, }; use ahash::AHashSet; -use common::listener::SessionStream; +use common::{ipc::PushNotification, listener::SessionStream}; use directory::Permission; use imap_proto::{ Command, StatusResponse, @@ -48,10 +48,10 @@ impl Session { let is_utf8 = self.is_utf8; let is_qresync = self.is_qresync; - // Register with state manager - let mut change_rx = self + // Register with push manager + let mut push_rx = self .server - .subscribe_state_manager(data.account_id, types) + .subscribe_push_manager(&data.access_token, types) .await .imap_ctx(&request.tag, trc::location!())?; @@ -92,21 +92,30 @@ impl Session { } } } - state_change = change_rx.recv() => { - if let Some(state_change) = state_change { + push_notification = push_rx.recv() => { + if let Some(push_notification) = push_notification { let mut has_mailbox_changes = false; let mut has_email_changes = false; - for type_state in state_change.types { - match type_state { - DataType::Email | DataType::EmailDelivery => { - has_email_changes = true; + match push_notification { + PushNotification::StateChange(state_change) => { + for type_state in state_change.types { + match type_state { + DataType::Email | DataType::EmailDelivery => { + has_email_changes = true; + } + DataType::Mailbox => { + has_mailbox_changes = true; + } + _ => {} + } } - DataType::Mailbox => { - has_mailbox_changes = true; - } - _ => {} - } + }, + PushNotification::EmailPush(_) => { + has_email_changes = true; + has_mailbox_changes = true; + }, + PushNotification::CalendarAlert(_) => (), } if has_mailbox_changes || has_email_changes { diff --git a/crates/jmap-proto/src/request/websocket.rs b/crates/jmap-proto/src/request/websocket.rs index 8de95652..8726bcd8 100644 --- a/crates/jmap-proto/src/request/websocket.rs +++ b/crates/jmap-proto/src/request/websocket.rs @@ -9,16 +9,14 @@ use crate::{ error::request::{RequestError, RequestErrorType, RequestLimitError}, object::AnyId, request::{Call, deserialize::DeserializeArguments}, - response::{Response, ResponseMethod, serialize::serialize_hex}, - types::state::State, + response::{Response, ResponseMethod, serialize::serialize_hex, status::PushObject}, }; use serde::{ Deserialize, Deserializer, de::{self, MapAccess, Visitor}, }; use std::{borrow::Cow, collections::HashMap, fmt}; -use types::{id::Id, type_state::DataType}; -use utils::map::vec_map::VecMap; +use types::type_state::DataType; #[derive(Debug)] pub struct WebSocketRequest<'x> { @@ -66,18 +64,13 @@ pub enum WebSocketMessage<'x> { } #[derive(serde::Serialize, Debug)] -pub enum WebSocketStateChangeType { - StateChange, -} +pub struct WebSocketPushObject { + #[serde(flatten)] + pub push: PushObject, -#[derive(serde::Serialize, Debug)] -pub struct WebSocketStateChange { - #[serde(rename = "@type")] - pub type_: WebSocketStateChangeType, - pub changed: VecMap>, #[serde(rename = "pushState")] #[serde(skip_serializing_if = "Option::is_none")] - push_state: Option, + pub push_state: Option, } #[derive(Debug, serde::Serialize)] @@ -248,15 +241,7 @@ impl<'x> WebSocketResponse<'x> { } } -impl WebSocketStateChange { - pub fn new(push_state: Option) -> Self { - WebSocketStateChange { - type_: WebSocketStateChangeType::StateChange, - changed: VecMap::new(), - push_state, - } - } - +impl WebSocketPushObject { pub fn to_json(&self) -> String { serde_json::to_string(self).unwrap() } diff --git a/crates/jmap-proto/src/response/status.rs b/crates/jmap-proto/src/response/status.rs index 8b51d89b..463e62aa 100644 --- a/crates/jmap-proto/src/response/status.rs +++ b/crates/jmap-proto/src/response/status.rs @@ -9,28 +9,33 @@ use types::{id::Id, type_state::DataType}; use utils::map::vec_map::VecMap; #[derive(serde::Serialize, serde::Deserialize, Debug)] -pub enum StateChangeType { - StateChange, +#[serde(tag = "@type")] +pub enum PushObject { + StateChange { + changed: VecMap>, + }, + EmailPush { + #[serde(rename = "accountId")] + account_id: Id, + email: EmailPushObject, + }, + CalendarAlert { + #[serde(rename = "accountId")] + account_id: Id, + #[serde(rename = "calendarEventId")] + calendar_event_id: Id, + uid: String, + #[serde(rename = "recurrenceId")] + recurrence_id: Option, + #[serde(rename = "alertId")] + alert_id: String, + }, + Group { + entries: Vec, + }, } #[derive(serde::Serialize, serde::Deserialize, Debug)] -pub struct StateChangeResponse { - #[serde(rename = "@type")] - pub type_: StateChangeType, - pub changed: VecMap>, -} - -impl StateChangeResponse { - pub fn new() -> Self { - Self { - type_: StateChangeType::StateChange, - changed: VecMap::new(), - } - } -} - -impl Default for StateChangeResponse { - fn default() -> Self { - Self::new() - } +pub struct EmailPushObject { + pub subject: String, } diff --git a/crates/jmap/src/api/event_source.rs b/crates/jmap/src/api/event_source.rs index 721331ba..e14afda1 100644 --- a/crates/jmap/src/api/event_source.rs +++ b/crates/jmap/src/api/event_source.rs @@ -4,21 +4,22 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{LONG_1D_SLUMBER, Server, auth::AccessToken}; +use crate::api::IntoPushObject; +use common::{LONG_1D_SLUMBER, Server, auth::AccessToken, ipc::PushNotification}; use http_body_util::{StreamBody, combinators::BoxBody}; use http_proto::*; use hyper::{ StatusCode, body::{Bytes, Frame}, }; -use jmap_proto::response::status::StateChangeResponse; +use jmap_proto::{response::status::PushObject, types::state::State}; use std::{future::Future, str::FromStr}; use std::{ sync::Arc, time::{Duration, Instant}, }; -use types::type_state::DataType; -use utils::map::bitmap::Bitmap; +use types::{id::Id, type_state::DataType}; +use utils::map::{bitmap::Bitmap, vec_map::VecMap}; struct Ping { interval: Duration, @@ -96,13 +97,11 @@ impl EventSourceHandler for Server { } else { None }; - let mut response = StateChangeResponse::new(); - let throttle = self.core.jmap.event_source_throttle; - // Register with state manager - let mut change_rx = self - .subscribe_state_manager(access_token.primary_id(), types) - .await?; + // Register with push manager + let mut push_rx = self.subscribe_push_manager(&access_token, types).await?; + let mut changed: VecMap> = VecMap::new(); + let throttle = self.core.jmap.event_source_throttle; Ok(HttpResponse::new(StatusCode::OK) .with_content_type("text/event-stream") @@ -113,13 +112,30 @@ impl EventSourceHandler for Server { ping.as_ref().map(|p| p.interval).unwrap_or(LONG_1D_SLUMBER); loop { - match tokio::time::timeout(timeout, change_rx.recv()).await { - Ok(Some(state_change)) => { - for type_state in state_change.types { - response - .changed - .get_mut_or_insert(state_change.account_id.into()) - .set(type_state, state_change.change_id.into()); + match tokio::time::timeout(timeout, push_rx.recv()).await { + Ok(Some(notification)) => { + match notification { + PushNotification::StateChange(state_change) => { + for type_state in state_change.types { + changed + .get_mut_or_insert(state_change.account_id.into()) + .set(type_state, (state_change.change_id).into()); + } + } + PushNotification::CalendarAlert(calendar_alert) => { + yield Ok(Frame::data(Bytes::from(format!( + "event: calendarAlert\ndata: {}\n\n", + serde_json::to_string(&calendar_alert.into_push_object()).unwrap() + )))); + } + PushNotification::EmailPush(email_push) => { + let state_change = email_push.to_state_change(); + for type_state in state_change.types { + changed + .get_mut_or_insert(state_change.account_id.into()) + .set(type_state, state_change.change_id.into()); + } + } } } Ok(None) => { @@ -128,10 +144,13 @@ impl EventSourceHandler for Server { Err(_) => (), } - timeout = if !response.changed.is_empty() { + timeout = if !changed.is_empty() { let elapsed = last_message.elapsed(); if elapsed >= throttle { last_message = Instant::now(); + let response = + PushObject::StateChange { changed: std::mem::take(&mut changed) }; + yield Ok(Frame::data(Bytes::from(format!( "event: state\ndata: {}\n\n", serde_json::to_string(&response).unwrap() @@ -141,8 +160,7 @@ impl EventSourceHandler for Server { break; } - response.changed.clear(); - ping.as_ref().map(|p| p.interval).unwrap_or(LONG_1D_SLUMBER) + ping.as_ref().map(|p| p.interval).unwrap_or(LONG_1D_SLUMBER) } else { throttle - elapsed } diff --git a/crates/jmap/src/api/mod.rs b/crates/jmap/src/api/mod.rs index 7558933a..e034d990 100644 --- a/crates/jmap/src/api/mod.rs +++ b/crates/jmap/src/api/mod.rs @@ -5,13 +5,18 @@ */ use crate::blob::UploadResponse; +use calcard::jscalendar::JSCalendarDateTime; +use common::ipc::{CalendarAlert, PushNotification}; use http_proto::{HttpResponse, JsonResponse, ToHttpResponse}; use hyper::StatusCode; use jmap_proto::{ error::request::{RequestError, RequestLimitError}, request::capability::Session, - response::Response, + response::{Response, status::PushObject}, + types::state::State, }; +use types::{id::Id, type_state::DataType}; +use utils::map::vec_map::VecMap; pub mod acl; pub mod auth; @@ -119,3 +124,63 @@ impl ToRequestError for trc::Error { } } } + +pub(crate) trait IntoPushObject { + fn into_push_object(self) -> PushObject; +} + +impl IntoPushObject for Vec { + fn into_push_object(self) -> PushObject { + let mut changed: VecMap> = VecMap::new(); + let mut objects = Vec::with_capacity(self.len()); + for notification in self { + match notification { + PushNotification::StateChange(state_change) => { + for type_state in state_change.types { + changed + .get_mut_or_insert(state_change.account_id.into()) + .set(type_state, (state_change.change_id).into()); + } + } + PushNotification::CalendarAlert(calendar_alert) => { + objects.push(calendar_alert.into_push_object()); + } + PushNotification::EmailPush(email_push) => { + let state_change = email_push.to_state_change(); + for type_state in state_change.types { + changed + .get_mut_or_insert(state_change.account_id.into()) + .set(type_state, state_change.change_id.into()); + } + } + } + } + + if !objects.is_empty() { + if changed.is_empty() { + objects.push(PushObject::StateChange { changed }); + } + if objects.len() > 1 { + PushObject::Group { entries: objects } + } else { + objects.into_iter().next().unwrap() + } + } else { + PushObject::StateChange { changed } + } + } +} + +impl IntoPushObject for CalendarAlert { + fn into_push_object(self) -> PushObject { + PushObject::CalendarAlert { + account_id: self.account_id.into(), + calendar_event_id: self.event_id.into(), + uid: self.uid, + recurrence_id: self + .recurrence_id + .map(|timestamp| JSCalendarDateTime::new(timestamp, true).to_rfc3339()), + alert_id: self.alert_id, + } + } +} diff --git a/crates/jmap/src/email/set.rs b/crates/jmap/src/email/set.rs index fb465551..494ce751 100644 --- a/crates/jmap/src/email/set.rs +++ b/crates/jmap/src/email/set.rs @@ -11,7 +11,9 @@ use crate::{ changes::state::JmapCacheState, email::{PatchResult, handle_email_patch, ingested_into_object}, }; -use common::{Server, auth::AccessToken, storage::index::ObjectIndexBuilder}; +use common::{ + Server, auth::AccessToken, ipc::PushNotification, storage::index::ObjectIndexBuilder, +}; use email::{ cache::{MessageCacheFetch, email::MessageCacheAccess, mailbox::MailboxCacheAccess}, mailbox::UidMailbox, @@ -1107,12 +1109,13 @@ impl EmailSet for Server { if let Some(change_id) = last_change_id { if response.updated.is_empty() && response.destroyed.is_empty() { // Message ingest does not broadcast state changes - self.broadcast_state_change( - StateChange::new(account_id, change_id) + self.broadcast_push_notification(PushNotification::StateChange( + StateChange::new(account_id) + .with_change_id(change_id) .with_change(DataType::Email) .with_change(DataType::Mailbox) .with_change(DataType::Thread), - ) + )) .await; } diff --git a/crates/jmap/src/push/get.rs b/crates/jmap/src/push/get.rs index b7217f1d..19c986e2 100644 --- a/crates/jmap/src/push/get.rs +++ b/crates/jmap/src/push/get.rs @@ -4,11 +4,8 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{ - Server, - auth::AccessToken, - ipc::{EncryptionKeys, PushSubscription, StateEvent, UpdateSubscription}, -}; +use common::{Server, auth::AccessToken, ipc::PushEvent}; +use email::push::PushSubscriptions; use jmap_proto::{ method::get::{GetRequest, GetResponse}, object::push_subscription::{self, PushSubscriptionProperty, PushSubscriptionValue}, @@ -17,11 +14,11 @@ use jmap_proto::{ use jmap_tools::{Map, Value}; use std::future::Future; use store::{ - BitmapKey, ValueKey, - write::{AlignedBytes, Archive, ValueClass, now}, + Serialize, + write::{Archiver, BatchBuilder, now}, }; use trc::{AddContext, ServerEvent}; -use types::{collection::Collection, field::Field}; +use types::{collection::Collection, field::PrincipalField, id::Id}; use utils::map::bitmap::Bitmap; pub trait PushSubscriptionFetch: Sync + Send { @@ -30,13 +27,6 @@ pub trait PushSubscriptionFetch: Sync + Send { request: GetRequest, access_token: &AccessToken, ) -> impl Future>> + Send; - - fn fetch_push_subscriptions( - &self, - account_id: u32, - ) -> impl Future> + Send; - - fn update_push_subscriptions(&self, account_id: u32) -> impl Future + Send; } impl PushSubscriptionFetch for Server { @@ -53,46 +43,59 @@ impl PushSubscriptionFetch for Server { PushSubscriptionProperty::Expires, PushSubscriptionProperty::Types, ]); + let account_id = access_token.primary_id(); - let push_ids = self - .get_document_ids(account_id, Collection::PushSubscription) + + let mut response = GetResponse { + account_id: request.account_id.into(), + state: None, + list: Vec::new(), + not_found: vec![], + }; + + let Some(subscriptions_) = self + .get_archive_by_property( + account_id, + Collection::Principal, + 0, + PrincipalField::PushSubscriptions.into(), + ) .await? - .unwrap_or_default(); + else { + for id in ids.unwrap_or_default() { + response.not_found.push(id); + } + return Ok(response); + }; + let subscriptions = subscriptions_ + .to_unarchived::() + .caused_by(trc::location!())?; + let ids = if let Some(ids) = ids { ids } else { - push_ids + subscriptions + .inner + .subscriptions .iter() .take(self.core.jmap.get_max_objects) - .map(Into::into) + .map(|s| Id::from(s.id.to_native())) .collect::>() }; - let mut response = GetResponse { - account_id: None, - state: None, - list: Vec::with_capacity(ids.len()), - not_found: vec![], - }; for id in ids { // Obtain the push subscription object let document_id = id.document_id(); - if !push_ids.contains(document_id) { - response.not_found.push(id); - continue; - } - let push_ = if let Some(push) = self - .get_archive(account_id, Collection::PushSubscription, document_id) - .await? - { - push - } else { + let Some(push) = subscriptions + .inner + .subscriptions + .iter() + .find(|p| p.id.to_native() == document_id) + else { response.not_found.push(id); continue; }; - let push = push_ - .unarchive::() - .caused_by(trc::location!())?; + let mut result = Map::with_capacity(properties.len()); for property in &properties { match property { @@ -138,104 +141,67 @@ impl PushSubscriptionFetch for Server { response.list.push(result.into()); } - Ok(response) - } - - async fn fetch_push_subscriptions(&self, account_id: u32) -> trc::Result { - let mut subscriptions = Vec::new(); - let document_ids = self - .core - .storage - .data - .get_bitmap(BitmapKey::document_ids( - account_id, - Collection::PushSubscription, - )) - .await? - .unwrap_or_default(); - + // Purge old subscriptions let current_time = now(); + if subscriptions + .inner + .subscriptions + .iter() + .any(|s| s.expires.to_native() < current_time) + { + let mut updated_subscriptions = subscriptions.deserialize::()?; + updated_subscriptions + .subscriptions + .retain(|s| s.expires >= current_time); + let mut batch = BatchBuilder::new(); - for document_id in document_ids { - let subscription = self - .core - .storage - .data - .get_value::>(ValueKey { - account_id, - collection: Collection::PushSubscription.into(), - document_id, - class: ValueClass::Property(Field::ARCHIVE.into()), - }) - .await? - .ok_or_else(|| { - trc::StoreEvent::NotFound - .into_err() - .caused_by(trc::location!()) - .document_id(document_id) - })? - .deserialize::() - .caused_by(trc::location!())?; - - if subscription.expires > current_time { - if subscription.verified { - // Add verified subscription - subscriptions.push(UpdateSubscription::Verified(PushSubscription { - id: document_id, - url: subscription.url, - expires: subscription.expires, - types: subscription.types, - keys: subscription.keys.map(|keys| EncryptionKeys { - p256dh: keys.p256dh, - auth: keys.auth, - }), - })); - } else { - // Add unverified subscription - subscriptions.push(UpdateSubscription::Unverified { - id: document_id, - url: subscription.url, - code: subscription.verification_code, - keys: subscription.keys.map(|keys| EncryptionKeys { - p256dh: keys.p256dh, - auth: keys.auth, - }), - }); - } + if updated_subscriptions.subscriptions.is_empty() { + batch + .with_account_id(u32::MAX) + .with_collection(Collection::PushSubscription) + .delete_document(account_id); } - } - Ok(StateEvent::UpdateSubscriptions { - account_id, - subscriptions, - }) - } + batch + .with_account_id(account_id) + .with_collection(Collection::Principal) + .update_document(0) + .assert_value(PrincipalField::PushSubscriptions, subscriptions); - async fn update_push_subscriptions(&self, account_id: u32) -> bool { - let push_subs = match self.fetch_push_subscriptions(account_id).await { - Ok(push_subs) => push_subs, - Err(err) => { - trc::error!( - err.account_id(account_id) - .details("Failed to fetch push subscriptions") + if !updated_subscriptions.subscriptions.is_empty() { + batch.set( + PrincipalField::PushSubscriptions, + Archiver::new(updated_subscriptions) + .serialize() + .caused_by(trc::location!())?, ); - return false; + } else { + batch.clear(PrincipalField::PushSubscriptions); } - }; - let state_tx = self.inner.ipc.state_tx.clone(); - for event in [StateEvent::UpdateSharedAccounts { account_id }, push_subs] { - if state_tx.send(event).await.is_err() { + self.commit_batch(batch).await.caused_by(trc::location!())?; + + // Update push servers + if self + .inner + .ipc + .push_tx + .clone() + .send(PushEvent::PushServerUpdate { + account_id, + broadcast: true, + }) + .await + .is_err() + { trc::event!( Server(ServerEvent::ThreadError), - Details = "Error sending state change.", + Details = "Error sending push updates.", CausedBy = trc::location!() ); - - return false; } } - true + Ok(response) } } diff --git a/crates/jmap/src/push/set.rs b/crates/jmap/src/push/set.rs index 0b28f5df..e6e7bdb1 100644 --- a/crates/jmap/src/push/set.rs +++ b/crates/jmap/src/push/set.rs @@ -4,10 +4,9 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::get::PushSubscriptionFetch; use base64::{Engine, engine::general_purpose}; -use common::{Server, auth::AccessToken}; -use email::push::{Keys, PushSubscription}; +use common::{Server, auth::AccessToken, ipc::PushEvent}; +use email::push::{Keys, PushSubscription, PushSubscriptions}; use jmap_proto::{ error::set::SetError, method::set::{SetRequest, SetResponse}, @@ -24,8 +23,8 @@ use store::{ rand::{Rng, rng}, write::{Archiver, BatchBuilder, now}, }; -use trc::AddContext; -use types::{collection::Collection, field::Field}; +use trc::{AddContext, ServerEvent}; +use types::{collection::Collection, field::PrincipalField}; use utils::map::bitmap::Bitmap; const EXPIRES_MAX: i64 = 7 * 24 * 3600; // 7 days @@ -45,20 +44,43 @@ impl PushSubscriptionSet for Server { mut request: SetRequest<'_, push_subscription::PushSubscription>, access_token: &AccessToken, ) -> trc::Result> { + // Load existing push subscriptions let account_id = access_token.primary_id(); - let push_ids = self - .get_document_ids(account_id, Collection::PushSubscription) - .await? - .unwrap_or_default(); + let subscriptions_archive = self + .get_archive_by_property( + account_id, + Collection::Principal, + 0, + PrincipalField::PushSubscriptions.into(), + ) + .await?; + let mut subscriptions = if let Some(subscriptions) = &subscriptions_archive { + subscriptions + .deserialize::() + .caused_by(trc::location!())? + } else { + PushSubscriptions::default() + }; + + let num_subscriptions = subscriptions.subscriptions.len(); + let mut max_id = 0; + let current_time = now(); + subscriptions.subscriptions.retain(|s| { + max_id = max_id.max(s.id); + + s.expires > current_time + }); + let mut has_changes = num_subscriptions != subscriptions.subscriptions.len(); + + // Prepare response let mut response = SetResponse::from_request(&request, self.core.jmap.set_max_objects)?; let will_destroy = request.unwrap_destroy().into_valid().collect::>(); // Process creates - let mut batch = BatchBuilder::new(); 'create: for (id, object) in request.unwrap_create() { let mut push = PushSubscription::default(); - if push_ids.len() as usize >= self.core.jmap.push_max_total { + if subscriptions.subscriptions.len() >= self.core.jmap.push_max_total { response.not_created.append(id, SetError::forbidden().with_description( "There are too many subscriptions, please delete some before adding a new one.", )); @@ -101,23 +123,13 @@ impl PushSubscriptionSet for Server { .map(char::from) .collect::(); + // Set id + max_id += 1; + let document_id = max_id; + push.id = document_id; + // Insert record - let document_id = self - .store() - .assign_document_ids(account_id, Collection::PushSubscription, 1) - .await - .caused_by(trc::location!())?; - batch - .with_account_id(account_id) - .with_collection(Collection::PushSubscription) - .create_document(document_id) - .set( - Field::ARCHIVE, - Archiver::new(push) - .serialize() - .caused_by(trc::location!())?, - ) - .commit_point(); + subscriptions.subscriptions.push(push); response.created.insert( id, Map::with_capacity(1) @@ -132,6 +144,7 @@ impl PushSubscriptionSet for Server { ) .into(), ); + has_changes = true; } // Process updates @@ -144,13 +157,11 @@ impl PushSubscriptionSet for Server { // Obtain push subscription let document_id = id.document_id(); - let mut push = if let Some(push) = self - .get_archive(account_id, Collection::PushSubscription, document_id) - .await? - { - push.deserialize::() - .caused_by(trc::location!())? - } else { + let Some(push) = subscriptions + .subscriptions + .iter_mut() + .find(|p| p.id == document_id) + else { response.not_updated.append(id, SetError::not_found()); continue 'update; }; @@ -158,53 +169,91 @@ impl PushSubscriptionSet for Server { for (property, mut value) in object.into_expanded_object() { if let Err(err) = response .resolve_self_references(&mut value) - .and_then(|_| validate_push_value(&property, value, &mut push, false)) + .and_then(|_| validate_push_value(&property, value, push, false)) { response.not_updated.append(id, err); continue 'update; } } - // Update record - batch - .with_account_id(account_id) - .with_collection(Collection::PushSubscription) - .update_document(document_id) - .set( - Field::ARCHIVE, - Archiver::new(push) - .serialize() - .caused_by(trc::location!())?, - ) - .commit_point(); + has_changes = true; response.updated.append(id, None); } // Process deletions for id in will_destroy { let document_id = id.document_id(); - if push_ids.contains(document_id) { - // Update record - batch - .with_account_id(account_id) - .with_collection(Collection::PushSubscription) - .delete_document(document_id) - .clear(Field::ARCHIVE) - .commit_point(); + if let Some(idx) = subscriptions + .subscriptions + .iter() + .position(|p| p.id == document_id) + { + subscriptions.subscriptions.swap_remove(idx); + has_changes = true; response.destroyed.push(id); } else { response.not_destroyed.append(id, SetError::not_found()); } } - // Write changes - if !batch.is_empty() { - self.commit_batch(batch).await.caused_by(trc::location!())?; - } - // Update push subscriptions - if response.has_changes() { - self.update_push_subscriptions(account_id).await; + if has_changes { + // Save changes + let mut batch = BatchBuilder::new(); + + if subscriptions_archive.is_none() { + batch + .with_account_id(u32::MAX) + .with_collection(Collection::PushSubscription) + .create_document(account_id); + } else if subscriptions.subscriptions.is_empty() { + batch + .with_account_id(u32::MAX) + .with_collection(Collection::PushSubscription) + .delete_document(account_id); + } + + batch + .with_account_id(account_id) + .with_collection(Collection::Principal) + .update_document(0); + + if let Some(subscriptions_archive) = subscriptions_archive { + batch.assert_value(PrincipalField::PushSubscriptions, subscriptions_archive); + } + + if !subscriptions.subscriptions.is_empty() { + batch.set( + PrincipalField::PushSubscriptions, + Archiver::new(subscriptions) + .serialize() + .caused_by(trc::location!())?, + ); + } else { + batch.clear(PrincipalField::PushSubscriptions); + } + + self.commit_batch(batch).await.caused_by(trc::location!())?; + + // Notify push manager + if self + .inner + .ipc + .push_tx + .clone() + .send(PushEvent::PushServerUpdate { + account_id, + broadcast: true, + }) + .await + .is_err() + { + trc::event!( + Server(ServerEvent::ThreadError), + Details = "Error sending push updates.", + CausedBy = trc::location!() + ); + } } Ok(response) diff --git a/crates/jmap/src/websocket/stream.rs b/crates/jmap/src/websocket/stream.rs index 96fb1a29..c448652e 100644 --- a/crates/jmap/src/websocket/stream.rs +++ b/crates/jmap/src/websocket/stream.rs @@ -4,8 +4,8 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::api::{ToRequestError, request::RequestHandler}; -use common::{Server, auth::AccessToken}; +use crate::api::{IntoPushObject, ToRequestError, request::RequestHandler}; +use common::{Server, auth::AccessToken, ipc::PushNotification}; use futures_util::{SinkExt, StreamExt}; use http_proto::HttpSessionData; use hyper::upgrade::Upgraded; @@ -13,7 +13,7 @@ use hyper_util::rt::TokioIo; use jmap_proto::{ error::request::RequestError, request::websocket::{ - WebSocketMessage, WebSocketRequestError, WebSocketResponse, WebSocketStateChange, + WebSocketMessage, WebSocketPushObject, WebSocketRequestError, WebSocketResponse, }, }; use std::future::Future; @@ -21,7 +21,7 @@ use std::{sync::Arc, time::Instant}; use tokio_tungstenite::WebSocketStream; use trc::JmapEvent; use tungstenite::Message; -use types::type_state::DataType; +use types::type_state::{DataType, StateChange}; use utils::map::bitmap::Bitmap; pub trait WebSocketHandler: Sync + Send { @@ -56,15 +56,15 @@ impl WebSocketHandler for Server { let mut last_heartbeat = Instant::now() - heartbeat; let mut next_event = heartbeat; - // Register with state manager - let mut change_rx = match self - .subscribe_state_manager(access_token.primary_id(), Bitmap::all()) + // Register with push manager + let mut push_rx = match self + .subscribe_push_manager(&access_token, Bitmap::all()) .await { - Ok(change_rx) => change_rx, + Ok(push_rx) => push_rx, Err(err) => { trc::error!( - err.details("Failed to subscribe to state manager") + err.details("Failed to subscribe to push manager") .span_id(session.session_id) ); @@ -79,7 +79,7 @@ impl WebSocketHandler for Server { } }; - let mut changes = WebSocketStateChange::new(None); + let mut notifications = Vec::new(); let mut change_types: Bitmap = Bitmap::new(); loop { @@ -102,7 +102,6 @@ impl WebSocketHandler for Server { &session, ) .await; - WebSocketResponse::from_response(response, request.id) .to_json() } @@ -174,17 +173,47 @@ impl WebSocketHandler for Server { } } } - state_change = change_rx.recv() => { - if let Some(state_change) = state_change { - let mut types = state_change.types; - types.intersection(&change_types); + push_notification = push_rx.recv() => { + if let Some(push_notification) = push_notification { + match push_notification { + PushNotification::StateChange(state_change) => { + let mut types = state_change.types; + types.intersection(&change_types); - for type_state in types { - changes - .changed - .get_mut_or_insert(state_change.account_id.into()) - .set(type_state, state_change.change_id.into()); + if !types.is_empty() { + notifications.push(PushNotification::StateChange( + StateChange { + account_id: state_change.account_id, + types, + change_id: state_change.change_id, + } + )); + } + }, + PushNotification::EmailPush(email_push) => { + let state_change = email_push.to_state_change(); + let mut types = state_change.types; + types.intersection(&change_types); + + if !types.is_empty() { + notifications.push(PushNotification::StateChange( + StateChange { + account_id: state_change.account_id, + types, + change_id: state_change.change_id, + } + )); + } + }, + PushNotification::CalendarAlert(calendar_alert) => { + if change_types.contains(DataType::CalendarAlert) { + notifications.push(PushNotification::CalendarAlert( + calendar_alert + )); + } + }, } + } else { trc::event!( Jmap(JmapEvent::WebsocketStop), @@ -197,11 +226,15 @@ impl WebSocketHandler for Server { } } - if !changes.changed.is_empty() { + if !notifications.is_empty() { // Send any queued changes let elapsed = last_changes_sent.elapsed(); if elapsed >= throttle { - if let Err(err) = stream.send(Message::Text(changes.to_json().into())).await { + let payload = WebSocketPushObject { + push: std::mem::take(&mut notifications).into_push_object(), + push_state: None, + }; + if let Err(err) = stream.send(Message::Text(payload.to_json().into())).await { trc::event!( Jmap(JmapEvent::WebsocketError), Details = "Failed to send state change message.", @@ -209,7 +242,6 @@ impl WebSocketHandler for Server { Reason = err.to_string() ); } - changes.changed.clear(); last_changes_sent = Instant::now(); last_heartbeat = Instant::now(); next_event = heartbeat; diff --git a/crates/migration/src/push.rs b/crates/migration/src/push.rs index 2e86a7aa..ed5730b9 100644 --- a/crates/migration/src/push.rs +++ b/crates/migration/src/push.rs @@ -21,6 +21,7 @@ pub(crate) async fn migrate_push_subscriptions( account_id: u32, ) -> trc::Result { // Obtain email ids + let todo = "fix"; let push_subscription_ids = server .get_document_ids(account_id, Collection::PushSubscription) .await @@ -121,6 +122,7 @@ impl FromLegacy for PushSubscription { .unwrap_or_default(); PushSubscription { + id: 0, url: legacy .get(&Property::Url) .as_string() @@ -147,6 +149,7 @@ impl FromLegacy for PushSubscription { .filter_map(|v| v.as_string().and_then(DataType::parse)) .collect(), keys: convert_keys(legacy.get(&Property::Keys)), + email_push: vec![], } } } diff --git a/crates/services/src/broadcast/mod.rs b/crates/services/src/broadcast/mod.rs index e90b2a24..c41f5a68 100644 --- a/crates/services/src/broadcast/mod.rs +++ b/crates/services/src/broadcast/mod.rs @@ -4,8 +4,8 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::ipc::BroadcastEvent; -use std::borrow::Borrow; +use common::ipc::{BroadcastEvent, CalendarAlert, EmailPush, PushNotification}; +use std::{borrow::Borrow, io::Write}; use types::type_state::StateChange; use utils::{ codec::leb128::{Leb128Iterator, Leb128Writer}, @@ -41,31 +41,54 @@ impl BroadcastBatch> { let _ = serialized.write_leb128(node_id); for message in &self.messages { match message { - BroadcastEvent::StateChange(state_change) => { - serialized.push(0u8); - let _ = serialized.write_leb128(state_change.change_id); - let _ = serialized.write_leb128(*state_change.types.as_ref()); - let _ = serialized.write_leb128(state_change.account_id); - } + BroadcastEvent::PushNotification(notification) => match notification { + PushNotification::StateChange(state_change) => { + serialized.push(0u8); + let _ = serialized.write_leb128(state_change.change_id); + let _ = serialized.write_leb128(*state_change.types.as_ref()); + let _ = serialized.write_leb128(state_change.account_id); + } + PushNotification::CalendarAlert(calendar_alert) => { + serialized.push(1u8); + let _ = serialized.write_leb128(calendar_alert.account_id); + let _ = serialized.write_leb128(calendar_alert.event_id); + let _ = serialized + .write_leb128(calendar_alert.recurrence_id.unwrap_or_default() as u64); + let _ = serialized.write_leb128(calendar_alert.uid.len()); + let _ = serialized.write(calendar_alert.uid.as_bytes()); + let _ = serialized.write_leb128(calendar_alert.alert_id.len()); + let _ = serialized.write(calendar_alert.alert_id.as_bytes()); + } + PushNotification::EmailPush(email_push) => { + serialized.push(2u8); + let _ = serialized.write_leb128(email_push.account_id); + let _ = serialized.write_leb128(email_push.email_id); + let _ = serialized.write_leb128(email_push.change_id); + } + }, BroadcastEvent::InvalidateAccessTokens(items) => { - serialized.push(1u8); + serialized.push(3u8); let _ = serialized.write_leb128(items.len()); for item in items { let _ = serialized.write_leb128(*item); } } - BroadcastEvent::InvalidateDavCache(items) => { - serialized.push(2u8); + BroadcastEvent::InvalidateGroupwareCache(items) => { + serialized.push(4u8); let _ = serialized.write_leb128(items.len()); for item in items { let _ = serialized.write_leb128(*item); } } BroadcastEvent::ReloadSettings => { - serialized.push(3u8); + serialized.push(5u8); } BroadcastEvent::ReloadBlockedIps => { - serialized.push(4u8); + serialized.push(6u8); + } + BroadcastEvent::ReloadPushServers(account_id) => { + serialized.push(7u8); + let _ = serialized.write_leb128(*account_id); } } } @@ -89,12 +112,54 @@ where pub fn next_event(&mut self) -> Result, ()> { if let Some(id) = self.messages.next() { match id.borrow() { - 0 => Ok(Some(BroadcastEvent::StateChange(StateChange { - change_id: self.messages.next_leb128().ok_or(())?, - types: Bitmap::from(self.messages.next_leb128::().ok_or(())?), - account_id: self.messages.next_leb128().ok_or(())?, - }))), + 0 => Ok(Some(BroadcastEvent::PushNotification( + PushNotification::StateChange(StateChange { + change_id: self.messages.next_leb128().ok_or(())?, + types: Bitmap::from(self.messages.next_leb128::().ok_or(())?), + account_id: self.messages.next_leb128().ok_or(())?, + }), + ))), + 1 => { + let account_id = self.messages.next_leb128().ok_or(())?; + let event_id = self.messages.next_leb128().ok_or(())?; + let recurrence_id = self.messages.next_leb128::().ok_or(())? as i64; + let uid_len = self.messages.next_leb128::().ok_or(())?; + let mut uid_bytes = vec![0u8; uid_len]; + for byte in uid_bytes.iter_mut() { + *byte = self.messages.next().ok_or(())?.borrow().to_owned(); + } + let uid = String::from_utf8(uid_bytes).map_err(|_| ())?; + let alert_id_len = self.messages.next_leb128::().ok_or(())?; + let mut alert_id_bytes = vec![0u8; alert_id_len]; + for byte in alert_id_bytes.iter_mut() { + *byte = self.messages.next().ok_or(())?.borrow().to_owned(); + } + let alert_id = String::from_utf8(alert_id_bytes).map_err(|_| ())?; + Ok(Some(BroadcastEvent::PushNotification( + PushNotification::CalendarAlert(CalendarAlert { + account_id, + event_id, + recurrence_id: if recurrence_id == 0 { + None + } else { + Some(recurrence_id) + }, + uid, + alert_id, + }), + ))) + } + + 2 => Ok(Some(BroadcastEvent::PushNotification( + PushNotification::EmailPush(EmailPush { + account_id: self.messages.next_leb128().ok_or(())?, + email_id: self.messages.next_leb128().ok_or(())?, + change_id: self.messages.next_leb128().ok_or(())?, + }), + ))), + + 3 => { let count = self.messages.next_leb128::().ok_or(())?; let mut items = Vec::with_capacity(count); for _ in 0..count { @@ -102,17 +167,24 @@ where } Ok(Some(BroadcastEvent::InvalidateAccessTokens(items))) } - 2 => { + + 4 => { let count = self.messages.next_leb128::().ok_or(())?; let mut items = Vec::with_capacity(count); for _ in 0..count { items.push(self.messages.next_leb128().ok_or(())?); } - Ok(Some(BroadcastEvent::InvalidateDavCache(items))) + Ok(Some(BroadcastEvent::InvalidateGroupwareCache(items))) } - 3 => Ok(Some(BroadcastEvent::ReloadSettings)), - 4 => Ok(Some(BroadcastEvent::ReloadBlockedIps)), + 5 => Ok(Some(BroadcastEvent::ReloadSettings)), + + 6 => Ok(Some(BroadcastEvent::ReloadBlockedIps)), + + 7 => { + let account_id = self.messages.next_leb128().ok_or(())?; + Ok(Some(BroadcastEvent::ReloadPushServers(account_id))) + } _ => Err(()), } diff --git a/crates/services/src/broadcast/subscriber.rs b/crates/services/src/broadcast/subscriber.rs index 41fd3567..ae958fba 100644 --- a/crates/services/src/broadcast/subscriber.rs +++ b/crates/services/src/broadcast/subscriber.rs @@ -8,7 +8,7 @@ use crate::broadcast::{BROADCAST_TOPIC, BroadcastBatch}; use common::{ Inner, core::BuildServer, - ipc::{BroadcastEvent, HousekeeperEvent, StateEvent}, + ipc::{BroadcastEvent, HousekeeperEvent, PushEvent, PushNotification}, }; use compact_str::CompactString; use std::{sync::Arc, time::Duration}; @@ -94,8 +94,6 @@ pub fn spawn_broadcast_subscriber(inner: Arc, mut shutdown_rx: watch::Rec } }; - let mut max_timestamp = 0; - loop { match batch.next_event() { Ok(Some(event)) => { @@ -106,14 +104,12 @@ pub fn spawn_broadcast_subscriber(inner: Arc, mut shutdown_rx: watch::Rec Details = log_event(&event), ); match event { - BroadcastEvent::StateChange(state_change) => { - max_timestamp = - std::cmp::max(max_timestamp, state_change.change_id); + BroadcastEvent::PushNotification(notification) => { if inner .ipc - .state_tx - .send(StateEvent::Publish { - state_change, + .push_tx + .send(PushEvent::Publish { + notification, broadcast: false, }) .await @@ -121,7 +117,22 @@ pub fn spawn_broadcast_subscriber(inner: Arc, mut shutdown_rx: watch::Rec { trc::event!( Server(ServerEvent::ThreadError), - Details = "Error sending state change.", + Details = "Error sending push notification.", + CausedBy = trc::location!() + ); + } + } + BroadcastEvent::ReloadPushServers(account_id) => { + if inner + .ipc + .push_tx + .send(PushEvent::PushServerUpdate { account_id, broadcast: false }) + .await + .is_err() + { + trc::event!( + Server(ServerEvent::ThreadError), + Details = "Error sending reload request.", CausedBy = trc::location!() ); } @@ -132,7 +143,7 @@ pub fn spawn_broadcast_subscriber(inner: Arc, mut shutdown_rx: watch::Rec inner.cache.access_tokens.remove(id); } } - BroadcastEvent::InvalidateDavCache(ids) => { + BroadcastEvent::InvalidateGroupwareCache(ids) => { for id in &ids { inner.cache.files.remove(id); inner.cache.contacts.remove(id); @@ -210,12 +221,28 @@ pub fn spawn_broadcast_subscriber(inner: Arc, mut shutdown_rx: watch::Rec fn log_event(event: &BroadcastEvent) -> trc::Value { match event { - BroadcastEvent::StateChange(state_change) => trc::Value::Array(vec![ - "StateChange".into(), - state_change.account_id.into(), - state_change.change_id.into(), - (*state_change.types.as_ref()).into(), - ]), + BroadcastEvent::PushNotification(notification) => match notification { + PushNotification::StateChange(state_change) => trc::Value::Array(vec![ + "StateChange".into(), + state_change.account_id.into(), + state_change.change_id.into(), + (*state_change.types.as_ref()).into(), + ]), + PushNotification::CalendarAlert(calendar_alert) => trc::Value::Array(vec![ + "CalendarAlert".into(), + calendar_alert.account_id.into(), + calendar_alert.event_id.into(), + calendar_alert.recurrence_id.into(), + calendar_alert.uid.clone().into(), + calendar_alert.alert_id.clone().into(), + ]), + PushNotification::EmailPush(email_push) => trc::Value::Array(vec![ + "EmailPush".into(), + email_push.account_id.into(), + email_push.email_id.into(), + email_push.change_id.into(), + ]), + }, BroadcastEvent::ReloadSettings => CompactString::const_new("ReloadSettings").into(), BroadcastEvent::ReloadBlockedIps => CompactString::const_new("ReloadBlockedIps").into(), BroadcastEvent::InvalidateAccessTokens(items) => { @@ -226,13 +253,16 @@ fn log_event(event: &BroadcastEvent) -> trc::Value { } trc::Value::Array(array) } - BroadcastEvent::InvalidateDavCache(items) => { + BroadcastEvent::InvalidateGroupwareCache(items) => { let mut array = Vec::with_capacity(items.len() + 1); - array.push("InvalidateDavCache".into()); + array.push("InvalidateGroupwareCache".into()); for item in items { array.push((*item).into()); } trc::Value::Array(array) } + BroadcastEvent::ReloadPushServers(account_id) => { + trc::Value::Array(vec!["ReloadPushServers".into(), (*account_id).into()]) + } } } diff --git a/crates/services/src/housekeeper/mod.rs b/crates/services/src/housekeeper/mod.rs index dcb5721b..7a5f1590 100644 --- a/crates/services/src/housekeeper/mod.rs +++ b/crates/services/src/housekeeper/mod.rs @@ -80,7 +80,7 @@ pub fn spawn_housekeeper(inner: Arc, mut rx: mpsc::Receiver, mut rx: mpsc::Receiver, mut rx: mpsc::Receiver, mut rx: mpsc::Receiver { @@ -436,7 +436,7 @@ pub fn spawn_housekeeper(inner: Arc, mut rx: mpsc::Receiver // SPDX-License-Identifier: LicenseRef-SEL diff --git a/crates/services/src/lib.rs b/crates/services/src/lib.rs index b0bfff96..6fe30e98 100644 --- a/crates/services/src/lib.rs +++ b/crates/services/src/lib.rs @@ -10,7 +10,7 @@ use common::{ manager::boot::{BootManager, IpcReceivers}, }; use housekeeper::spawn_housekeeper; -use state_manager::manager::spawn_state_manager; +use state_manager::manager::spawn_push_router; use std::sync::Arc; use task_manager::spawn_task_manager; @@ -50,8 +50,8 @@ impl StartServices for BootManager { impl SpawnServices for IpcReceivers { fn spawn_services(&mut self, inner: Arc) { - // Spawn state manager - spawn_state_manager(inner.clone(), self.state_rx.take().unwrap()); + // Spawn push manager + spawn_push_router(inner.clone(), self.push_rx.take().unwrap()); // Spawn housekeeper spawn_housekeeper(inner.clone(), self.housekeeper_rx.take().unwrap()); diff --git a/crates/services/src/state_manager/http.rs b/crates/services/src/state_manager/http.rs index fea74b26..153dc755 100644 --- a/crates/services/src/state_manager/http.rs +++ b/crates/services/src/state_manager/http.rs @@ -4,49 +4,90 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::{Event, PushServer, ece::ece_encrypt}; +use super::{Event, ece::ece_encrypt}; +use crate::state_manager::PushRegistration; use base64::Engine; -use common::ipc::EncryptionKeys; -use jmap_proto::response::status::StateChangeResponse; +use calcard::jscalendar::JSCalendarDateTime; +use common::ipc::PushNotification; +use email::push::PushSubscription; +use jmap_proto::{ + response::status::{EmailPushObject, PushObject}, + types::state::State, +}; use reqwest::header::{CONTENT_ENCODING, CONTENT_TYPE}; use std::time::{Duration, Instant}; use tokio::sync::mpsc; use trc::PushSubscriptionEvent; -use types::id::Id; +use types::{id::Id, type_state::DataType}; +use utils::map::vec_map::VecMap; -impl PushServer { +impl PushRegistration { pub fn send(&mut self, id: Id, push_tx: mpsc::Sender, push_timeout: Duration) { - let url = self.url.clone(); - let keys = self.keys.clone(); - let state_changes = std::mem::take(&mut self.state_changes); + let server = self.server.clone(); + let notifications = std::mem::take(&mut self.notifications); self.in_flight = true; self.last_request = Instant::now(); tokio::spawn(async move { - let mut response = StateChangeResponse::new(); - for state_change in &state_changes { - for type_state in state_change.types { - response - .changed - .get_mut_or_insert(state_change.account_id.into()) - .set(type_state, (state_change.change_id).into()); + let mut changed: VecMap> = VecMap::new(); + let mut objects = Vec::with_capacity(notifications.len()); + for notification in ¬ifications { + match notification { + PushNotification::StateChange(state_change) => { + for type_state in state_change.types { + changed + .get_mut_or_insert(state_change.account_id.into()) + .set(type_state, (state_change.change_id).into()); + } + } + PushNotification::CalendarAlert(calendar_alert) => { + objects.push(PushObject::CalendarAlert { + account_id: calendar_alert.account_id.into(), + calendar_event_id: calendar_alert.event_id.into(), + uid: calendar_alert.uid.clone(), + recurrence_id: calendar_alert.recurrence_id.map(|timestamp| { + JSCalendarDateTime::new(timestamp, true).to_rfc3339() + }), + alert_id: calendar_alert.alert_id.clone(), + }); + } + PushNotification::EmailPush(email_push) => { + objects.push(PushObject::EmailPush { + account_id: email_push.account_id.into(), + email: EmailPushObject { + subject: Default::default(), + }, + }); + } } } + let response = if !objects.is_empty() { + if changed.is_empty() { + objects.push(PushObject::StateChange { changed }); + } + if objects.len() > 1 { + PushObject::Group { entries: objects } + } else { + objects.into_iter().next().unwrap() + } + } else { + PushObject::StateChange { changed } + }; + push_tx .send( if http_request( - url, + &server, serde_json::to_string(&response).unwrap(), - keys, push_timeout, ) .await { Event::DeliverySuccess { id } } else { - Event::DeliveryFailure { id, state_changes } + Event::DeliveryFailure { id, notifications } }, ) .await @@ -56,9 +97,8 @@ impl PushServer { } pub(crate) async fn http_request( - url: String, + details: &PushSubscription, mut body: String, - keys: Option, push_timeout: Duration, ) -> bool { let client_builder = reqwest::Client::builder().timeout(push_timeout); @@ -69,11 +109,11 @@ pub(crate) async fn http_request( let mut client = client_builder .build() .unwrap_or_default() - .post(url.as_str()) + .post(details.url.as_str()) .header(CONTENT_TYPE, "application/json") .header("TTL", "86400"); - if let Some(keys) = keys { + if let Some(keys) = &details.keys { match ece_encrypt(&keys.p256dh, &keys.auth, body.as_bytes()) .map(|b| base64::engine::general_purpose::URL_SAFE.encode(b)) { @@ -87,7 +127,7 @@ pub(crate) async fn http_request( trc::event!( PushSubscription(PushSubscriptionEvent::Error), Details = "Failed to encrypt push subscription", - Url = url, + Url = details.url.to_string(), Reason = err ); return true; @@ -98,14 +138,17 @@ pub(crate) async fn http_request( match client.body(body).send().await { Ok(response) => { if response.status().is_success() { - trc::event!(PushSubscription(PushSubscriptionEvent::Success), Url = url,); + trc::event!( + PushSubscription(PushSubscriptionEvent::Success), + Url = details.url.to_string() + ); true } else { trc::event!( PushSubscription(PushSubscriptionEvent::Error), Details = "HTTP POST failed", - Url = url, + Url = details.url.to_string(), Code = response.status().as_u16(), ); @@ -116,7 +159,7 @@ pub(crate) async fn http_request( trc::event!( PushSubscription(PushSubscriptionEvent::Error), Details = "HTTP POST failed", - Url = url, + Url = details.url.to_string(), Reason = err.to_string() ); diff --git a/crates/services/src/state_manager/manager.rs b/crates/services/src/state_manager/manager.rs index 1cfe6560..865be340 100644 --- a/crates/services/src/state_manager/manager.rs +++ b/crates/services/src/state_manager/manager.rs @@ -4,46 +4,36 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::{ - Event, PURGE_EVERY, PushUpdate, SEND_TIMEOUT, Subscriber, SubscriberId, SubscriberType, - push::spawn_push_manager, -}; +use super::{Event, PURGE_EVERY, SEND_TIMEOUT, push::spawn_push_manager}; +use crate::state_manager::IpcSubscriber; use common::{ Inner, - core::BuildServer, - ipc::{BroadcastEvent, PushSubscription, StateEvent, UpdateSubscription}, + ipc::{BroadcastEvent, PushEvent}, }; -use std::{ - sync::Arc, - time::{Instant, SystemTime}, -}; -use store::{ahash::AHashMap, rand}; +use std::{sync::Arc, time::Instant}; +use store::ahash::AHashMap; use tokio::sync::mpsc; use trc::ServerEvent; -use types::{ - id::Id, - type_state::{DataType, StateChange}, -}; -use utils::map::bitmap::Bitmap; + +#[derive(Default)] +struct Subscriber { + ipc: Vec, + is_push: bool, +} #[allow(clippy::unwrap_or_default)] -pub fn spawn_state_manager(inner: Arc, mut change_rx: mpsc::Receiver) { +pub fn spawn_push_router(inner: Arc, mut change_rx: mpsc::Receiver) { let push_tx = spawn_push_manager(inner.clone()); tokio::spawn(async move { - let mut subscribers: AHashMap> = - AHashMap::default(); - let mut shared_accounts: AHashMap> = AHashMap::default(); - let mut shared_accounts_map: AHashMap>> = - AHashMap::default(); - + let mut subscribers: AHashMap = AHashMap::default(); let mut last_purge = Instant::now(); while let Some(event) = change_rx.recv().await { let mut purge_needed = last_purge.elapsed() >= PURGE_EVERY; match event { - StateEvent::Stop => { + PushEvent::Stop => { if push_tx.send(Event::Reset).await.is_err() { trc::event!( Server(ServerEvent::ThreadError), @@ -53,96 +43,50 @@ pub fn spawn_state_manager(inner: Arc, mut change_rx: mpsc::Receiver { - // Obtain account membership and shared mailboxes - let acl = match inner.build_server().get_access_token(account_id).await { - Ok(result) => result, - Err(err) => { - trc::error!( - err.account_id(account_id) - .details("Failed to obtain access token.") - ); - continue; - } - }; - - // Delete any removed sharings - if let Some(shared_account_ids) = shared_accounts.get(&account_id) { - for shared_account_id in shared_account_ids { - if *shared_account_id != acl.primary_id - && !acl.member_of.contains(shared_account_id) - && !acl - .access_to - .iter() - .any(|(id, _)| *id == *shared_account_id) - && let Some(shared_list) = - shared_accounts_map.get_mut(shared_account_id) - { - shared_list.remove(&account_id); - if shared_list.is_empty() { - shared_accounts_map.remove(shared_account_id); - } - } - } - } - - // Update lists - let mut shared_account_ids = - Vec::with_capacity(acl.member_of.len() + 1 + acl.access_to.len()); - for member_id in [acl.primary_id].iter().chain(acl.member_of.iter()) { - shared_account_ids.push(*member_id); - shared_accounts_map - .entry(*member_id) - .or_insert_with(AHashMap::new) - .insert(account_id, Bitmap::all()); - } - for (shared_account_id, shared_collections) in acl.access_to.iter() { - let mut types: Bitmap = Bitmap::new(); - for collection in *shared_collections { - if let Ok(type_state) = DataType::try_from(collection) { - types.insert(type_state); - if type_state == DataType::Email { - types.insert(DataType::EmailDelivery); - types.insert(DataType::Thread); - } - } - } - if !types.is_empty() { - shared_account_ids.push(*shared_account_id); - shared_accounts_map - .entry(*shared_account_id) - .or_insert_with(AHashMap::new) - .insert(account_id, types); - } - } - shared_accounts.insert(account_id, shared_account_ids); - } - StateEvent::Subscribe { - account_id, + PushEvent::Subscribe { + account_ids, types, tx, } => { - subscribers - .entry(account_id) - .or_insert_with(AHashMap::default) - .insert( - SubscriberId::Ipc(rand::random()), - Subscriber { + for account_id in account_ids { + subscribers + .entry(account_id) + .or_default() + .ipc + .push(IpcSubscriber { types, - subscription: SubscriberType::Ipc { tx }, - }, - ); + tx: tx.clone(), + }); + } } - StateEvent::Publish { - state_change, + + PushEvent::PushServerRegister { activate, expired } => { + for account_id in activate { + subscribers.entry(account_id).or_default().is_push = true; + } + + for account_id in expired { + let mut remove_account = false; + if let Some(subscriber_list) = subscribers.get_mut(&account_id) { + subscriber_list.is_push = false; + remove_account = subscriber_list.ipc.is_empty(); + } + if remove_account { + subscribers.remove(&account_id); + } + } + } + + PushEvent::Publish { + notification, broadcast, } => { // Publish event to cluster if broadcast && let Some(broadcast_tx) = &inner.ipc.broadcast_tx.clone() && broadcast_tx - .send(BroadcastEvent::StateChange(state_change)) + .send(BroadcastEvent::PushNotification(notification.clone())) .await .is_err() { @@ -153,77 +97,37 @@ pub fn spawn_state_manager(inner: Arc, mut change_rx: mpsc::Receiver { - let subscriber_tx = tx.clone(); - - tokio::spawn(async move { - // Timeout after 500ms in case there is a blocked client - if subscriber_tx - .send_timeout( - StateChange { - account_id: state_change.account_id, - change_id: state_change.change_id, - types, - }, - SEND_TIMEOUT, - ) - .await - .is_err() - { - trc::event!( - Server(ServerEvent::ThreadError), - Details = "Error sending state change to subscriber.", - CausedBy = trc::location!() - ); - } - }); - } - SubscriberType::Push { expires } - if expires > ¤t_time => - { - push_ids.push(Id::from_parts( - *owner_account_id, - (*subscriber_id).into(), - )); - } - _ => { - purge_needed = true; - } - } - } + }); + } else { + purge_needed = true; } } } - if !push_ids.is_empty() - && push_tx - .send(Event::Push { - ids: push_ids, - state_change, - }) - .await - .is_err() + if subscribers.is_push + && push_tx.send(Event::Push { notification }).await.is_err() { trc::event!( Server(ServerEvent::ThreadError), @@ -233,84 +137,28 @@ pub fn spawn_state_manager(inner: Arc, mut change_rx: mpsc::Receiver { - let mut updated_ids = Vec::with_capacity(subscriptions.len()); - let mut push_updates = Vec::with_capacity(subscriptions.len()); - - if let Some(subscribers) = subscribers.get_mut(&account_id) { - let mut remove_ids = Vec::new(); - - for subscriber_id in subscribers.keys() { - if let SubscriberId::Push(push_id) = subscriber_id - && !subscriptions.iter().any(|s| { - matches!(s, UpdateSubscription::Verified( - PushSubscription { id, .. } - ) if id == push_id) - }) - { - remove_ids.push(*subscriber_id); - } - } - - for remove_id in remove_ids { - push_updates.push(PushUpdate::Unregister { - id: Id::from_parts(account_id, remove_id.into()), - }); - subscribers.remove(&remove_id); - } - } - - for subscription in subscriptions { - match subscription { - UpdateSubscription::Unverified { - id, - url, - code, - keys, - } => { - push_updates.push(PushUpdate::Verify { - id, - account_id, - url, - code, - keys, - }); - } - UpdateSubscription::Verified(verified) => { - updated_ids.push(verified.id); - subscribers - .entry(account_id) - .or_insert_with(AHashMap::default) - .insert( - SubscriberId::Push(verified.id), - Subscriber { - types: verified.types, - subscription: SubscriberType::Push { - expires: verified.expires, - }, - }, - ); - - push_updates.push(PushUpdate::Register { - id: Id::from_parts(account_id, verified.id), - url: verified.url, - keys: verified.keys, - }); - } - } - } - - if !push_updates.is_empty() - && push_tx - .send(Event::Update { - updates: push_updates, - }) + // Publish event to cluster + if broadcast + && let Some(broadcast_tx) = &inner.ipc.broadcast_tx.clone() + && broadcast_tx + .send(BroadcastEvent::ReloadPushServers(account_id)) .await .is_err() { + trc::event!( + Server(trc::ServerEvent::ThreadError), + Details = "Error sending broadcast event.", + CausedBy = trc::location!() + ); + } + + // Notify push manager + if push_tx.send(Event::Update { account_id }).await.is_err() { trc::event!( Server(ServerEvent::ThreadError), Details = "Error sending push updates.", @@ -322,26 +170,12 @@ pub fn spawn_state_manager(inner: Arc, mut change_rx: mpsc::Receiver, - subscription: SubscriberType, + tx: mpsc::Sender, } #[derive(Debug)] -pub enum SubscriberType { - Ipc { tx: mpsc::Sender }, - Push { expires: u64 }, -} - -#[derive(Debug)] -pub struct PushServer { - url: String, - keys: Option, +pub struct PushRegistration { + server: Arc, + member_account_ids: Vec, num_attempts: u32, last_request: Instant, - state_changes: Vec, + notifications: Vec, in_flight: bool, } #[derive(Debug)] pub enum Event { - Update { - updates: Vec, - }, Push { - ids: Vec, - state_change: StateChange, + notification: PushNotification, + }, + Update { + account_id: u32, }, DeliverySuccess { id: Id, }, DeliveryFailure { id: Id, - state_changes: Vec, + notifications: Vec, }, Reset, } -#[derive(Debug)] -pub enum PushUpdate { - Verify { - id: u32, - account_id: u32, - url: String, - code: String, - keys: Option, - }, - Register { - id: Id, - url: String, - keys: Option, - }, - Unregister { - id: Id, - }, -} - -impl Subscriber { - fn is_valid(&self, current_time: u64) -> bool { - match &self.subscription { - SubscriberType::Ipc { tx } => !tx.is_closed(), - SubscriberType::Push { expires } => expires > ¤t_time, - } - } -} - -const PURGE_EVERY: Duration = Duration::from_secs(3600); -const SEND_TIMEOUT: Duration = Duration::from_millis(500); - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -enum SubscriberId { - Ipc(u32), - Push(u32), -} - -impl From for u32 { - fn from(subscriber_id: SubscriberId) -> u32 { - match subscriber_id { - SubscriberId::Ipc(id) => id, - SubscriberId::Push(id) => id, - } +impl IpcSubscriber { + fn is_valid(&self) -> bool { + !self.tx.is_closed() } } diff --git a/crates/services/src/state_manager/push.rs b/crates/services/src/state_manager/push.rs index 705441dd..84fb335a 100644 --- a/crates/services/src/state_manager/push.rs +++ b/crates/services/src/state_manager/push.rs @@ -4,29 +4,123 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::{Event, PushServer, PushUpdate, http::http_request}; -use common::{IPC_CHANNEL_BUFFER, Inner, LONG_1Y_SLUMBER, core::BuildServer}; +use super::{Event, http::http_request}; +use crate::state_manager::PushRegistration; +use common::{ + IPC_CHANNEL_BUFFER, Inner, LONG_1Y_SLUMBER, Server, + core::BuildServer, + ipc::{PushEvent, PushNotification}, +}; +use email::push::PushSubscriptions; use std::{ collections::hash_map::Entry, sync::Arc, time::{Duration, Instant}, }; -use store::ahash::{AHashMap, AHashSet}; +use store::{ + ahash::{AHashMap, AHashSet}, + write::now, +}; use tokio::sync::mpsc; -use trc::PushSubscriptionEvent; -use types::id::Id; +use trc::{AddContext, PushSubscriptionEvent, ServerEvent}; +use types::{collection::Collection, field::PrincipalField, id::Id}; pub fn spawn_push_manager(inner: Arc) -> mpsc::Sender { let (push_tx_, mut push_rx) = mpsc::channel::(IPC_CHANNEL_BUFFER); let push_tx = push_tx_.clone(); + tokio::spawn(async move {}); + tokio::spawn(async move { - let mut subscriptions = AHashMap::default(); + let mut push_servers: AHashMap = AHashMap::default(); + let mut account_push_ids: AHashMap> = AHashMap::default(); let mut last_verify: AHashMap = AHashMap::default(); let mut last_retry = Instant::now(); let mut retry_timeout = LONG_1Y_SLUMBER; let mut retry_ids = AHashSet::default(); + // Load active subscriptions on startup + { + let server = inner.build_server(); + match server + .get_document_ids(u32::MAX, Collection::PushSubscription) + .await + { + Ok(Some(account_ids)) => { + for account_id in account_ids { + if server + .core + .network + .roles + .push_notifications + .is_enabled_for_account(account_id) + { + // Load push subscriptions for account + let (subscriptions, member_account_ids) = + match load_push_subscriptions(&server, account_id).await { + Ok(subscriptions) => subscriptions, + Err(err) => { + trc::error!(err.caused_by(trc::location!())); + continue; + } + }; + let current_time = now(); + for subscription in subscriptions + .subscriptions + .into_iter() + .filter(|s| s.verified && s.expires > current_time) + { + let id = Id::from_parts(subscription.id, account_id); + let subscription = Arc::new(subscription); + + for account_id in &member_account_ids { + account_push_ids.entry(*account_id).or_default().insert(id); + } + push_servers.insert( + id, + PushRegistration { + member_account_ids: member_account_ids.clone(), + num_attempts: 0, + last_request: Instant::now() + - (server.core.jmap.push_throttle + + Duration::from_millis(1)), + notifications: Vec::new(), + server: subscription.clone(), + in_flight: false, + }, + ); + } + } + } + } + Ok(None) => {} + Err(err) => { + trc::error!(err.caused_by(trc::location!())); + } + } + + // Subscribe to push events + if !account_push_ids.is_empty() + && server + .inner + .ipc + .push_tx + .clone() + .send(PushEvent::PushServerRegister { + activate: account_push_ids.keys().copied().collect(), + expired: vec![], + }) + .await + .is_err() + { + trc::event!( + Server(ServerEvent::ThreadError), + Details = "Error sending state change.", + CausedBy = trc::location!() + ); + } + } + loop { // Wait for the next event or timeout let event_or_timeout = tokio::time::timeout(retry_timeout, push_rx.recv()).await; @@ -42,125 +136,273 @@ pub fn spawn_push_manager(inner: Arc) -> mpsc::Sender { match event_or_timeout { Ok(Some(event)) => match event { - Event::Update { updates } => { - for update in updates { - match update { - PushUpdate::Verify { - id, - account_id, - url, - code, - keys, - } => { - let current_time = Instant::now(); + Event::Update { account_id } => { + if !server + .core + .network + .roles + .push_notifications + .is_enabled_for_account(account_id) + { + continue; + } - #[cfg(feature = "test_mode")] - if url.contains("skip_checks") { - last_verify.insert( - account_id, - current_time - - (push_verify_timeout + Duration::from_millis(1)), - ); - } - - if last_verify - .get(&account_id) - .map(|last_verify| { - current_time - *last_verify > push_verify_timeout - }) - .unwrap_or(true) - { - tokio::spawn(async move { - http_request( - url, - format!( - concat!( - "{{\"@type\":\"PushVerification\",", - "\"pushSubscriptionId\":\"{}\",", - "\"verificationCode\":\"{}\"}}" - ), - Id::from(id), - code - ), - keys, - push_timeout, - ) - .await; - }); - - last_verify.insert(account_id, current_time); - } else { - trc::event!( - PushSubscription(PushSubscriptionEvent::Error), - Details = "Failed to verify push subscription", - Url = url.clone(), - AccountId = account_id, - Reason = "Too many requests" - ); - - continue; - } + // Load push subscriptions for account + let (subscriptions, member_account_ids) = + match load_push_subscriptions(&server, account_id).await { + Ok(subscriptions) => subscriptions, + Err(err) => { + trc::error!(err.caused_by(trc::location!())); + continue; } - PushUpdate::Register { id, url, keys } => { - if let Entry::Vacant(entry) = subscriptions.entry(id) { - entry.insert(PushServer { - url, - keys, + }; + let old_account_push_ids = account_push_ids + .remove(&account_id) + .filter(|v| !v.is_empty()); + + // Process subscriptions + let current_time = now(); + for subscription in subscriptions + .subscriptions + .into_iter() + .filter(|s| s.expires > current_time) + { + let id = Id::from_parts(subscription.id, account_id); + let subscription = Arc::new(subscription); + + if subscription.verified { + for account_id in &member_account_ids { + account_push_ids.entry(*account_id).or_default().insert(id); + } + + match push_servers.entry(id) { + Entry::Occupied(mut entry) => { + // Update existing subscription + let entry = entry.get_mut(); + entry.server = subscription.clone(); + entry.member_account_ids = member_account_ids.clone(); + } + Entry::Vacant(entry) => { + entry.insert(PushRegistration { + member_account_ids: member_account_ids.clone(), num_attempts: 0, last_request: Instant::now() - (push_throttle + Duration::from_millis(1)), - state_changes: Vec::new(), + notifications: Vec::new(), + server: subscription.clone(), in_flight: false, }); } } - PushUpdate::Unregister { id } => { - subscriptions.remove(&id); + } else { + let current_time = Instant::now(); + + #[cfg(feature = "test_mode")] + if subscription.url.contains("skip_checks") { + last_verify.insert( + account_id, + current_time + - (push_verify_timeout + Duration::from_millis(1)), + ); + } + + if last_verify + .get(&account_id) + .map(|last_verify| { + current_time - *last_verify > push_verify_timeout + }) + .unwrap_or(true) + { + tokio::spawn(async move { + http_request( + &subscription, + format!( + concat!( + "{{\"@type\":\"PushVerification\",", + "\"pushSubscriptionId\":\"{}\",", + "\"verificationCode\":\"{}\"}}" + ), + Id::from(subscription.id), + subscription.verification_code + ), + push_timeout, + ) + .await; + }); + + last_verify.insert(account_id, current_time); + } else { + trc::event!( + PushSubscription(PushSubscriptionEvent::Error), + Details = "Failed to verify push subscription", + Url = subscription.url.clone(), + AccountId = account_id, + Reason = "Too many requests" + ); + + continue; } } } - } - Event::Push { ids, state_change } => { - for id in ids { - if let Some(subscription) = subscriptions.get_mut(&id) { - subscription.state_changes.push(state_change); - let last_request = subscription.last_request.elapsed(); - if !subscription.in_flight - && ((subscription.num_attempts == 0 - && last_request > push_throttle) - || ((1..push_attempts_max) - .contains(&subscription.num_attempts) - && last_request > push_attempt_interval)) - { - subscription.send(id, push_tx.clone(), push_timeout); - retry_ids.remove(&id); - } else { - retry_ids.insert(id); + // Update subscriptions + let mut remove_push_ids = AHashSet::new(); + let mut active_account_ids = Vec::new(); + let mut inactive_account_ids = Vec::new(); + match (old_account_push_ids, account_push_ids.get(&account_id)) { + (Some(old), Some(current)) if &old != current => { + for id in old.difference(current) { + remove_push_ids.insert(*id); + } + active_account_ids = member_account_ids; + } + (Some(old), None) => { + remove_push_ids = old; + } + (None, Some(_)) => { + active_account_ids = member_account_ids; + } + _ => {} + } + + // Update push server registrations + if !remove_push_ids.is_empty() { + for id in remove_push_ids { + if let Some(subscription) = push_servers.remove(&id) { + for account_id in &subscription.member_account_ids { + if let Some(ids) = account_push_ids.get_mut(account_id) { + ids.remove(&id); + if ids.is_empty() { + account_push_ids.remove(account_id); + inactive_account_ids.push(*account_id); + } + } + } + } + } + } + if (!active_account_ids.is_empty() || !inactive_account_ids.is_empty()) + && server + .inner + .ipc + .push_tx + .clone() + .send(PushEvent::PushServerRegister { + activate: active_account_ids, + expired: inactive_account_ids, + }) + .await + .is_err() + { + trc::event!( + Server(ServerEvent::ThreadError), + Details = "Error sending state change.", + CausedBy = trc::location!() + ); + } + } + Event::Push { notification } => { + let account_id = notification.account_id(); + if let Some(ids) = account_push_ids.get_mut(&account_id) { + let current_time = now(); + let mut remove_ids = Vec::new(); + + for id in ids.iter() { + if let Some(subscription) = push_servers.get_mut(id) { + if subscription.server.expires > current_time { + if let Some(mut notification) = + notification.filter_types(&subscription.server.types) + { + // Build email push notification + if let PushNotification::EmailPush(email_push) = + ¬ification + { + if let Some(_email_push) = subscription + .server + .email_push + .iter() + .find(|ep| ep.account_id == account_id) + { + // TODO: Apply filters once RFC is finalized + } else { + notification = PushNotification::StateChange( + email_push.to_state_change(), + ); + } + } + + subscription.notifications.push(notification); + let last_request = subscription.last_request.elapsed(); + + if !subscription.in_flight + && ((subscription.num_attempts == 0 + && last_request > push_throttle) + || ((1..push_attempts_max) + .contains(&subscription.num_attempts) + && last_request > push_attempt_interval)) + { + subscription.send( + *id, + push_tx.clone(), + push_timeout, + ); + retry_ids.remove(id); + } else { + retry_ids.insert(*id); + } + } + } else { + push_servers.remove(id); + } + } else { + remove_ids.push(*id); + } + } + + if !remove_ids.is_empty() { + for remove_id in remove_ids { + ids.remove(&remove_id); + } + if ids.is_empty() { + account_push_ids.remove(&account_id); + if server + .inner + .ipc + .push_tx + .clone() + .send(PushEvent::PushServerRegister { + activate: vec![], + expired: vec![account_id], + }) + .await + .is_err() + { + trc::event!( + Server(ServerEvent::ThreadError), + Details = "Error sending state change.", + CausedBy = trc::location!() + ); + } } - } else { - trc::event!( - PushSubscription(PushSubscriptionEvent::NotFound), - Id = id.document_id(), - ); } } } Event::Reset => { - subscriptions.clear(); + push_servers.clear(); + account_push_ids.clear(); } Event::DeliverySuccess { id } => { - if let Some(subscription) = subscriptions.get_mut(&id) { + if let Some(subscription) = push_servers.get_mut(&id) { subscription.num_attempts = 0; subscription.in_flight = false; retry_ids.remove(&id); } } - Event::DeliveryFailure { id, state_changes } => { - if let Some(subscription) = subscriptions.get_mut(&id) { + Event::DeliveryFailure { id, notifications } => { + if let Some(subscription) = push_servers.get_mut(&id) { subscription.last_request = Instant::now(); subscription.num_attempts += 1; - subscription.state_changes.extend(state_changes); + subscription.notifications.extend(notifications); subscription.in_flight = false; retry_ids.insert(id); } @@ -179,7 +421,7 @@ pub fn spawn_push_manager(inner: Arc) -> mpsc::Sender { let mut remove_ids = Vec::with_capacity(retry_ids.len()); for retry_id in &retry_ids { - if let Some(subscription) = subscriptions.get_mut(retry_id) { + if let Some(subscription) = push_servers.get_mut(retry_id) { let last_request = subscription.last_request.elapsed(); if !subscription.in_flight @@ -194,11 +436,11 @@ pub fn spawn_push_manager(inner: Arc) -> mpsc::Sender { trc::event!( PushSubscription(PushSubscriptionEvent::Error), Details = "Failed to deliver push subscription", - Url = subscription.url.clone(), - Reason = "Too many attempts" + Url = subscription.server.url.clone(), + Reason = "Too many failed attempts" ); - subscription.state_changes.clear(); + subscription.notifications.clear(); subscription.num_attempts = 0; } remove_ids.push(*retry_id); @@ -229,3 +471,32 @@ pub fn spawn_push_manager(inner: Arc) -> mpsc::Sender { push_tx_ } + +async fn load_push_subscriptions( + server: &Server, + account_id: u32, +) -> trc::Result<(PushSubscriptions, Vec)> { + let member_of = server + .get_access_token(account_id) + .await + .caused_by(trc::location!())? + .member_ids() + .collect::>(); + + if let Some(push_subscriptions) = server + .get_archive_by_property( + account_id, + Collection::Principal, + 0, + PrincipalField::PushSubscriptions.into(), + ) + .await? + { + push_subscriptions + .deserialize::() + .map(|push_subscriptions| (push_subscriptions, member_of)) + .caused_by(trc::location!()) + } else { + Ok((PushSubscriptions::default(), member_of)) + } +} diff --git a/crates/services/src/task_manager/alarm.rs b/crates/services/src/task_manager/alarm.rs index a2416118..2ed43911 100644 --- a/crates/services/src/task_manager/alarm.rs +++ b/crates/services/src/task_manager/alarm.rs @@ -4,10 +4,9 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::Task; use calcard::{ common::timezone::Tz, - icalendar::{ArchivedICalendarParameterName, ArchivedICalendarProperty}, + icalendar::{ArchivedICalendarParameterName, ArchivedICalendarProperty, ICalendarProperty}, }; use chrono::{DateTime, Locale}; use common::{ @@ -15,10 +14,14 @@ use common::{ auth::AccessToken, config::groupware::CalendarTemplateVariable, i18n, + ipc::{CalendarAlert, PushNotification}, listener::{ServerInstance, stream::NullIo}, }; use directory::Permission; -use groupware::calendar::{ArchivedCalendarEvent, CalendarEvent, alarm::CalendarAlarm}; +use groupware::calendar::{ + ArchivedCalendarEvent, CalendarEvent, + alarm::{CalendarAlarm, CalendarAlarmType}, +}; use mail_builder::{ MessageBuilder, headers::{HeaderType, content_type::ContentType}, @@ -36,7 +39,8 @@ use utils::{sanitize_email, template::Variables}; pub trait SendAlarmTask: Sync + Send { fn send_alarm( &self, - task: &Task, + account_id: u32, + document_id: u32, alarm: &CalendarAlarm, server_instance: Arc, ) -> impl Future + Send; @@ -45,34 +49,55 @@ pub trait SendAlarmTask: Sync + Send { impl SendAlarmTask for Server { async fn send_alarm( &self, - task: &Task, + account_id: u32, + document_id: u32, alarm: &CalendarAlarm, server_instance: Arc, ) -> bool { - match send_alarm(self, task, alarm, server_instance).await { - Ok(result) => result, - Err(err) => { - trc::error!( - err.account_id(task.account_id) - .document_id(task.document_id) - .caused_by(trc::location!()) - .details("Failed to process alarm") - ); - false + match &alarm.typ { + CalendarAlarmType::Display { .. } => { + match send_display_alarm(self, account_id, document_id, alarm).await { + Ok(result) => result, + Err(err) => { + trc::error!( + err.account_id(account_id) + .document_id(document_id) + .caused_by(trc::location!()) + .details("Failed to process e-mail alarm") + ); + false + } + } + } + CalendarAlarmType::Email { .. } => { + match send_email_alarm(self, account_id, document_id, alarm, server_instance).await + { + Ok(result) => result, + Err(err) => { + trc::error!( + err.account_id(account_id) + .document_id(document_id) + .caused_by(trc::location!()) + .details("Failed to process e-mail alarm") + ); + false + } + } } } } } -async fn send_alarm( +async fn send_email_alarm( server: &Server, - task: &Task, + account_id: u32, + document_id: u32, alarm: &CalendarAlarm, server_instance: Arc, ) -> trc::Result { // Obtain access token let access_token = server - .get_access_token(task.account_id) + .get_access_token(account_id) .await .caused_by(trc::location!())?; @@ -80,31 +105,31 @@ async fn send_alarm( trc::event!( Calendar(trc::CalendarEvent::AlarmSkipped), Reason = "Account does not have permission to send calendar alarms", - AccountId = task.account_id, - DocumentId = task.document_id, + AccountId = account_id, + DocumentId = document_id, ); return Ok(true); } else if access_token.emails.is_empty() { trc::event!( Calendar(trc::CalendarEvent::AlarmFailed), Reason = "Account does not have any email addresses", - AccountId = task.account_id, - DocumentId = task.document_id, + AccountId = account_id, + DocumentId = document_id, ); return Ok(true); } // Fetch event let Some(event_) = server - .get_archive(task.account_id, Collection::CalendarEvent, task.document_id) + .get_archive(account_id, Collection::CalendarEvent, document_id) .await .caused_by(trc::location!())? else { trc::event!( TaskQueue(TaskQueueEvent::MetadataNotFound), Details = "Calendar Event metadata not found", - AccountId = task.account_id, - DocumentId = task.document_id, + AccountId = account_id, + DocumentId = document_id, ); return Ok(true); @@ -119,7 +144,16 @@ async fn send_alarm( let account_main_email = access_token.emails.first().unwrap(); let account_main_domain = account_main_email.rsplit('@').next().unwrap_or("localhost"); let logo_cid = format!("logo.{}@{account_main_domain}", now()); - let Some(tpl) = build_template(server, &access_token, task, alarm, event, &logo_cid).await? + let Some(tpl) = build_template( + server, + &access_token, + account_id, + document_id, + alarm, + event, + &logo_cid, + ) + .await? else { return Ok(true); }; @@ -240,16 +274,16 @@ async fn send_alarm( Ok(Ok(queue_id)) => { trc::event!( Calendar(trc::CalendarEvent::AlarmSent), - AccountId = task.account_id, - DocumentId = task.document_id, + AccountId = account_id, + DocumentId = document_id, QueueId = queue_id, ); } Ok(Err(err)) => { trc::event!( Calendar(trc::CalendarEvent::AlarmFailed), - AccountId = task.account_id, - DocumentId = task.document_id, + AccountId = account_id, + DocumentId = document_id, Reason = err, ); } @@ -257,14 +291,88 @@ async fn send_alarm( trc::event!( Server(trc::ServerEvent::ThreadError), Details = "Join Error", - AccountId = task.account_id, - DocumentId = task.document_id, + AccountId = account_id, + DocumentId = document_id, CausedBy = trc::location!(), ); return Ok(false); } } + write_next_alarm(server, account_id, document_id, event).await +} + +async fn send_display_alarm( + server: &Server, + account_id: u32, + document_id: u32, + alarm: &CalendarAlarm, +) -> trc::Result { + // Fetch event + let Some(event_) = server + .get_archive(account_id, Collection::CalendarEvent, document_id) + .await + .caused_by(trc::location!())? + else { + trc::event!( + TaskQueue(TaskQueueEvent::MetadataNotFound), + Details = "Calendar Event metadata not found", + AccountId = account_id, + DocumentId = document_id, + ); + + return Ok(true); + }; + + // Unarchive event + let event = event_ + .unarchive::() + .caused_by(trc::location!())?; + + let recurrence_id = match &alarm.typ { + CalendarAlarmType::Display { recurrence_id } => *recurrence_id, + _ => None, + }; + + let ical = &event.data.event; + server + .broadcast_push_notification(PushNotification::CalendarAlert(CalendarAlert { + account_id, + event_id: document_id, + recurrence_id, + uid: ical.uids().next().unwrap_or_default().to_string(), + alert_id: ical + .components + .get(alarm.alarm_id as usize) + .and_then(|c| c.property(&ICalendarProperty::Jsid)) + .and_then(|v| v.values.first()) + .and_then(|v| v.as_text()) + .map(|v| v.to_string()) + .unwrap_or_else(|| { + format!( + "k{}", + ical.components + .get(alarm.event_id as usize) + .and_then(|c| c + .component_ids + .iter() + .position(|id| id.to_native() == alarm.alarm_id as u32)) + .unwrap_or_default() + + 1 + ) + }), + })) + .await; + + write_next_alarm(server, account_id, document_id, event).await +} + +async fn write_next_alarm( + server: &Server, + account_id: u32, + document_id: u32, + event: &ArchivedCalendarEvent, +) -> trc::Result { // Find next alarm time and write to task queue let now = now() as i64; if let Some(next_alarm) = @@ -279,8 +387,8 @@ async fn send_alarm( Calendar(trc::CalendarEvent::AlarmSkipped), Reason = "Next alarm skipped due to minimum interval", Details = next_alarm.alarm_time - now, - AccountId = task.account_id, - DocumentId = task.document_id, + AccountId = account_id, + DocumentId = document_id, ); event.data.next_alarm(max_next_alarm, Default::default()) } else { @@ -290,15 +398,16 @@ async fn send_alarm( { let mut batch = BatchBuilder::new(); batch - .with_account_id(task.account_id) + .with_account_id(account_id) .with_collection(Collection::CalendarEvent) - .update_document(task.document_id); + .update_document(document_id); next_alarm.write_task(&mut batch); server .store() .write(batch.build_all()) .await .caused_by(trc::location!())?; + server.notify_task_queue(); } Ok(true) @@ -313,7 +422,8 @@ struct Details { async fn build_template( server: &Server, access_token: &AccessToken, - task: &Task, + account_id: u32, + document_id: u32, alarm: &CalendarAlarm, event: &ArchivedCalendarEvent, logo_cid: &str, @@ -325,8 +435,8 @@ async fn build_template( trc::event!( TaskQueue(TaskQueueEvent::MetadataNotFound), Details = "Calendar Alarm component not found", - AccountId = task.account_id, - DocumentId = task.document_id, + AccountId = account_id, + DocumentId = document_id, ); return Ok(None); }; @@ -336,8 +446,8 @@ async fn build_template( Ok(uri) => uri, Err(err) => { trc::error!( - err.account_id(task.account_id) - .document_id(task.document_id) + err.account_id(account_id) + .document_id(document_id) .caused_by(trc::location!()) .details("Failed to generate webcal URI") ); @@ -421,8 +531,8 @@ async fn build_template( Calendar(trc::CalendarEvent::AlarmRecipientOverride), Reason = "External recipient not allowed for calendar alarms", Details = rcpt_to, - AccountId = task.account_id, - DocumentId = task.document_id, + AccountId = account_id, + DocumentId = document_id, ); access_token.emails.first().unwrap().to_string() @@ -451,22 +561,32 @@ async fn build_template( .as_deref() .and_then(|locale| Locale::from_str(locale).ok()) .unwrap_or(Locale::en_US); + let (event_start, event_start_tz, event_end, event_end_tz) = match alarm.typ { + CalendarAlarmType::Email { + event_start, + event_start_tz, + event_end, + event_end_tz, + } => (event_start, event_start_tz, event_end, event_end_tz), + CalendarAlarmType::Display { .. } => unreachable!(), + }; + let start = format!( "{} ({})", - DateTime::from_timestamp(alarm.event_start, 0) + DateTime::from_timestamp(event_start, 0) .unwrap_or_default() .format_localized(locale.calendar_date_template, chrono_locale), - Tz::from_id(alarm.event_start_tz) + Tz::from_id(event_start_tz) .unwrap_or(Tz::UTC) .name() .unwrap_or_default() ); let end = format!( "{} ({})", - DateTime::from_timestamp(alarm.event_end, 0) + DateTime::from_timestamp(event_end, 0) .unwrap_or_default() .format_localized(locale.calendar_date_template, chrono_locale), - Tz::from_id(alarm.event_end_tz) + Tz::from_id(event_end_tz) .unwrap_or(Tz::UTC) .name() .unwrap_or_default() diff --git a/crates/services/src/task_manager/bayes.rs b/crates/services/src/task_manager/bayes.rs index 474bb7dc..e37ded20 100644 --- a/crates/services/src/task_manager/bayes.rs +++ b/crates/services/src/task_manager/bayes.rs @@ -4,7 +4,6 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::Task; use common::Server; use email::message::bayes::EmailBayesTrain; use mail_parser::MessageParser; @@ -15,14 +14,21 @@ use types::{blob_hash::BlobHash, collection::Collection}; pub trait BayesTrainTask: Sync + Send { fn bayes_train( &self, - task: &Task, + account_id: u32, + document_id: u32, hash: &BlobHash, learn_spam: bool, ) -> impl Future + Send; } impl BayesTrainTask for Server { - async fn bayes_train(&self, task: &Task, hash: &BlobHash, learn_spam: bool) -> bool { + async fn bayes_train( + &self, + account_id: u32, + document_id: u32, + hash: &BlobHash, + learn_spam: bool, + ) -> bool { let op_start = Instant::now(); // Obtain raw message if let Ok(Some(raw_message)) = self @@ -32,7 +38,7 @@ impl BayesTrainTask for Server { { // Train bayes classifier for account self.email_bayes_train( - task.account_id, + account_id, 0, MessageParser::new().parse(&raw_message).unwrap_or_default(), learn_spam, @@ -41,9 +47,9 @@ impl BayesTrainTask for Server { trc::event!( Spam(SpamEvent::TrainAccount), - AccountId = task.account_id, + AccountId = account_id, Collection = Collection::Email, - DocumentId = task.document_id, + DocumentId = document_id, Details = if learn_spam { "spam" } else { "ham" }, Elapsed = op_start.elapsed(), ); @@ -51,8 +57,8 @@ impl BayesTrainTask for Server { } else { trc::event!( TaskQueue(TaskQueueEvent::BlobNotFound), - AccountId = task.account_id, - DocumentId = task.document_id, + AccountId = account_id, + DocumentId = document_id, BlobId = hash.as_slice(), ); false diff --git a/crates/services/src/task_manager/fts.rs b/crates/services/src/task_manager/fts.rs index 6fd87355..a17540a5 100644 --- a/crates/services/src/task_manager/fts.rs +++ b/crates/services/src/task_manager/fts.rs @@ -4,7 +4,6 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::Task; use common::Server; use directory::{Type, backend::internal::manage::ManageDirectory}; use email::message::{index::IndexMessageText, metadata::MessageMetadata}; @@ -24,7 +23,12 @@ use types::{ }; pub trait FtsIndexTask: Sync + Send { - fn fts_index(&self, task: &Task, hash: &BlobHash) -> impl Future + Send; + fn fts_index( + &self, + account_id: u32, + document_id: u32, + hash: &BlobHash, + ) -> impl Future + Send; fn fts_reindex( &self, account_id: Option, @@ -33,7 +37,7 @@ pub trait FtsIndexTask: Sync + Send { } impl FtsIndexTask for Server { - async fn fts_index(&self, task: &Task, hash: &BlobHash) -> bool { + async fn fts_index(&self, account_id: u32, document_id: u32, hash: &BlobHash) -> bool { // Obtain raw message let op_start = Instant::now(); let raw_message = if let Ok(Some(raw_message)) = self @@ -45,8 +49,8 @@ impl FtsIndexTask for Server { } else { trc::event!( TaskQueue(TaskQueueEvent::BlobNotFound), - AccountId = task.account_id, - DocumentId = task.document_id, + AccountId = account_id, + DocumentId = document_id, BlobId = hash.as_slice(), ); return false; @@ -54,9 +58,9 @@ impl FtsIndexTask for Server { match self .get_archive_by_property( - task.account_id, + account_id, Collection::Email, - task.document_id, + document_id, EmailField::Metadata.into(), ) .await @@ -67,14 +71,14 @@ impl FtsIndexTask for Server { // Index message let document = FtsDocument::with_default_language(self.core.jmap.default_language) - .with_account_id(task.account_id) + .with_account_id(account_id) .with_collection(Collection::Email) - .with_document_id(task.document_id) + .with_document_id(document_id) .index_message(metadata, &raw_message); if let Err(err) = self.core.storage.fts.index(document).await { trc::error!( - err.account_id(task.account_id) - .document_id(task.document_id) + err.account_id(account_id) + .document_id(document_id) .details("Failed to index email in FTS index") ); @@ -83,16 +87,16 @@ impl FtsIndexTask for Server { trc::event!( MessageIngest(MessageIngestEvent::FtsIndex), - AccountId = task.account_id, + AccountId = account_id, Collection = Collection::Email, - DocumentId = task.document_id, + DocumentId = document_id, Elapsed = op_start.elapsed(), ); } Err(err) => { trc::error!( - err.account_id(task.account_id) - .document_id(task.document_id) + err.account_id(account_id) + .document_id(document_id) .details("Failed to unarchive email metadata") ); } @@ -102,8 +106,8 @@ impl FtsIndexTask for Server { trc::event!( TaskQueue(TaskQueueEvent::MetadataNotFound), Details = "E-mail blob hash mismatch", - AccountId = task.account_id, - DocumentId = task.document_id, + AccountId = account_id, + DocumentId = document_id, ); } } @@ -112,8 +116,8 @@ impl FtsIndexTask for Server { } Err(err) => { trc::error!( - err.account_id(task.account_id) - .document_id(task.document_id) + err.account_id(account_id) + .document_id(document_id) .caused_by(trc::location!()) .details("Failed to retrieve email metadata") ); @@ -125,8 +129,8 @@ impl FtsIndexTask for Server { trc::event!( TaskQueue(TaskQueueEvent::MetadataNotFound), Details = "E-mail metadata not found", - AccountId = task.account_id, - DocumentId = task.document_id, + AccountId = account_id, + DocumentId = document_id, ); true } diff --git a/crates/services/src/task_manager/imip.rs b/crates/services/src/task_manager/imip.rs index 39c7dda1..abb411a3 100644 --- a/crates/services/src/task_manager/imip.rs +++ b/crates/services/src/task_manager/imip.rs @@ -4,7 +4,6 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::task_manager::Task; use calcard::{ common::timezone::Tz, icalendar::{ @@ -46,19 +45,27 @@ use utils::template::{Variable, Variables}; pub trait SendImipTask: Sync + Send { fn send_imip( &self, - task: &Task, + account_id: u32, + document_id: u32, + due: u64, server_instance: Arc, ) -> impl Future + Send; } impl SendImipTask for Server { - async fn send_imip(&self, task: &Task, server_instance: Arc) -> bool { - match send_imip(self, task, server_instance).await { + async fn send_imip( + &self, + account_id: u32, + document_id: u32, + due: u64, + server_instance: Arc, + ) -> bool { + match send_imip(self, account_id, document_id, due, server_instance).await { Ok(result) => result, Err(err) => { trc::error!( - err.account_id(task.account_id) - .document_id(task.document_id) + err.account_id(account_id) + .document_id(document_id) .caused_by(trc::location!()) .details("Failed to process alarm") ); @@ -70,12 +77,14 @@ impl SendImipTask for Server { async fn send_imip( server: &Server, - task: &Task, + account_id: u32, + document_id: u32, + due: u64, server_instance: Arc, ) -> trc::Result { // Obtain access token let access_token = server - .get_access_token(task.account_id) + .get_access_token(account_id) .await .caused_by(trc::location!())?; @@ -83,11 +92,11 @@ async fn send_imip( let Some(archive) = server .store() .get_value::>(ValueKey { - account_id: task.account_id, + account_id, collection: 0, - document_id: task.document_id, + document_id, class: ValueClass::TaskQueue(TaskQueueClass::SendImip { - due: task.due, + due, is_payload: true, }), }) @@ -96,8 +105,8 @@ async fn send_imip( else { trc::event!( Calendar(trc::CalendarEvent::ItipMessageError), - AccountId = task.account_id, - DocumentId = task.document_id, + AccountId = account_id, + DocumentId = document_id, Reason = "Missing iMIP payload", ); return Ok(true); @@ -146,7 +155,8 @@ async fn send_imip( let tpl = build_itip_template( server, &access_token, - task, + account_id, + document_id, itip_message.from.as_str(), recipient.as_str(), &itip_message.summary, @@ -211,8 +221,6 @@ async fn send_imip( let access_token = access_token.clone(); let from = itip_message.from.to_string(); let to = recipient.to_string(); - let account_id = task.account_id; - let document_id = task.document_id; tokio::spawn(async move { let mut session = Session::::local( server_, @@ -301,10 +309,12 @@ pub struct Details { pub body: String, } +#[allow(clippy::too_many_arguments)] pub async fn build_itip_template( server: &Server, access_token: &AccessToken, - task: &Task, + account_id: u32, + document_id: u32, from: &str, to: &str, summary: &ArchivedItipSummary, @@ -513,9 +523,7 @@ pub async fn build_itip_template( if matches!( summary, ArchivedItipSummary::Invite(_) | ArchivedItipSummary::Update { .. } - ) && let Some(rsvp_url) = server - .http_rsvp_url(task.account_id, task.document_id, to) - .await + ) && let Some(rsvp_url) = server.http_rsvp_url(account_id, document_id, to).await { variables.insert_single( CalendarTemplateVariable::Rsvp, diff --git a/crates/services/src/task_manager/mod.rs b/crates/services/src/task_manager/mod.rs index 2ea8371c..d2cc145d 100644 --- a/crates/services/src/task_manager/mod.rs +++ b/crates/services/src/task_manager/mod.rs @@ -13,7 +13,7 @@ use common::listener::limiter::ConcurrencyLimiter; use common::listener::{ServerInstance, TcpAcceptor}; use common::{Inner, KV_LOCK_TASK, Server, core::BuildServer}; use fts::FtsIndexTask; -use groupware::calendar::alarm::CalendarAlarm; +use groupware::calendar::alarm::{CalendarAlarm, CalendarAlarmType}; use std::collections::hash_map::Entry; use std::future::Future; use std::time::Duration; @@ -95,21 +95,33 @@ pub fn spawn_task_manager(inner: Arc) { for mut rx_index in [rx_index_1, rx_index_2, rx_index_3, rx_index_4] { let inner = inner.clone(); let server_instance = server_instance.clone(); + let todo = "shard tasks based on config"; tokio::spawn(async move { - while let Some(task) = rx_index.recv().await { + while let Some(mut task) = rx_index.recv().await { let server = inner.build_server(); // Lock task if server.try_lock_task(&task).await { - let success = match &task.action { - TaskAction::Index { hash } => server.fts_index(&task, hash).await, + let success = match &mut task.action { + TaskAction::Index { hash } => { + server + .fts_index(task.account_id, task.document_id, hash) + .await + } TaskAction::BayesTrain { hash, learn_spam } => { - server.bayes_train(&task, hash, *learn_spam).await + server + .bayes_train(task.account_id, task.document_id, hash, *learn_spam) + .await } TaskAction::SendAlarm { alarm } => { if server.core.groupware.alarms_enabled { server - .send_alarm(&task, alarm, server_instance.clone()) + .send_alarm( + task.account_id, + task.document_id, + alarm, + server_instance.clone(), + ) .await } else { true @@ -117,7 +129,14 @@ pub fn spawn_task_manager(inner: Arc) { } TaskAction::SendImip => { if server.core.groupware.itip_enabled { - server.send_imip(&task, server_instance.clone()).await + server + .send_imip( + task.account_id, + task.document_id, + task.due, + server_instance.clone(), + ) + .await } else { true } @@ -206,9 +225,7 @@ impl TaskQueueManager for Server { let mut next_event = None; ipc.revision += 1; let _ = self - .core - .storage - .data + .store() .iterate( IterateParams::new(from_key, to_key).ascending(), |key, value| { @@ -386,6 +403,7 @@ impl Task { event_id: alarm.event_id, alarm_id: alarm.alarm_id, due: self.due, + is_email_alert: matches!(alarm.typ, CalendarAlarmType::Email { .. }), }, TaskAction::SendImip => TaskQueueClass::SendImip { due: self.due, @@ -418,18 +436,8 @@ impl Task { .and_then(|bytes| BlobHash::try_from_hash_slice(bytes).ok()) .ok_or_else(|| trc::Error::corrupted_key(key, None, trc::location!()))?, }, - Some(1) => TaskAction::BayesTrain { - learn_spam: true, - hash: key - .get( - U64_LEN + U32_LEN + U32_LEN + 1 - ..U64_LEN + U32_LEN + U32_LEN + BLOB_HASH_LEN + 1, - ) - .and_then(|bytes| BlobHash::try_from_hash_slice(bytes).ok()) - .ok_or_else(|| trc::Error::corrupted_key(key, None, trc::location!()))?, - }, - Some(2) => TaskAction::BayesTrain { - learn_spam: false, + Some(v @ (1 | 2)) => TaskAction::BayesTrain { + learn_spam: *v == 1, hash: key .get( U64_LEN + U32_LEN + U32_LEN + 1 @@ -443,13 +451,34 @@ impl Task { event_id: key.deserialize_be_u16(U64_LEN + U32_LEN + U32_LEN + 1)?, alarm_id: key .deserialize_be_u16(U64_LEN + U32_LEN + U32_LEN + U16_LEN + 1)?, - event_start: value.deserialize_be_u64(0)? as i64, - event_end: value.deserialize_be_u64(U64_LEN)? as i64, - event_start_tz: value.deserialize_be_u16(U64_LEN * 2)?, - event_end_tz: value.deserialize_be_u16((U64_LEN * 2) + U16_LEN)?, alarm_time: 0, + typ: CalendarAlarmType::Email { + event_start: value.deserialize_be_u64(0)? as i64, + event_end: value.deserialize_be_u64(U64_LEN)? as i64, + event_start_tz: value.deserialize_be_u16(U64_LEN * 2)?, + event_end_tz: value.deserialize_be_u16((U64_LEN * 2) + U16_LEN)?, + }, }, }, + Some(6) => { + let recurrence_id = value.deserialize_be_u64(0)? as i64; + + TaskAction::SendAlarm { + alarm: CalendarAlarm { + event_id: key.deserialize_be_u16(U64_LEN + U32_LEN + U32_LEN + 1)?, + alarm_id: key + .deserialize_be_u16(U64_LEN + U32_LEN + U32_LEN + U16_LEN + 1)?, + alarm_time: 0, + typ: CalendarAlarmType::Display { + recurrence_id: if recurrence_id != 0 { + Some(recurrence_id) + } else { + None + }, + }, + }, + } + } Some(4) => TaskAction::SendImip, _ => return Err(trc::Error::corrupted_key(key, None, trc::location!())), }, diff --git a/crates/store/src/write/batch.rs b/crates/store/src/write/batch.rs index 93cfb77a..0c20c367 100644 --- a/crates/store/src/write/batch.rs +++ b/crates/store/src/write/batch.rs @@ -379,6 +379,24 @@ impl BatchBuilder { self } + pub fn log_share_notification( + &mut self, + notification_id: u64, + notify_account_id: u32, + value: impl SerializeInfallible, + ) -> &mut Self { + self.changed_collections + .get_mut_or_insert(notify_account_id) + .share_notification_id = Some(notification_id); + self.set( + ValueClass::ShareNotification { + notification_id, + notify_account_id, + }, + value.serialize(), + ) + } + fn serialize_changes(&mut self) { if !self.changes.is_empty() { for (account_id, changelog) in std::mem::take(&mut self.changes) { diff --git a/crates/store/src/write/key.rs b/crates/store/src/write/key.rs index 1cc7fb18..15c3369e 100644 --- a/crates/store/src/write/key.rs +++ b/crates/store/src/write/key.rs @@ -302,10 +302,11 @@ impl ValueClass { due, event_id, alarm_id, + is_email_alert, } => serializer .write(*due) .write(account_id) - .write(3u8) + .write(if *is_email_alert { 3u8 } else { 6u8 }) .write(document_id) .write(*event_id) .write(*alarm_id), diff --git a/crates/store/src/write/mod.rs b/crates/store/src/write/mod.rs index c4dc1ae5..5f9dd211 100644 --- a/crates/store/src/write/mod.rs +++ b/crates/store/src/write/mod.rs @@ -119,6 +119,7 @@ pub struct BatchBuilder { pub struct ChangedCollection { pub changed_containers: Bitmap, pub changed_items: Bitmap, + pub share_notification_id: Option, } #[derive(Debug, PartialEq, Eq, Hash)] @@ -217,6 +218,7 @@ pub enum TaskQueueClass { due: u64, event_id: u16, alarm_id: u16, + is_email_alert: bool, }, SendImip { due: u64, diff --git a/crates/types/src/field.rs b/crates/types/src/field.rs index 87affd46..87bcc0a7 100644 --- a/crates/types/src/field.rs +++ b/crates/types/src/field.rs @@ -88,6 +88,7 @@ pub enum PrincipalField { DefaultCalendarId, DefaultAddressBookId, ActiveScriptId, + PushSubscriptions, } impl From for u8 { @@ -177,6 +178,7 @@ impl From for u8 { PrincipalField::DefaultCalendarId => 47, PrincipalField::DefaultAddressBookId => 48, PrincipalField::ActiveScriptId => 49, + PrincipalField::PushSubscriptions => 44, PrincipalField::Archive => ARCHIVE_FIELD, } } diff --git a/crates/types/src/type_state.rs b/crates/types/src/type_state.rs index bc49ba7f..b4f8b703 100644 --- a/crates/types/src/type_state.rs +++ b/crates/types/src/type_state.rs @@ -57,7 +57,9 @@ pub enum DataType { ShareNotification = 20, #[serde(rename = "ParticipantIdentity")] ParticipantIdentity = 21, - None = 22, + #[serde(rename = "CalendarAlert")] + CalendarAlert = 22, + None = 23, } #[derive(Debug, Clone, Copy)] @@ -68,10 +70,10 @@ pub struct StateChange { } impl StateChange { - pub fn new(account_id: u32, change_id: u64) -> Self { + pub fn new(account_id: u32) -> Self { Self { account_id, - change_id, + change_id: 0, types: Default::default(), } } @@ -85,6 +87,11 @@ impl StateChange { self } + pub fn with_change_id(mut self, change_id: u64) -> Self { + self.change_id = change_id; + self + } + pub fn has_changes(&self) -> bool { !self.types.is_empty() } @@ -125,6 +132,7 @@ impl From for DataType { 19 => DataType::Principal, 20 => DataType::ShareNotification, 21 => DataType::ParticipantIdentity, + 22 => DataType::CalendarAlert, _ => { debug_assert!(false, "Invalid type_state value: {}", value); DataType::None @@ -183,6 +191,7 @@ impl DataType { b"Principal" => DataType::Principal, b"ShareNotification" => DataType::ShareNotification, b"ParticipantIdentity" => DataType::ParticipantIdentity, + b"CalendarAlert" => DataType::CalendarAlert, ) } @@ -210,6 +219,7 @@ impl DataType { DataType::Principal => "Principal", DataType::ShareNotification => "ShareNotification", DataType::ParticipantIdentity => "ParticipantIdentity", + DataType::CalendarAlert => "CalendarAlert", DataType::None => "", } } diff --git a/tests/Cargo.toml b/tests/Cargo.toml index bf13d18e..49641da3 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -48,7 +48,7 @@ mail-send = { version = "0.5", default-features = false, features = ["cram-md5", mail-auth = { version = "0.7.1", features = ["test"] } sieve-rs = { version = "0.7", features = ["rkyv"] } utils = { path = "../crates/utils", features = ["test_mode"] } -jmap-client = { version = "0.3", features = ["websockets", "debug", "async"] } +jmap-client = { version = "0.4", features = ["websockets", "debug", "async"] } mail-parser = { version = "0.11", features = ["full_encoding", "rkyv"] } tokio = { version = "1.47", features = ["full"] } tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "tls12"] } diff --git a/tests/src/jmap/calendar/alarm.rs b/tests/src/jmap/calendar/alarm.rs new file mode 100644 index 00000000..c5323a66 --- /dev/null +++ b/tests/src/jmap/calendar/alarm.rs @@ -0,0 +1,171 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use futures::StreamExt; +use jmap_client::{ + CalendarAlert, PushObject, client_ws::WebSocketMessage, event_source::PushNotification, +}; +use jmap_proto::request::method::MethodObject; +use mail_parser::DateTime; +use serde_json::json; +use std::time::Instant; +use store::write::now; +use tokio::sync::mpsc; + +use crate::jmap::{IntoJmapSet, JMAPTest, JmapUtils}; + +pub async fn test(params: &mut JMAPTest) { + println!("Running Calendar Alarm tests..."); + let account = params.account("jdoe@example.com"); + let account_id = account.id_string(); + let client = account.client(); + let client_ws = account.client_owned().await; + + // Create test calendar + let response = account + .jmap_create( + MethodObject::Calendar, + [json!({ + "name": "Alarming Calendar", + })], + Vec::<(&str, &str)>::new(), + ) + .await; + let calendar_id = response.created(0).id().to_string(); + + // Connect to EventSource + let (event_tx, mut event_rx) = mpsc::channel::(100); + let mut notifications = client + .event_source(None::>, false, 1.into(), None) + .await + .unwrap(); + tokio::spawn(async move { + while let Some(notification) = notifications.next().await { + if let Err(_err) = event_tx.send(notification.unwrap()).await { + break; + } + } + }); + + // Connect to WebSocket + let mut ws_stream = client_ws.connect_ws().await.unwrap(); + let (stream_tx, mut stream_rx) = mpsc::channel::(100); + tokio::spawn(async move { + while let Some(change) = ws_stream.next().await { + stream_tx.send(change.unwrap()).await.unwrap(); + } + }); + client_ws + .enable_push_ws(None::>, None::<&str>) + .await + .unwrap(); + + // Create test event + let response = account + .jmap_create( + MethodObject::CalendarEvent, + [json!({ + "@type": "Event", + "calendarIds": ([calendar_id.as_str()].into_jmap_set()), + "description": "What mirror where?!", + "timeZone": "Etc/UTC", + "start": DateTime::from_timestamp(now() as i64 + 5) + .to_rfc3339().trim_end_matches("Z").to_string(), + "title": "See the pretty girl in that mirror there", + "alerts": { + "k1": { + "@type": "Alert", + "trigger": { + "@type": "OffsetTrigger", + "offset": "-PT2S" + }, + "action": "display" + }, + "k2": { + "trigger": { + "@type": "OffsetTrigger", + "offset": "-PT4S" + }, + "action": "display", + "@type": "Alert" + } + }, + "locations": { + "0b7168ae-ed3e-5eae-9540-89ba3a469b16": { + "name": "West Side", + "@type": "Location" + } + }, + "uid": "2371c2d9-a136-43b0-bba3-f6ab249ad46e", + "duration": "P1D" + })], + Vec::<(&str, &str)>::new(), + ) + .await; + let event_id = response.created(0).id().to_string(); + + // Wait for alarm notifications + let start = Instant::now(); + let mut ws_events = Vec::new(); + let mut es_events = Vec::new(); + + while start.elapsed().as_secs() < 7 && (ws_events.len() < 2 || es_events.len() < 2) { + tokio::select! { + Some(notification) = event_rx.recv() => { + if let PushNotification::CalendarAlert(alert) = notification { + es_events.push(alert); + } + } + Some(message) = stream_rx.recv() => { + match message { + WebSocketMessage::PushNotification(PushObject::CalendarAlert(alert)) => { + ws_events.push(alert); + } + WebSocketMessage::PushNotification(PushObject::Group {entries} ) => { + ws_events.extend(entries.into_iter().filter_map(|entry| { + if let PushObject::CalendarAlert(alert) = entry { + Some(alert) + } else { + None + } + })); + } + _ => {} + } + } + _ = tokio::time::sleep(std::time::Duration::from_secs(6)) => { + break; + } + } + } + + let expected_alerts = vec![ + CalendarAlert { + account_id: account_id.to_string(), + calendar_event_id: event_id.clone(), + uid: "2371c2d9-a136-43b0-bba3-f6ab249ad46e".to_string(), + recurrence_id: None, + alert_id: "k2".to_string(), + }, + CalendarAlert { + account_id: account_id.to_string(), + calendar_event_id: event_id.clone(), + uid: "2371c2d9-a136-43b0-bba3-f6ab249ad46e".to_string(), + recurrence_id: None, + alert_id: "k1".to_string(), + }, + ]; + + assert_eq!( + es_events, expected_alerts, + "EventSource alarms do not match" + ); + assert_eq!(ws_events, expected_alerts, "WebSocket alarms do not match"); + + // Cleanup + account.destroy_all_calendars().await; + params.assert_is_empty().await; +} diff --git a/tests/src/jmap/calendar/mod.rs b/tests/src/jmap/calendar/mod.rs index 62921ddd..58c48fde 100644 --- a/tests/src/jmap/calendar/mod.rs +++ b/tests/src/jmap/calendar/mod.rs @@ -5,6 +5,7 @@ */ pub mod acl; +pub mod alarm; pub mod calendars; pub mod event; pub mod identity; diff --git a/tests/src/jmap/core/event_source.rs b/tests/src/jmap/core/event_source.rs index ddb3403f..8f53e284 100644 --- a/tests/src/jmap/core/event_source.rs +++ b/tests/src/jmap/core/event_source.rs @@ -7,7 +7,11 @@ use crate::jmap::{JMAPTest, mail::delivery::SmtpConnection}; use email::mailbox::INBOX_ID; use futures::StreamExt; -use jmap_client::{TypeState, event_source::Changes, mailbox::Role}; +use jmap_client::{ + DataType, + event_source::{Changes, PushNotification}, + mailbox::Role, +}; use std::time::Duration; use store::ahash::AHashSet; use tokio::sync::mpsc; @@ -29,7 +33,13 @@ pub async fn test(params: &mut JMAPTest) { tokio::spawn(async move { while let Some(change) = changes.next().await { - if let Err(_err) = event_tx.send(change.unwrap()).await { + if let Err(_err) = event_tx + .send(match change.unwrap() { + PushNotification::StateChange(changes) => changes, + PushNotification::CalendarAlert(_) => unreachable!(), + }) + .await + { //println!("Error sending event: {}", _err); break; } @@ -44,7 +54,7 @@ pub async fn test(params: &mut JMAPTest) { .await .unwrap() .take_id(); - assert_state(&mut event_rx, account.id_string(), &[TypeState::Mailbox]).await; + assert_state(&mut event_rx, account.id_string(), &[DataType::Mailbox]).await; // Multiple changes should be grouped and delivered in intervals for num in 0..5 { @@ -53,7 +63,7 @@ pub async fn test(params: &mut JMAPTest) { .await .unwrap(); } - assert_state(&mut event_rx, account.id_string(), &[TypeState::Mailbox]).await; + assert_state(&mut event_rx, account.id_string(), &[DataType::Mailbox]).await; assert_ping(&mut event_rx).await; // Pings are only received in cfg(test) // Ingest email and expect state change @@ -77,10 +87,10 @@ pub async fn test(params: &mut JMAPTest) { &mut event_rx, account.id_string(), &[ - TypeState::EmailDelivery, - TypeState::Email, - TypeState::Thread, - TypeState::Mailbox, + DataType::EmailDelivery, + DataType::Email, + DataType::Thread, + DataType::Mailbox, ], ) .await; @@ -88,7 +98,7 @@ pub async fn test(params: &mut JMAPTest) { // Destroy mailbox client.mailbox_destroy(&mailbox_id, true).await.unwrap(); - assert_state(&mut event_rx, account.id_string(), &[TypeState::Mailbox]).await; + assert_state(&mut event_rx, account.id_string(), &[DataType::Mailbox]).await; // Destroy Inbox client @@ -98,7 +108,7 @@ pub async fn test(params: &mut JMAPTest) { assert_state( &mut event_rx, account.id_string(), - &[TypeState::Email, TypeState::Thread, TypeState::Mailbox], + &[DataType::Email, DataType::Thread, DataType::Mailbox], ) .await; assert_ping(&mut event_rx).await; @@ -111,7 +121,7 @@ pub async fn test(params: &mut JMAPTest) { async fn assert_state( event_rx: &mut mpsc::Receiver, account_id: &str, - state: &[TypeState], + state: &[DataType], ) { match tokio::time::timeout(Duration::from_millis(700), event_rx.recv()).await { Ok(Some(changes)) => { @@ -120,8 +130,8 @@ async fn assert_state( .changes(account_id) .unwrap() .map(|x| x.0) - .collect::>(), - state.iter().collect::>() + .collect::>(), + state.iter().collect::>() ); } result => { diff --git a/tests/src/jmap/core/push_subscription.rs b/tests/src/jmap/core/push_subscription.rs index 7a4c9c25..0ce3a565 100644 --- a/tests/src/jmap/core/push_subscription.rs +++ b/tests/src/jmap/core/push_subscription.rs @@ -12,7 +12,7 @@ use http_proto::{HtmlResponse, ToHttpResponse, request::fetch_body}; use hyper::{StatusCode, body, header::CONTENT_ENCODING, server::conn::http1, service::service_fn}; use hyper_util::rt::TokioIo; use jmap_client::{mailbox::Role, push_subscription::Keys}; -use jmap_proto::response::status::StateChangeResponse; +use jmap_proto::{response::status::PushObject, types::state::State}; use services::state_manager::ece::ece_encrypt; use std::{ sync::{ @@ -24,7 +24,7 @@ use std::{ use store::ahash::AHashSet; use tokio::sync::mpsc; use types::{id::Id, type_state::DataType}; -use utils::config::Config; +use utils::{config::Config, map::vec_map::VecMap}; const SERVER: &str = r#" [server] @@ -125,7 +125,7 @@ pub async fn test(params: &mut JMAPTest) { // Receive states just for the requested types client - .push_subscription_update_types(&push_id, [jmap_client::TypeState::Email].into()) + .push_subscription_update_types(&push_id, [jmap_client::DataType::Email].into()) .await .unwrap(); client @@ -224,15 +224,15 @@ pub struct PushServer { #[derive(serde::Deserialize, Debug)] #[serde(untagged)] enum PushMessage { - StateChange(StateChangeResponse), + PushObject(PushObject), Verification(PushVerification), } impl PushMessage { - pub fn unwrap_state_change(self) -> StateChangeResponse { + pub fn unwrap_state_change(self) -> VecMap> { match self { - PushMessage::StateChange(state_change) => state_change, - _ => panic!("Expected StateChange"), + PushMessage::PushObject(PushObject::StateChange { changed }) => changed, + _ => panic!("Expected PushObject"), } } @@ -348,7 +348,6 @@ async fn assert_state(event_rx: &mut mpsc::Receiver, id: &Id, state expect_push(event_rx) .await .unwrap_state_change() - .changed .get(id) .unwrap() .iter() diff --git a/tests/src/jmap/core/websocket.rs b/tests/src/jmap/core/websocket.rs index 4af0a5c7..9d0f8a57 100644 --- a/tests/src/jmap/core/websocket.rs +++ b/tests/src/jmap/core/websocket.rs @@ -8,7 +8,7 @@ use crate::jmap::JMAPTest; use ahash::AHashSet; use futures::StreamExt; use jmap_client::{ - TypeState, + DataType, PushObject, client_ws::WebSocketMessage, core::{ response::{Response, TaggedMethodResponse}, @@ -66,7 +66,7 @@ pub async fn test(params: &mut JMAPTest) { .mailbox_update_sort_order(&mailbox_id, 1) .await .unwrap(); - assert_state(&mut stream_rx, account.id_string(), &[TypeState::Mailbox]).await; + assert_state(&mut stream_rx, account.id_string(), &[DataType::Mailbox]).await; // Multiple changes should be grouped and delivered in intervals for num in 0..5 { @@ -76,7 +76,7 @@ pub async fn test(params: &mut JMAPTest) { .unwrap(); } tokio::time::sleep(Duration::from_millis(500)).await; - assert_state(&mut stream_rx, account.id_string(), &[TypeState::Mailbox]).await; + assert_state(&mut stream_rx, account.id_string(), &[DataType::Mailbox]).await; expect_nothing(&mut stream_rx).await; // Disable push notifications @@ -117,18 +117,18 @@ async fn expect_response( async fn assert_state( stream_rx: &mut mpsc::Receiver, id: &str, - state: &[TypeState], + state: &[DataType], ) { match tokio::time::timeout(Duration::from_millis(700), stream_rx.recv()).await { Ok(Some(message)) => match message { - WebSocketMessage::StateChange(changes) => { + WebSocketMessage::PushNotification(PushObject::StateChange { changed }) => { assert_eq!( - changes - .changes(id) + changed + .get(id) .unwrap() - .map(|x| x.0) - .collect::>(), - state.iter().collect::>() + .keys() + .collect::>(), + state.iter().collect::>() ); } _ => panic!("Expected state change, got: {:?}", message), diff --git a/tests/src/jmap/mod.rs b/tests/src/jmap/mod.rs index 3ccd91fc..dd6d478a 100644 --- a/tests/src/jmap/mod.rs +++ b/tests/src/jmap/mod.rs @@ -78,11 +78,12 @@ async fn jmap_tests() { ) .await; - /*server::webhooks::test(&mut params).await; - mail::query::test(&mut params, delete).await; + server::webhooks::test(&mut params).await; + mail::get::test(&mut params).await; mail::set::test(&mut params).await; mail::parse::test(&mut params).await; + mail::query::test(&mut params, delete).await; mail::search_snippet::test(&mut params).await; mail::changes::test(&mut params).await; mail::query_changes::test(&mut params).await; @@ -92,20 +93,20 @@ async fn jmap_tests() { mail::mailbox::test(&mut params).await; mail::delivery::test(&mut params).await; mail::acl::test(&mut params).await; - auth::limits::test(&mut params).await; - auth::oauth::test(&mut params).await; - core::event_source::test(&mut params).await; - core::push_subscription::test(&mut params).await; mail::sieve_script::test(&mut params).await; mail::vacation_response::test(&mut params).await; mail::submission::test(&mut params).await; - core::websocket::test(&mut params).await; - auth::quota::test(&mut params).await; mail::crypto::test(&mut params).await; + + core::event_source::test(&mut params).await; + core::websocket::test(&mut params).await; + core::push_subscription::test(&mut params).await; core::blob::test(&mut params).await; + + auth::limits::test(&mut params).await; + auth::oauth::test(&mut params).await; + auth::quota::test(&mut params).await; auth::permissions::test(¶ms).await; - server::purge::test(&mut params).await; - server::enterprise::test(&mut params).await;*/ contacts::addressbook::test(&mut params).await; contacts::contact::test(&mut params).await; @@ -117,12 +118,17 @@ async fn jmap_tests() { calendar::calendars::test(&mut params).await; calendar::event::test(&mut params).await; calendar::notification::test(&mut params).await; + calendar::alarm::test(&mut params).await; + calendar::identity::test(&mut params).await; calendar::acl::test(&mut params).await; principal::get::test(&mut params).await; principal::availability::test(&mut params).await; + server::purge::test(&mut params).await; + server::enterprise::test(&mut params).await; + if delete { params.temp_dir.delete(); } @@ -1691,6 +1697,9 @@ enable = true [sharing] allow-directory-query = true +[calendar.alarms] +minimum-interval = "1s" + [tracer.console] type = "console" level = "{LEVEL}" diff --git a/tests/src/webdav/cal_alarm.rs b/tests/src/webdav/cal_alarm.rs index 8f9ae1e7..43203f64 100644 --- a/tests/src/webdav/cal_alarm.rs +++ b/tests/src/webdav/cal_alarm.rs @@ -4,9 +4,8 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::jmap::mail::mailbox::destroy_all_mailboxes_for_account; - use super::WebDavTest; +use crate::jmap::mail::mailbox::destroy_all_mailboxes_for_account; use email::cache::MessageCacheFetch; use hyper::StatusCode; use mail_parser::{DateTime, MessageParser}; @@ -45,8 +44,6 @@ pub async fn test(test: &WebDavTest) { .fetch_email(client.account_id, message.document_id) .await; - //let t = std::fs::write(format!("message_{}.eml", message.document_id), &contents).unwrap(); - let message = MessageParser::new().parse(&contents).unwrap(); let contents = message .html_bodies() diff --git a/tests/src/webdav/cal_scheduling.rs b/tests/src/webdav/cal_scheduling.rs index 5f360d6f..142f5715 100644 --- a/tests/src/webdav/cal_scheduling.rs +++ b/tests/src/webdav/cal_scheduling.rs @@ -27,7 +27,7 @@ use groupware::{ }; use hyper::StatusCode; use mail_parser::{DateTime, MessageParser}; -use services::task_manager::{Task, TaskAction, imip::build_itip_template}; +use services::task_manager::imip::build_itip_template; use std::str::FromStr; use store::write::now; use types::collection::SyncCollection; @@ -594,12 +594,6 @@ async fn fetch_icals(client: &DummyWebDavClient) -> Vec { pub async fn test_build_itip_templates(server: &Server) { let dummy_access_token = AccessToken::from_id(0); - let dummy_task = Task { - account_id: 123, - document_id: 156, - due: 0, - action: TaskAction::SendImip, - }; for (idx, summary) in [ ItipSummary::Invite(vec![ @@ -796,7 +790,8 @@ pub async fn test_build_itip_templates(server: &Server) { let html = build_itip_template( server, &dummy_access_token, - &dummy_task, + 0, + 1, "john.doe@example.org", "jane.smith@example.net", summary,