From 2eb388674df2ea0d8b6fc848e705db2681d67630 Mon Sep 17 00:00:00 2001 From: mdecimus Date: Fri, 17 Jan 2025 15:29:55 +0100 Subject: [PATCH] Refactored local delivery to avoid mpsc channel --- Cargo.lock | 70 +- Cargo.toml | 1 + crates/common/src/config/smtp/queue.rs | 6 - crates/common/src/core.rs | 423 ++++++++++- crates/common/src/ipc.rs | 36 +- crates/common/src/lib.rs | 8 +- crates/common/src/manager/boot.rs | 18 +- crates/email/Cargo.toml | 39 + crates/{jmap/src/email => email/src}/cache.rs | 2 - crates/email/src/crypto.rs | 670 +++++++++++++++++ .../ingest.rs => email/src/delivery.rs} | 207 ++++-- crates/{jmap/src/email => email/src}/index.rs | 0 .../{jmap/src/email => email/src}/ingest.rs | 25 +- crates/email/src/lib.rs | 14 + crates/email/src/mailbox.rs | 495 +++++++++++++ .../{jmap/src/email => email/src}/metadata.rs | 0 .../sieve/ingest.rs => email/src/sieve.rs} | 386 +++++++++- crates/imap/Cargo.toml | 1 + crates/imap/src/core/mailbox.rs | 3 +- crates/imap/src/core/message.rs | 2 +- crates/imap/src/op/acl.rs | 9 +- crates/imap/src/op/append.rs | 8 +- crates/imap/src/op/copy_move.rs | 16 +- crates/imap/src/op/create.rs | 12 +- crates/imap/src/op/delete.rs | 2 +- crates/imap/src/op/expunge.rs | 19 +- crates/imap/src/op/fetch.rs | 19 +- crates/imap/src/op/rename.rs | 13 +- crates/imap/src/op/status.rs | 1 - crates/imap/src/op/store.rs | 16 +- crates/imap/src/op/subscribe.rs | 12 +- crates/imap/src/op/thread.rs | 2 +- crates/jmap/Cargo.toml | 20 +- crates/jmap/src/api/form.rs | 16 +- .../src/api/management/enterprise/undelete.rs | 7 +- crates/jmap/src/api/management/stores.rs | 13 +- crates/jmap/src/api/request.rs | 1 - crates/jmap/src/auth/acl.rs | 2 - crates/jmap/src/blob/copy.rs | 8 +- crates/jmap/src/blob/download.rs | 1 + crates/jmap/src/blob/get.rs | 2 +- crates/jmap/src/blob/upload.rs | 67 +- crates/jmap/src/changes/mod.rs | 1 - crates/jmap/src/changes/write.rs | 112 --- crates/jmap/src/email/bayes.rs | 16 +- crates/jmap/src/email/body.rs | 6 +- crates/jmap/src/email/copy.rs | 23 +- crates/jmap/src/email/crypto.rs | 677 +----------------- crates/jmap/src/email/delete.rs | 13 +- crates/jmap/src/email/get.rs | 9 +- crates/jmap/src/email/import.rs | 10 +- crates/jmap/src/email/mod.rs | 4 - crates/jmap/src/email/parse.rs | 2 +- crates/jmap/src/email/query.rs | 3 +- crates/jmap/src/email/set.rs | 16 +- crates/jmap/src/email/snippet.rs | 4 +- crates/jmap/src/identity/get.rs | 2 +- crates/jmap/src/identity/set.rs | 19 +- crates/jmap/src/lib.rs | 326 +-------- crates/jmap/src/mailbox/get.rs | 218 +----- crates/jmap/src/mailbox/mod.rs | 78 -- crates/jmap/src/mailbox/query.rs | 3 +- crates/jmap/src/mailbox/set.rs | 185 +---- crates/jmap/src/principal/get.rs | 2 - crates/jmap/src/push/get.rs | 2 - crates/jmap/src/push/set.rs | 19 +- crates/jmap/src/quota/get.rs | 2 - crates/jmap/src/services/delivery.rs | 57 -- crates/jmap/src/services/gossip/ping.rs | 2 - crates/jmap/src/services/index.rs | 13 +- crates/jmap/src/services/mod.rs | 2 - crates/jmap/src/services/state.rs | 27 - crates/jmap/src/sieve/get.rs | 206 +----- crates/jmap/src/sieve/mod.rs | 130 ---- crates/jmap/src/sieve/set.rs | 19 +- crates/jmap/src/submission/get.rs | 2 +- crates/jmap/src/submission/set.rs | 23 +- crates/jmap/src/thread/get.rs | 2 +- crates/jmap/src/vacation/set.rs | 16 +- crates/jmap/src/websocket/stream.rs | 9 +- crates/managesieve/src/op/deletescript.rs | 2 +- crates/managesieve/src/op/getscript.rs | 2 +- crates/managesieve/src/op/havespace.rs | 1 - crates/managesieve/src/op/listscripts.rs | 1 - crates/managesieve/src/op/putscript.rs | 7 +- crates/managesieve/src/op/renamescript.rs | 5 +- crates/managesieve/src/op/setactive.rs | 2 +- crates/pop3/Cargo.toml | 1 + crates/pop3/src/mailbox.rs | 5 +- crates/pop3/src/op/delete.rs | 4 +- crates/pop3/src/op/fetch.rs | 3 +- crates/smtp/Cargo.toml | 1 + crates/smtp/src/core/mod.rs | 35 +- crates/smtp/src/inbound/spawn.rs | 6 - crates/smtp/src/outbound/delivery.rs | 2 +- crates/smtp/src/outbound/local.rs | 124 ++-- crates/store/src/dispatch/store.rs | 16 +- tests/Cargo.toml | 1 + tests/src/imap/mod.rs | 2 +- tests/src/jmap/auth_acl.rs | 2 +- tests/src/jmap/blob.rs | 2 +- tests/src/jmap/crypto.rs | 2 +- tests/src/jmap/delivery.rs | 5 +- tests/src/jmap/email_get.rs | 2 +- tests/src/jmap/email_query.rs | 2 +- tests/src/jmap/email_query_changes.rs | 3 +- tests/src/jmap/email_search_snippet.rs | 3 +- tests/src/jmap/email_set.rs | 3 +- tests/src/jmap/event_source.rs | 3 +- tests/src/jmap/mod.rs | 6 +- tests/src/jmap/permissions.rs | 22 +- tests/src/jmap/purge.rs | 7 +- tests/src/jmap/quota.rs | 3 +- tests/src/jmap/stress_test.rs | 2 +- tests/src/jmap/thread_merge.rs | 6 +- tests/src/smtp/mod.rs | 4 +- tests/src/smtp/queue/concurrent.rs | 10 +- 117 files changed, 2611 insertions(+), 2628 deletions(-) create mode 100644 crates/email/Cargo.toml rename crates/{jmap/src/email => email/src}/cache.rs (98%) create mode 100644 crates/email/src/crypto.rs rename crates/{jmap/src/services/ingest.rs => email/src/delivery.rs} (58%) rename crates/{jmap/src/email => email/src}/index.rs (100%) rename crates/{jmap/src/email => email/src}/ingest.rs (98%) create mode 100644 crates/email/src/lib.rs create mode 100644 crates/email/src/mailbox.rs rename crates/{jmap/src/email => email/src}/metadata.rs (100%) rename crates/{jmap/src/sieve/ingest.rs => email/src/sieve.rs} (65%) delete mode 100644 crates/jmap/src/changes/write.rs delete mode 100644 crates/jmap/src/services/delivery.rs diff --git a/Cargo.lock b/Cargo.lock index 6b72abb9..80fa4c7c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2058,6 +2058,37 @@ dependencies = [ "zeroize", ] +[[package]] +name = "email" +version = "0.11.2" +dependencies = [ + "aes", + "aes-gcm", + "aes-gcm-siv", + "bincode", + "cbc", + "common", + "directory", + "jmap_proto", + "mail-builder", + "mail-parser", + "nlp", + "rasn", + "rasn-cms", + "rasn-pkix", + "rsa", + "sequoia-openpgp", + "serde", + "serde_json", + "sieve-rs", + "smtp-proto", + "spam-filter", + "store", + "tokio", + "trc", + "utils", +] + [[package]] name = "ena" version = "0.14.3" @@ -3225,6 +3256,7 @@ dependencies = [ "common", "dashmap", "directory", + "email", "imap_proto", "jmap", "jmap_proto", @@ -3471,18 +3503,16 @@ dependencies = [ name = "jmap" version = "0.11.2" dependencies = [ - "aes", "aes-gcm", "aes-gcm-siv", "async-stream", - "async-trait", "base64 0.22.1", "bincode", - "cbc", "chrono", "common", "dashmap", "directory", + "email", "form-data", "form_urlencoded", "futures-util", @@ -3501,15 +3531,11 @@ dependencies = [ "nlp", "p256", "pkcs8", - "quick-xml 0.36.2", + "quick-xml 0.37.1", "rand 0.8.5", - "rasn", - "rasn-cms", - "rasn-pkix", "reqwest 0.12.9", "rev_lines", "rsa", - "sequoia-openpgp", "serde", "serde_json", "sha1", @@ -3520,9 +3546,9 @@ dependencies = [ "spam-filter", "store", "tokio", - "tokio-tungstenite 0.24.0", + "tokio-tungstenite 0.26.1", "trc", - "tungstenite 0.24.0", + "tungstenite 0.26.1", "utils", "x509-parser 0.16.0", ] @@ -4786,6 +4812,7 @@ version = "0.11.2" dependencies = [ "common", "directory", + "email", "imap", "jmap", "jmap_proto", @@ -5070,15 +5097,6 @@ dependencies = [ "serde", ] -[[package]] -name = "quick-xml" -version = "0.36.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7649a7b4df05aed9ea7ec6f628c67c9953a43869b8bc50929569b2999d443fe" -dependencies = [ - "memchr", -] - [[package]] name = "quick-xml" version = "0.37.1" @@ -6443,6 +6461,7 @@ dependencies = [ "common", "dashmap", "directory", + "email", "form_urlencoded", "http-body-util", "hyper 1.5.2", @@ -6830,6 +6849,7 @@ dependencies = [ "dashmap", "directory", "ece", + "email", "flate2", "form_urlencoded", "futures", @@ -7091,14 +7111,14 @@ dependencies = [ [[package]] name = "tokio-tungstenite" -version = "0.24.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9" +checksum = "be4bf6fecd69fcdede0ec680aaf474cdab988f9de6bc73d3758f0160e3b7025a" dependencies = [ "futures-util", "log", "tokio", - "tungstenite 0.24.0", + "tungstenite 0.26.1", ] [[package]] @@ -7312,9 +7332,9 @@ dependencies = [ [[package]] name = "tungstenite" -version = "0.24.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a" +checksum = "413083a99c579593656008130e29255e54dcaae495be556cc26888f211648c24" dependencies = [ "byteorder", "bytes", @@ -7324,7 +7344,7 @@ dependencies = [ "log", "rand 0.8.5", "sha1", - "thiserror 1.0.69", + "thiserror 2.0.9", "utf-8", ] diff --git a/Cargo.toml b/Cargo.toml index 9d0d8357..f055474f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ members = [ "crates/main", "crates/jmap", "crates/jmap-proto", + "crates/email", "crates/imap", "crates/imap-proto", "crates/smtp", diff --git a/crates/common/src/config/smtp/queue.rs b/crates/common/src/config/smtp/queue.rs index 6764084c..17c37a29 100644 --- a/crates/common/src/config/smtp/queue.rs +++ b/crates/common/src/config/smtp/queue.rs @@ -85,7 +85,6 @@ pub struct QueueOutboundTimeout { #[derive(Debug, Clone)] pub struct QueueThrottle { pub outbound_concurrency: usize, - pub local_concurrency: usize, pub sender: Vec, pub rcpt: Vec, pub host: Vec, @@ -204,7 +203,6 @@ impl Default for QueueConfig { }, throttle: QueueThrottle { outbound_concurrency: 25, - local_concurrency: 10, sender: Default::default(), rcpt: Default::default(), host: Default::default(), @@ -392,10 +390,6 @@ fn parse_queue_throttle(config: &mut Config) -> QueueThrottle { .property_or_default::("queue.threads.remote", "25") .unwrap_or(25) .max(1), - local_concurrency: config - .property_or_default::("queue.threads.local", "10") - .unwrap_or(10) - .max(1), }; let all_throttles = parse_throttle( diff --git a/crates/common/src/core.rs b/crates/common/src/core.rs index 01608417..5ef3dab4 100644 --- a/crates/common/src/core.rs +++ b/crates/common/src/core.rs @@ -4,21 +4,33 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::sync::Arc; +use std::{sync::Arc, time::Duration}; -use directory::{backend::internal::manage::ManageDirectory, Directory, Type}; +use directory::{backend::internal::manage::ManageDirectory, Directory, QueryBy, Type}; +use jmap_proto::types::{ + blob::BlobId, collection::Collection, property::Property, state::StateChange, +}; use sieve::Sieve; use store::{ - write::{QueueClass, ValueClass}, - BlobStore, FtsStore, InMemoryStore, IterateParams, Store, ValueKey, + dispatch::DocumentSet, + roaring::RoaringBitmap, + write::{ + key::DeserializeBigEndian, log::ChangeLogBuilder, now, BatchBuilder, BitmapClass, BlobOp, + DirectoryClass, QueueClass, TagValue, ValueClass, + }, + BitmapKey, BlobClass, BlobStore, Deserialize, FtsStore, InMemoryStore, IterateParams, LogKey, + Serialize, Store, ValueKey, U32_LEN, }; use trc::AddContext; +use utils::BlobHash; use crate::{ + auth::{AccessToken, ResourceToken, TenantInfo}, config::smtp::{ auth::{ArcSealer, DkimSigner}, queue::RelayHost, }, + ipc::StateEvent, ImapId, Inner, MailboxState, Server, }; @@ -170,6 +182,249 @@ impl Server { }) } + pub async fn get_used_quota(&self, account_id: u32) -> trc::Result { + self.core + .storage + .data + .get_counter(DirectoryClass::UsedQuota(account_id)) + .await + .add_context(|err| err.caused_by(trc::location!()).account_id(account_id)) + } + + pub async fn has_available_quota( + &self, + quotas: &ResourceToken, + item_size: u64, + ) -> trc::Result<()> { + if quotas.quota != 0 { + let used_quota = self.get_used_quota(quotas.account_id).await? as u64; + + if used_quota + item_size > quotas.quota { + return Err(trc::LimitEvent::Quota + .into_err() + .ctx(trc::Key::Limit, quotas.quota) + .ctx(trc::Key::Size, used_quota)); + } + } + + // SPDX-SnippetBegin + // SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + // SPDX-License-Identifier: LicenseRef-SEL + + #[cfg(feature = "enterprise")] + if self.core.is_enterprise_edition() { + if let Some(tenant) = quotas.tenant.filter(|tenant| tenant.quota != 0) { + let used_quota = self.get_used_quota(tenant.id).await? as u64; + + if used_quota + item_size > tenant.quota { + return Err(trc::LimitEvent::TenantQuota + .into_err() + .ctx(trc::Key::Limit, tenant.quota) + .ctx(trc::Key::Size, used_quota)); + } + } + } + + // SPDX-SnippetEnd + + Ok(()) + } + + pub async fn get_resource_token( + &self, + access_token: &AccessToken, + account_id: u32, + ) -> trc::Result { + Ok(if access_token.primary_id == account_id { + ResourceToken { + account_id, + quota: access_token.quota, + tenant: access_token.tenant, + } + } else { + let mut quotas = ResourceToken { + account_id, + ..Default::default() + }; + + if let Some(principal) = self + .core + .storage + .directory + .query(QueryBy::Id(account_id), false) + .await + .add_context(|err| err.caused_by(trc::location!()).account_id(account_id))? + { + quotas.quota = principal.quota(); + + // SPDX-SnippetBegin + // SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + // SPDX-License-Identifier: LicenseRef-SEL + + #[cfg(feature = "enterprise")] + if self.core.is_enterprise_edition() { + if let Some(tenant_id) = principal.tenant() { + quotas.tenant = TenantInfo { + id: tenant_id, + quota: self + .core + .storage + .directory + .query(QueryBy::Id(tenant_id), false) + .await + .add_context(|err| { + err.caused_by(trc::location!()).account_id(tenant_id) + })? + .map(|tenant| tenant.quota()) + .unwrap_or_default(), + } + .into(); + } + } + + // SPDX-SnippetEnd + } + + quotas + }) + } + + pub async fn get_property( + &self, + account_id: u32, + collection: Collection, + document_id: u32, + property: impl AsRef + Sync + Send, + ) -> trc::Result> + where + U: Deserialize + 'static, + { + let property = property.as_ref(); + + self.core + .storage + .data + .get_value::(ValueKey { + account_id, + collection: collection.into(), + document_id, + class: ValueClass::Property(property.into()), + }) + .await + .add_context(|err| { + err.caused_by(trc::location!()) + .account_id(account_id) + .collection(collection) + .document_id(document_id) + .id(property.to_string()) + }) + } + + pub async fn get_properties( + &self, + account_id: u32, + collection: Collection, + iterate: &I, + property: P, + ) -> trc::Result> + where + I: DocumentSet + Send + Sync, + P: AsRef + Sync + Send, + U: Deserialize + 'static, + { + let property: u8 = property.as_ref().into(); + let collection: u8 = collection.into(); + let expected_results = iterate.len(); + let mut results = Vec::with_capacity(expected_results); + + self.core + .storage + .data + .iterate( + IterateParams::new( + ValueKey { + account_id, + collection, + document_id: iterate.min(), + class: ValueClass::Property(property), + }, + ValueKey { + account_id, + collection, + document_id: iterate.max(), + class: ValueClass::Property(property), + }, + ), + |key, value| { + let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; + if iterate.contains(document_id) { + results.push((document_id, U::deserialize(value)?)); + Ok(expected_results == 0 || results.len() < expected_results) + } else { + Ok(true) + } + }, + ) + .await + .add_context(|err| { + err.caused_by(trc::location!()) + .account_id(account_id) + .collection(collection) + .id(property.to_string()) + }) + .map(|_| results) + } + + pub async fn get_document_ids( + &self, + account_id: u32, + collection: Collection, + ) -> trc::Result> { + self.core + .storage + .data + .get_bitmap(BitmapKey::document_ids(account_id, collection)) + .await + .add_context(|err| { + err.caused_by(trc::location!()) + .account_id(account_id) + .collection(collection) + }) + } + + pub async fn get_tag( + &self, + account_id: u32, + collection: Collection, + property: impl AsRef + Sync + Send, + value: impl Into> + Sync + Send, + ) -> trc::Result> { + let property = property.as_ref(); + self.core + .storage + .data + .get_bitmap(BitmapKey { + account_id, + collection: collection.into(), + class: BitmapClass::Tag { + field: property.into(), + value: value.into(), + }, + document_id: 0, + }) + .await + .add_context(|err| { + err.caused_by(trc::location!()) + .account_id(account_id) + .collection(collection) + .id(property.to_string()) + }) + } + + pub fn notify_task_queue(&self) { + self.inner.ipc.index_tx.notify_one(); + } + pub async fn total_queued_messages(&self) -> trc::Result { let mut total = 0; self.store() @@ -190,6 +445,166 @@ impl Server { .map(|_| total) } + pub fn begin_changes(&self, account_id: u32) -> trc::Result { + self.assign_change_id(account_id) + .map(ChangeLogBuilder::with_change_id) + } + + #[inline(always)] + pub fn assign_change_id(&self, _: u32) -> trc::Result { + self.generate_snowflake_id() + } + + pub fn generate_snowflake_id(&self) -> trc::Result { + self.inner.data.jmap_id_gen.generate().ok_or_else(|| { + trc::StoreEvent::UnexpectedError + .into_err() + .caused_by(trc::location!()) + .ctx(trc::Key::Reason, "Failed to generate snowflake id.") + }) + } + + pub async fn commit_changes( + &self, + account_id: u32, + mut changes: ChangeLogBuilder, + ) -> trc::Result { + if changes.change_id == u64::MAX || changes.change_id == 0 { + changes.change_id = self.assign_change_id(account_id)?; + } + let state = changes.change_id; + + let mut builder = BatchBuilder::new(); + builder.with_account_id(account_id).custom(changes); + self.core + .storage + .data + .write(builder.build()) + .await + .caused_by(trc::location!()) + .map(|_| state) + } + + pub async fn delete_changes(&self, account_id: u32, before: Duration) -> trc::Result<()> { + let reference_cid = self.inner.data.jmap_id_gen.past_id(before).ok_or_else(|| { + trc::StoreEvent::UnexpectedError + .caused_by(trc::location!()) + .ctx(trc::Key::Reason, "Failed to generate reference change id.") + })?; + + for collection in [ + Collection::Email, + Collection::Mailbox, + Collection::Thread, + Collection::Identity, + Collection::EmailSubmission, + ] { + self.core + .storage + .data + .delete_range( + LogKey { + account_id, + collection: collection.into(), + change_id: 0, + }, + LogKey { + account_id, + collection: collection.into(), + change_id: reference_cid, + }, + ) + .await?; + } + + Ok(()) + } + + pub async fn broadcast_state_change(&self, state_change: StateChange) -> bool { + match self + .inner + .ipc + .state_tx + .clone() + .send(StateEvent::Publish { state_change }) + .await + { + Ok(_) => true, + Err(_) => { + trc::event!( + Server(trc::ServerEvent::ThreadError), + Details = "Error sending state change.", + CausedBy = trc::location!() + ); + + false + } + } + } + + #[allow(clippy::blocks_in_conditions)] + pub async fn put_blob( + &self, + account_id: u32, + data: &[u8], + set_quota: bool, + ) -> trc::Result { + // First reserve the hash + let hash = BlobHash::from(data); + let mut batch = BatchBuilder::new(); + let until = now() + self.core.jmap.upload_tmp_ttl; + + batch.with_account_id(account_id).set( + BlobOp::Reserve { + hash: hash.clone(), + until, + }, + (if set_quota { data.len() as u32 } else { 0u32 }).serialize(), + ); + self.core + .storage + .data + .write(batch.build()) + .await + .caused_by(trc::location!())?; + + if !self + .core + .storage + .data + .blob_exists(&hash) + .await + .caused_by(trc::location!())? + { + // Upload blob to store + self.core + .storage + .blob + .put_blob(hash.as_ref(), data) + .await + .caused_by(trc::location!())?; + + // Commit blob + let mut batch = BatchBuilder::new(); + batch.set(BlobOp::Commit { hash: hash.clone() }, Vec::new()); + self.core + .storage + .data + .write(batch.build()) + .await + .caused_by(trc::location!())?; + } + + Ok(BlobId { + hash, + class: BlobClass::Reserved { + account_id, + expires: until, + }, + section: None, + }) + } + pub async fn total_accounts(&self) -> trc::Result { self.store() .count_principals(None, Type::Individual.into(), None) diff --git a/crates/common/src/ipc.rs b/crates/common/src/ipc.rs index c0980d8a..7abb3ef8 100644 --- a/crates/common/src/ipc.rs +++ b/crates/common/src/ipc.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{borrow::Cow, sync::Arc, time::Instant}; +use std::{sync::Arc, time::Instant}; use ahash::RandomState; use jmap_proto::types::{state::StateChange, type_state::DataType}; @@ -14,8 +14,8 @@ use mail_auth::{ report::{tlsrpt::FailureDetails, Record}, }; use store::{BlobStore, InMemoryStore, Store}; -use tokio::sync::{mpsc, oneshot}; -use utils::{map::bitmap::Bitmap, BlobHash}; +use tokio::sync::mpsc; +use utils::map::bitmap::Bitmap; use crate::{ config::smtp::{ @@ -25,36 +25,6 @@ use crate::{ listener::limiter::ConcurrencyLimiter, }; -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum DeliveryResult { - Success, - TemporaryFailure { - reason: Cow<'static, str>, - }, - PermanentFailure { - code: [u8; 3], - reason: Cow<'static, str>, - }, -} - -#[derive(Debug)] -pub enum DeliveryEvent { - Ingest { - message: IngestMessage, - result_tx: oneshot::Sender>, - }, - Stop, -} - -#[derive(Debug)] -pub struct IngestMessage { - pub sender_address: String, - pub recipients: Vec, - pub message_blob: BlobHash, - pub message_size: usize, - pub session_id: u64, -} - pub enum HousekeeperEvent { AcmeReschedule { provider_id: String, diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index fdeae3b0..db6b15cd 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -33,7 +33,7 @@ use config::{ use dashmap::DashMap; use imap_proto::protocol::list::Attribute; -use ipc::{DeliveryEvent, HousekeeperEvent, QueueEvent, ReportingEvent, StateEvent}; +use ipc::{HousekeeperEvent, QueueEvent, ReportingEvent, StateEvent}; use listener::{ asn::AsnGeoLookupData, blocked::Security, limiter::ConcurrencyLimiter, tls::AcmeProviders, }; @@ -43,7 +43,7 @@ use manager::webadmin::{Resource, WebAdminManager}; use nlp::bayes::{TokenHash, Weights}; use parking_lot::{Mutex, RwLock}; use rustls::sign::CertifiedKey; -use tokio::sync::{mpsc, Notify}; +use tokio::sync::{mpsc, Notify, Semaphore}; use tokio_rustls::TlsConnector; use utils::{ cache::{Cache, CacheItemWeight, CacheWithTtl}, @@ -167,10 +167,10 @@ pub struct HttpAuthCache { pub struct Ipc { pub state_tx: mpsc::Sender, pub housekeeper_tx: mpsc::Sender, - pub delivery_tx: mpsc::Sender, pub index_tx: Arc, pub queue_tx: mpsc::Sender, pub report_tx: mpsc::Sender, + pub local_delivery_sm: Arc, } pub struct TlsConnectors { @@ -442,10 +442,10 @@ impl Default for Ipc { Self { state_tx: mpsc::channel(IPC_CHANNEL_BUFFER).0, housekeeper_tx: mpsc::channel(IPC_CHANNEL_BUFFER).0, - delivery_tx: mpsc::channel(IPC_CHANNEL_BUFFER).0, index_tx: Default::default(), queue_tx: mpsc::channel(IPC_CHANNEL_BUFFER).0, report_tx: mpsc::channel(IPC_CHANNEL_BUFFER).0, + local_delivery_sm: Arc::new(Semaphore::new(10)), } } } diff --git a/crates/common/src/manager/boot.rs b/crates/common/src/manager/boot.rs index dec4e16f..cec049bd 100644 --- a/crates/common/src/manager/boot.rs +++ b/crates/common/src/manager/boot.rs @@ -16,7 +16,7 @@ use store::{ rand::{distributions::Alphanumeric, thread_rng, Rng}, Stores, }; -use tokio::sync::{mpsc, Notify}; +use tokio::sync::{mpsc, Notify, Semaphore}; use utils::{ config::{Config, ConfigKey}, failed, Semver, UnwrapFailure, @@ -25,7 +25,7 @@ use utils::{ use crate::{ config::{network::AsnGeoLookupConfig, server::Listeners, telemetry::Telemetry}, core::BuildServer, - ipc::{DeliveryEvent, HousekeeperEvent, QueueEvent, ReportingEvent, StateEvent}, + ipc::{HousekeeperEvent, QueueEvent, ReportingEvent, StateEvent}, Caches, Core, Data, Inner, Ipc, IPC_CHANNEL_BUFFER, }; @@ -46,7 +46,6 @@ pub struct BootManager { pub struct IpcReceivers { pub state_rx: Option>, pub housekeeper_rx: Option>, - pub delivery_rx: Option>, pub queue_rx: Option>, pub report_rx: Option>, } @@ -427,7 +426,7 @@ impl BootManager { core.network.asn_geo_lookup, AsnGeoLookupConfig::Resource { .. } ); - let (ipc, ipc_rxs) = build_ipc(); + let (ipc, ipc_rxs) = build_ipc(&mut config); let inner = Arc::new(Inner { shared_core: ArcSwap::from_pointee(core), data, @@ -484,9 +483,8 @@ impl BootManager { } } -pub fn build_ipc() -> (Ipc, IpcReceivers) { +pub fn build_ipc(config: &mut Config) -> (Ipc, IpcReceivers) { // Build ipc receivers - let (delivery_tx, delivery_rx) = mpsc::channel(IPC_CHANNEL_BUFFER); let (state_tx, state_rx) = mpsc::channel(IPC_CHANNEL_BUFFER); let (housekeeper_tx, housekeeper_rx) = mpsc::channel(IPC_CHANNEL_BUFFER); let (queue_tx, queue_rx) = mpsc::channel(IPC_CHANNEL_BUFFER); @@ -495,15 +493,19 @@ pub fn build_ipc() -> (Ipc, IpcReceivers) { Ipc { state_tx, housekeeper_tx, - delivery_tx, queue_tx, report_tx, index_tx: Arc::new(Notify::new()), + local_delivery_sm: Arc::new(Semaphore::new( + config + .property_or_default::("queue.threads.local", "10") + .unwrap_or(10) + .max(1), + )), }, IpcReceivers { state_rx: Some(state_rx), housekeeper_rx: Some(housekeeper_rx), - delivery_rx: Some(delivery_rx), queue_rx: Some(queue_rx), report_rx: Some(report_rx), }, diff --git a/crates/email/Cargo.toml b/crates/email/Cargo.toml new file mode 100644 index 00000000..5051732d --- /dev/null +++ b/crates/email/Cargo.toml @@ -0,0 +1,39 @@ +[package] +name = "email" +version = "0.11.2" +edition = "2021" +resolver = "2" + +[dependencies] +utils = { path = "../utils" } +nlp = { path = "../nlp" } +store = { path = "../store" } +trc = { path = "../trc" } +jmap_proto = { path = "../jmap-proto" } +common = { path = "../common" } +directory = { path = "../directory" } +spam-filter = { path = "../spam-filter" } +smtp-proto = { version = "0.1", features = ["serde_support"] } +mail-parser = { version = "0.9", features = ["full_encoding", "ludicrous_mode"] } +mail-builder = { version = "0.3", features = ["ludicrous_mode"] } +sieve-rs = { version = "0.5" } +tokio = { version = "1.23", features = ["net", "macros"] } +serde = { version = "1.0", features = ["derive"]} +serde_json = "1.0" +bincode = "1.3.3" +aes = "0.8.3" +aes-gcm = "0.10.1" +aes-gcm-siv = "0.11.1" +cbc = { version = "0.1.2", features = ["alloc"] } +rasn = "0.10" +rasn-cms = "0.10" +rasn-pkix = "0.10" +rsa = "0.9.2" +sequoia-openpgp = { version = "1.16", default-features = false, features = ["crypto-rust", "allow-experimental-crypto", "allow-variable-time-crypto"] } + +[features] +test_mode = [] +enterprise = [] + +[dev-dependencies] +tokio = { version = "1.23", features = ["full"] } diff --git a/crates/jmap/src/email/cache.rs b/crates/email/src/cache.rs similarity index 98% rename from crates/jmap/src/email/cache.rs rename to crates/email/src/cache.rs index 8521120b..a1f447f3 100644 --- a/crates/jmap/src/email/cache.rs +++ b/crates/email/src/cache.rs @@ -11,8 +11,6 @@ use jmap_proto::types::{collection::Collection, property::Property}; use std::future::Future; use trc::AddContext; -use crate::JmapMethods; - pub trait ThreadCache: Sync + Send { fn get_cached_thread_ids( &self, diff --git a/crates/email/src/crypto.rs b/crates/email/src/crypto.rs new file mode 100644 index 00000000..5a6e3c46 --- /dev/null +++ b/crates/email/src/crypto.rs @@ -0,0 +1,670 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::{borrow::Cow, collections::BTreeSet, fmt::Display, io::Cursor}; + +use aes::cipher::{block_padding::Pkcs7, BlockEncryptMut, KeyIvInit}; + +use mail_builder::{encoders::base64::base64_encode_mime, mime::make_boundary}; +use mail_parser::{decoders::base64::base64_decode, Message, MimeHeaders, PartType}; +use openpgp::{ + parse::Parse, + serialize::stream, + types::{KeyFlags, SymmetricAlgorithm}, +}; +use rasn::types::{ObjectIdentifier, OctetString}; +use rasn_cms::{ + algorithms::{AES128_CBC, AES256_CBC, RSA}, + pkcs7_compat::EncapsulatedContentInfo, + AlgorithmIdentifier, EncryptedContent, EncryptedContentInfo, EncryptedKey, EnvelopedData, + IssuerAndSerialNumber, KeyTransRecipientInfo, RecipientIdentifier, RecipientInfo, CONTENT_DATA, + CONTENT_ENVELOPED_DATA, +}; +use rsa::{pkcs1::DecodeRsaPublicKey, Pkcs1v15Encrypt, RsaPublicKey}; +use sequoia_openpgp as openpgp; +use store::rand::{rngs::StdRng, RngCore, SeedableRng}; +use store::{ + write::{Bincode, ToBitmaps}, + Deserialize, Serialize, +}; + +const P: openpgp::policy::StandardPolicy<'static> = openpgp::policy::StandardPolicy::new(); + +#[derive(Debug)] +pub enum EncryptMessageError { + AlreadyEncrypted, + Error(String), +} + +#[derive(Debug, Clone, Copy, serde::Serialize, serde::Deserialize)] +pub enum Algorithm { + Aes128, + Aes256, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub enum EncryptionMethod { + PGP, + SMIME, +} + +#[derive(Debug, serde::Serialize, serde::Deserialize)] +pub struct EncryptionParams { + pub method: EncryptionMethod, + pub algo: Algorithm, + pub certs: Vec>, +} + +#[derive(Debug, serde::Serialize, serde::Deserialize, Default)] +#[serde(tag = "type")] +#[serde(rename_all = "camelCase")] +pub enum EncryptionType { + PGP { + algo: Algorithm, + certs: String, + }, + SMIME { + algo: Algorithm, + certs: String, + }, + #[default] + Disabled, +} + +#[allow(async_fn_in_trait)] +pub trait EncryptMessage { + async fn encrypt(&self, params: &EncryptionParams) -> Result, EncryptMessageError>; + fn is_encrypted(&self) -> bool; +} + +impl EncryptMessage for Message<'_> { + async fn encrypt(&self, params: &EncryptionParams) -> Result, EncryptMessageError> { + let root = self.root_part(); + let raw_message = self.raw_message(); + let mut outer_message = Vec::with_capacity((raw_message.len() as f64 * 1.5) as usize); + let mut inner_message = Vec::with_capacity(raw_message.len()); + + // Move MIME headers and body to inner message + for header in root.headers() { + (if header.name.is_mime_header() { + &mut inner_message + } else { + &mut outer_message + }) + .extend_from_slice(&raw_message[header.offset_field()..header.offset_end()]); + } + inner_message.extend_from_slice(b"\r\n"); + inner_message.extend_from_slice(&raw_message[root.raw_body_offset()..]); + + // Encrypt inner message + match params.method { + EncryptionMethod::PGP => { + // Prepare encrypted message + let boundary = make_boundary("_"); + outer_message.extend_from_slice( + concat!( + "Content-Type: multipart/encrypted;\r\n\t", + "protocol=\"application/pgp-encrypted\";\r\n\t", + "boundary=\"" + ) + .as_bytes(), + ); + outer_message.extend_from_slice(boundary.as_bytes()); + outer_message.extend_from_slice( + concat!( + "\"\r\n\r\n", + "OpenPGP/MIME message (Automatically encrypted by Stalwart)\r\n\r\n", + "--" + ) + .as_bytes(), + ); + outer_message.extend_from_slice(boundary.as_bytes()); + outer_message.extend_from_slice( + concat!( + "\r\nContent-Type: application/pgp-encrypted\r\n\r\n", + "Version: 1\r\n\r\n--" + ) + .as_bytes(), + ); + outer_message.extend_from_slice(boundary.as_bytes()); + outer_message.extend_from_slice( + concat!( + "\r\nContent-Type: application/octet-stream; name=\"encrypted.asc\"\r\n", + "Content-Disposition: inline; filename=\"encrypted.asc\"\r\n\r\n" + ) + .as_bytes(), + ); + + let certs = params + .certs + .iter() + .map(openpgp::Cert::from_bytes) + .collect::, _>>() + .map_err(|err| { + EncryptMessageError::Error(format!( + "Failed to parse OpenPGP public key: {}", + err + )) + })?; + + // Encrypt contents (TODO: use rayon) + let algo = params.algo; + let encrypted_contents = tokio::task::spawn_blocking(move || { + // Parse public key + let mut keys = Vec::with_capacity(certs.len()); + let policy = openpgp::policy::StandardPolicy::new(); + + for cert in &certs { + for key in cert + .keys() + .with_policy(&policy, None) + .supported() + .alive() + .revoked(false) + .key_flags(KeyFlags::empty().set_transport_encryption()) + { + keys.push(key); + } + } + + // Compose a writer stack corresponding to the output format and + // packet structure we want. + let mut sink = Vec::with_capacity(inner_message.len()); + + // Stream an OpenPGP message. + let message = stream::Armorer::new(stream::Message::new(&mut sink)) + .build() + .map_err(|err| { + EncryptMessageError::Error(format!("Failed to create armorer: {}", err)) + })?; + let message = stream::Encryptor2::for_recipients(message, keys) + .symmetric_algo(match algo { + Algorithm::Aes128 => SymmetricAlgorithm::AES128, + Algorithm::Aes256 => SymmetricAlgorithm::AES256, + }) + .build() + .map_err(|err| { + EncryptMessageError::Error(format!( + "Failed to build encryptor: {}", + err + )) + })?; + let mut message = + stream::LiteralWriter::new(message).build().map_err(|err| { + EncryptMessageError::Error(format!( + "Failed to create literal writer: {}", + err + )) + })?; + std::io::copy(&mut Cursor::new(inner_message), &mut message).map_err( + |err| { + EncryptMessageError::Error(format!( + "Failed to encrypt message: {}", + err + )) + }, + )?; + message.finalize().map_err(|err| { + EncryptMessageError::Error(format!("Failed to finalize message: {}", err)) + })?; + + String::from_utf8(sink).map_err(|err| { + EncryptMessageError::Error(format!( + "Failed to convert encrypted message to UTF-8: {}", + err + )) + }) + }) + .await + .map_err(|err| { + EncryptMessageError::Error(format!("Failed to encrypt message: {}", err)) + })??; + outer_message.extend_from_slice(encrypted_contents.as_bytes()); + outer_message.extend_from_slice(b"\r\n--"); + outer_message.extend_from_slice(boundary.as_bytes()); + outer_message.extend_from_slice(b"--\r\n"); + } + EncryptionMethod::SMIME => { + // Generate random IV + let mut rng = StdRng::from_entropy(); + let mut iv = vec![0u8; 16]; + rng.fill_bytes(&mut iv); + + // Generate random key + let mut key = vec![0u8; params.algo.key_size()]; + rng.fill_bytes(&mut key); + + // Encrypt contents (TODO: use rayon) + let algo = params.algo; + let (encrypted_contents, key, iv) = tokio::task::spawn_blocking(move || { + (algo.encrypt(&key, &iv, &inner_message), key, iv) + }) + .await + .map_err(|err| { + EncryptMessageError::Error(format!("Failed to encrypt message: {}", err)) + })?; + + // Encrypt key using public keys + #[allow(clippy::mutable_key_type)] + let mut recipient_infos = BTreeSet::new(); + for cert in ¶ms.certs { + let cert = + rasn::der::decode::(cert).map_err(|err| { + EncryptMessageError::Error(format!( + "Failed to parse certificate: {}", + err + )) + })?; + + let public_key = RsaPublicKey::from_pkcs1_der( + cert.tbs_certificate + .subject_public_key_info + .subject_public_key + .as_raw_slice(), + ) + .map_err(|err| { + EncryptMessageError::Error(format!("Failed to parse public key: {}", err)) + })?; + let encrypted_key = public_key + .encrypt(&mut rng, Pkcs1v15Encrypt, &key[..]) + .map_err(|err| { + EncryptMessageError::Error(format!("Failed to encrypt key: {}", err)) + }) + .unwrap(); + + recipient_infos.insert(RecipientInfo::KeyTransRecipientInfo( + KeyTransRecipientInfo { + version: 0.into(), + rid: RecipientIdentifier::IssuerAndSerialNumber( + IssuerAndSerialNumber { + issuer: cert.tbs_certificate.issuer, + serial_number: cert.tbs_certificate.serial_number, + }, + ), + key_encryption_algorithm: AlgorithmIdentifier { + algorithm: RSA.into(), + parameters: Some( + rasn::der::encode(&()) + .map_err(|err| { + EncryptMessageError::Error(format!( + "Failed to encode RSA algorithm identifier: {}", + err + )) + })? + .into(), + ), + }, + encrypted_key: EncryptedKey::from(encrypted_key), + }, + )); + } + + let pkcs7 = rasn::der::encode(&EncapsulatedContentInfo { + content_type: CONTENT_ENVELOPED_DATA.into(), + content: Some( + rasn::der::encode(&EnvelopedData { + version: 0.into(), + originator_info: None, + recipient_infos, + encrypted_content_info: EncryptedContentInfo { + content_type: CONTENT_DATA.into(), + content_encryption_algorithm: AlgorithmIdentifier { + algorithm: params.algo.to_algorithm_identifier(), + parameters: Some( + rasn::der::encode(&OctetString::from(iv)) + .map_err(|err| { + EncryptMessageError::Error(format!( + "Failed to encode IV: {}", + err + )) + })? + .into(), + ), + }, + encrypted_content: Some(EncryptedContent::from(encrypted_contents)), + }, + unprotected_attrs: None, + }) + .map_err(|err| { + EncryptMessageError::Error(format!( + "Failed to encode EnvelopedData: {}", + err + )) + })? + .into(), + ), + }) + .map_err(|err| { + EncryptMessageError::Error(format!("Failed to encode ContentInfo: {}", err)) + })?; + + // Generate message + outer_message.extend_from_slice( + concat!( + "Content-Type: application/pkcs7-mime;\r\n", + "\tname=\"smime.p7m\";\r\n", + "\tsmime-type=enveloped-data\r\n", + "Content-Disposition: attachment;\r\n", + "\tfilename=\"smime.p7m\"\r\n", + "Content-Transfer-Encoding: base64\r\n\r\n" + ) + .as_bytes(), + ); + base64_encode_mime(&pkcs7, &mut outer_message, false).map_err(|err| { + EncryptMessageError::Error(format!("Failed to base64 encode PKCS7: {}", err)) + })?; + } + } + + Ok(outer_message) + } + + fn is_encrypted(&self) -> bool { + if self.content_type().is_some_and(|ct| { + let main_type = ct.c_type.as_ref(); + let sub_type = ct + .c_subtype + .as_ref() + .map(|s| s.as_ref()) + .unwrap_or_default(); + + (main_type.eq_ignore_ascii_case("application") + && (sub_type.eq_ignore_ascii_case("pkcs7-mime") + || sub_type.eq_ignore_ascii_case("pkcs7-signature") + || (sub_type.eq_ignore_ascii_case("octet-stream") + && self.attachment_name().is_some_and(|name| { + name.rsplit_once('.') + .is_some_and(|(_, ext)| ["p7m", "p7s", "p7c", "p7z"].contains(&ext)) + })))) + || (main_type.eq_ignore_ascii_case("multipart") + && sub_type.eq_ignore_ascii_case("encrypted")) + }) { + return true; + } + + if self.parts.len() <= 2 { + let mut text_part = None; + let mut is_multipart = false; + + for part in &self.parts { + match &part.body { + PartType::Text(text) => { + text_part = Some(text.as_ref()); + } + PartType::Multipart(_) => { + is_multipart = true; + } + _ => (), + } + } + + match text_part { + Some(text) if self.parts.len() == 1 || is_multipart => { + if text.trim_start().starts_with("-----BEGIN PGP MESSAGE-----") { + return true; + } + } + _ => (), + } + } + + false + } +} + +impl Algorithm { + fn key_size(&self) -> usize { + match self { + Algorithm::Aes128 => 16, + Algorithm::Aes256 => 32, + } + } + + fn to_algorithm_identifier(self) -> ObjectIdentifier { + match self { + Algorithm::Aes128 => AES128_CBC.into(), + Algorithm::Aes256 => AES256_CBC.into(), + } + } + + fn encrypt(&self, key: &[u8], iv: &[u8], contents: &[u8]) -> Vec { + match self { + Algorithm::Aes128 => cbc::Encryptor::::new(key.into(), iv.into()) + .encrypt_padded_vec_mut::(contents), + Algorithm::Aes256 => cbc::Encryptor::::new(key.into(), iv.into()) + .encrypt_padded_vec_mut::(contents), + } + } +} + +pub fn try_parse_certs( + expected_method: EncryptionMethod, + cert: Vec, +) -> Result>, Cow<'static, str>> { + // Check if it's a PEM file + let (method, certs) = if let Some(result) = try_parse_pem(&cert)? { + result + } else if rasn::der::decode::(&cert[..]).is_ok() { + (EncryptionMethod::SMIME, vec![cert]) + } else if let Ok(cert_) = openpgp::Cert::from_bytes(&cert[..]) { + if !has_pgp_keys(cert_) { + (EncryptionMethod::PGP, vec![cert]) + } else { + return Err("Could not find any suitable keys in certificate".into()); + } + } else { + return Err("Could not find any valid certificates".into()); + }; + + if method == expected_method { + Ok(certs) + } else { + Err("No valid certificates found for the selected encryption".into()) + } +} + +fn has_pgp_keys(cert: openpgp::Cert) -> bool { + cert.keys() + .with_policy(&P, None) + .supported() + .alive() + .revoked(false) + .key_flags(KeyFlags::empty().set_transport_encryption()) + .next() + .is_some() +} + +#[allow(clippy::type_complexity)] +fn try_parse_pem( + bytes_: &[u8], +) -> Result>)>, Cow<'static, str>> { + if let Some(internal) = std::str::from_utf8(bytes_) + .ok() + .and_then(|cert| cert.strip_prefix("-----STALWART CERTIFICATE-----")) + { + return base64_decode(internal.as_bytes()) + .ok_or(Cow::from("Failed to decode base64")) + .and_then(|bytes| { + Bincode::::deserialize(&bytes) + .map_err(|_| Cow::from("Failed to deserialize internal certificate")) + }) + .map(|params| Some((params.inner.method, params.inner.certs))); + } + + let mut bytes = bytes_.iter().enumerate(); + let mut buf = vec![]; + let mut method = None; + let mut certs = vec![]; + + loop { + // Find start of PEM block + let mut start_pos = 0; + for (pos, &ch) in bytes.by_ref() { + if ch.is_ascii_whitespace() { + continue; + } else if ch == b'-' { + start_pos = pos; + break; + } else { + return Ok(None); + } + } + + // Find block type + for (_, &ch) in bytes.by_ref() { + match ch { + b'-' => (), + b'\n' => break, + _ => { + if ch.is_ascii() { + buf.push(ch.to_ascii_uppercase()); + } else { + return Ok(None); + } + } + } + } + if buf.is_empty() { + break; + } + + // Find type + let tag = std::str::from_utf8(&buf).unwrap(); + if tag.contains("CERTIFICATE") { + if method.is_some_and(|m| m == EncryptionMethod::PGP) { + return Err("Cannot mix OpenPGP and S/MIME certificates".into()); + } else { + method = Some(EncryptionMethod::SMIME); + } + } else if tag.contains("PGP") { + if method.is_some_and(|m| m == EncryptionMethod::SMIME) { + return Err("Cannot mix OpenPGP and S/MIME certificates".into()); + } else { + method = Some(EncryptionMethod::PGP); + } + } else { + // Ignore block + let mut found_end = false; + for (_, &ch) in bytes.by_ref() { + if ch == b'-' { + found_end = true; + } else if ch == b'\n' && found_end { + break; + } + } + buf.clear(); + continue; + } + + // Collect base64 + buf.clear(); + let mut found_end = false; + let mut end_pos = 0; + for (pos, &ch) in bytes.by_ref() { + match ch { + b'-' => { + found_end = true; + } + b'\n' => { + if found_end { + end_pos = pos; + break; + } + } + _ => { + if !ch.is_ascii_whitespace() { + buf.push(ch); + } + } + } + } + + // Decode base64 + let cert = + base64_decode(&buf).ok_or_else(|| Cow::from("Failed to decode base64 certificate."))?; + match method.unwrap() { + EncryptionMethod::PGP => match openpgp::Cert::from_bytes(bytes_) { + Ok(cert) => { + if !has_pgp_keys(cert) { + return Err("Could not find any suitable keys in OpenPGP public key".into()); + } + certs.push( + bytes_ + .get(start_pos..end_pos + 1) + .unwrap_or_default() + .to_vec(), + ); + } + Err(err) => { + return Err(format!("Failed to decode OpenPGP public key: {err}").into()) + } + }, + EncryptionMethod::SMIME => { + if let Err(err) = rasn::der::decode::(&cert) { + return Err(format!("Failed to decode X509 certificate: {err}").into()); + } + certs.push(cert); + } + } + buf.clear(); + } + + Ok(method.map(|method| (method, certs))) +} + +impl Serialize for &EncryptionParams { + fn serialize(self) -> Vec { + let len = bincode::serialized_size(&self).unwrap_or_default(); + let mut buf = Vec::with_capacity(len as usize + 1); + buf.push(1); + let _ = bincode::serialize_into(&mut buf, &self); + buf + } +} + +impl Deserialize for EncryptionParams { + fn deserialize(bytes: &[u8]) -> trc::Result { + let version = *bytes + .first() + .ok_or_else(|| trc::StoreEvent::DataCorruption.caused_by(trc::location!()))?; + match version { + 1 if bytes.len() > 1 => bincode::deserialize(&bytes[1..]).map_err(|err| { + trc::EventType::Store(trc::StoreEvent::DeserializeError) + .from_bincode_error(err) + .caused_by(trc::location!()) + }), + + _ => Err(trc::StoreEvent::DeserializeError + .into_err() + .caused_by(trc::location!()) + .ctx(trc::Key::Value, version as u64)), + } + } +} + +impl ToBitmaps for &EncryptionParams { + fn to_bitmaps(&self, _: &mut Vec, _: u8, _: bool) { + unreachable!() + } +} + +impl Display for EncryptionMethod { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + EncryptionMethod::PGP => write!(f, "OpenPGP"), + EncryptionMethod::SMIME => write!(f, "S/MIME"), + } + } +} + +impl Display for Algorithm { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Algorithm::Aes128 => write!(f, "AES-128"), + Algorithm::Aes256 => write!(f, "AES-256"), + } + } +} diff --git a/crates/jmap/src/services/ingest.rs b/crates/email/src/delivery.rs similarity index 58% rename from crates/jmap/src/services/ingest.rs rename to crates/email/src/delivery.rs index 980f7d14..ea829a9c 100644 --- a/crates/jmap/src/services/ingest.rs +++ b/crates/email/src/delivery.rs @@ -4,36 +4,128 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{ - ipc::{DeliveryResult, IngestMessage}, - Server, -}; +use common::Server; use directory::Permission; use jmap_proto::types::{state::StateChange, type_state::DataType}; use mail_parser::MessageParser; -use std::future::Future; +use std::{borrow::Cow, future::Future}; use store::ahash::AHashMap; +use utils::BlobHash; use crate::{ - email::{ - bayes::EmailBayesTrain, - ingest::{EmailIngest, IngestEmail, IngestSource}, - }, + ingest::{EmailIngest, IngestEmail, IngestSource}, mailbox::INBOX_ID, - sieve::{get::SieveScriptGet, ingest::SieveScriptIngest}, + sieve::SieveScriptIngest, }; -use super::state::StateManager; +#[derive(Debug)] +pub struct IngestMessage { + pub sender_address: String, + pub recipients: Vec, + pub message_blob: BlobHash, + pub message_size: usize, + pub session_id: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum LocalDeliveryStatus { + Success, + TemporaryFailure { + reason: Cow<'static, str>, + }, + PermanentFailure { + code: [u8; 3], + reason: Cow<'static, str>, + }, +} + +pub struct LocalDeliveryResult { + pub status: Vec, + pub autogenerated: Vec, +} + +pub struct AutogeneratedMessage { + pub sender_address: String, + pub recipients: Vec, + pub message: Vec, +} pub trait MailDelivery: Sync + Send { fn deliver_message( &self, message: IngestMessage, - ) -> impl Future> + Send; + ) -> impl Future + Send; } +/* + +let semaphore = Arc::new(Semaphore::new( + inner + .shared_core + .load() + .smtp + .queue + .throttle + .local_concurrency, + )); + + loop { + let permit = match semaphore.clone().acquire_owned().await { + Ok(permit) => permit, + Err(_) => { + trc::error!(trc::StoreEvent::UnexpectedError + .into_err() + .details("Semaphore error") + .caused_by(trc::location!())); + break; + } + }; + + match delivery_rx.recv().await { + Some(event) => match event { + DeliveryEvent::Ingest { message, result_tx } => { + let server = inner.build_server(); + + tokio::spawn(async move { + result_tx.send(server.deliver_message(message).await).ok(); + + drop(permit); + }); + } + DeliveryEvent::Stop => break, + }, + None => { + break; + } + } + } + +*/ + impl MailDelivery for Server { - async fn deliver_message(&self, message: IngestMessage) -> Vec { + async fn deliver_message(&self, message: IngestMessage) -> LocalDeliveryResult { + // Obtain permit + let _permit = match self.inner.ipc.local_delivery_sm.acquire().await { + Ok(permit) => permit, + Err(_) => { + trc::error!( + trc::Error::new(trc::EventType::Server(trc::ServerEvent::ThreadError)) + .details("Failed to obtain semaphore permit.") + .span_id(message.session_id) + .caused_by(trc::location!()) + ); + + return LocalDeliveryResult { + status: (0..message.recipients.len()) + .map(|_| LocalDeliveryStatus::TemporaryFailure { + reason: "Temporary I/O error.".into(), + }) + .collect::>(), + autogenerated: vec![], + }; + } + }; + // Read message let raw_message = match self .core @@ -51,11 +143,14 @@ impl MailDelivery for Server { CausedBy = trc::location!() ); - return (0..message.recipients.len()) - .map(|_| DeliveryResult::TemporaryFailure { - reason: "Blob not found.".into(), - }) - .collect::>(); + return LocalDeliveryResult { + status: (0..message.recipients.len()) + .map(|_| LocalDeliveryStatus::TemporaryFailure { + reason: "Blob not found.".into(), + }) + .collect::>(), + autogenerated: vec![], + }; } Err(err) => { trc::error!(err @@ -63,17 +158,24 @@ impl MailDelivery for Server { .span_id(message.session_id) .caused_by(trc::location!())); - return (0..message.recipients.len()) - .map(|_| DeliveryResult::TemporaryFailure { - reason: "Temporary I/O error.".into(), - }) - .collect::>(); + return LocalDeliveryResult { + status: (0..message.recipients.len()) + .map(|_| LocalDeliveryStatus::TemporaryFailure { + reason: "Temporary I/O error.".into(), + }) + .collect::>(), + autogenerated: vec![], + }; } }; // Obtain the UIDs for each recipient let mut uids: AHashMap = AHashMap::with_capacity(message.recipients.len()); - let mut results = Vec::with_capacity(message.recipients.len()); + let mut result = LocalDeliveryResult { + status: Vec::with_capacity(message.recipients.len()), + autogenerated: Vec::new(), + }; + for rcpt in message.recipients { let uid = match self .email_to_id(&self.core.storage.directory, &rcpt, message.session_id) @@ -82,7 +184,7 @@ impl MailDelivery for Server { Ok(Some(uid)) => uid, Ok(None) => { // Something went wrong - results.push(DeliveryResult::PermanentFailure { + result.status.push(LocalDeliveryStatus::PermanentFailure { code: [5, 5, 0], reason: "Mailbox not found.".into(), }); @@ -94,19 +196,19 @@ impl MailDelivery for Server { .ctx(trc::Key::To, rcpt) .span_id(message.session_id) .caused_by(trc::location!())); - results.push(DeliveryResult::TemporaryFailure { + result.status.push(LocalDeliveryStatus::TemporaryFailure { reason: "Address lookup failed.".into(), }); continue; } }; - if let Some(result) = uids.get(&uid).and_then(|pos| results.get(*pos)) { - results.push(result.clone()); + if let Some(status) = uids.get(&uid).and_then(|pos| result.status.get(*pos)) { + result.status.push(status.clone()); continue; } // Obtain access token - let result = match self.get_access_token(uid).await.and_then(|token| { + let status = match self.get_access_token(uid).await.and_then(|token| { token .assert_has_permission(Permission::EmailReceive) .map(|_| token) @@ -114,17 +216,6 @@ impl MailDelivery for Server { Ok(access_token) => { // Check if there is an active sieve script match self.sieve_script_get_active(uid).await { - Ok(Some(active_script)) => { - self.sieve_script_ingest( - &access_token, - &raw_message, - &message.sender_address, - &rcpt, - message.session_id, - active_script, - ) - .await - } Ok(None) => { // Ingest message self.email_ingest(IngestEmail { @@ -142,6 +233,18 @@ impl MailDelivery for Server { }) .await } + Ok(Some(active_script)) => { + self.sieve_script_ingest( + &access_token, + &raw_message, + &message.sender_address, + &rcpt, + message.session_id, + active_script, + &mut result.autogenerated, + ) + .await + } Err(err) => Err(err), } } @@ -149,7 +252,7 @@ impl MailDelivery for Server { Err(err) => Err(err), }; - let result = match result { + let status = match status { Ok(ingested_message) => { // Notify state change if ingested_message.change_id != u64::MAX { @@ -163,28 +266,28 @@ impl MailDelivery for Server { .await; } - DeliveryResult::Success + LocalDeliveryStatus::Success } Err(err) => { - let result = match err.as_ref() { + let status = match err.as_ref() { trc::EventType::Limit(trc::LimitEvent::Quota) => { - DeliveryResult::TemporaryFailure { + LocalDeliveryStatus::TemporaryFailure { reason: "Mailbox over quota.".into(), } } trc::EventType::Limit(trc::LimitEvent::TenantQuota) => { - DeliveryResult::TemporaryFailure { + LocalDeliveryStatus::TemporaryFailure { reason: "Organization over quota.".into(), } } trc::EventType::Security(trc::SecurityEvent::Unauthorized) => { - DeliveryResult::PermanentFailure { + LocalDeliveryStatus::PermanentFailure { code: [5, 5, 0], reason: "This account is not authorized to receive email.".into(), } } trc::EventType::MessageIngest(trc::MessageIngestEvent::Error) => { - DeliveryResult::PermanentFailure { + LocalDeliveryStatus::PermanentFailure { code: err .value(trc::Key::Code) .and_then(|v| v.to_uint()) @@ -199,7 +302,7 @@ impl MailDelivery for Server { .into(), } } - _ => DeliveryResult::TemporaryFailure { + _ => LocalDeliveryStatus::TemporaryFailure { reason: "Transient server failure.".into(), }, }; @@ -208,16 +311,16 @@ impl MailDelivery for Server { .ctx(trc::Key::To, rcpt.to_string()) .span_id(message.session_id)); - result + status } }; // Cache response for UID to avoid duplicate deliveries - uids.insert(uid, results.len()); + uids.insert(uid, result.status.len()); - results.push(result); + result.status.push(status); } - results + result } } diff --git a/crates/jmap/src/email/index.rs b/crates/email/src/index.rs similarity index 100% rename from crates/jmap/src/email/index.rs rename to crates/email/src/index.rs diff --git a/crates/jmap/src/email/ingest.rs b/crates/email/src/ingest.rs similarity index 98% rename from crates/jmap/src/email/ingest.rs rename to crates/email/src/ingest.rs index 51822756..494b2ba8 100644 --- a/crates/jmap/src/email/ingest.rs +++ b/crates/email/src/ingest.rs @@ -10,7 +10,11 @@ use std::{ time::{Duration, Instant}, }; -use common::{auth::ResourceToken, Server}; +use common::{ + auth::{AccessToken, ResourceToken}, + Server, +}; +use directory::Permission; use jmap_proto::{ object::Object, types::{ @@ -23,11 +27,11 @@ use mail_parser::{ PartType, }; -use rand::Rng; use spam_filter::{ analysis::init::SpamFilterInit, modules::bayes::BayesClassifier, SpamFilterInput, }; use std::future::Future; +use store::rand::Rng; use store::{ ahash::AHashSet, query::Filter, @@ -42,12 +46,8 @@ use trc::{AddContext, MessageIngestEvent}; use utils::map::vec_map::VecMap; use crate::{ - blob::upload::BlobUpload, - changes::write::ChangeLog, - email::index::{IndexMessage, VisitValues, MAX_ID_LENGTH}, + index::{IndexMessage, VisitValues, MAX_ID_LENGTH}, mailbox::{UidMailbox, INBOX_ID, JUNK_ID}, - services::index::Indexer, - JmapMethods, }; use super::{ @@ -104,6 +104,7 @@ pub trait EmailIngest: Sync + Send { account_id: u32, mailbox_id: u32, ) -> impl Future> + Send; + fn email_bayes_can_train(&self, access_token: &AccessToken) -> bool; } impl EmailIngest for Server { @@ -437,7 +438,6 @@ impl EmailIngest for Server { // Obtain a documentId and changeId let change_id = self .assign_change_id(account_id) - .await .caused_by(trc::location!())?; // Store blob @@ -646,7 +646,6 @@ impl EmailIngest for Server { let mut batch = BatchBuilder::new(); let change_id = self .assign_change_id(account_id) - .await .caused_by(trc::location!())?; let mut changes = ChangeLogBuilder::with_change_id(change_id); batch @@ -702,7 +701,7 @@ impl EmailIngest for Server { match self.core.storage.data.write(batch.build()).await { Ok(_) => return Ok(Some(thread_id)), Err(err) if err.is_assertion_failure() && try_count < MAX_RETRIES => { - let backoff = rand::thread_rng().gen_range(50..=300); + let backoff = store::rand::thread_rng().gen_range(50..=300); tokio::time::sleep(Duration::from_millis(backoff)).await; try_count += 1; } @@ -728,6 +727,12 @@ impl EmailIngest for Server { .await .and_then(|v| v.last_counter_id().map(|id| id as u32)) } + + fn email_bayes_can_train(&self, access_token: &AccessToken) -> bool { + self.core.spam.bayes.as_ref().is_some_and(|bayes| { + bayes.account_classify && access_token.has_permission(Permission::SpamFilterTrain) + }) + } } pub struct LogEmailInsert(Option); diff --git a/crates/email/src/lib.rs b/crates/email/src/lib.rs new file mode 100644 index 00000000..bf9ff2fa --- /dev/null +++ b/crates/email/src/lib.rs @@ -0,0 +1,14 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub mod cache; +pub mod crypto; +pub mod delivery; +pub mod index; +pub mod ingest; +pub mod mailbox; +pub mod metadata; +pub mod sieve; diff --git a/crates/email/src/mailbox.rs b/crates/email/src/mailbox.rs new file mode 100644 index 00000000..20d27156 --- /dev/null +++ b/crates/email/src/mailbox.rs @@ -0,0 +1,495 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::{future::Future, slice::Iter}; + +use common::{config::jmap::settings::SpecialUse, Server}; +use jmap_proto::{ + object::{ + index::{IndexAs, IndexProperty, ObjectIndexBuilder}, + Object, + }, + types::{collection::Collection, id::Id, keyword::Keyword, property::Property, value::Value}, +}; +use store::{ + ahash::AHashSet, + query::Filter, + rand, + roaring::RoaringBitmap, + write::{ + BatchBuilder, BitmapClass, DeserializeFrom, MaybeDynamicId, Operation, SerializeInto, + TagValue, ToBitmaps, + }, + Serialize, U32_LEN, +}; +use trc::AddContext; +use utils::codec::leb128::{Leb128Iterator, Leb128Vec}; + +use crate::cache::ThreadCache; + +pub const INBOX_ID: u32 = 0; +pub const TRASH_ID: u32 = 1; +pub const JUNK_ID: u32 = 2; +pub const DRAFTS_ID: u32 = 3; +pub const SENT_ID: u32 = 4; +pub const ARCHIVE_ID: u32 = 5; +pub const TOMBSTONE_ID: u32 = u32::MAX - 1; + +#[derive(Debug)] +pub struct ExpandPath<'x> { + pub path: Vec<&'x str>, + pub found_names: Vec<(String, u32, u32)>, +} + +pub static SCHEMA: &[IndexProperty] = &[ + IndexProperty::new(Property::Name) + .index_as(IndexAs::Text { + tokenize: true, + index: true, + }) + .required(), + IndexProperty::new(Property::Role).index_as(IndexAs::Text { + tokenize: false, + index: true, + }), + IndexProperty::new(Property::Role).index_as(IndexAs::HasProperty), + IndexProperty::new(Property::ParentId).index_as(IndexAs::Integer), + IndexProperty::new(Property::SortOrder).index_as(IndexAs::Integer), + IndexProperty::new(Property::IsSubscribed).index_as(IndexAs::IntegerList), + IndexProperty::new(Property::Acl).index_as(IndexAs::Acl), +]; + +#[derive(Debug, Clone, Copy)] +pub struct UidMailbox { + pub mailbox_id: u32, + pub uid: u32, +} + +pub trait MailboxFnc: Sync + Send { + fn mailbox_get_or_create( + &self, + account_id: u32, + ) -> impl Future> + Send; + + fn mailbox_create_path( + &self, + account_id: u32, + path: &str, + ) -> impl Future)>>> + Send; + + fn mailbox_count_threads( + &self, + account_id: u32, + document_ids: Option, + ) -> impl Future> + Send; + + fn mailbox_unread_tags( + &self, + account_id: u32, + document_id: u32, + message_ids: &Option, + ) -> impl Future>> + Send; + + fn mailbox_expand_path<'x>( + &self, + account_id: u32, + path: &'x str, + exact_match: bool, + ) -> impl Future>>> + Send; + + fn mailbox_get_by_name( + &self, + account_id: u32, + path: &str, + ) -> impl Future>> + Send; + + fn mailbox_get_by_role( + &self, + account_id: u32, + role: &str, + ) -> impl Future>> + Send; +} + +impl MailboxFnc for Server { + async fn mailbox_get_or_create(&self, account_id: u32) -> trc::Result { + let mut mailbox_ids = self + .get_document_ids(account_id, Collection::Mailbox) + .await? + .unwrap_or_default(); + if !mailbox_ids.is_empty() { + return Ok(mailbox_ids); + } + + #[cfg(feature = "test_mode")] + if mailbox_ids.is_empty() && account_id == 0 { + return Ok(mailbox_ids); + } + + let mut batch = BatchBuilder::new(); + batch + .with_account_id(account_id) + .with_collection(Collection::Mailbox); + + // Create mailboxes + let mut last_document_id = ARCHIVE_ID; + for folder in &self.core.jmap.default_folders { + let (role, document_id) = match folder.special_use { + SpecialUse::Inbox => ("inbox", INBOX_ID), + SpecialUse::Trash => ("trash", TRASH_ID), + SpecialUse::Junk => ("junk", JUNK_ID), + SpecialUse::Drafts => ("drafts", DRAFTS_ID), + SpecialUse::Sent => ("sent", SENT_ID), + SpecialUse::Archive => ("archive", ARCHIVE_ID), + SpecialUse::None => { + last_document_id += 1; + ("", last_document_id) + } + SpecialUse::Shared => unreachable!(), + }; + + let mut object = Object::with_capacity(4) + .with_property(Property::Name, folder.name.clone()) + .with_property(Property::ParentId, Value::Id(0u64.into())) + .with_property( + Property::Cid, + Value::UnsignedInt(rand::random::() as u64), + ); + if !role.is_empty() { + object.set(Property::Role, role); + } + if folder.subscribe { + object.set( + Property::IsSubscribed, + Value::List(vec![Value::Id(account_id.into())]), + ); + } + batch + .create_document_with_id(document_id) + .custom(ObjectIndexBuilder::new(SCHEMA).with_changes(object)); + mailbox_ids.insert(document_id); + } + + self.core + .storage + .data + .write(batch.build()) + .await + .caused_by(trc::location!()) + .map(|_| mailbox_ids) + } + + async fn mailbox_create_path( + &self, + account_id: u32, + path: &str, + ) -> trc::Result)>> { + let expanded_path = + if let Some(expand_path) = self.mailbox_expand_path(account_id, path, false).await? { + expand_path + } else { + return Ok(None); + }; + + let mut next_parent_id = 0; + let mut path = expanded_path.path.into_iter().enumerate().peekable(); + 'outer: while let Some((pos, name)) = path.peek() { + let is_inbox = *pos == 0 && name.eq_ignore_ascii_case("inbox"); + + for (part, parent_id, document_id) in &expanded_path.found_names { + if (part.eq(name) || (is_inbox && part.eq_ignore_ascii_case("inbox"))) + && *parent_id == next_parent_id + { + next_parent_id = *document_id; + path.next(); + continue 'outer; + } + } + break; + } + + // Create missing folders + if path.peek().is_some() { + let mut changes = self.begin_changes(account_id)?; + + for (_, name) in path { + if name.len() > self.core.jmap.mailbox_name_max_len { + return Ok(None); + } + let mut batch = BatchBuilder::new(); + batch + .with_account_id(account_id) + .with_collection(Collection::Mailbox) + .create_document() + .custom( + ObjectIndexBuilder::new(SCHEMA).with_changes( + Object::with_capacity(3) + .with_property(Property::Name, name) + .with_property( + Property::ParentId, + Value::Id(Id::from(next_parent_id)), + ) + .with_property( + Property::Cid, + Value::UnsignedInt(rand::random::() as u64), + ), + ), + ); + let document_id = self + .store() + .write_expect_id(batch) + .await + .caused_by(trc::location!())?; + changes.log_insert(Collection::Mailbox, document_id); + next_parent_id = document_id + 1; + } + let change_id = changes.change_id; + let mut batch = BatchBuilder::new(); + + batch + .with_account_id(account_id) + .with_collection(Collection::Mailbox) + .custom(changes); + self.store() + .write(batch.build()) + .await + .caused_by(trc::location!())?; + + Ok(Some((next_parent_id - 1, Some(change_id)))) + } else { + Ok(Some((next_parent_id - 1, None))) + } + } + + async fn mailbox_count_threads( + &self, + account_id: u32, + document_ids: Option, + ) -> trc::Result { + if let Some(document_ids) = document_ids { + let mut thread_ids = AHashSet::default(); + self.get_cached_thread_ids(account_id, document_ids.into_iter()) + .await + .caused_by(trc::location!())? + .into_iter() + .for_each(|(_, thread_id)| { + thread_ids.insert(thread_id); + }); + Ok(thread_ids.len()) + } else { + Ok(0) + } + } + + async fn mailbox_unread_tags( + &self, + account_id: u32, + document_id: u32, + message_ids: &Option, + ) -> trc::Result> { + if let (Some(message_ids), Some(mailbox_message_ids)) = ( + message_ids, + self.get_tag( + account_id, + Collection::Email, + Property::MailboxIds, + document_id, + ) + .await?, + ) { + if let Some(mut seen) = self + .get_tag( + account_id, + Collection::Email, + Property::Keywords, + Keyword::Seen, + ) + .await? + { + seen ^= message_ids; + seen &= &mailbox_message_ids; + if !seen.is_empty() { + Ok(Some(seen)) + } else { + Ok(None) + } + } else { + Ok(mailbox_message_ids.into()) + } + } else { + Ok(None) + } + } + + async fn mailbox_expand_path<'x>( + &self, + account_id: u32, + path: &'x str, + exact_match: bool, + ) -> trc::Result>> { + let path = path + .split('/') + .filter_map(|p| { + let p = p.trim(); + if !p.is_empty() { + p.into() + } else { + None + } + }) + .collect::>(); + if path.is_empty() || path.len() > self.core.jmap.mailbox_max_depth { + return Ok(None); + } + + let mut filter = Vec::with_capacity(path.len() + 2); + let mut has_inbox = false; + filter.push(Filter::Or); + for (pos, item) in path.iter().enumerate() { + if pos == 0 && item.eq_ignore_ascii_case("inbox") { + has_inbox = true; + } else { + filter.push(Filter::eq(Property::Name, *item)); + } + } + filter.push(Filter::End); + + let mut document_ids = if filter.len() > 2 { + self.store() + .filter(account_id, Collection::Mailbox, filter) + .await + .caused_by(trc::location!())? + .results + } else { + RoaringBitmap::new() + }; + if has_inbox { + document_ids.insert(INBOX_ID); + } + if exact_match && (document_ids.len() as usize) < path.len() { + return Ok(None); + } + + let mut found_names = Vec::new(); + for document_id in document_ids { + if let Some(mut obj) = self + .get_property::>( + account_id, + Collection::Mailbox, + document_id, + Property::Value, + ) + .await? + { + if let Some(Value::Text(value)) = obj.properties.remove(&Property::Name) { + found_names.push(( + value, + if let Some(Value::Id(value)) = obj.properties.remove(&Property::ParentId) { + value.document_id() + } else { + 0 + }, + document_id + 1, + )); + } else { + return Ok(None); + } + } else { + return Ok(None); + } + } + + Ok(Some(ExpandPath { path, found_names })) + } + + async fn mailbox_get_by_name(&self, account_id: u32, path: &str) -> trc::Result> { + Ok(self + .mailbox_expand_path(account_id, path, true) + .await? + .and_then(|ep| { + let mut next_parent_id = 0; + 'outer: for (pos, name) in ep.path.iter().enumerate() { + let is_inbox = pos == 0 && name.eq_ignore_ascii_case("inbox"); + + for (part, parent_id, document_id) in &ep.found_names { + if (part.eq(name) || (is_inbox && part.eq_ignore_ascii_case("inbox"))) + && *parent_id == next_parent_id + { + next_parent_id = *document_id; + continue 'outer; + } + } + return None; + } + Some(next_parent_id - 1) + })) + } + + async fn mailbox_get_by_role(&self, account_id: u32, role: &str) -> trc::Result> { + self.store() + .filter( + account_id, + Collection::Mailbox, + vec![Filter::eq(Property::Role, role)], + ) + .await + .caused_by(trc::location!()) + .map(|r| r.results.min()) + } +} + +impl PartialEq for UidMailbox { + fn eq(&self, other: &Self) -> bool { + self.mailbox_id == other.mailbox_id + } +} + +impl Eq for UidMailbox {} + +impl ToBitmaps for UidMailbox { + fn to_bitmaps(&self, ops: &mut Vec, field: u8, set: bool) { + ops.push(Operation::Bitmap { + class: BitmapClass::Tag { + field, + value: TagValue::Id(MaybeDynamicId::Static(self.mailbox_id)), + }, + set, + }); + } +} + +impl SerializeInto for UidMailbox { + fn serialize_into(&self, buf: &mut Vec) { + buf.push_leb128(self.mailbox_id); + buf.push_leb128(self.uid); + } +} + +impl DeserializeFrom for UidMailbox { + fn deserialize_from(bytes: &mut Iter<'_, u8>) -> Option { + Some(UidMailbox { + mailbox_id: bytes.next_leb128()?, + uid: bytes.next_leb128()?, + }) + } +} + +impl Serialize for UidMailbox { + fn serialize(self) -> Vec { + let mut buf = Vec::with_capacity(U32_LEN * 2); + self.serialize_into(&mut buf); + buf + } +} + +impl UidMailbox { + pub fn new(mailbox_id: u32, uid: u32) -> Self { + UidMailbox { mailbox_id, uid } + } + + pub fn new_unassigned(mailbox_id: u32) -> Self { + UidMailbox { mailbox_id, uid: 0 } + } +} diff --git a/crates/jmap/src/email/metadata.rs b/crates/email/src/metadata.rs similarity index 100% rename from crates/jmap/src/email/metadata.rs rename to crates/email/src/metadata.rs diff --git a/crates/jmap/src/sieve/ingest.rs b/crates/email/src/sieve.rs similarity index 65% rename from crates/jmap/src/sieve/ingest.rs rename to crates/email/src/sieve.rs index b4731610..f2731fd1 100644 --- a/crates/jmap/src/sieve/ingest.rs +++ b/crates/email/src/sieve.rs @@ -4,33 +4,31 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::borrow::Cow; +use std::{borrow::Cow, sync::Arc}; -use common::{ - auth::AccessToken, listener::stream::NullIo, scripts::plugins::PluginContext, Server, +use crate::{ + delivery::AutogeneratedMessage, + ingest::{EmailIngest, IngestEmail, IngestSource, IngestedEmail}, + mailbox::{MailboxFnc, INBOX_ID, TRASH_ID}, }; +use common::{auth::AccessToken, scripts::plugins::PluginContext, Server}; use directory::{backend::internal::PrincipalField, Permission, QueryBy}; -use jmap_proto::types::{collection::Collection, id::Id, keyword::Keyword, property::Property}; +use jmap_proto::{ + object::Object, + types::{collection::Collection, id::Id, keyword::Keyword, property::Property, value::Value}, +}; use mail_parser::MessageParser; -use sieve::{Envelope, Event, Input, Mailbox, Recipient}; -use smtp::core::{Session, SessionAddress}; +use serde::ser::SerializeSeq; +use sieve::{Envelope, Event, Input, Mailbox, Recipient, Sieve}; use store::{ ahash::AHashSet, - write::{now, BatchBuilder, Bincode, F_VALUE}, + blake3, + query::Filter, + write::{assert::HashedValue, now, BatchBuilder, Bincode, BlobOp, F_VALUE}, + Deserialize, Serialize, }; use trc::{AddContext, SieveEvent}; -use crate::{ - email::{ - bayes::EmailBayesTrain, - ingest::{EmailIngest, IngestEmail, IngestSource, IngestedEmail}, - }, - mailbox::{get::MailboxGet, set::MailboxSet, INBOX_ID, TRASH_ID}, - sieve::SeenIdHash, - JmapMethods, -}; - -use super::{get::SieveScriptGet, ActiveScript}; use std::future::Future; struct SieveMessage<'x> { @@ -39,7 +37,27 @@ struct SieveMessage<'x> { pub flags: Vec, } +pub struct ActiveScript { + pub document_id: u32, + pub script_name: String, + pub script: Arc, + pub seen_ids: SeenIds, +} + +#[derive(Debug, Clone)] +pub struct SeenIdHash { + hash: [u8; 32], + expiry: u64, +} + +#[derive(Debug, Clone, Default)] +pub struct SeenIds { + pub ids: AHashSet, + pub has_changes: bool, +} + pub trait SieveScriptIngest: Sync + Send { + #[allow(clippy::too_many_arguments)] fn sieve_script_ingest( &self, access_token: &AccessToken, @@ -48,7 +66,25 @@ pub trait SieveScriptIngest: Sync + Send { envelope_to: &str, session_id: u64, active_script: ActiveScript, + autogenerated: &mut Vec, ) -> impl Future> + Send; + + fn sieve_script_get_active( + &self, + account_id: u32, + ) -> impl Future>> + Send; + + fn sieve_script_get_by_name( + &self, + account_id: u32, + name: &str, + ) -> impl Future>> + Send; + + fn sieve_script_compile( + &self, + account_id: u32, + document_id: u32, + ) -> impl Future)>> + Send; } impl SieveScriptIngest for Server { @@ -61,6 +97,7 @@ impl SieveScriptIngest for Server { envelope_to: &str, session_id: u64, mut active_script: ActiveScript, + autogenerated: &mut Vec, ) -> trc::Result { // Parse message let message = if let Some(message) = MessageParser::new().parse(raw_message) { @@ -347,10 +384,8 @@ impl SieveScriptIngest for Server { input = true.into(); if let Some(message) = messages.get(message_id) { let recipients = match recipient { - Recipient::Address(rcpt) => vec![SessionAddress::new(rcpt)], - Recipient::Group(rcpts) => { - rcpts.into_iter().map(SessionAddress::new).collect() - } + Recipient::Address(rcpt) => vec![rcpt], + Recipient::Group(rcpts) => rcpts, Recipient::List(_) => { // Not yet implemented continue; @@ -363,28 +398,24 @@ impl SieveScriptIngest for Server { From = mail_from.clone(), To = recipients .iter() - .map(|r| trc::Value::String(r.address_lcase.clone())) + .map(|r| trc::Value::String(r.clone())) .collect::>(), Size = message.raw_message.len(), SpanId = session_id ); - Session::::sieve( - self.clone(), - SessionAddress::new(mail_from.clone()), + autogenerated.push(AutogeneratedMessage { + sender_address: mail_from.clone(), recipients, - message.raw_message.to_vec(), - 0, - ) - .queue_message() - .await; + message: message.raw_message.to_vec(), + }); } else { trc::event!( Sieve(SieveEvent::MessageTooLarge), From = mail_from.clone(), To = recipients .iter() - .map(|r| trc::Value::String(r.address_lcase.clone())) + .map(|r| trc::Value::String(r.clone())) .collect::>(), Size = message.raw_message.len(), Limit = self.core.jmap.mail_max_size, @@ -521,7 +552,9 @@ impl SieveScriptIngest for Server { Bincode::new(active_script.seen_ids), F_VALUE, ); - let _ = self.write_batch(batch).await; + if let Err(err) = self.store().write(batch).await.caused_by(trc::location!()) { + trc::error!(err.details("Failed to save Sieve seen ids changes.")); + } } if let Some(reject_reason) = reject_reason { @@ -538,6 +571,189 @@ impl SieveScriptIngest for Server { Err(last_temp_error.unwrap()) } } + + async fn sieve_script_get_active(&self, account_id: u32) -> trc::Result> { + // Find the currently active script + if let Some(document_id) = self + .store() + .filter( + account_id, + Collection::SieveScript, + vec![Filter::eq(Property::IsActive, 1u32)], + ) + .await + .caused_by(trc::location!())? + .results + .min() + { + let (script, mut script_object) = + self.sieve_script_compile(account_id, document_id).await?; + Ok(Some(ActiveScript { + document_id, + script: Arc::new(script), + script_name: script_object + .properties + .remove(&Property::Name) + .and_then(|name| name.try_unwrap_string()) + .unwrap_or_else(|| account_id.to_string()), + seen_ids: self + .get_property::>( + account_id, + Collection::SieveScript, + document_id, + Property::EmailIds, + ) + .await? + .map(|seen_ids| seen_ids.inner) + .unwrap_or_default(), + })) + } else { + Ok(None) + } + } + + async fn sieve_script_get_by_name( + &self, + account_id: u32, + name: &str, + ) -> trc::Result> { + // Find the script by name + if let Some(document_id) = self + .store() + .filter( + account_id, + Collection::SieveScript, + vec![Filter::eq(Property::Name, name)], + ) + .await + .caused_by(trc::location!())? + .results + .min() + { + self.sieve_script_compile(account_id, document_id) + .await + .map(|(sieve, _)| Some(sieve)) + } else { + Ok(None) + } + } + + #[allow(clippy::blocks_in_conditions)] + async fn sieve_script_compile( + &self, + account_id: u32, + document_id: u32, + ) -> trc::Result<(Sieve, Object)> { + // Obtain script object + let script_object = self + .get_property::>>( + account_id, + Collection::SieveScript, + document_id, + Property::Value, + ) + .await? + .ok_or_else(|| { + trc::StoreEvent::NotFound + .into_err() + .caused_by(trc::location!()) + .document_id(document_id) + })?; + + // Obtain the sieve script length + let (script_offset, blob_id) = script_object + .inner + .properties + .get(&Property::BlobId) + .and_then(|v| v.as_blob_id()) + .and_then(|v| (v.section.as_ref()?.size, v).into()) + .ok_or_else(|| { + trc::StoreEvent::NotFound + .into_err() + .caused_by(trc::location!()) + .document_id(document_id) + })?; + + // Obtain the sieve script blob + let script_bytes = self + .core + .storage + .blob + .get_blob(blob_id.hash.as_ref(), 0..usize::MAX) + .await + .caused_by(trc::location!())? + .ok_or_else(|| { + trc::StoreEvent::NotFound + .into_err() + .caused_by(trc::location!()) + .document_id(document_id) + })?; + + // Obtain the precompiled script + if let Some(sieve) = script_bytes + .get(script_offset..) + .and_then(|bytes| Bincode::::deserialize(bytes).ok()) + { + Ok((sieve.inner, script_object.inner)) + } else { + // Deserialization failed, probably because the script compiler version changed + match self.core.sieve.untrusted_compiler.compile( + script_bytes.get(0..script_offset).ok_or_else(|| { + trc::StoreEvent::NotFound + .into_err() + .caused_by(trc::location!()) + .document_id(document_id) + })?, + ) { + Ok(sieve) => { + // Store updated compiled sieve script + let sieve = Bincode::new(sieve); + let compiled_bytes = (&sieve).serialize(); + let mut updated_sieve_bytes = + Vec::with_capacity(script_offset + compiled_bytes.len()); + updated_sieve_bytes.extend_from_slice(&script_bytes[0..script_offset]); + updated_sieve_bytes.extend_from_slice(&compiled_bytes); + + // Store updated blob + let mut new_blob_id = blob_id.clone(); + new_blob_id.hash = self + .put_blob(account_id, &updated_sieve_bytes, false) + .await? + .hash; + let mut new_script_object = script_object.inner.clone(); + new_script_object.set(Property::BlobId, new_blob_id.clone()); + + // Update script object + let mut batch = BatchBuilder::new(); + batch + .with_account_id(account_id) + .with_collection(Collection::SieveScript) + .update_document(document_id) + .assert_value(Property::Value, &script_object) + .set(Property::Value, (&new_script_object).serialize()) + .clear(BlobOp::Link { + hash: blob_id.hash.clone(), + }) + .set( + BlobOp::Link { + hash: new_blob_id.hash, + }, + Vec::new(), + ); + self.store() + .write(batch.build()) + .await + .caused_by(trc::location!())?; + + Ok((sieve.inner, new_script_object)) + } + Err(error) => Err(trc::StoreEvent::UnexpectedError + .caused_by(trc::location!()) + .reason(error) + .details("Failed to compile Sieve script")), + } + } + } } #[inline(always)] @@ -554,3 +770,107 @@ pub fn is_valid_role(role: &str) -> bool { ] .contains(&role) } + +impl SeenIdHash { + pub fn new(id: &str, expiry: u64) -> Self { + let mut hasher = blake3::Hasher::new(); + hasher.update(id.as_bytes()); + SeenIdHash { + hash: hasher.finalize().into(), + expiry, + } + } +} + +impl PartialOrd for SeenIdHash { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for SeenIdHash { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + self.expiry.cmp(&other.expiry) + } +} + +impl std::hash::Hash for SeenIdHash { + fn hash(&self, state: &mut H) { + self.hash.hash(state); + } +} + +impl PartialEq for SeenIdHash { + fn eq(&self, other: &Self) -> bool { + self.hash == other.hash + } +} + +impl Eq for SeenIdHash {} + +// SeenIds serializer +impl serde::Serialize for SeenIds { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + let mut seq = serializer.serialize_seq((self.ids.len() * 2).into())?; + for id in &self.ids { + seq.serialize_element(&id.expiry)?; + seq.serialize_element(&id.hash)?; + } + + seq.end() + } +} + +impl<'de> serde::Deserialize<'de> for SeenIds { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + deserializer.deserialize_seq(SeenIdsVisitor) + } +} + +struct SeenIdsVisitor; + +impl<'de> serde::de::Visitor<'de> for SeenIdsVisitor { + type Value = SeenIds; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + formatter.write_str("invalid SeenIds") + } + + fn visit_seq(self, mut seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + let num_entries = seq.size_hint().unwrap_or(0) / 2; + let mut seen_ids = SeenIds { + ids: AHashSet::with_capacity(num_entries), + has_changes: false, + }; + let now = now(); + + for _ in 0..num_entries { + let expiry = seq + .next_element::()? + .ok_or_else(|| serde::de::Error::custom("Expected expiry."))?; + if expiry > now { + seen_ids.ids.insert(SeenIdHash { + hash: seq + .next_element()? + .ok_or_else(|| serde::de::Error::custom("Expected hash."))?, + expiry, + }); + } else { + seq.next_element::<[u8; 32]>()? + .ok_or_else(|| serde::de::Error::custom("Expected hash."))?; + seen_ids.has_changes = true; + } + } + + Ok(seen_ids) + } +} diff --git a/crates/imap/Cargo.toml b/crates/imap/Cargo.toml index 04c2f007..377baf0e 100644 --- a/crates/imap/Cargo.toml +++ b/crates/imap/Cargo.toml @@ -12,6 +12,7 @@ directory = { path = "../directory" } trc = { path = "../trc" } store = { path = "../store" } common = { path = "../common" } +email = { path = "../email" } nlp = { path = "../nlp" } utils = { path = "../utils" } mail-parser = { version = "0.9", features = ["full_encoding", "ludicrous_mode"] } diff --git a/crates/imap/src/core/mailbox.rs b/crates/imap/src/core/mailbox.rs index a9aa11c9..63b5c761 100644 --- a/crates/imap/src/core/mailbox.rs +++ b/crates/imap/src/core/mailbox.rs @@ -11,12 +11,11 @@ use common::{ AccountId, Mailbox, }; use directory::{backend::internal::PrincipalField, QueryBy}; +use email::mailbox::{MailboxFnc, INBOX_ID}; use imap_proto::protocol::list::Attribute; use jmap::{ auth::acl::{AclMethods, EffectiveAcl}, changes::get::ChangesLookup, - mailbox::{get::MailboxGet, set::MailboxSet, INBOX_ID}, - JmapMethods, }; use jmap_proto::{ object::Object, diff --git a/crates/imap/src/core/message.rs b/crates/imap/src/core/message.rs index a3873091..0721ba74 100644 --- a/crates/imap/src/core/message.rs +++ b/crates/imap/src/core/message.rs @@ -8,8 +8,8 @@ use std::{collections::BTreeMap, sync::Arc}; use ahash::AHashMap; use common::{listener::SessionStream, NextMailboxState}; +use email::mailbox::UidMailbox; use imap_proto::protocol::{expunge, select::Exists, Sequence}; -use jmap::{mailbox::UidMailbox, JmapMethods}; use jmap_proto::{ object::Object, types::{collection::Collection, property::Property, value::Value}, diff --git a/crates/imap/src/op/acl.rs b/crates/imap/src/op/acl.rs index 25b72128..cc71f237 100644 --- a/crates/imap/src/op/acl.rs +++ b/crates/imap/src/op/acl.rs @@ -11,6 +11,7 @@ use directory::{ backend::internal::{manage::ChangedPrincipals, PrincipalField}, Permission, QueryBy, Type, }; +use email::mailbox::SCHEMA; use imap_proto::{ protocol::acl::{ Arguments, GetAclResponse, ListRightsResponse, ModRightsOp, MyRightsResponse, Rights, @@ -19,10 +20,7 @@ use imap_proto::{ Command, ResponseCode, StatusResponse, }; -use jmap::{ - auth::acl::EffectiveAcl, changes::write::ChangeLog, mailbox::set::SCHEMA, - services::state::StateManager, JmapMethods, -}; +use jmap::auth::acl::EffectiveAcl; use jmap_proto::{ object::{index::ObjectIndexBuilder, Object}, types::{ @@ -352,7 +350,8 @@ impl Session { ); if !batch.is_empty() { data.server - .write_batch(batch) + .store() + .write(batch) .await .imap_ctx(&arguments.tag, trc::location!())?; let mut changes = ChangeLogBuilder::new(); diff --git a/crates/imap/src/op/append.rs b/crates/imap/src/op/append.rs index 9ef60cbe..73c54c4c 100644 --- a/crates/imap/src/op/append.rs +++ b/crates/imap/src/op/append.rs @@ -7,6 +7,7 @@ use std::{sync::Arc, time::Instant}; use directory::Permission; +use email::ingest::{EmailIngest, IngestEmail, IngestSource}; use imap_proto::{ protocol::{append::Arguments, select::HighestModSeq}, receiver::Request, @@ -18,13 +19,6 @@ use crate::{ spawn_op, }; use common::{listener::SessionStream, MailboxId}; -use jmap::{ - email::{ - bayes::EmailBayesTrain, - ingest::{EmailIngest, IngestEmail, IngestSource}, - }, - services::state::StateManager, -}; use jmap_proto::types::{acl::Acl, keyword::Keyword, state::StateChange, type_state::DataType}; use mail_parser::MessageParser; diff --git a/crates/imap/src/op/copy_move.rs b/crates/imap/src/op/copy_move.rs index 79b02514..c531513e 100644 --- a/crates/imap/src/op/copy_move.rs +++ b/crates/imap/src/op/copy_move.rs @@ -7,6 +7,10 @@ use std::{sync::Arc, time::Instant}; use directory::Permission; +use email::{ + ingest::EmailIngest, + mailbox::{UidMailbox, JUNK_ID}, +}; use imap_proto::{ protocol::copy_move::Arguments, receiver::Request, Command, ResponseCode, ResponseType, StatusResponse, @@ -17,13 +21,7 @@ use crate::{ spawn_op, }; use common::{listener::SessionStream, MailboxId}; -use jmap::{ - changes::write::ChangeLog, - email::{bayes::EmailBayesTrain, copy::EmailCopy, ingest::EmailIngest, set::TagManager}, - mailbox::{UidMailbox, JUNK_ID}, - services::{index::Indexer, state::StateManager}, - JmapMethods, -}; +use jmap::email::{bayes::EmailBayesTrain, copy::EmailCopy, set::TagManager}; use jmap_proto::{ error::set::SetErrorType, types::{ @@ -236,7 +234,6 @@ impl SessionData { changelog.change_id = self .server .assign_change_id(account_id) - .await .imap_ctx(&arguments.tag, trc::location!())?; } batch.value(Property::Cid, changelog.change_id, F_VALUE); @@ -270,7 +267,8 @@ impl SessionData { // Write changes self.server - .write_batch(batch) + .store() + .write(batch) .await .imap_ctx(&arguments.tag, trc::location!())?; diff --git a/crates/imap/src/op/create.rs b/crates/imap/src/op/create.rs index 2889cb5f..981f1f44 100644 --- a/crates/imap/src/op/create.rs +++ b/crates/imap/src/op/create.rs @@ -13,14 +13,13 @@ use crate::{ }; use common::{listener::SessionStream, Account, Mailbox}; use directory::Permission; +use email::mailbox::SCHEMA; use imap_proto::{ protocol::{create::Arguments, list::Attribute}, receiver::Request, Command, ResponseCode, StatusResponse, }; -use jmap::{ - changes::write::ChangeLog, mailbox::set::SCHEMA, services::state::StateManager, JmapMethods, -}; +use jmap::JmapMethods; use jmap_proto::{ object::{index::ObjectIndexBuilder, Object}, types::{ @@ -79,7 +78,6 @@ impl SessionData { let mut changes = self .server .begin_changes(params.account_id) - .await .imap_ctx(&arguments.tag, trc::location!())?; let mut parent_id = params.parent_mailbox_id.map(|id| id + 1).unwrap_or(0); @@ -105,7 +103,8 @@ impl SessionData { .custom(ObjectIndexBuilder::new(SCHEMA).with_changes(mailbox)); let mailbox_id = self .server - .write_batch_expect_id(batch) + .store() + .write_expect_id(batch) .await .imap_ctx(&arguments.tag, trc::location!())?; changes.log_insert(Collection::Mailbox, mailbox_id); @@ -121,7 +120,8 @@ impl SessionData { .with_collection(Collection::Mailbox) .custom(changes); self.server - .write_batch(batch) + .store() + .write(batch) .await .imap_ctx(&arguments.tag, trc::location!())?; diff --git a/crates/imap/src/op/delete.rs b/crates/imap/src/op/delete.rs index 0d9ecec8..f208d9a7 100644 --- a/crates/imap/src/op/delete.rs +++ b/crates/imap/src/op/delete.rs @@ -15,7 +15,7 @@ use directory::Permission; use imap_proto::{ protocol::delete::Arguments, receiver::Request, Command, ResponseCode, StatusResponse, }; -use jmap::{changes::write::ChangeLog, mailbox::set::MailboxSet, services::state::StateManager}; +use jmap::mailbox::set::MailboxSet; use jmap_proto::types::{state::StateChange, type_state::DataType}; use store::write::log::ChangeLogBuilder; diff --git a/crates/imap/src/op/expunge.rs b/crates/imap/src/op/expunge.rs index 519c2808..a32599c3 100644 --- a/crates/imap/src/op/expunge.rs +++ b/crates/imap/src/op/expunge.rs @@ -8,6 +8,7 @@ use std::{sync::Arc, time::Instant}; use ahash::AHashMap; use directory::Permission; +use email::mailbox::UidMailbox; use imap_proto::{ parser::parse_sequence_set, receiver::{Request, Token}, @@ -17,13 +18,7 @@ use trc::AddContext; use crate::core::{SavedSearch, SelectedMailbox, Session, SessionData}; use common::{listener::SessionStream, ImapId}; -use jmap::{ - changes::write::ChangeLog, - email::{delete::EmailDeletion, set::TagManager}, - mailbox::UidMailbox, - services::state::StateManager, - JmapMethods, -}; +use jmap::email::{delete::EmailDeletion, set::TagManager}; use jmap_proto::types::{ acl::Acl, collection::Collection, id::Id, keyword::Keyword, property::Property, state::StateChange, type_state::DataType, @@ -251,10 +246,16 @@ impl SessionData { mailboxes.update_batch(&mut batch, Property::MailboxIds); keywords.update_batch(&mut batch, Property::Keywords); if changelog.change_id == u64::MAX { - changelog.change_id = self.server.assign_change_id(account_id).await? + changelog.change_id = self.server.assign_change_id(account_id)? } batch.value(Property::Cid, changelog.change_id, F_VALUE); - match self.server.write_batch(batch).await { + match self + .server + .store() + .write(batch) + .await + .caused_by(trc::location!()) + { Ok(_) => { changelog.log_update(Collection::Email, Id::from_parts(thread_id, id)); changelog.log_child_update(Collection::Mailbox, mailbox_id.mailbox_id); diff --git a/crates/imap/src/op/fetch.rs b/crates/imap/src/op/fetch.rs index fa0ae8ee..7df4bcc1 100644 --- a/crates/imap/src/op/fetch.rs +++ b/crates/imap/src/op/fetch.rs @@ -13,6 +13,7 @@ use crate::{ use ahash::AHashMap; use common::listener::SessionStream; use directory::Permission; +use email::metadata::MessageMetadata; use imap_proto::{ parser::PushUnique, protocol::{ @@ -26,13 +27,7 @@ use imap_proto::{ receiver::Request, Command, ResponseCode, ResponseType, StatusResponse, }; -use jmap::{ - blob::download::BlobDownload, - changes::{get::ChangesLookup, write::ChangeLog}, - email::metadata::MessageMetadata, - services::state::StateManager, - JmapMethods, -}; +use jmap::{blob::download::BlobDownload, changes::get::ChangesLookup}; use jmap_proto::types::{ acl::Acl, collection::Collection, id::Id, keyword::Keyword, property::Property, state::StateChange, type_state::DataType, @@ -42,6 +37,7 @@ use store::{ query::log::{Change, Query}, write::{assert::HashedValue, BatchBuilder, Bincode, F_BITMAP, F_VALUE}, }; +use trc::AddContext; use super::{FromModSeq, ImapContext}; @@ -532,7 +528,6 @@ impl SessionData { let mut changelog = self .server .begin_changes(account_id) - .await .imap_ctx(&arguments.tag, trc::location!())?; for (id, mut keywords) in set_seen_ids { keywords.inner.push(Keyword::Seen); @@ -545,7 +540,13 @@ impl SessionData { .value(Property::Keywords, keywords.inner, F_VALUE) .value(Property::Keywords, Keyword::Seen, F_BITMAP) .value(Property::Cid, changelog.change_id, F_VALUE); - match self.server.write_batch(batch).await { + match self + .server + .store() + .write(batch) + .await + .caused_by(trc::location!()) + { Ok(_) => { changelog.log_update(Collection::Email, id); } diff --git a/crates/imap/src/op/rename.rs b/crates/imap/src/op/rename.rs index 9daa7fd7..b0f53f43 100644 --- a/crates/imap/src/op/rename.rs +++ b/crates/imap/src/op/rename.rs @@ -12,13 +12,11 @@ use crate::{ }; use common::listener::SessionStream; use directory::Permission; +use email::mailbox::SCHEMA; use imap_proto::{ protocol::rename::Arguments, receiver::Request, Command, ResponseCode, StatusResponse, }; -use jmap::{ - auth::acl::EffectiveAcl, changes::write::ChangeLog, mailbox::set::SCHEMA, - services::state::StateManager, JmapMethods, -}; +use jmap::auth::acl::EffectiveAcl; use jmap_proto::{ object::{index::ObjectIndexBuilder, Object}, types::{ @@ -138,7 +136,6 @@ impl SessionData { let mut changes = self .server .begin_changes(params.account_id) - .await .imap_ctx(&arguments.tag, trc::location!())?; let mut parent_id = params.parent_mailbox_id.map(|id| id + 1).unwrap_or(0); @@ -163,7 +160,8 @@ impl SessionData { let mailbox_id = self .server - .write_batch_expect_id(batch) + .store() + .write_expect_id(batch) .await .imap_ctx(&arguments.tag, trc::location!())?; @@ -195,7 +193,8 @@ impl SessionData { let change_id = changes.change_id; batch.custom(changes); self.server - .write_batch(batch) + .store() + .write(batch) .await .imap_ctx(&arguments.tag, trc::location!())?; diff --git a/crates/imap/src/op/status.rs b/crates/imap/src/op/status.rs index 75ac01da..21de341c 100644 --- a/crates/imap/src/op/status.rs +++ b/crates/imap/src/op/status.rs @@ -19,7 +19,6 @@ use imap_proto::{ receiver::Request, Command, ResponseCode, StatusResponse, }; -use jmap::JmapMethods; use jmap_proto::{ object::Object, types::{collection::Collection, id::Id, keyword::Keyword, property::Property, value::Value}, diff --git a/crates/imap/src/op/store.rs b/crates/imap/src/op/store.rs index 309b6785..a77fd8ac 100644 --- a/crates/imap/src/op/store.rs +++ b/crates/imap/src/op/store.rs @@ -13,6 +13,7 @@ use crate::{ use ahash::AHashSet; use common::listener::SessionStream; use directory::Permission; +use email::{ingest::EmailIngest, mailbox::UidMailbox}; use imap_proto::{ protocol::{ fetch::{DataItem, FetchItem}, @@ -23,11 +24,8 @@ use imap_proto::{ Command, ResponseCode, ResponseType, StatusResponse, }; use jmap::{ - changes::{get::ChangesLookup, write::ChangeLog}, + changes::get::ChangesLookup, email::{bayes::EmailBayesTrain, set::TagManager}, - mailbox::UidMailbox, - services::{index::Indexer, state::StateManager}, - JmapMethods, }; use jmap_proto::types::{ acl::Acl, collection::Collection, id::Id, keyword::Keyword, property::Property, @@ -37,6 +35,7 @@ use store::{ query::log::{Change, Query}, write::{assert::HashedValue, log::ChangeLogBuilder, BatchBuilder, ValueClass, F_VALUE}, }; +use trc::AddContext; use super::{FromModSeq, ImapContext}; @@ -291,7 +290,6 @@ impl SessionData { changelog.change_id = self .server .assign_change_id(account_id) - .await .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())? } batch.value(Property::Cid, changelog.change_id, F_VALUE); @@ -310,7 +308,13 @@ impl SessionData { has_spam_train_tasks = true; } - match self.server.write_batch(batch).await { + match self + .server + .store() + .write(batch) + .await + .caused_by(trc::location!()) + { Ok(_) => { // Set all current mailboxes as changed if the Seen tag changed if seen_changed { diff --git a/crates/imap/src/op/subscribe.rs b/crates/imap/src/op/subscribe.rs index 79b426d1..671db792 100644 --- a/crates/imap/src/op/subscribe.rs +++ b/crates/imap/src/op/subscribe.rs @@ -12,13 +12,9 @@ use crate::{ }; use common::listener::SessionStream; use directory::Permission; +use email::mailbox::SCHEMA; use imap_proto::{receiver::Request, Command, ResponseCode, StatusResponse}; -use jmap::{ - changes::write::ChangeLog, - mailbox::set::{MailboxSubscribe, SCHEMA}, - services::state::StateManager, - JmapMethods, -}; +use jmap::mailbox::set::MailboxSubscribe; use jmap_proto::{ object::{index::ObjectIndexBuilder, Object}, types::{ @@ -129,7 +125,6 @@ impl SessionData { let mut changes = self .server .begin_changes(account_id) - .await .imap_ctx(&tag, trc::location!())?; let mut batch = BatchBuilder::new(); batch @@ -148,7 +143,8 @@ impl SessionData { let change_id = changes.change_id; batch.custom(changes); self.server - .write_batch(batch) + .store() + .write(batch) .await .imap_ctx(&tag, trc::location!())?; diff --git a/crates/imap/src/op/thread.rs b/crates/imap/src/op/thread.rs index 94d8da12..f65395ba 100644 --- a/crates/imap/src/op/thread.rs +++ b/crates/imap/src/op/thread.rs @@ -13,6 +13,7 @@ use crate::{ use ahash::AHashMap; use common::listener::SessionStream; use directory::Permission; +use email::cache::ThreadCache; use imap_proto::{ protocol::{ thread::{Arguments, Response}, @@ -21,7 +22,6 @@ use imap_proto::{ receiver::Request, Command, StatusResponse, }; -use jmap::email::cache::ThreadCache; use trc::AddContext; impl Session { diff --git a/crates/jmap/Cargo.toml b/crates/jmap/Cargo.toml index 02929751..cdfbb6fd 100644 --- a/crates/jmap/Cargo.toml +++ b/crates/jmap/Cargo.toml @@ -14,6 +14,7 @@ common = { path = "../common" } directory = { path = "../directory" } trc = { path = "../trc" } spam-filter = { path = "../spam-filter" } +email = { path = "../email" } smtp-proto = { version = "0.1" } mail-parser = { version = "0.9", features = ["full_encoding", "serde_support", "ludicrous_mode"] } mail-builder = { version = "0.3", features = ["ludicrous_mode"] } @@ -27,8 +28,6 @@ hyper-util = { version = "0.1.1", features = ["tokio"] } http-body-util = "0.1.0" form_urlencoded = "1.1.0" tokio = { version = "1.23", features = ["rt"] } -aes-gcm = "0.10.1" -aes-gcm-siv = "0.11.1" bincode = "1.3.3" form-data = { version = "0.5.0", features = ["sync"], default-features = false } mime = "0.3.17" @@ -40,25 +39,20 @@ hkdf = "0.12.3" sha1 = "0.10" sha2 = "0.10" reqwest = { version = "0.12", default-features = false, features = ["rustls-tls-webpki-roots", "http2"]} -tokio-tungstenite = "0.24" -tungstenite = "0.24" +tokio-tungstenite = "0.26" +tungstenite = "0.26" chrono = "0.4" dashmap = "6.0" -aes = "0.8.3" -cbc = { version = "0.1.2", features = ["alloc"] } -sequoia-openpgp = { version = "1.16", default-features = false, features = ["crypto-rust", "allow-experimental-crypto", "allow-variable-time-crypto"] } rand = "0.8.5" pkcs8 = { version = "0.10.2", features = ["alloc", "std"] } -rasn = "0.10" -rasn-cms = "0.10" -rasn-pkix = "0.10" -rsa = "0.9.2" -async-trait = "0.1.68" lz4_flex = { version = "0.11", default-features = false } rev_lines = "0.3.0" x509-parser = "0.16.0" -quick-xml = "0.36" +quick-xml = "0.37" memory-stats = "1.2.0" +aes-gcm = "0.10.1" +aes-gcm-siv = "0.11.1" +rsa = "0.9.2" [features] test_mode = [] diff --git a/crates/jmap/src/api/form.rs b/crates/jmap/src/api/form.rs index 34e87d79..48284b2b 100644 --- a/crates/jmap/src/api/form.rs +++ b/crates/jmap/src/api/form.rs @@ -9,10 +9,9 @@ use std::{borrow::Cow, fmt::Write, future::Future}; use chrono::Utc; use common::{ config::network::{ContactForm, FieldOrDefault}, - ip_to_bytes, - ipc::{DeliveryResult, IngestMessage}, - psl, Server, KV_RATE_LIMIT_CONTACT, + ip_to_bytes, psl, Server, KV_RATE_LIMIT_CONTACT, }; +use email::delivery::{IngestMessage, LocalDeliveryStatus, MailDelivery}; use hyper::StatusCode; use mail_auth::common::cache::NoCache; use mail_builder::{ @@ -32,7 +31,7 @@ use trc::AddContext; use utils::BlobHash; use x509_parser::nom::AsBytes; -use crate::{auth::oauth::FormData, services::ingest::MailDelivery}; +use crate::auth::oauth::FormData; use super::{ http::{HttpSessionData, ToHttpResponse}, @@ -210,13 +209,16 @@ impl FormHandler for Server { session_id: session.session_id, }) .await + .status { match result { - DeliveryResult::Success => { + LocalDeliveryStatus::Success => { has_success = true; } - DeliveryResult::TemporaryFailure { reason } - | DeliveryResult::PermanentFailure { reason, .. } => failure = Some(reason), + LocalDeliveryStatus::TemporaryFailure { reason } + | LocalDeliveryStatus::PermanentFailure { reason, .. } => { + failure = Some(reason) + } } } diff --git a/crates/jmap/src/api/management/enterprise/undelete.rs b/crates/jmap/src/api/management/enterprise/undelete.rs index c5cef6c6..ecbad57b 100644 --- a/crates/jmap/src/api/management/enterprise/undelete.rs +++ b/crates/jmap/src/api/management/enterprise/undelete.rs @@ -13,6 +13,10 @@ use std::str::FromStr; use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; use common::{auth::AccessToken, enterprise::undelete::DeletedBlob, Server}; use directory::backend::internal::manage::ManageDirectory; +use email::{ + ingest::{EmailIngest, IngestEmail, IngestSource}, + mailbox::INBOX_ID, +}; use hyper::Method; use jmap_proto::types::collection::Collection; use mail_parser::{DateTime, MessageParser}; @@ -29,9 +33,6 @@ use crate::{ HttpRequest, HttpResponse, JsonResponse, }, blob::download::BlobDownload, - email::ingest::{EmailIngest, IngestEmail, IngestSource}, - mailbox::INBOX_ID, - JmapMethods, }; #[derive(serde::Deserialize, serde::Serialize)] diff --git a/crates/jmap/src/api/management/stores.rs b/crates/jmap/src/api/management/stores.rs index 6eea3384..715cda17 100644 --- a/crates/jmap/src/api/management/stores.rs +++ b/crates/jmap/src/api/management/stores.rs @@ -15,6 +15,10 @@ use directory::{ backend::internal::manage::{self, ManageDirectory}, Permission, }; +use email::{ + ingest::EmailIngest, + mailbox::{UidMailbox, SCHEMA}, +}; use hyper::Method; use jmap_proto::{ object::{index::ObjectIndexBuilder, Object}, @@ -30,10 +34,7 @@ use crate::{ http::{HttpSessionData, ToHttpResponse}, HttpRequest, HttpResponse, JsonResponse, }, - email::ingest::EmailIngest, - mailbox::{set::SCHEMA, UidMailbox}, services::index::Indexer, - JmapMethods, }; use super::decode_path_element; @@ -344,7 +345,8 @@ pub async fn reset_imap_uids(server: &Server, account_id: u32) -> trc::Result<(u ) .clear(Property::EmailIds); server - .write_batch(batch) + .store() + .write(batch) .await .caused_by(trc::location!())?; mailbox_count += 1; @@ -388,7 +390,8 @@ pub async fn reset_imap_uids(server: &Server, account_id: u32) -> trc::Result<(u .assert_value(ValueClass::Property(Property::MailboxIds.into()), &uids) .value(Property::MailboxIds, uids.inner, F_VALUE); server - .write_batch(batch) + .store() + .write(batch) .await .caused_by(trc::location!())?; email_count += 1; diff --git a/crates/jmap/src/api/request.rs b/crates/jmap/src/api/request.rs index 4b2a184a..e9d6eef4 100644 --- a/crates/jmap/src/api/request.rs +++ b/crates/jmap/src/api/request.rs @@ -30,7 +30,6 @@ use crate::{ principal::{get::PrincipalGet, query::PrincipalQuery}, push::{get::PushSubscriptionFetch, set::PushSubscriptionSet}, quota::{get::QuotaGet, query::QuotaQuery}, - services::state::StateManager, sieve::{ get::SieveScriptGet, query::SieveScriptQuery, set::SieveScriptSet, validate::SieveScriptValidate, diff --git a/crates/jmap/src/auth/acl.rs b/crates/jmap/src/auth/acl.rs index fc5f14ab..f80425b5 100644 --- a/crates/jmap/src/auth/acl.rs +++ b/crates/jmap/src/auth/acl.rs @@ -30,8 +30,6 @@ use store::{ use trc::AddContext; use utils::map::bitmap::Bitmap; -use crate::JmapMethods; - pub trait AclMethods: Sync + Send { fn shared_documents( &self, diff --git a/crates/jmap/src/blob/copy.rs b/crates/jmap/src/blob/copy.rs index 19aa6df0..c53f5714 100644 --- a/crates/jmap/src/blob/copy.rs +++ b/crates/jmap/src/blob/copy.rs @@ -10,6 +10,7 @@ use jmap_proto::{ method::copy::{CopyBlobRequest, CopyBlobResponse}, types::blob::BlobId, }; +use trc::AddContext; use std::future::Future; use store::{ @@ -18,8 +19,6 @@ use store::{ }; use utils::map::vec_map::VecMap; -use crate::JmapMethods; - use super::download::BlobDownload; pub trait BlobCopy: Sync + Send { @@ -55,7 +54,10 @@ impl BlobCopy for Server { }, 0u32.serialize(), ); - self.write_batch(batch).await?; + self.store() + .write(batch) + .await + .caused_by(trc::location!())?; let dest_blob_id = BlobId { hash: blob_id.hash.clone(), class: BlobClass::Reserved { diff --git a/crates/jmap/src/blob/download.rs b/crates/jmap/src/blob/download.rs index 19534b07..ceac9561 100644 --- a/crates/jmap/src/blob/download.rs +++ b/crates/jmap/src/blob/download.rs @@ -129,6 +129,7 @@ impl BlobDownload for Server { })) } + #[inline(always)] async fn get_blob(&self, hash: &BlobHash, range: Range) -> trc::Result>> { self.core .storage diff --git a/crates/jmap/src/blob/get.rs b/crates/jmap/src/blob/get.rs index b73e05c5..cdc3b115 100644 --- a/crates/jmap/src/blob/get.rs +++ b/crates/jmap/src/blob/get.rs @@ -5,6 +5,7 @@ */ use common::{auth::AccessToken, Server}; +use email::mailbox::UidMailbox; use jmap_proto::{ method::{ get::{GetRequest, GetResponse}, @@ -26,7 +27,6 @@ use sha2::{Sha256, Sha512}; use store::BlobClass; use utils::map::vec_map::VecMap; -use crate::{mailbox::UidMailbox, JmapMethods}; use std::future::Future; use super::download::BlobDownload; diff --git a/crates/jmap/src/blob/upload.rs b/crates/jmap/src/blob/upload.rs index 49255196..196dbfea 100644 --- a/crates/jmap/src/blob/upload.rs +++ b/crates/jmap/src/blob/upload.rs @@ -14,16 +14,12 @@ use jmap_proto::{ BlobUploadRequest, BlobUploadResponse, BlobUploadResponseObject, DataSourceObject, }, request::reference::MaybeReference, - types::{blob::BlobId, id::Id}, + types::id::Id, }; -use store::{ - write::{now, BatchBuilder, BlobOp}, - BlobClass, Serialize, -}; -use trc::AddContext; -use utils::BlobHash; -use crate::{auth::rate_limit::RateLimiter, JmapMethods}; +use trc::AddContext; + +use crate::auth::rate_limit::RateLimiter; use super::{download::BlobDownload, UploadResponse}; use std::future::Future; @@ -46,13 +42,6 @@ pub trait BlobUpload: Sync + Send { data: &[u8], access_token: Arc, ) -> impl Future> + Send; - - fn put_blob( - &self, - account_id: u32, - data: &[u8], - set_quota: bool, - ) -> impl Future> + Send; } impl BlobUpload for Server { @@ -261,52 +250,4 @@ impl BlobUpload for Server { size: data.len(), }) } - - #[allow(clippy::blocks_in_conditions)] - async fn put_blob(&self, account_id: u32, data: &[u8], set_quota: bool) -> trc::Result { - // First reserve the hash - let hash = BlobHash::from(data); - let mut batch = BatchBuilder::new(); - let until = now() + self.core.jmap.upload_tmp_ttl; - - batch.with_account_id(account_id).set( - BlobOp::Reserve { - hash: hash.clone(), - until, - }, - (if set_quota { data.len() as u32 } else { 0u32 }).serialize(), - ); - self.write_batch(batch).await?; - - if !self - .core - .storage - .data - .blob_exists(&hash) - .await - .caused_by(trc::location!())? - { - // Upload blob to store - self.core - .storage - .blob - .put_blob(hash.as_ref(), data) - .await - .caused_by(trc::location!())?; - - // Commit blob - let mut batch = BatchBuilder::new(); - batch.set(BlobOp::Commit { hash: hash.clone() }, Vec::new()); - self.write_batch(batch).await?; - } - - Ok(BlobId { - hash, - class: BlobClass::Reserved { - account_id, - expires: until, - }, - section: None, - }) - } } diff --git a/crates/jmap/src/changes/mod.rs b/crates/jmap/src/changes/mod.rs index 6c14d1b7..1785abf9 100644 --- a/crates/jmap/src/changes/mod.rs +++ b/crates/jmap/src/changes/mod.rs @@ -7,4 +7,3 @@ pub mod get; pub mod query; pub mod state; -pub mod write; diff --git a/crates/jmap/src/changes/write.rs b/crates/jmap/src/changes/write.rs deleted file mode 100644 index 7074cc21..00000000 --- a/crates/jmap/src/changes/write.rs +++ /dev/null @@ -1,112 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use std::time::Duration; - -use common::Server; -use jmap_proto::types::collection::Collection; -use std::future::Future; -use store::{ - write::{log::ChangeLogBuilder, BatchBuilder}, - LogKey, -}; -use trc::AddContext; - -pub trait ChangeLog: Sync + Send { - fn begin_changes( - &self, - account_id: u32, - ) -> impl Future> + Send; - fn assign_change_id(&self, account_id: u32) -> impl Future> + Send; - fn generate_snowflake_id(&self) -> trc::Result; - fn commit_changes( - &self, - account_id: u32, - changes: ChangeLogBuilder, - ) -> impl Future> + Send; - fn delete_changes( - &self, - account_id: u32, - before: Duration, - ) -> impl Future> + Send; -} - -impl ChangeLog for Server { - async fn begin_changes(&self, account_id: u32) -> trc::Result { - self.assign_change_id(account_id) - .await - .map(ChangeLogBuilder::with_change_id) - } - - async fn assign_change_id(&self, _: u32) -> trc::Result { - self.generate_snowflake_id() - } - - fn generate_snowflake_id(&self) -> trc::Result { - self.inner.data.jmap_id_gen.generate().ok_or_else(|| { - trc::StoreEvent::UnexpectedError - .into_err() - .caused_by(trc::location!()) - .ctx(trc::Key::Reason, "Failed to generate snowflake id.") - }) - } - - async fn commit_changes( - &self, - account_id: u32, - mut changes: ChangeLogBuilder, - ) -> trc::Result { - if changes.change_id == u64::MAX || changes.change_id == 0 { - changes.change_id = self.assign_change_id(account_id).await?; - } - let state = changes.change_id; - - let mut builder = BatchBuilder::new(); - builder.with_account_id(account_id).custom(changes); - self.core - .storage - .data - .write(builder.build()) - .await - .caused_by(trc::location!()) - .map(|_| state) - } - - async fn delete_changes(&self, account_id: u32, before: Duration) -> trc::Result<()> { - let reference_cid = self.inner.data.jmap_id_gen.past_id(before).ok_or_else(|| { - trc::StoreEvent::UnexpectedError - .caused_by(trc::location!()) - .ctx(trc::Key::Reason, "Failed to generate reference change id.") - })?; - - for collection in [ - Collection::Email, - Collection::Mailbox, - Collection::Thread, - Collection::Identity, - Collection::EmailSubmission, - ] { - self.core - .storage - .data - .delete_range( - LogKey { - account_id, - collection: collection.into(), - change_id: 0, - }, - LogKey { - account_id, - collection: collection.into(), - change_id: reference_cid, - }, - ) - .await?; - } - - Ok(()) - } -} diff --git a/crates/jmap/src/email/bayes.rs b/crates/jmap/src/email/bayes.rs index 629ab110..9d8dd4c8 100644 --- a/crates/jmap/src/email/bayes.rs +++ b/crates/jmap/src/email/bayes.rs @@ -6,8 +6,8 @@ use std::future::Future; -use common::{auth::AccessToken, Server}; -use directory::Permission; +use common::Server; +use email::metadata::MessageMetadata; use jmap_proto::types::{collection::Collection, property::Property}; use mail_parser::Message; use spam_filter::{ @@ -16,10 +16,6 @@ use spam_filter::{ use store::write::{Bincode, TaskQueueClass}; use trc::StoreEvent; -use crate::{changes::write::ChangeLog, JmapMethods}; - -use super::metadata::MessageMetadata; - pub trait EmailBayesTrain: Sync + Send { fn email_bayes_train( &self, @@ -35,8 +31,6 @@ pub trait EmailBayesTrain: Sync + Send { document_id: u32, learn_spam: bool, ) -> impl Future> + Send; - - fn email_bayes_can_train(&self, access_token: &AccessToken) -> bool; } impl EmailBayesTrain for Server { @@ -83,10 +77,4 @@ impl EmailBayesTrain for Server { learn_spam, }) } - - fn email_bayes_can_train(&self, access_token: &AccessToken) -> bool { - self.core.spam.bayes.as_ref().is_some_and(|bayes| { - bayes.account_classify && access_token.has_permission(Permission::SpamFilterTrain) - }) - } } diff --git a/crates/jmap/src/email/body.rs b/crates/jmap/src/email/body.rs index 9a48615e..31860dd0 100644 --- a/crates/jmap/src/email/body.rs +++ b/crates/jmap/src/email/body.rs @@ -4,16 +4,14 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use email::metadata::{MessageMetadataContents, MetadataPartType}; use jmap_proto::{ object::Object, types::{blob::BlobId, property::Property, value::Value}, }; use mail_parser::{HeaderValue, MessagePart, MimeHeaders, PartType}; -use super::{ - headers::HeaderToValue, - metadata::{MessageMetadataContents, MetadataPartType}, -}; +use super::headers::HeaderToValue; pub trait ToBodyPart { fn to_body_part( diff --git a/crates/jmap/src/email/copy.rs b/crates/jmap/src/email/copy.rs index c71435fe..7be91279 100644 --- a/crates/jmap/src/email/copy.rs +++ b/crates/jmap/src/email/copy.rs @@ -8,6 +8,12 @@ use common::{ auth::{AccessToken, ResourceToken}, Server, }; +use email::{ + index::{EmailIndexBuilder, TrimTextValue, VisitValues, MAX_ID_LENGTH, MAX_SORT_FIELD_LENGTH}, + ingest::{EmailIngest, IngestedEmail, LogEmailInsert}, + mailbox::{MailboxFnc, UidMailbox}, + metadata::MessageMetadata, +}; use jmap_proto::{ error::set::SetError, method::{ @@ -45,22 +51,9 @@ use store::{ use trc::AddContext; use utils::map::vec_map::VecMap; -use crate::{ - api::http::HttpSessionData, - auth::acl::AclMethods, - changes::{state::StateManager, write::ChangeLog}, - mailbox::{set::MailboxSet, UidMailbox}, - services::index::Indexer, - JmapMethods, -}; +use crate::{api::http::HttpSessionData, auth::acl::AclMethods, changes::state::StateManager}; use std::future::Future; -use super::{ - index::{EmailIndexBuilder, TrimTextValue, VisitValues, MAX_ID_LENGTH, MAX_SORT_FIELD_LENGTH}, - ingest::{EmailIngest, IngestedEmail, LogEmailInsert}, - metadata::MessageMetadata, -}; - pub trait EmailCopy: Sync + Send { fn email_copy( &self, @@ -413,7 +406,7 @@ impl EmailCopy for Server { } // Prepare batch - let change_id = self.assign_change_id(account_id).await?; + let change_id = self.assign_change_id(account_id)?; let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) diff --git a/crates/jmap/src/email/crypto.rs b/crates/jmap/src/email/crypto.rs index fcdd826a..fc6f5129 100644 --- a/crates/jmap/src/email/crypto.rs +++ b/crates/jmap/src/email/crypto.rs @@ -4,661 +4,20 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{ - borrow::Cow, collections::BTreeSet, fmt::Display, future::Future, io::Cursor, sync::Arc, -}; +use std::{future::Future, sync::Arc}; -use crate::{ - api::{http::ToHttpResponse, HttpResponse, JsonResponse}, - JmapMethods, -}; -use aes::cipher::{block_padding::Pkcs7, BlockEncryptMut, KeyIvInit}; +use crate::api::{http::ToHttpResponse, HttpResponse, JsonResponse}; use common::{auth::AccessToken, Server}; use directory::backend::internal::manage; +use email::crypto::{ + try_parse_certs, EncryptMessage, EncryptMessageError, EncryptionMethod, EncryptionParams, + EncryptionType, +}; use jmap_proto::types::{collection::Collection, property::Property}; -use mail_builder::{encoders::base64::base64_encode_mime, mime::make_boundary}; -use mail_parser::{decoders::base64::base64_decode, Message, MessageParser, MimeHeaders, PartType}; -use openpgp::{ - parse::Parse, - serialize::stream, - types::{KeyFlags, SymmetricAlgorithm}, -}; -use rand::{rngs::StdRng, RngCore, SeedableRng}; -use rasn::types::{ObjectIdentifier, OctetString}; -use rasn_cms::{ - algorithms::{AES128_CBC, AES256_CBC, RSA}, - pkcs7_compat::EncapsulatedContentInfo, - AlgorithmIdentifier, EncryptedContent, EncryptedContentInfo, EncryptedKey, EnvelopedData, - IssuerAndSerialNumber, KeyTransRecipientInfo, RecipientIdentifier, RecipientInfo, CONTENT_DATA, - CONTENT_ENVELOPED_DATA, -}; -use rsa::{pkcs1::DecodeRsaPublicKey, Pkcs1v15Encrypt, RsaPublicKey}; -use sequoia_openpgp as openpgp; +use mail_builder::encoders::base64::base64_encode_mime; +use mail_parser::MessageParser; use serde_json::json; -use store::{ - write::{BatchBuilder, Bincode, ToBitmaps, F_CLEAR, F_VALUE}, - Deserialize, Serialize, -}; - -const P: openpgp::policy::StandardPolicy<'static> = openpgp::policy::StandardPolicy::new(); - -#[derive(Debug)] -pub enum EncryptMessageError { - AlreadyEncrypted, - Error(String), -} - -#[derive(Debug, Clone, Copy, serde::Serialize, serde::Deserialize)] -pub enum Algorithm { - Aes128, - Aes256, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -pub enum EncryptionMethod { - PGP, - SMIME, -} - -#[derive(Debug, serde::Serialize, serde::Deserialize)] -pub struct EncryptionParams { - pub method: EncryptionMethod, - pub algo: Algorithm, - pub certs: Vec>, -} - -#[derive(Debug, serde::Serialize, serde::Deserialize, Default)] -#[serde(tag = "type")] -#[serde(rename_all = "camelCase")] -pub enum EncryptionType { - PGP { - algo: Algorithm, - certs: String, - }, - SMIME { - algo: Algorithm, - certs: String, - }, - #[default] - Disabled, -} - -#[allow(async_fn_in_trait)] -pub trait EncryptMessage { - async fn encrypt(&self, params: &EncryptionParams) -> Result, EncryptMessageError>; - fn is_encrypted(&self) -> bool; -} - -impl EncryptMessage for Message<'_> { - async fn encrypt(&self, params: &EncryptionParams) -> Result, EncryptMessageError> { - let root = self.root_part(); - let raw_message = self.raw_message(); - let mut outer_message = Vec::with_capacity((raw_message.len() as f64 * 1.5) as usize); - let mut inner_message = Vec::with_capacity(raw_message.len()); - - // Move MIME headers and body to inner message - for header in root.headers() { - (if header.name.is_mime_header() { - &mut inner_message - } else { - &mut outer_message - }) - .extend_from_slice(&raw_message[header.offset_field()..header.offset_end()]); - } - inner_message.extend_from_slice(b"\r\n"); - inner_message.extend_from_slice(&raw_message[root.raw_body_offset()..]); - - // Encrypt inner message - match params.method { - EncryptionMethod::PGP => { - // Prepare encrypted message - let boundary = make_boundary("_"); - outer_message.extend_from_slice( - concat!( - "Content-Type: multipart/encrypted;\r\n\t", - "protocol=\"application/pgp-encrypted\";\r\n\t", - "boundary=\"" - ) - .as_bytes(), - ); - outer_message.extend_from_slice(boundary.as_bytes()); - outer_message.extend_from_slice( - concat!( - "\"\r\n\r\n", - "OpenPGP/MIME message (Automatically encrypted by Stalwart)\r\n\r\n", - "--" - ) - .as_bytes(), - ); - outer_message.extend_from_slice(boundary.as_bytes()); - outer_message.extend_from_slice( - concat!( - "\r\nContent-Type: application/pgp-encrypted\r\n\r\n", - "Version: 1\r\n\r\n--" - ) - .as_bytes(), - ); - outer_message.extend_from_slice(boundary.as_bytes()); - outer_message.extend_from_slice( - concat!( - "\r\nContent-Type: application/octet-stream; name=\"encrypted.asc\"\r\n", - "Content-Disposition: inline; filename=\"encrypted.asc\"\r\n\r\n" - ) - .as_bytes(), - ); - - let certs = params - .certs - .iter() - .map(openpgp::Cert::from_bytes) - .collect::, _>>() - .map_err(|err| { - EncryptMessageError::Error(format!( - "Failed to parse OpenPGP public key: {}", - err - )) - })?; - - // Encrypt contents (TODO: use rayon) - let algo = params.algo; - let encrypted_contents = tokio::task::spawn_blocking(move || { - // Parse public key - let mut keys = Vec::with_capacity(certs.len()); - let policy = openpgp::policy::StandardPolicy::new(); - - for cert in &certs { - for key in cert - .keys() - .with_policy(&policy, None) - .supported() - .alive() - .revoked(false) - .key_flags(KeyFlags::empty().set_transport_encryption()) - { - keys.push(key); - } - } - - // Compose a writer stack corresponding to the output format and - // packet structure we want. - let mut sink = Vec::with_capacity(inner_message.len()); - - // Stream an OpenPGP message. - let message = stream::Armorer::new(stream::Message::new(&mut sink)) - .build() - .map_err(|err| { - EncryptMessageError::Error(format!("Failed to create armorer: {}", err)) - })?; - let message = stream::Encryptor2::for_recipients(message, keys) - .symmetric_algo(match algo { - Algorithm::Aes128 => SymmetricAlgorithm::AES128, - Algorithm::Aes256 => SymmetricAlgorithm::AES256, - }) - .build() - .map_err(|err| { - EncryptMessageError::Error(format!( - "Failed to build encryptor: {}", - err - )) - })?; - let mut message = - stream::LiteralWriter::new(message).build().map_err(|err| { - EncryptMessageError::Error(format!( - "Failed to create literal writer: {}", - err - )) - })?; - std::io::copy(&mut Cursor::new(inner_message), &mut message).map_err( - |err| { - EncryptMessageError::Error(format!( - "Failed to encrypt message: {}", - err - )) - }, - )?; - message.finalize().map_err(|err| { - EncryptMessageError::Error(format!("Failed to finalize message: {}", err)) - })?; - - String::from_utf8(sink).map_err(|err| { - EncryptMessageError::Error(format!( - "Failed to convert encrypted message to UTF-8: {}", - err - )) - }) - }) - .await - .map_err(|err| { - EncryptMessageError::Error(format!("Failed to encrypt message: {}", err)) - })??; - outer_message.extend_from_slice(encrypted_contents.as_bytes()); - outer_message.extend_from_slice(b"\r\n--"); - outer_message.extend_from_slice(boundary.as_bytes()); - outer_message.extend_from_slice(b"--\r\n"); - } - EncryptionMethod::SMIME => { - // Generate random IV - let mut rng = StdRng::from_entropy(); - let mut iv = vec![0u8; 16]; - rng.fill_bytes(&mut iv); - - // Generate random key - let mut key = vec![0u8; params.algo.key_size()]; - rng.fill_bytes(&mut key); - - // Encrypt contents (TODO: use rayon) - let algo = params.algo; - let (encrypted_contents, key, iv) = tokio::task::spawn_blocking(move || { - (algo.encrypt(&key, &iv, &inner_message), key, iv) - }) - .await - .map_err(|err| { - EncryptMessageError::Error(format!("Failed to encrypt message: {}", err)) - })?; - - // Encrypt key using public keys - #[allow(clippy::mutable_key_type)] - let mut recipient_infos = BTreeSet::new(); - for cert in ¶ms.certs { - let cert = - rasn::der::decode::(cert).map_err(|err| { - EncryptMessageError::Error(format!( - "Failed to parse certificate: {}", - err - )) - })?; - - let public_key = RsaPublicKey::from_pkcs1_der( - cert.tbs_certificate - .subject_public_key_info - .subject_public_key - .as_raw_slice(), - ) - .map_err(|err| { - EncryptMessageError::Error(format!("Failed to parse public key: {}", err)) - })?; - let encrypted_key = public_key - .encrypt(&mut rng, Pkcs1v15Encrypt, &key[..]) - .map_err(|err| { - EncryptMessageError::Error(format!("Failed to encrypt key: {}", err)) - }) - .unwrap(); - - recipient_infos.insert(RecipientInfo::KeyTransRecipientInfo( - KeyTransRecipientInfo { - version: 0.into(), - rid: RecipientIdentifier::IssuerAndSerialNumber( - IssuerAndSerialNumber { - issuer: cert.tbs_certificate.issuer, - serial_number: cert.tbs_certificate.serial_number, - }, - ), - key_encryption_algorithm: AlgorithmIdentifier { - algorithm: RSA.into(), - parameters: Some( - rasn::der::encode(&()) - .map_err(|err| { - EncryptMessageError::Error(format!( - "Failed to encode RSA algorithm identifier: {}", - err - )) - })? - .into(), - ), - }, - encrypted_key: EncryptedKey::from(encrypted_key), - }, - )); - } - - let pkcs7 = rasn::der::encode(&EncapsulatedContentInfo { - content_type: CONTENT_ENVELOPED_DATA.into(), - content: Some( - rasn::der::encode(&EnvelopedData { - version: 0.into(), - originator_info: None, - recipient_infos, - encrypted_content_info: EncryptedContentInfo { - content_type: CONTENT_DATA.into(), - content_encryption_algorithm: AlgorithmIdentifier { - algorithm: params.algo.to_algorithm_identifier(), - parameters: Some( - rasn::der::encode(&OctetString::from(iv)) - .map_err(|err| { - EncryptMessageError::Error(format!( - "Failed to encode IV: {}", - err - )) - })? - .into(), - ), - }, - encrypted_content: Some(EncryptedContent::from(encrypted_contents)), - }, - unprotected_attrs: None, - }) - .map_err(|err| { - EncryptMessageError::Error(format!( - "Failed to encode EnvelopedData: {}", - err - )) - })? - .into(), - ), - }) - .map_err(|err| { - EncryptMessageError::Error(format!("Failed to encode ContentInfo: {}", err)) - })?; - - // Generate message - outer_message.extend_from_slice( - concat!( - "Content-Type: application/pkcs7-mime;\r\n", - "\tname=\"smime.p7m\";\r\n", - "\tsmime-type=enveloped-data\r\n", - "Content-Disposition: attachment;\r\n", - "\tfilename=\"smime.p7m\"\r\n", - "Content-Transfer-Encoding: base64\r\n\r\n" - ) - .as_bytes(), - ); - base64_encode_mime(&pkcs7, &mut outer_message, false).map_err(|err| { - EncryptMessageError::Error(format!("Failed to base64 encode PKCS7: {}", err)) - })?; - } - } - - Ok(outer_message) - } - - fn is_encrypted(&self) -> bool { - if self.content_type().is_some_and(|ct| { - let main_type = ct.c_type.as_ref(); - let sub_type = ct - .c_subtype - .as_ref() - .map(|s| s.as_ref()) - .unwrap_or_default(); - - (main_type.eq_ignore_ascii_case("application") - && (sub_type.eq_ignore_ascii_case("pkcs7-mime") - || sub_type.eq_ignore_ascii_case("pkcs7-signature") - || (sub_type.eq_ignore_ascii_case("octet-stream") - && self.attachment_name().is_some_and(|name| { - name.rsplit_once('.') - .is_some_and(|(_, ext)| ["p7m", "p7s", "p7c", "p7z"].contains(&ext)) - })))) - || (main_type.eq_ignore_ascii_case("multipart") - && sub_type.eq_ignore_ascii_case("encrypted")) - }) { - return true; - } - - if self.parts.len() <= 2 { - let mut text_part = None; - let mut is_multipart = false; - - for part in &self.parts { - match &part.body { - PartType::Text(text) => { - text_part = Some(text.as_ref()); - } - PartType::Multipart(_) => { - is_multipart = true; - } - _ => (), - } - } - - match text_part { - Some(text) if self.parts.len() == 1 || is_multipart => { - if text.trim_start().starts_with("-----BEGIN PGP MESSAGE-----") { - return true; - } - } - _ => (), - } - } - - false - } -} - -impl Algorithm { - fn key_size(&self) -> usize { - match self { - Algorithm::Aes128 => 16, - Algorithm::Aes256 => 32, - } - } - - fn to_algorithm_identifier(self) -> ObjectIdentifier { - match self { - Algorithm::Aes128 => AES128_CBC.into(), - Algorithm::Aes256 => AES256_CBC.into(), - } - } - - fn encrypt(&self, key: &[u8], iv: &[u8], contents: &[u8]) -> Vec { - match self { - Algorithm::Aes128 => cbc::Encryptor::::new(key.into(), iv.into()) - .encrypt_padded_vec_mut::(contents), - Algorithm::Aes256 => cbc::Encryptor::::new(key.into(), iv.into()) - .encrypt_padded_vec_mut::(contents), - } - } -} - -pub fn try_parse_certs( - expected_method: EncryptionMethod, - cert: Vec, -) -> Result>, Cow<'static, str>> { - // Check if it's a PEM file - let (method, certs) = if let Some(result) = try_parse_pem(&cert)? { - result - } else if rasn::der::decode::(&cert[..]).is_ok() { - (EncryptionMethod::SMIME, vec![cert]) - } else if let Ok(cert_) = openpgp::Cert::from_bytes(&cert[..]) { - if !has_pgp_keys(cert_) { - (EncryptionMethod::PGP, vec![cert]) - } else { - return Err("Could not find any suitable keys in certificate".into()); - } - } else { - return Err("Could not find any valid certificates".into()); - }; - - if method == expected_method { - Ok(certs) - } else { - Err("No valid certificates found for the selected encryption".into()) - } -} - -fn has_pgp_keys(cert: openpgp::Cert) -> bool { - cert.keys() - .with_policy(&P, None) - .supported() - .alive() - .revoked(false) - .key_flags(KeyFlags::empty().set_transport_encryption()) - .next() - .is_some() -} - -#[allow(clippy::type_complexity)] -fn try_parse_pem( - bytes_: &[u8], -) -> Result>)>, Cow<'static, str>> { - if let Some(internal) = std::str::from_utf8(bytes_) - .ok() - .and_then(|cert| cert.strip_prefix("-----STALWART CERTIFICATE-----")) - { - return base64_decode(internal.as_bytes()) - .ok_or(Cow::from("Failed to decode base64")) - .and_then(|bytes| { - Bincode::::deserialize(&bytes) - .map_err(|_| Cow::from("Failed to deserialize internal certificate")) - }) - .map(|params| Some((params.inner.method, params.inner.certs))); - } - - let mut bytes = bytes_.iter().enumerate(); - let mut buf = vec![]; - let mut method = None; - let mut certs = vec![]; - - loop { - // Find start of PEM block - let mut start_pos = 0; - for (pos, &ch) in bytes.by_ref() { - if ch.is_ascii_whitespace() { - continue; - } else if ch == b'-' { - start_pos = pos; - break; - } else { - return Ok(None); - } - } - - // Find block type - for (_, &ch) in bytes.by_ref() { - match ch { - b'-' => (), - b'\n' => break, - _ => { - if ch.is_ascii() { - buf.push(ch.to_ascii_uppercase()); - } else { - return Ok(None); - } - } - } - } - if buf.is_empty() { - break; - } - - // Find type - let tag = std::str::from_utf8(&buf).unwrap(); - if tag.contains("CERTIFICATE") { - if method.is_some_and(|m| m == EncryptionMethod::PGP) { - return Err("Cannot mix OpenPGP and S/MIME certificates".into()); - } else { - method = Some(EncryptionMethod::SMIME); - } - } else if tag.contains("PGP") { - if method.is_some_and(|m| m == EncryptionMethod::SMIME) { - return Err("Cannot mix OpenPGP and S/MIME certificates".into()); - } else { - method = Some(EncryptionMethod::PGP); - } - } else { - // Ignore block - let mut found_end = false; - for (_, &ch) in bytes.by_ref() { - if ch == b'-' { - found_end = true; - } else if ch == b'\n' && found_end { - break; - } - } - buf.clear(); - continue; - } - - // Collect base64 - buf.clear(); - let mut found_end = false; - let mut end_pos = 0; - for (pos, &ch) in bytes.by_ref() { - match ch { - b'-' => { - found_end = true; - } - b'\n' => { - if found_end { - end_pos = pos; - break; - } - } - _ => { - if !ch.is_ascii_whitespace() { - buf.push(ch); - } - } - } - } - - // Decode base64 - let cert = - base64_decode(&buf).ok_or_else(|| Cow::from("Failed to decode base64 certificate."))?; - match method.unwrap() { - EncryptionMethod::PGP => match openpgp::Cert::from_bytes(bytes_) { - Ok(cert) => { - if !has_pgp_keys(cert) { - return Err("Could not find any suitable keys in OpenPGP public key".into()); - } - certs.push( - bytes_ - .get(start_pos..end_pos + 1) - .unwrap_or_default() - .to_vec(), - ); - } - Err(err) => { - return Err(format!("Failed to decode OpenPGP public key: {err}").into()) - } - }, - EncryptionMethod::SMIME => { - if let Err(err) = rasn::der::decode::(&cert) { - return Err(format!("Failed to decode X509 certificate: {err}").into()); - } - certs.push(cert); - } - } - buf.clear(); - } - - Ok(method.map(|method| (method, certs))) -} - -impl Serialize for &EncryptionParams { - fn serialize(self) -> Vec { - let len = bincode::serialized_size(&self).unwrap_or_default(); - let mut buf = Vec::with_capacity(len as usize + 1); - buf.push(1); - let _ = bincode::serialize_into(&mut buf, &self); - buf - } -} - -impl Deserialize for EncryptionParams { - fn deserialize(bytes: &[u8]) -> trc::Result { - let version = *bytes - .first() - .ok_or_else(|| trc::StoreEvent::DataCorruption.caused_by(trc::location!()))?; - match version { - 1 if bytes.len() > 1 => bincode::deserialize(&bytes[1..]).map_err(|err| { - trc::EventType::Store(trc::StoreEvent::DeserializeError) - .from_bincode_error(err) - .caused_by(trc::location!()) - }), - - _ => Err(trc::StoreEvent::DeserializeError - .into_err() - .caused_by(trc::location!()) - .ctx(trc::Key::Value, version as u64)), - } - } -} - -impl ToBitmaps for &EncryptionParams { - fn to_bitmaps(&self, _: &mut Vec, _: u8, _: bool) { - unreachable!() - } -} +use store::{write::{BatchBuilder, Bincode, F_CLEAR, F_VALUE}, Serialize}; pub trait CryptoHandler: Sync + Send { fn handle_crypto_get( @@ -777,21 +136,3 @@ impl CryptoHandler for Server { .into_http_response()) } } - -impl Display for EncryptionMethod { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - EncryptionMethod::PGP => write!(f, "OpenPGP"), - EncryptionMethod::SMIME => write!(f, "S/MIME"), - } - } -} - -impl Display for Algorithm { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Algorithm::Aes128 => write!(f, "AES-128"), - Algorithm::Aes256 => write!(f, "AES-256"), - } - } -} diff --git a/crates/jmap/src/email/delete.rs b/crates/jmap/src/email/delete.rs index 996a1e49..54f43dc5 100644 --- a/crates/jmap/src/email/delete.rs +++ b/crates/jmap/src/email/delete.rs @@ -7,6 +7,11 @@ use std::time::Duration; use common::{Server, KV_LOCK_PURGE_ACCOUNT}; +use email::{ + index::EmailIndexBuilder, + mailbox::{UidMailbox, JUNK_ID, TOMBSTONE_ID, TRASH_ID}, + metadata::MessageMetadata, +}; use jmap_proto::types::{ collection::Collection, id::Id, keyword::Keyword, property::Property, state::StateChange, type_state::DataType, @@ -23,14 +28,6 @@ use store::{ use trc::{AddContext, StoreEvent}; use utils::codec::leb128::Leb128Reader; -use crate::{ - changes::write::ChangeLog, - mailbox::{UidMailbox, JUNK_ID, TOMBSTONE_ID, TRASH_ID}, - services::state::StateManager, - JmapMethods, -}; - -use super::{index::EmailIndexBuilder, metadata::MessageMetadata}; use rand::prelude::SliceRandom; use std::future::Future; diff --git a/crates/jmap/src/email/get.rs b/crates/jmap/src/email/get.rs index 921ecd5f..4684c890 100644 --- a/crates/jmap/src/email/get.rs +++ b/crates/jmap/src/email/get.rs @@ -5,6 +5,11 @@ */ use common::{auth::AccessToken, Server}; +use email::{ + cache::ThreadCache, + mailbox::UidMailbox, + metadata::{MessageMetadata, MetadataPartType}, +}; use jmap_proto::{ method::get::{GetRequest, GetResponse}, object::{email::GetArguments, Object}, @@ -25,15 +30,13 @@ use trc::{AddContext, StoreEvent}; use crate::{ auth::acl::AclMethods, blob::download::BlobDownload, changes::state::StateManager, - email::headers::HeaderToValue, mailbox::UidMailbox, JmapMethods, + email::headers::HeaderToValue, }; use std::future::Future; use super::{ body::{ToBodyPart, TruncateBody}, - cache::ThreadCache, headers::IntoForm, - metadata::{MessageMetadata, MetadataPartType}, }; pub trait EmailGet: Sync + Send { diff --git a/crates/jmap/src/email/import.rs b/crates/jmap/src/email/import.rs index 8ddc972f..1aa12b79 100644 --- a/crates/jmap/src/email/import.rs +++ b/crates/jmap/src/email/import.rs @@ -5,6 +5,10 @@ */ use common::{auth::AccessToken, Server}; +use email::{ + ingest::{EmailIngest, IngestEmail, IngestSource}, + mailbox::MailboxFnc, +}; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::import::{ImportEmailRequest, ImportEmailResponse}, @@ -22,13 +26,9 @@ use utils::map::vec_map::VecMap; use crate::{ api::http::HttpSessionData, auth::acl::AclMethods, blob::download::BlobDownload, - changes::state::StateManager, mailbox::set::MailboxSet, JmapMethods, + changes::state::StateManager, }; -use super::{ - bayes::EmailBayesTrain, - ingest::{EmailIngest, IngestEmail, IngestSource}, -}; use std::future::Future; pub trait EmailImport: Sync + Send { diff --git a/crates/jmap/src/email/mod.rs b/crates/jmap/src/email/mod.rs index c2ddb5da..0993bbc4 100644 --- a/crates/jmap/src/email/mod.rs +++ b/crates/jmap/src/email/mod.rs @@ -6,16 +6,12 @@ pub mod bayes; pub mod body; -pub mod cache; pub mod copy; pub mod crypto; pub mod delete; pub mod get; pub mod headers; pub mod import; -pub mod index; -pub mod ingest; -pub mod metadata; pub mod parse; pub mod query; pub mod set; diff --git a/crates/jmap/src/email/parse.rs b/crates/jmap/src/email/parse.rs index d0b33751..3b55a484 100644 --- a/crates/jmap/src/email/parse.rs +++ b/crates/jmap/src/email/parse.rs @@ -5,6 +5,7 @@ */ use common::{auth::AccessToken, Server}; +use email::index::PREVIEW_LENGTH; use jmap_proto::{ method::parse::{ParseEmailRequest, ParseEmailResponse}, object::Object, @@ -21,7 +22,6 @@ use crate::blob::download::BlobDownload; use super::{ body::{ToBodyPart, TruncateBody}, headers::HeaderToValue, - index::PREVIEW_LENGTH, }; pub trait EmailParse: Sync + Send { diff --git a/crates/jmap/src/email/query.rs b/crates/jmap/src/email/query.rs index 1a56c33f..f75bc62b 100644 --- a/crates/jmap/src/email/query.rs +++ b/crates/jmap/src/email/query.rs @@ -5,6 +5,7 @@ */ use common::{auth::AccessToken, Server}; +use email::cache::ThreadCache; use jmap_proto::{ method::query::{Comparator, Filter, QueryRequest, QueryResponse, SortProperty}, object::email::QueryArguments, @@ -23,8 +24,6 @@ use store::{ use crate::{auth::acl::AclMethods, JmapMethods}; -use super::cache::ThreadCache; - pub trait EmailQuery: Sync + Send { fn email_query( &self, diff --git a/crates/jmap/src/email/set.rs b/crates/jmap/src/email/set.rs index df12ea6a..44460821 100644 --- a/crates/jmap/src/email/set.rs +++ b/crates/jmap/src/email/set.rs @@ -7,6 +7,10 @@ use std::{borrow::Cow, collections::HashMap, slice::IterMut}; use common::{auth::AccessToken, Server}; +use email::{ + ingest::{EmailIngest, IngestEmail, IngestSource}, + mailbox::{MailboxFnc, UidMailbox}, +}; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::set::{RequestArguments, SetRequest, SetResponse}, @@ -42,20 +46,14 @@ use store::{ use trc::AddContext; use crate::{ - api::http::HttpSessionData, - auth::acl::AclMethods, - blob::download::BlobDownload, - changes::{state::StateManager, write::ChangeLog}, - mailbox::{set::MailboxSet, UidMailbox}, - JmapMethods, + api::http::HttpSessionData, auth::acl::AclMethods, blob::download::BlobDownload, + changes::state::StateManager, JmapMethods, }; use std::future::Future; use super::{ - bayes::EmailBayesTrain, delete::EmailDeletion, headers::{BuildHeader, ValueToHeader}, - ingest::{EmailIngest, IngestEmail, IngestSource}, }; pub trait EmailSet: Sync + Send { @@ -892,7 +890,7 @@ impl EmailSet for Server { // Update last change id if changes.change_id == u64::MAX { - changes.change_id = self.assign_change_id(account_id).await?; + changes.change_id = self.assign_change_id(account_id)?; } batch.value(Property::Cid, changes.change_id, F_VALUE); } diff --git a/crates/jmap/src/email/snippet.rs b/crates/jmap/src/email/snippet.rs index bace9f17..a9b302f7 100644 --- a/crates/jmap/src/email/snippet.rs +++ b/crates/jmap/src/email/snippet.rs @@ -5,6 +5,7 @@ */ use common::{auth::AccessToken, Server}; +use email::metadata::{MessageMetadata, MetadataPartType}; use jmap_proto::{ method::{ query::Filter, @@ -16,9 +17,8 @@ use mail_parser::{decoders::html::html_to_text, GetHeader, HeaderName, PartType} use nlp::language::{search_snippet::generate_snippet, stemmer::Stemmer, Language}; use store::{backend::MAX_TOKEN_LENGTH, write::Bincode}; -use crate::{auth::acl::AclMethods, blob::download::BlobDownload, JmapMethods}; +use crate::{auth::acl::AclMethods, blob::download::BlobDownload}; -use super::metadata::{MessageMetadata, MetadataPartType}; use std::future::Future; pub trait EmailSearchSnippet: Sync + Send { diff --git a/crates/jmap/src/identity/get.rs b/crates/jmap/src/identity/get.rs index 1f255d7e..1e75eeed 100644 --- a/crates/jmap/src/identity/get.rs +++ b/crates/jmap/src/identity/get.rs @@ -18,7 +18,7 @@ use store::{ use trc::AddContext; use utils::sanitize_email; -use crate::{changes::state::StateManager, JmapMethods}; +use crate::changes::state::StateManager; use std::future::Future; diff --git a/crates/jmap/src/identity/set.rs b/crates/jmap/src/identity/set.rs index cfc65b24..aad44a2c 100644 --- a/crates/jmap/src/identity/set.rs +++ b/crates/jmap/src/identity/set.rs @@ -19,10 +19,9 @@ use jmap_proto::{ }; use std::future::Future; use store::write::{log::ChangeLogBuilder, BatchBuilder, F_CLEAR, F_VALUE}; +use trc::AddContext; use utils::sanitize_email; -use crate::{changes::write::ChangeLog, JmapMethods}; - pub trait IdentitySet: Sync + Send { fn identity_set( &self, @@ -102,7 +101,11 @@ impl IdentitySet for Server { .with_collection(Collection::Identity) .create_document() .value(Property::Value, identity, F_VALUE); - let document_id = self.write_batch_expect_id(batch).await?; + let document_id = self + .store() + .write_expect_id(batch) + .await + .caused_by(trc::location!())?; identity_ids.insert(document_id); changes.log_insert(Collection::Identity, document_id); response.created(id, document_id); @@ -158,7 +161,10 @@ impl IdentitySet for Server { .with_collection(Collection::Identity) .update_document(document_id) .value(Property::Value, identity, F_VALUE); - self.write_batch(batch).await?; + self.store() + .write(batch) + .await + .caused_by(trc::location!())?; changes.log_update(Collection::Identity, document_id); response.updated.append(id, None); } @@ -174,7 +180,10 @@ impl IdentitySet for Server { .with_collection(Collection::Identity) .delete_document(document_id) .value(Property::Value, (), F_VALUE | F_CLEAR); - self.write_batch(batch).await?; + self.store() + .write(batch) + .await + .caused_by(trc::location!())?; changes.log_delete(Collection::Identity, document_id); response.destroyed.push(id); } else { diff --git a/crates/jmap/src/lib.rs b/crates/jmap/src/lib.rs index 578619ff..62a9b484 100644 --- a/crates/jmap/src/lib.rs +++ b/crates/jmap/src/lib.rs @@ -8,33 +8,24 @@ use std::{fmt::Display, future::Future, sync::Arc, time::Duration}; use changes::state::StateManager; use common::{ - auth::{AccessToken, ResourceToken, TenantInfo}, manager::boot::{BootManager, IpcReceivers}, Inner, Server, }; -use directory::QueryBy; use jmap_proto::{ method::{ query::{QueryRequest, QueryResponse}, set::{SetRequest, SetResponse}, }, - types::{collection::Collection, property::Property}, + types::collection::Collection, }; use services::{ - delivery::spawn_delivery_manager, housekeeper::spawn_housekeeper, - index::spawn_email_queue_task, state::spawn_state_manager, + housekeeper::spawn_housekeeper, index::spawn_email_queue_task, state::spawn_state_manager, }; use store::{ - dispatch::DocumentSet, fts::FtsFilter, query::{sort::Pagination, Comparator, Filter, ResultSet, SortedResultSet}, roaring::RoaringBitmap, - write::{ - key::DeserializeBigEndian, AssignedIds, BatchBuilder, BitmapClass, DirectoryClass, - TagValue, ValueClass, - }, - BitmapKey, Deserialize, IterateParams, ValueKey, U32_LEN, }; use trc::AddContext; @@ -88,9 +79,6 @@ impl StartServices for BootManager { impl SpawnServices for IpcReceivers { fn spawn_services(&mut self, inner: Arc) { - // Spawn delivery manager - spawn_delivery_manager(inner.clone(), self.delivery_rx.take().unwrap()); - // Spawn state manager spawn_state_manager(inner.clone(), self.state_rx.take().unwrap()); @@ -103,138 +91,6 @@ impl SpawnServices for IpcReceivers { } impl JmapMethods for Server { - async fn get_property( - &self, - account_id: u32, - collection: Collection, - document_id: u32, - property: impl AsRef + Sync + Send, - ) -> trc::Result> - where - U: Deserialize + 'static, - { - let property = property.as_ref(); - - self.core - .storage - .data - .get_value::(ValueKey { - account_id, - collection: collection.into(), - document_id, - class: ValueClass::Property(property.into()), - }) - .await - .add_context(|err| { - err.caused_by(trc::location!()) - .account_id(account_id) - .collection(collection) - .document_id(document_id) - .id(property.to_string()) - }) - } - - async fn get_properties( - &self, - account_id: u32, - collection: Collection, - iterate: &I, - property: P, - ) -> trc::Result> - where - I: DocumentSet + Send + Sync, - P: AsRef + Sync + Send, - U: Deserialize + 'static, - { - let property: u8 = property.as_ref().into(); - let collection: u8 = collection.into(); - let expected_results = iterate.len(); - let mut results = Vec::with_capacity(expected_results); - - self.core - .storage - .data - .iterate( - IterateParams::new( - ValueKey { - account_id, - collection, - document_id: iterate.min(), - class: ValueClass::Property(property), - }, - ValueKey { - account_id, - collection, - document_id: iterate.max(), - class: ValueClass::Property(property), - }, - ), - |key, value| { - let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; - if iterate.contains(document_id) { - results.push((document_id, U::deserialize(value)?)); - Ok(expected_results == 0 || results.len() < expected_results) - } else { - Ok(true) - } - }, - ) - .await - .add_context(|err| { - err.caused_by(trc::location!()) - .account_id(account_id) - .collection(collection) - .id(property.to_string()) - }) - .map(|_| results) - } - - async fn get_document_ids( - &self, - account_id: u32, - collection: Collection, - ) -> trc::Result> { - self.core - .storage - .data - .get_bitmap(BitmapKey::document_ids(account_id, collection)) - .await - .add_context(|err| { - err.caused_by(trc::location!()) - .account_id(account_id) - .collection(collection) - }) - } - - async fn get_tag( - &self, - account_id: u32, - collection: Collection, - property: impl AsRef + Sync + Send, - value: impl Into> + Sync + Send, - ) -> trc::Result> { - let property = property.as_ref(); - self.core - .storage - .data - .get_bitmap(BitmapKey { - account_id, - collection: collection.into(), - class: BitmapClass::Tag { - field: property.into(), - value: value.into(), - }, - document_id: 0, - }) - .await - .add_context(|err| { - err.caused_by(trc::location!()) - .account_id(account_id) - .collection(collection) - .id(property.to_string()) - }) - } - async fn prepare_set_response( &self, request: &SetRequest, @@ -252,109 +108,6 @@ impl JmapMethods for Server { ) } - async fn get_resource_token( - &self, - access_token: &AccessToken, - account_id: u32, - ) -> trc::Result { - Ok(if access_token.primary_id == account_id { - ResourceToken { - account_id, - quota: access_token.quota, - tenant: access_token.tenant, - } - } else { - let mut quotas = ResourceToken { - account_id, - ..Default::default() - }; - - if let Some(principal) = self - .core - .storage - .directory - .query(QueryBy::Id(account_id), false) - .await - .add_context(|err| err.caused_by(trc::location!()).account_id(account_id))? - { - quotas.quota = principal.quota(); - - // SPDX-SnippetBegin - // SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd - // SPDX-License-Identifier: LicenseRef-SEL - - #[cfg(feature = "enterprise")] - if self.core.is_enterprise_edition() { - if let Some(tenant_id) = principal.tenant() { - quotas.tenant = TenantInfo { - id: tenant_id, - quota: self - .core - .storage - .directory - .query(QueryBy::Id(tenant_id), false) - .await - .add_context(|err| { - err.caused_by(trc::location!()).account_id(tenant_id) - })? - .map(|tenant| tenant.quota()) - .unwrap_or_default(), - } - .into(); - } - } - - // SPDX-SnippetEnd - } - - quotas - }) - } - - async fn get_used_quota(&self, account_id: u32) -> trc::Result { - self.core - .storage - .data - .get_counter(DirectoryClass::UsedQuota(account_id)) - .await - .add_context(|err| err.caused_by(trc::location!()).account_id(account_id)) - } - - async fn has_available_quota(&self, quotas: &ResourceToken, item_size: u64) -> trc::Result<()> { - if quotas.quota != 0 { - let used_quota = self.get_used_quota(quotas.account_id).await? as u64; - - if used_quota + item_size > quotas.quota { - return Err(trc::LimitEvent::Quota - .into_err() - .ctx(trc::Key::Limit, quotas.quota) - .ctx(trc::Key::Size, used_quota)); - } - } - - // SPDX-SnippetBegin - // SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd - // SPDX-License-Identifier: LicenseRef-SEL - - #[cfg(feature = "enterprise")] - if self.core.is_enterprise_edition() { - if let Some(tenant) = quotas.tenant.filter(|tenant| tenant.quota != 0) { - let used_quota = self.get_used_quota(tenant.id).await? as u64; - - if used_quota + item_size > tenant.quota { - return Err(trc::LimitEvent::TenantQuota - .into_err() - .ctx(trc::Key::Limit, tenant.quota) - .ctx(trc::Key::Size, used_quota)); - } - } - } - - // SPDX-SnippetEnd - - Ok(()) - } - async fn filter( &self, account_id: u32, @@ -466,21 +219,6 @@ impl JmapMethods for Server { Ok(response) } - async fn write_batch(&self, batch: BatchBuilder) -> trc::Result { - self.core - .storage - .data - .write(batch.build()) - .await - .caused_by(trc::location!()) - } - - async fn write_batch_expect_id(&self, batch: BatchBuilder) -> trc::Result { - self.write_batch(batch) - .await - .and_then(|ids| ids.last_document_id().caused_by(trc::location!())) - } - fn increment_config_version(&self) { self.inner .data @@ -490,62 +228,12 @@ impl JmapMethods for Server { } pub trait JmapMethods: Sync + Send { - fn get_property( - &self, - account_id: u32, - collection: Collection, - document_id: u32, - property: impl AsRef + Sync + Send, - ) -> impl Future>> + Send - where - U: Deserialize + 'static; - - fn get_properties( - &self, - account_id: u32, - collection: Collection, - iterate: &I, - property: P, - ) -> impl Future>> + Send - where - I: DocumentSet + Send + Sync, - P: AsRef + Sync + Send, - U: Deserialize + 'static; - - fn get_document_ids( - &self, - account_id: u32, - collection: Collection, - ) -> impl Future>> + Send; - - fn get_tag( - &self, - account_id: u32, - collection: Collection, - property: impl AsRef + Sync + Send, - value: impl Into> + Sync + Send, - ) -> impl Future>> + Send; - fn prepare_set_response( &self, request: &SetRequest, collection: Collection, ) -> impl Future> + Send; - fn get_resource_token( - &self, - access_token: &AccessToken, - account_id: u32, - ) -> impl Future> + Send; - - fn get_used_quota(&self, account_id: u32) -> impl Future> + Send; - - fn has_available_quota( - &self, - quotas: &ResourceToken, - item_size: u64, - ) -> impl Future> + Send; - fn filter( &self, account_id: u32, @@ -574,16 +262,6 @@ pub trait JmapMethods: Sync + Send { response: QueryResponse, ) -> impl Future> + Send; - fn write_batch( - &self, - batch: BatchBuilder, - ) -> impl Future> + Send; - - fn write_batch_expect_id( - &self, - batch: BatchBuilder, - ) -> impl Future> + Send; - fn increment_config_version(&self); } diff --git a/crates/jmap/src/mailbox/get.rs b/crates/jmap/src/mailbox/get.rs index 94f93304..05283e21 100644 --- a/crates/jmap/src/mailbox/get.rs +++ b/crates/jmap/src/mailbox/get.rs @@ -5,22 +5,18 @@ */ use common::{auth::AccessToken, Server}; +use email::mailbox::MailboxFnc; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, object::Object, - types::{acl::Acl, collection::Collection, keyword::Keyword, property::Property, value::Value}, + types::{acl::Acl, collection::Collection, property::Property, value::Value}, }; -use store::{ahash::AHashSet, query::Filter, roaring::RoaringBitmap}; -use trc::AddContext; use crate::{ auth::acl::{AclMethods, EffectiveAcl}, changes::state::StateManager, - email::cache::ThreadCache, - JmapMethods, }; -use super::{set::MailboxSet, INBOX_ID}; use std::future::Future; pub trait MailboxGet: Sync + Send { @@ -29,38 +25,6 @@ pub trait MailboxGet: Sync + Send { request: GetRequest, access_token: &AccessToken, ) -> impl Future> + Send; - - fn mailbox_count_threads( - &self, - account_id: u32, - document_ids: Option, - ) -> impl Future> + Send; - - fn mailbox_unread_tags( - &self, - account_id: u32, - document_id: u32, - message_ids: &Option, - ) -> impl Future>> + Send; - - fn mailbox_expand_path<'x>( - &self, - account_id: u32, - path: &'x str, - exact_match: bool, - ) -> impl Future>>> + Send; - - fn mailbox_get_by_name( - &self, - account_id: u32, - path: &str, - ) -> impl Future>> + Send; - - fn mailbox_get_by_role( - &self, - account_id: u32, - role: &str, - ) -> impl Future>> + Send; } impl MailboxGet for Server { @@ -282,182 +246,4 @@ impl MailboxGet for Server { } Ok(response) } - - async fn mailbox_count_threads( - &self, - account_id: u32, - document_ids: Option, - ) -> trc::Result { - if let Some(document_ids) = document_ids { - let mut thread_ids = AHashSet::default(); - self.get_cached_thread_ids(account_id, document_ids.into_iter()) - .await - .caused_by(trc::location!())? - .into_iter() - .for_each(|(_, thread_id)| { - thread_ids.insert(thread_id); - }); - Ok(thread_ids.len()) - } else { - Ok(0) - } - } - - async fn mailbox_unread_tags( - &self, - account_id: u32, - document_id: u32, - message_ids: &Option, - ) -> trc::Result> { - if let (Some(message_ids), Some(mailbox_message_ids)) = ( - message_ids, - self.get_tag( - account_id, - Collection::Email, - Property::MailboxIds, - document_id, - ) - .await?, - ) { - if let Some(mut seen) = self - .get_tag( - account_id, - Collection::Email, - Property::Keywords, - Keyword::Seen, - ) - .await? - { - seen ^= message_ids; - seen &= &mailbox_message_ids; - if !seen.is_empty() { - Ok(Some(seen)) - } else { - Ok(None) - } - } else { - Ok(mailbox_message_ids.into()) - } - } else { - Ok(None) - } - } - - async fn mailbox_expand_path<'x>( - &self, - account_id: u32, - path: &'x str, - exact_match: bool, - ) -> trc::Result>> { - let path = path - .split('/') - .filter_map(|p| { - let p = p.trim(); - if !p.is_empty() { - p.into() - } else { - None - } - }) - .collect::>(); - if path.is_empty() || path.len() > self.core.jmap.mailbox_max_depth { - return Ok(None); - } - - let mut filter = Vec::with_capacity(path.len() + 2); - let mut has_inbox = false; - filter.push(Filter::Or); - for (pos, item) in path.iter().enumerate() { - if pos == 0 && item.eq_ignore_ascii_case("inbox") { - has_inbox = true; - } else { - filter.push(Filter::eq(Property::Name, *item)); - } - } - filter.push(Filter::End); - - let mut document_ids = if filter.len() > 2 { - self.filter(account_id, Collection::Mailbox, filter) - .await? - .results - } else { - RoaringBitmap::new() - }; - if has_inbox { - document_ids.insert(INBOX_ID); - } - if exact_match && (document_ids.len() as usize) < path.len() { - return Ok(None); - } - - let mut found_names = Vec::new(); - for document_id in document_ids { - if let Some(mut obj) = self - .get_property::>( - account_id, - Collection::Mailbox, - document_id, - Property::Value, - ) - .await? - { - if let Some(Value::Text(value)) = obj.properties.remove(&Property::Name) { - found_names.push(( - value, - if let Some(Value::Id(value)) = obj.properties.remove(&Property::ParentId) { - value.document_id() - } else { - 0 - }, - document_id + 1, - )); - } else { - return Ok(None); - } - } else { - return Ok(None); - } - } - - Ok(Some(ExpandPath { path, found_names })) - } - - async fn mailbox_get_by_name(&self, account_id: u32, path: &str) -> trc::Result> { - Ok(self - .mailbox_expand_path(account_id, path, true) - .await? - .and_then(|ep| { - let mut next_parent_id = 0; - 'outer: for (pos, name) in ep.path.iter().enumerate() { - let is_inbox = pos == 0 && name.eq_ignore_ascii_case("inbox"); - - for (part, parent_id, document_id) in &ep.found_names { - if (part.eq(name) || (is_inbox && part.eq_ignore_ascii_case("inbox"))) - && *parent_id == next_parent_id - { - next_parent_id = *document_id; - continue 'outer; - } - } - return None; - } - Some(next_parent_id - 1) - })) - } - - async fn mailbox_get_by_role(&self, account_id: u32, role: &str) -> trc::Result> { - self.filter( - account_id, - Collection::Mailbox, - vec![Filter::eq(Property::Role, role)], - ) - .await - .map(|r| r.results.min()) - } -} - -#[derive(Debug)] -pub struct ExpandPath<'x> { - pub path: Vec<&'x str>, - pub found_names: Vec<(String, u32, u32)>, } diff --git a/crates/jmap/src/mailbox/mod.rs b/crates/jmap/src/mailbox/mod.rs index ed0deb83..b459a10c 100644 --- a/crates/jmap/src/mailbox/mod.rs +++ b/crates/jmap/src/mailbox/mod.rs @@ -4,84 +4,6 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::slice::Iter; - -use store::{ - write::{ - BitmapClass, DeserializeFrom, MaybeDynamicId, Operation, SerializeInto, TagValue, ToBitmaps, - }, - Serialize, U32_LEN, -}; -use utils::codec::leb128::{Leb128Iterator, Leb128Vec}; - pub mod get; pub mod query; pub mod set; - -pub const INBOX_ID: u32 = 0; -pub const TRASH_ID: u32 = 1; -pub const JUNK_ID: u32 = 2; -pub const DRAFTS_ID: u32 = 3; -pub const SENT_ID: u32 = 4; -pub const ARCHIVE_ID: u32 = 5; -pub const TOMBSTONE_ID: u32 = u32::MAX - 1; - -#[derive(Debug, Clone, Copy)] -pub struct UidMailbox { - pub mailbox_id: u32, - pub uid: u32, -} - -impl PartialEq for UidMailbox { - fn eq(&self, other: &Self) -> bool { - self.mailbox_id == other.mailbox_id - } -} - -impl Eq for UidMailbox {} - -impl ToBitmaps for UidMailbox { - fn to_bitmaps(&self, ops: &mut Vec, field: u8, set: bool) { - ops.push(Operation::Bitmap { - class: BitmapClass::Tag { - field, - value: TagValue::Id(MaybeDynamicId::Static(self.mailbox_id)), - }, - set, - }); - } -} - -impl SerializeInto for UidMailbox { - fn serialize_into(&self, buf: &mut Vec) { - buf.push_leb128(self.mailbox_id); - buf.push_leb128(self.uid); - } -} - -impl DeserializeFrom for UidMailbox { - fn deserialize_from(bytes: &mut Iter<'_, u8>) -> Option { - Some(UidMailbox { - mailbox_id: bytes.next_leb128()?, - uid: bytes.next_leb128()?, - }) - } -} - -impl Serialize for UidMailbox { - fn serialize(self) -> Vec { - let mut buf = Vec::with_capacity(U32_LEN * 2); - self.serialize_into(&mut buf); - buf - } -} - -impl UidMailbox { - pub fn new(mailbox_id: u32, uid: u32) -> Self { - UidMailbox { mailbox_id, uid } - } - - pub fn new_unassigned(mailbox_id: u32) -> Self { - UidMailbox { mailbox_id, uid: 0 } - } -} diff --git a/crates/jmap/src/mailbox/query.rs b/crates/jmap/src/mailbox/query.rs index 46f1c3eb..4f039e06 100644 --- a/crates/jmap/src/mailbox/query.rs +++ b/crates/jmap/src/mailbox/query.rs @@ -5,6 +5,7 @@ */ use common::{auth::AccessToken, Server}; +use email::mailbox::MailboxFnc; use jmap_proto::{ method::query::{Comparator, Filter, QueryRequest, QueryResponse, SortProperty}, object::{mailbox::QueryArguments, Object}, @@ -19,8 +20,6 @@ use store::{ use crate::{auth::acl::AclMethods, JmapMethods, UpdateResults}; use std::future::Future; -use super::set::MailboxSet; - pub trait MailboxQuery: Sync + Send { fn mailbox_query( &self, diff --git a/crates/jmap/src/mailbox/set.rs b/crates/jmap/src/mailbox/set.rs index 4e8ba7d0..bdd45b7b 100644 --- a/crates/jmap/src/mailbox/set.rs +++ b/crates/jmap/src/mailbox/set.rs @@ -4,16 +4,13 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{auth::AccessToken, config::jmap::settings::SpecialUse, Server}; +use common::{auth::AccessToken, Server}; use directory::Permission; +use email::mailbox::{MailboxFnc, SCHEMA}; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::set::{SetRequest, SetResponse}, - object::{ - index::{IndexAs, IndexProperty, ObjectIndexBuilder}, - mailbox::SetArguments, - Object, - }, + object::{index::ObjectIndexBuilder, mailbox::SetArguments, Object}, response::references::EvalObjectReferences, types::{ acl::Acl, @@ -34,18 +31,15 @@ use store::{ BatchBuilder, F_BITMAP, F_CLEAR, F_VALUE, }, }; -use trc::AddContext; use crate::{ auth::acl::{AclMethods, EffectiveAcl}, - changes::write::ChangeLog, email::delete::EmailDeletion, JmapMethods, }; -use super::{get::MailboxGet, ARCHIVE_ID, DRAFTS_ID, SENT_ID}; #[allow(unused_imports)] -use super::{UidMailbox, INBOX_ID, JUNK_ID, TRASH_ID}; +use email::mailbox::{UidMailbox, INBOX_ID, JUNK_ID, TRASH_ID}; use std::future::Future; pub struct SetContext<'x> { @@ -57,24 +51,6 @@ pub struct SetContext<'x> { will_destroy: Vec, } -pub static SCHEMA: &[IndexProperty] = &[ - IndexProperty::new(Property::Name) - .index_as(IndexAs::Text { - tokenize: true, - index: true, - }) - .required(), - IndexProperty::new(Property::Role).index_as(IndexAs::Text { - tokenize: false, - index: true, - }), - IndexProperty::new(Property::Role).index_as(IndexAs::HasProperty), - IndexProperty::new(Property::ParentId).index_as(IndexAs::Integer), - IndexProperty::new(Property::SortOrder).index_as(IndexAs::Integer), - IndexProperty::new(Property::IsSubscribed).index_as(IndexAs::IntegerList), - IndexProperty::new(Property::Acl).index_as(IndexAs::Acl), -]; - pub trait MailboxSet: Sync + Send { fn mailbox_set( &self, @@ -97,17 +73,6 @@ pub trait MailboxSet: Sync + Send { update: Option<(u32, HashedValue>)>, ctx: &SetContext, ) -> impl Future>> + Send; - - fn mailbox_get_or_create( - &self, - account_id: u32, - ) -> impl Future> + Send; - - fn mailbox_create_path( - &self, - account_id: u32, - path: &str, - ) -> impl Future)>>> + Send; } impl MailboxSet for Server { @@ -801,148 +766,6 @@ impl MailboxSet for Server { .with_current_opt(current) .validate()) } - - async fn mailbox_get_or_create(&self, account_id: u32) -> trc::Result { - let mut mailbox_ids = self - .get_document_ids(account_id, Collection::Mailbox) - .await? - .unwrap_or_default(); - if !mailbox_ids.is_empty() { - return Ok(mailbox_ids); - } - - #[cfg(feature = "test_mode")] - if mailbox_ids.is_empty() && account_id == 0 { - return Ok(mailbox_ids); - } - - let mut batch = BatchBuilder::new(); - batch - .with_account_id(account_id) - .with_collection(Collection::Mailbox); - - // Create mailboxes - let mut last_document_id = ARCHIVE_ID; - for folder in &self.core.jmap.default_folders { - let (role, document_id) = match folder.special_use { - SpecialUse::Inbox => ("inbox", INBOX_ID), - SpecialUse::Trash => ("trash", TRASH_ID), - SpecialUse::Junk => ("junk", JUNK_ID), - SpecialUse::Drafts => ("drafts", DRAFTS_ID), - SpecialUse::Sent => ("sent", SENT_ID), - SpecialUse::Archive => ("archive", ARCHIVE_ID), - SpecialUse::None => { - last_document_id += 1; - ("", last_document_id) - } - SpecialUse::Shared => unreachable!(), - }; - - let mut object = Object::with_capacity(4) - .with_property(Property::Name, folder.name.clone()) - .with_property(Property::ParentId, Value::Id(0u64.into())) - .with_property( - Property::Cid, - Value::UnsignedInt(rand::random::() as u64), - ); - if !role.is_empty() { - object.set(Property::Role, role); - } - if folder.subscribe { - object.set( - Property::IsSubscribed, - Value::List(vec![Value::Id(account_id.into())]), - ); - } - batch - .create_document_with_id(document_id) - .custom(ObjectIndexBuilder::new(SCHEMA).with_changes(object)); - mailbox_ids.insert(document_id); - } - - self.core - .storage - .data - .write(batch.build()) - .await - .caused_by(trc::location!()) - .map(|_| mailbox_ids) - } - - async fn mailbox_create_path( - &self, - account_id: u32, - path: &str, - ) -> trc::Result)>> { - let expanded_path = - if let Some(expand_path) = self.mailbox_expand_path(account_id, path, false).await? { - expand_path - } else { - return Ok(None); - }; - - let mut next_parent_id = 0; - let mut path = expanded_path.path.into_iter().enumerate().peekable(); - 'outer: while let Some((pos, name)) = path.peek() { - let is_inbox = *pos == 0 && name.eq_ignore_ascii_case("inbox"); - - for (part, parent_id, document_id) in &expanded_path.found_names { - if (part.eq(name) || (is_inbox && part.eq_ignore_ascii_case("inbox"))) - && *parent_id == next_parent_id - { - next_parent_id = *document_id; - path.next(); - continue 'outer; - } - } - break; - } - - // Create missing folders - if path.peek().is_some() { - let mut changes = self.begin_changes(account_id).await?; - - for (_, name) in path { - if name.len() > self.core.jmap.mailbox_name_max_len { - return Ok(None); - } - let mut batch = BatchBuilder::new(); - batch - .with_account_id(account_id) - .with_collection(Collection::Mailbox) - .create_document() - .custom( - ObjectIndexBuilder::new(SCHEMA).with_changes( - Object::with_capacity(3) - .with_property(Property::Name, name) - .with_property( - Property::ParentId, - Value::Id(Id::from(next_parent_id)), - ) - .with_property( - Property::Cid, - Value::UnsignedInt(rand::random::() as u64), - ), - ), - ); - let document_id = self.write_batch_expect_id(batch).await?; - changes.log_insert(Collection::Mailbox, document_id); - next_parent_id = document_id + 1; - } - let change_id = changes.change_id; - let mut batch = BatchBuilder::new(); - - batch - .with_account_id(account_id) - .with_collection(Collection::Mailbox) - .custom(changes); - self.write_batch(batch).await?; - - Ok(Some((next_parent_id - 1, Some(change_id)))) - } else { - Ok(Some((next_parent_id - 1, None))) - } - } } pub trait MailboxSubscribe { diff --git a/crates/jmap/src/principal/get.rs b/crates/jmap/src/principal/get.rs index f17d1198..0950ad58 100644 --- a/crates/jmap/src/principal/get.rs +++ b/crates/jmap/src/principal/get.rs @@ -13,8 +13,6 @@ use jmap_proto::{ }; use std::future::Future; -use crate::JmapMethods; - pub trait PrincipalGet: Sync + Send { fn principal_get( &self, diff --git a/crates/jmap/src/push/get.rs b/crates/jmap/src/push/get.rs index 3554748e..8ec88ede 100644 --- a/crates/jmap/src/push/get.rs +++ b/crates/jmap/src/push/get.rs @@ -21,8 +21,6 @@ use store::{ }; use utils::map::bitmap::Bitmap; -use crate::JmapMethods; - use super::{EncryptionKeys, PushSubscription}; use std::future::Future; diff --git a/crates/jmap/src/push/set.rs b/crates/jmap/src/push/set.rs index f07f12d8..78ca5540 100644 --- a/crates/jmap/src/push/set.rs +++ b/crates/jmap/src/push/set.rs @@ -24,8 +24,9 @@ use store::{ rand::{distributions::Alphanumeric, thread_rng, Rng}, write::{now, BatchBuilder, F_CLEAR, F_VALUE}, }; +use trc::AddContext; -use crate::{services::state::StateManager, JmapMethods}; +use crate::services::state::StateManager; const EXPIRES_MAX: i64 = 7 * 24 * 3600; // 7 days const VERIFICATION_CODE_LEN: usize = 32; @@ -119,7 +120,11 @@ impl PushSubscriptionSet for Server { .with_collection(Collection::PushSubscription) .create_document() .value(Property::Value, push, F_VALUE); - let document_id = self.write_batch_expect_id(batch).await?; + let document_id = self + .store() + .write_expect_id(batch) + .await + .caused_by(trc::location!())?; push_ids.insert(document_id); response.created.insert( id, @@ -180,7 +185,10 @@ impl PushSubscriptionSet for Server { .with_collection(Collection::PushSubscription) .update_document(document_id) .value(Property::Value, push, F_VALUE); - self.write_batch(batch).await?; + self.store() + .write(batch) + .await + .caused_by(trc::location!())?; response.updated.append(id, None); } @@ -195,7 +203,10 @@ impl PushSubscriptionSet for Server { .with_collection(Collection::PushSubscription) .delete_document(document_id) .value(Property::Value, (), F_VALUE | F_CLEAR); - self.write_batch(batch).await?; + self.store() + .write(batch) + .await + .caused_by(trc::location!())?; response.destroyed.push(id); } else { response.not_destroyed.append(id, SetError::not_found()); diff --git a/crates/jmap/src/quota/get.rs b/crates/jmap/src/quota/get.rs index a4edeb52..33f3d367 100644 --- a/crates/jmap/src/quota/get.rs +++ b/crates/jmap/src/quota/get.rs @@ -12,8 +12,6 @@ use jmap_proto::{ }; use std::future::Future; -use crate::JmapMethods; - pub trait QuotaGet: Sync + Send { fn quota_get( &self, diff --git a/crates/jmap/src/services/delivery.rs b/crates/jmap/src/services/delivery.rs deleted file mode 100644 index 17b9b542..00000000 --- a/crates/jmap/src/services/delivery.rs +++ /dev/null @@ -1,57 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use std::sync::Arc; - -use common::{core::BuildServer, ipc::DeliveryEvent, Inner}; -use tokio::sync::{mpsc, Semaphore}; - -use super::ingest::MailDelivery; - -pub fn spawn_delivery_manager(inner: Arc, mut delivery_rx: mpsc::Receiver) { - tokio::spawn(async move { - let semaphore = Arc::new(Semaphore::new( - inner - .shared_core - .load() - .smtp - .queue - .throttle - .local_concurrency, - )); - - loop { - let permit = match semaphore.clone().acquire_owned().await { - Ok(permit) => permit, - Err(_) => { - trc::error!(trc::StoreEvent::UnexpectedError - .into_err() - .details("Semaphore error") - .caused_by(trc::location!())); - break; - } - }; - - match delivery_rx.recv().await { - Some(event) => match event { - DeliveryEvent::Ingest { message, result_tx } => { - let server = inner.build_server(); - - tokio::spawn(async move { - result_tx.send(server.deliver_message(message).await).ok(); - - drop(permit); - }); - } - DeliveryEvent::Stop => break, - }, - None => { - break; - } - } - } - }); -} diff --git a/crates/jmap/src/services/gossip/ping.rs b/crates/jmap/src/services/gossip/ping.rs index 53c97c40..419f67cb 100644 --- a/crates/jmap/src/services/gossip/ping.rs +++ b/crates/jmap/src/services/gossip/ping.rs @@ -10,8 +10,6 @@ use common::{ }; use trc::ClusterEvent; -use crate::services::index::Indexer; - use super::{request::Request, Gossiper, PeerStatus}; impl Gossiper { diff --git a/crates/jmap/src/services/index.rs b/crates/jmap/src/services/index.rs index cf5767f2..803d5655 100644 --- a/crates/jmap/src/services/index.rs +++ b/crates/jmap/src/services/index.rs @@ -11,6 +11,7 @@ use directory::{ backend::internal::{manage::ManageDirectory, PrincipalField}, Type, }; +use email::{index::IndexMessageText, metadata::MessageMetadata}; use jmap_proto::types::{collection::Collection, property::Property}; use store::{ ahash::AHashMap, @@ -27,12 +28,7 @@ use std::future::Future; use trc::{AddContext, TaskQueueEvent}; use utils::{BlobHash, BLOB_HASH_LEN}; -use crate::{ - blob::download::BlobDownload, - changes::write::ChangeLog, - email::{bayes::EmailBayesTrain, index::IndexMessageText, metadata::MessageMetadata}, - JmapMethods, -}; +use crate::{blob::download::BlobDownload, email::bayes::EmailBayesTrain}; #[derive(Debug, Clone)] pub struct EmailTask { @@ -81,7 +77,6 @@ pub trait Indexer: Sync + Send { account_id: Option, tenant_id: Option, ) -> impl Future> + Send; - fn notify_task_queue(&self); } impl Indexer for Server { @@ -313,10 +308,6 @@ impl Indexer for Server { } } - fn notify_task_queue(&self) { - self.inner.ipc.index_tx.notify_one(); - } - async fn reindex(&self, account_id: Option, tenant_id: Option) -> trc::Result<()> { let accounts = if let Some(account_id) = account_id { RoaringBitmap::from_sorted_iter([account_id]).unwrap() diff --git a/crates/jmap/src/services/mod.rs b/crates/jmap/src/services/mod.rs index c2017b94..276c9749 100644 --- a/crates/jmap/src/services/mod.rs +++ b/crates/jmap/src/services/mod.rs @@ -4,9 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -pub mod delivery; pub mod gossip; pub mod housekeeper; pub mod index; -pub mod ingest; pub mod state; diff --git a/crates/jmap/src/services/state.rs b/crates/jmap/src/services/state.rs index 670f5ff9..0d1d4764 100644 --- a/crates/jmap/src/services/state.rs +++ b/crates/jmap/src/services/state.rs @@ -372,11 +372,6 @@ pub trait StateManager: Sync + Send { types: Bitmap, ) -> impl Future>> + Send; - fn broadcast_state_change( - &self, - state_change: StateChange, - ) -> impl Future + Send; - fn update_push_subscriptions(&self, account_id: u32) -> impl Future + Send; } @@ -407,28 +402,6 @@ impl StateManager for Server { Ok(change_rx) } - async fn broadcast_state_change(&self, state_change: StateChange) -> bool { - match self - .inner - .ipc - .state_tx - .clone() - .send(StateEvent::Publish { state_change }) - .await - { - Ok(_) => true, - Err(_) => { - trc::event!( - Server(ServerEvent::ThreadError), - Details = "Error sending state change.", - CausedBy = trc::location!() - ); - - false - } - } - } - async fn update_push_subscriptions(&self, account_id: u32) -> bool { let push_subs = match self.fetch_push_subscriptions(account_id).await { Ok(push_subs) => push_subs, diff --git a/crates/jmap/src/sieve/get.rs b/crates/jmap/src/sieve/get.rs index d40bea1d..6e199e21 100644 --- a/crates/jmap/src/sieve/get.rs +++ b/crates/jmap/src/sieve/get.rs @@ -4,29 +4,16 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::sync::Arc; - use common::Server; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, object::Object, types::{collection::Collection, property::Property, value::Value}, }; -use sieve::Sieve; -use store::{ - query::Filter, - write::{assert::HashedValue, BatchBuilder, Bincode, BlobOp}, - BlobClass, Deserialize, Serialize, -}; +use store::BlobClass; -use crate::{ - blob::{download::BlobDownload, upload::BlobUpload}, - changes::state::StateManager, - sieve::SeenIds, - JmapMethods, -}; +use crate::changes::state::StateManager; -use super::ActiveScript; use std::future::Future; pub trait SieveScriptGet: Sync + Send { @@ -34,23 +21,6 @@ pub trait SieveScriptGet: Sync + Send { &self, request: GetRequest, ) -> impl Future> + Send; - - fn sieve_script_get_active( - &self, - account_id: u32, - ) -> impl Future>> + Send; - - fn sieve_script_get_by_name( - &self, - account_id: u32, - name: &str, - ) -> impl Future>> + Send; - - fn sieve_script_compile( - &self, - account_id: u32, - document_id: u32, - ) -> impl Future)>> + Send; } impl SieveScriptGet for Server { @@ -145,176 +115,4 @@ impl SieveScriptGet for Server { Ok(response) } - - async fn sieve_script_get_active(&self, account_id: u32) -> trc::Result> { - // Find the currently active script - if let Some(document_id) = self - .filter( - account_id, - Collection::SieveScript, - vec![Filter::eq(Property::IsActive, 1u32)], - ) - .await? - .results - .min() - { - let (script, mut script_object) = - self.sieve_script_compile(account_id, document_id).await?; - Ok(Some(ActiveScript { - document_id, - script: Arc::new(script), - script_name: script_object - .properties - .remove(&Property::Name) - .and_then(|name| name.try_unwrap_string()) - .unwrap_or_else(|| account_id.to_string()), - seen_ids: self - .get_property::>( - account_id, - Collection::SieveScript, - document_id, - Property::EmailIds, - ) - .await? - .map(|seen_ids| seen_ids.inner) - .unwrap_or_default(), - })) - } else { - Ok(None) - } - } - - async fn sieve_script_get_by_name( - &self, - account_id: u32, - name: &str, - ) -> trc::Result> { - // Find the script by name - if let Some(document_id) = self - .filter( - account_id, - Collection::SieveScript, - vec![Filter::eq(Property::Name, name)], - ) - .await? - .results - .min() - { - self.sieve_script_compile(account_id, document_id) - .await - .map(|(sieve, _)| Some(sieve)) - } else { - Ok(None) - } - } - - #[allow(clippy::blocks_in_conditions)] - async fn sieve_script_compile( - &self, - account_id: u32, - document_id: u32, - ) -> trc::Result<(Sieve, Object)> { - // Obtain script object - let script_object = self - .get_property::>>( - account_id, - Collection::SieveScript, - document_id, - Property::Value, - ) - .await? - .ok_or_else(|| { - trc::StoreEvent::NotFound - .into_err() - .caused_by(trc::location!()) - .document_id(document_id) - })?; - - // Obtain the sieve script length - let (script_offset, blob_id) = script_object - .inner - .properties - .get(&Property::BlobId) - .and_then(|v| v.as_blob_id()) - .and_then(|v| (v.section.as_ref()?.size, v).into()) - .ok_or_else(|| { - trc::StoreEvent::NotFound - .into_err() - .caused_by(trc::location!()) - .document_id(document_id) - })?; - - // Obtain the sieve script blob - let script_bytes = self - .get_blob(&blob_id.hash, 0..usize::MAX) - .await? - .ok_or_else(|| { - trc::StoreEvent::NotFound - .into_err() - .caused_by(trc::location!()) - .document_id(document_id) - })?; - - // Obtain the precompiled script - if let Some(sieve) = script_bytes - .get(script_offset..) - .and_then(|bytes| Bincode::::deserialize(bytes).ok()) - { - Ok((sieve.inner, script_object.inner)) - } else { - // Deserialization failed, probably because the script compiler version changed - match self.core.sieve.untrusted_compiler.compile( - script_bytes.get(0..script_offset).ok_or_else(|| { - trc::StoreEvent::NotFound - .into_err() - .caused_by(trc::location!()) - .document_id(document_id) - })?, - ) { - Ok(sieve) => { - // Store updated compiled sieve script - let sieve = Bincode::new(sieve); - let compiled_bytes = (&sieve).serialize(); - let mut updated_sieve_bytes = - Vec::with_capacity(script_offset + compiled_bytes.len()); - updated_sieve_bytes.extend_from_slice(&script_bytes[0..script_offset]); - updated_sieve_bytes.extend_from_slice(&compiled_bytes); - - // Store updated blob - let mut new_blob_id = blob_id.clone(); - new_blob_id.hash = self - .put_blob(account_id, &updated_sieve_bytes, false) - .await? - .hash; - let mut new_script_object = script_object.inner.clone(); - new_script_object.set(Property::BlobId, new_blob_id.clone()); - - // Update script object - let mut batch = BatchBuilder::new(); - batch - .with_account_id(account_id) - .with_collection(Collection::SieveScript) - .update_document(document_id) - .assert_value(Property::Value, &script_object) - .set(Property::Value, (&new_script_object).serialize()) - .clear(BlobOp::Link { - hash: blob_id.hash.clone(), - }) - .set( - BlobOp::Link { - hash: new_blob_id.hash, - }, - Vec::new(), - ); - self.write_batch(batch).await?; - - Ok((sieve.inner, new_script_object)) - } - Err(error) => Err(trc::StoreEvent::UnexpectedError - .caused_by(trc::location!()) - .reason(error) - .details("Failed to compile Sieve script")), - } - } - } } diff --git a/crates/jmap/src/sieve/mod.rs b/crates/jmap/src/sieve/mod.rs index 79c36d4d..85e640bd 100644 --- a/crates/jmap/src/sieve/mod.rs +++ b/crates/jmap/src/sieve/mod.rs @@ -4,137 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::sync::Arc; - -use serde::ser::SerializeSeq; -use sieve::Sieve; -use store::{ahash::AHashSet, blake3, write::now}; - pub mod get; -pub mod ingest; pub mod query; pub mod set; pub mod validate; - -pub struct ActiveScript { - pub document_id: u32, - pub script_name: String, - pub script: Arc, - pub seen_ids: SeenIds, -} - -#[derive(Debug, Clone)] -pub struct SeenIdHash { - hash: [u8; 32], - expiry: u64, -} - -#[derive(Debug, Clone, Default)] -pub struct SeenIds { - pub ids: AHashSet, - pub has_changes: bool, -} - -impl SeenIdHash { - pub fn new(id: &str, expiry: u64) -> Self { - let mut hasher = blake3::Hasher::new(); - hasher.update(id.as_bytes()); - SeenIdHash { - hash: hasher.finalize().into(), - expiry, - } - } -} - -impl PartialOrd for SeenIdHash { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for SeenIdHash { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - self.expiry.cmp(&other.expiry) - } -} - -impl std::hash::Hash for SeenIdHash { - fn hash(&self, state: &mut H) { - self.hash.hash(state); - } -} - -impl PartialEq for SeenIdHash { - fn eq(&self, other: &Self) -> bool { - self.hash == other.hash - } -} - -impl Eq for SeenIdHash {} - -// SeenIds serializer -impl serde::Serialize for SeenIds { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - let mut seq = serializer.serialize_seq((self.ids.len() * 2).into())?; - for id in &self.ids { - seq.serialize_element(&id.expiry)?; - seq.serialize_element(&id.hash)?; - } - - seq.end() - } -} - -impl<'de> serde::Deserialize<'de> for SeenIds { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - deserializer.deserialize_seq(SeenIdsVisitor) - } -} - -struct SeenIdsVisitor; - -impl<'de> serde::de::Visitor<'de> for SeenIdsVisitor { - type Value = SeenIds; - - fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { - formatter.write_str("invalid SeenIds") - } - - fn visit_seq(self, mut seq: A) -> Result - where - A: serde::de::SeqAccess<'de>, - { - let num_entries = seq.size_hint().unwrap_or(0) / 2; - let mut seen_ids = SeenIds { - ids: AHashSet::with_capacity(num_entries), - has_changes: false, - }; - let now = now(); - - for _ in 0..num_entries { - let expiry = seq - .next_element::()? - .ok_or_else(|| serde::de::Error::custom("Expected expiry."))?; - if expiry > now { - seen_ids.ids.insert(SeenIdHash { - hash: seq - .next_element()? - .ok_or_else(|| serde::de::Error::custom("Expected hash."))?, - expiry, - }); - } else { - seq.next_element::<[u8; 32]>()? - .ok_or_else(|| serde::de::Error::custom("Expected hash."))?; - seen_ids.has_changes = true; - } - } - - Ok(seen_ids) - } -} diff --git a/crates/jmap/src/sieve/set.rs b/crates/jmap/src/sieve/set.rs index 2b6d0368..01254395 100644 --- a/crates/jmap/src/sieve/set.rs +++ b/crates/jmap/src/sieve/set.rs @@ -36,13 +36,9 @@ use store::{ }, BlobClass, }; +use trc::AddContext; -use crate::{ - api::http::HttpSessionData, - blob::{download::BlobDownload, upload::BlobUpload}, - changes::write::ChangeLog, - JmapMethods, -}; +use crate::{api::http::HttpSessionData, blob::download::BlobDownload, JmapMethods}; use std::future::Future; pub struct SetContext<'x> { @@ -152,7 +148,11 @@ impl SieveScriptSet for Server { } } - let document_id = self.write_batch_expect_id(batch).await?; + let document_id = self + .store() + .write_expect_id(batch) + .await + .caused_by(trc::location!())?; sieve_ids.insert(document_id); changes.log_insert(Collection::SieveScript, document_id); @@ -444,7 +444,10 @@ impl SieveScriptSet for Server { } } - self.write_batch(batch).await?; + self.store() + .write(batch) + .await + .caused_by(trc::location!())?; Ok(true) } diff --git a/crates/jmap/src/submission/get.rs b/crates/jmap/src/submission/get.rs index ef9ca794..6881e2da 100644 --- a/crates/jmap/src/submission/get.rs +++ b/crates/jmap/src/submission/get.rs @@ -13,7 +13,7 @@ use jmap_proto::{ use smtp::queue::{self, spool::SmtpSpool}; use std::future::Future; -use crate::{changes::state::StateManager, JmapMethods}; +use crate::changes::state::StateManager; pub trait EmailSubmissionGet: Sync + Send { fn email_submission_get( diff --git a/crates/jmap/src/submission/set.rs b/crates/jmap/src/submission/set.rs index 08ad593b..07b6992d 100644 --- a/crates/jmap/src/submission/set.rs +++ b/crates/jmap/src/submission/set.rs @@ -10,6 +10,7 @@ use common::{ listener::{stream::NullIo, ServerInstance}, Server, }; +use email::metadata::MessageMetadata; use jmap_proto::{ error::set::{SetError, SetErrorType}, method::set::{self, SetRequest, SetResponse}, @@ -38,12 +39,10 @@ use smtp::{ }; use smtp_proto::{request::parser::Rfc5321Parser, MailFrom, RcptTo}; use store::write::{assert::HashedValue, log::ChangeLogBuilder, now, BatchBuilder, Bincode}; +use trc::AddContext; use utils::{map::vec_map::VecMap, sanitize_email}; -use crate::{ - blob::download::BlobDownload, changes::write::ChangeLog, email::metadata::MessageMetadata, - JmapMethods, -}; +use crate::blob::download::BlobDownload; use std::future::Future; pub static SCHEMA: &[IndexProperty] = &[ @@ -107,7 +106,11 @@ impl EmailSubmissionSet for Server { .with_collection(Collection::EmailSubmission) .create_document() .custom(ObjectIndexBuilder::new(SCHEMA).with_changes(submission)); - let document_id = self.write_batch_expect_id(batch).await?; + let document_id = self + .store() + .write_expect_id(batch) + .await + .caused_by(trc::location!())?; changes.log_insert(Collection::EmailSubmission, document_id); response.created(id, document_id); } @@ -193,7 +196,10 @@ impl EmailSubmissionSet for Server { .with_property(Property::UndoStatus, undo_status), ), ); - self.write_batch(batch).await?; + self.store() + .write(batch) + .await + .caused_by(trc::location!())?; changes.log_update(Collection::EmailSubmission, document_id); response.updated.append(id, None); } else { @@ -242,7 +248,10 @@ impl EmailSubmissionSet for Server { .with_collection(Collection::EmailSubmission) .delete_document(document_id) .custom(ObjectIndexBuilder::new(SCHEMA).with_current(submission)); - self.write_batch(batch).await?; + self.store() + .write(batch) + .await + .caused_by(trc::location!())?; changes.log_delete(Collection::EmailSubmission, document_id); response.destroyed.push(id); } else { diff --git a/crates/jmap/src/thread/get.rs b/crates/jmap/src/thread/get.rs index c328da38..4d6d090e 100644 --- a/crates/jmap/src/thread/get.rs +++ b/crates/jmap/src/thread/get.rs @@ -14,7 +14,7 @@ use std::future::Future; use store::query::{sort::Pagination, Comparator, ResultSet}; use trc::AddContext; -use crate::{changes::state::StateManager, JmapMethods}; +use crate::changes::state::StateManager; pub trait ThreadGet: Sync + Send { fn thread_get( diff --git a/crates/jmap/src/vacation/set.rs b/crates/jmap/src/vacation/set.rs index f924a782..b0d0e2a2 100644 --- a/crates/jmap/src/vacation/set.rs +++ b/crates/jmap/src/vacation/set.rs @@ -28,10 +28,9 @@ use store::write::{ log::{Changes, LogInsert}, BatchBuilder, BlobOp, DirectoryClass, F_CLEAR, F_VALUE, }; +use trc::AddContext; use crate::{ - blob::upload::BlobUpload, - changes::write::ChangeLog, sieve::set::{ObjectBlobId, SieveScriptSet, SCHEMA}, JmapMethods, }; @@ -119,7 +118,7 @@ impl VacationResponseSet for Server { // Prepare write batch let mut batch = BatchBuilder::new(); - let change_id = self.assign_change_id(account_id).await?; + let change_id = self.assign_change_id(account_id)?; batch .with_change_id(change_id) .with_account_id(account_id) @@ -307,7 +306,11 @@ impl VacationResponseSet for Server { // Write changes batch.custom(obj); let document_id = if !batch.is_empty() { - let ids = self.write_batch(batch).await?; + let ids = self + .store() + .write(batch) + .await + .caused_by(trc::location!())?; response.new_state = Some(change_id.into()); match document_id { Some(document_id) => document_id, @@ -350,7 +353,10 @@ impl VacationResponseSet for Server { // Write changes if !batch.is_empty() { - self.write_batch(batch).await?; + self.store() + .write(batch) + .await + .caused_by(trc::location!())?; response.new_state = Some(change_id.into()); } } diff --git a/crates/jmap/src/websocket/stream.rs b/crates/jmap/src/websocket/stream.rs index 85fa8482..1a8b79ea 100644 --- a/crates/jmap/src/websocket/stream.rs +++ b/crates/jmap/src/websocket/stream.rs @@ -76,7 +76,8 @@ impl WebSocketHandler for Server { let _ = stream .send(Message::Text( WebSocketRequestError::from(RequestError::internal_server_error()) - .to_json(), + .to_json() + .into(), )) .await; return; @@ -128,7 +129,7 @@ impl WebSocketHandler for Server { response }, }; - if let Err(err) = stream.send(Message::Text(response)).await { + if let Err(err) = stream.send(Message::Text(response.into())).await { trc::event!(Jmap(JmapEvent::WebsocketError), Details = "Failed to send text message", SpanId = session.session_id, @@ -208,7 +209,7 @@ impl WebSocketHandler for Server { // Send any queued changes let elapsed = last_changes_sent.elapsed(); if elapsed >= throttle { - if let Err(err) = stream.send(Message::Text(changes.to_json())).await { + if let Err(err) = stream.send(Message::Text(changes.to_json().into())).await { trc::event!( Jmap(JmapEvent::WebsocketError), Details = "Failed to send state change message.", @@ -224,7 +225,7 @@ impl WebSocketHandler for Server { next_event = throttle - elapsed; } } else if last_heartbeat.elapsed() > heartbeat { - if let Err(err) = stream.send(Message::Ping(vec![])).await { + if let Err(err) = stream.send(Message::Ping(Vec::::new().into())).await { trc::event!( Jmap(JmapEvent::WebsocketError), Details = "Failed to send ping message.", diff --git a/crates/managesieve/src/op/deletescript.rs b/crates/managesieve/src/op/deletescript.rs index 1a2b218c..2a2e5b51 100644 --- a/crates/managesieve/src/op/deletescript.rs +++ b/crates/managesieve/src/op/deletescript.rs @@ -9,7 +9,7 @@ use std::time::Instant; use common::listener::SessionStream; use directory::Permission; use imap_proto::receiver::Request; -use jmap::{changes::write::ChangeLog, sieve::set::SieveScriptSet}; +use jmap::sieve::set::SieveScriptSet; use jmap_proto::types::collection::Collection; use store::write::log::ChangeLogBuilder; use trc::AddContext; diff --git a/crates/managesieve/src/op/getscript.rs b/crates/managesieve/src/op/getscript.rs index 8837521a..52c42fc9 100644 --- a/crates/managesieve/src/op/getscript.rs +++ b/crates/managesieve/src/op/getscript.rs @@ -9,7 +9,7 @@ use std::time::Instant; use common::listener::SessionStream; use directory::Permission; use imap_proto::receiver::Request; -use jmap::{blob::download::BlobDownload, sieve::set::ObjectBlobId, JmapMethods}; +use jmap::{blob::download::BlobDownload, sieve::set::ObjectBlobId}; use jmap_proto::{ object::Object, types::{collection::Collection, property::Property, value::Value}, diff --git a/crates/managesieve/src/op/havespace.rs b/crates/managesieve/src/op/havespace.rs index a2b1a212..f7128098 100644 --- a/crates/managesieve/src/op/havespace.rs +++ b/crates/managesieve/src/op/havespace.rs @@ -9,7 +9,6 @@ use std::time::Instant; use common::listener::SessionStream; use directory::Permission; use imap_proto::receiver::Request; -use jmap::JmapMethods; use trc::AddContext; use crate::core::{Command, ResponseCode, Session, StatusResponse}; diff --git a/crates/managesieve/src/op/listscripts.rs b/crates/managesieve/src/op/listscripts.rs index 35fd38ca..6437ebf6 100644 --- a/crates/managesieve/src/op/listscripts.rs +++ b/crates/managesieve/src/op/listscripts.rs @@ -8,7 +8,6 @@ use std::time::Instant; use common::listener::SessionStream; use directory::Permission; -use jmap::JmapMethods; use jmap_proto::{ object::Object, types::{collection::Collection, property::Property, value::Value}, diff --git a/crates/managesieve/src/op/putscript.rs b/crates/managesieve/src/op/putscript.rs index d7a36279..58de4279 100644 --- a/crates/managesieve/src/op/putscript.rs +++ b/crates/managesieve/src/op/putscript.rs @@ -10,7 +10,6 @@ use common::listener::SessionStream; use directory::Permission; use imap_proto::receiver::Request; use jmap::{ - blob::upload::BlobUpload, sieve::set::{ObjectBlobId, SCHEMA}, JmapMethods, }; @@ -188,7 +187,8 @@ impl Session { ), ); self.server - .write_batch(batch) + .store() + .write(batch) .await .caused_by(trc::location!())?; @@ -248,7 +248,8 @@ impl Session { let assigned_ids = self .server - .write_batch(batch) + .store() + .write(batch) .await .caused_by(trc::location!())?; diff --git a/crates/managesieve/src/op/renamescript.rs b/crates/managesieve/src/op/renamescript.rs index 8164768c..01e7f144 100644 --- a/crates/managesieve/src/op/renamescript.rs +++ b/crates/managesieve/src/op/renamescript.rs @@ -9,7 +9,7 @@ use std::time::Instant; use common::listener::SessionStream; use directory::Permission; use imap_proto::receiver::Request; -use jmap::{changes::write::ChangeLog, sieve::set::SCHEMA, JmapMethods}; +use jmap::sieve::set::SCHEMA; use jmap_proto::{ object::{index::ObjectIndexBuilder, Object}, types::{collection::Collection, property::Property, value::Value}, @@ -93,7 +93,8 @@ impl Session { ); if !batch.is_empty() { self.server - .write_batch(batch) + .store() + .write(batch) .await .caused_by(trc::location!())?; let mut changelog = ChangeLogBuilder::new(); diff --git a/crates/managesieve/src/op/setactive.rs b/crates/managesieve/src/op/setactive.rs index 4ea6283c..e4a8c7d5 100644 --- a/crates/managesieve/src/op/setactive.rs +++ b/crates/managesieve/src/op/setactive.rs @@ -9,7 +9,7 @@ use std::time::Instant; use common::listener::SessionStream; use directory::Permission; use imap_proto::receiver::Request; -use jmap::{changes::write::ChangeLog, sieve::set::SieveScriptSet}; +use jmap::sieve::set::SieveScriptSet; use jmap_proto::types::collection::Collection; use store::write::log::ChangeLogBuilder; use trc::AddContext; diff --git a/crates/pop3/Cargo.toml b/crates/pop3/Cargo.toml index 5de5e439..4550d3a2 100644 --- a/crates/pop3/Cargo.toml +++ b/crates/pop3/Cargo.toml @@ -13,6 +13,7 @@ imap = { path = "../imap" } utils = { path = "../utils" } trc = { path = "../trc" } jmap_proto = { path = "../jmap-proto" } +email = { path = "../email" } mail-parser = { version = "0.9", features = ["full_encoding", "ludicrous_mode"] } mail-send = { version = "0.4", default-features = false, features = ["cram-md5", "ring", "tls12"] } rustls = { version = "0.23.5", default-features = false, features = ["std", "ring", "tls12"] } diff --git a/crates/pop3/src/mailbox.rs b/crates/pop3/src/mailbox.rs index 708bd86d..d865916a 100644 --- a/crates/pop3/src/mailbox.rs +++ b/crates/pop3/src/mailbox.rs @@ -7,10 +7,7 @@ use std::collections::BTreeMap; use common::listener::SessionStream; -use jmap::{ - mailbox::{set::MailboxSet, UidMailbox, INBOX_ID}, - JmapMethods, -}; +use email::mailbox::{MailboxFnc, UidMailbox, INBOX_ID}; use jmap_proto::{ object::Object, types::{collection::Collection, property::Property, value::Value}, diff --git a/crates/pop3/src/op/delete.rs b/crates/pop3/src/op/delete.rs index e8b53c49..89990e58 100644 --- a/crates/pop3/src/op/delete.rs +++ b/crates/pop3/src/op/delete.rs @@ -8,9 +8,7 @@ use std::time::Instant; use common::listener::SessionStream; use directory::Permission; -use jmap::{ - changes::write::ChangeLog, email::delete::EmailDeletion, services::state::StateManager, -}; +use jmap::email::delete::EmailDeletion; use jmap_proto::types::{state::StateChange, type_state::DataType}; use store::roaring::RoaringBitmap; use trc::AddContext; diff --git a/crates/pop3/src/op/fetch.rs b/crates/pop3/src/op/fetch.rs index 9d58cf45..d37c631b 100644 --- a/crates/pop3/src/op/fetch.rs +++ b/crates/pop3/src/op/fetch.rs @@ -8,7 +8,8 @@ use std::time::Instant; use common::listener::SessionStream; use directory::Permission; -use jmap::{blob::download::BlobDownload, email::metadata::MessageMetadata, JmapMethods}; +use email::metadata::MessageMetadata; +use jmap::blob::download::BlobDownload; use jmap_proto::types::{collection::Collection, property::Property}; use store::write::Bincode; use trc::AddContext; diff --git a/crates/smtp/Cargo.toml b/crates/smtp/Cargo.toml index 76eaf9a2..15a6cc64 100644 --- a/crates/smtp/Cargo.toml +++ b/crates/smtp/Cargo.toml @@ -17,6 +17,7 @@ utils = { path = "../utils" } nlp = { path = "../nlp" } directory = { path = "../directory" } common = { path = "../common" } +email = { path = "../email" } spam-filter = { path = "../spam-filter" } trc = { path = "../trc" } mail-auth = { version = "0.6" } diff --git a/crates/smtp/src/core/mod.rs b/crates/smtp/src/core/mod.rs index 1cb76137..c6e04436 100644 --- a/crates/smtp/src/core/mod.rs +++ b/crates/smtp/src/core/mod.rs @@ -7,18 +7,14 @@ use std::{ hash::Hash, net::IpAddr, - sync::{Arc, LazyLock}, + sync::Arc, time::{Duration, Instant}, }; use common::{ auth::AccessToken, config::smtp::auth::VerifyStrategy, - listener::{ - asn::AsnGeoLookupResult, - limiter::{ConcurrencyLimiter, InFlight}, - ServerInstance, - }, + listener::{asn::AsnGeoLookupResult, limiter::InFlight, ServerInstance}, Inner, Server, }; use directory::Directory; @@ -27,7 +23,6 @@ use smtp_proto::request::receiver::{ BdatReceiver, DataReceiver, DummyDataReceiver, DummyLineReceiver, LineReceiver, RequestReceiver, }; use tokio::io::{AsyncRead, AsyncWrite}; -use utils::snowflake::SnowflakeIdGenerator; use crate::{ inbound::auth::SaslToken, @@ -219,18 +214,6 @@ impl PartialOrd for SessionAddress { } } -static SIEVE: LazyLock> = LazyLock::new(|| { - Arc::new(ServerInstance { - id: "sieve".to_string(), - protocol: common::config::server::ServerProtocol::Lmtp, - acceptor: common::listener::TcpAcceptor::Plain, - limiter: ConcurrencyLimiter::new(0), - shutdown_rx: tokio::sync::watch::channel(false).1, - proxy_networks: vec![], - span_id_gen: Arc::new(SnowflakeIdGenerator::new()), - }) -}); - impl Session { pub fn local( server: Server, @@ -268,20 +251,6 @@ impl Session { } } - pub fn sieve( - server: Server, - mail_from: SessionAddress, - rcpt_to: Vec, - message: Vec, - session_id: u64, - ) -> Self { - Self::local( - server, - SIEVE.clone(), - SessionData::local(mail_from.into(), rcpt_to, message, session_id), - ) - } - pub fn has_failed(&mut self) -> Option { if self.stream.tx_buf.first().map_or(true, |&c| c == b'2') { self.stream.tx_buf.clear(); diff --git a/crates/smtp/src/inbound/spawn.rs b/crates/smtp/src/inbound/spawn.rs index fecc097f..6aff9b04 100644 --- a/crates/smtp/src/inbound/spawn.rs +++ b/crates/smtp/src/inbound/spawn.rs @@ -68,12 +68,6 @@ impl SessionManager for SmtpSessionManager { .report_tx .send(common::ipc::ReportingEvent::Stop) .await; - let _ = self - .inner - .ipc - .delivery_tx - .send(common::ipc::DeliveryEvent::Stop) - .await; } } } diff --git a/crates/smtp/src/outbound/delivery.rs b/crates/smtp/src/outbound/delivery.rs index f1b2d000..762d6eaf 100644 --- a/crates/smtp/src/outbound/delivery.rs +++ b/crates/smtp/src/outbound/delivery.rs @@ -271,7 +271,7 @@ impl DeliveryAttempt { let delivery_result = message .deliver_local( recipients.iter_mut().filter(|r| r.domain_idx == domain_idx), - &server.inner.ipc.delivery_tx, + &server, ) .await; diff --git a/crates/smtp/src/outbound/local.rs b/crates/smtp/src/outbound/local.rs index 4d900967..aa5a72f3 100644 --- a/crates/smtp/src/outbound/local.rs +++ b/crates/smtp/src/outbound/local.rs @@ -4,20 +4,24 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::ipc::{DeliveryEvent, DeliveryResult, IngestMessage}; +use common::Server; +use email::delivery::{IngestMessage, LocalDeliveryStatus, MailDelivery}; use smtp_proto::Response; -use tokio::sync::{mpsc, oneshot}; -use trc::ServerEvent; +use trc::SieveEvent; -use crate::queue::{ - Error, ErrorDetails, HostResponse, Message, Recipient, Status, RCPT_STATUS_CHANGED, +use crate::{ + queue::{ + quota::HasQueueQuota, spool::SmtpSpool, DomainPart, Error, ErrorDetails, HostResponse, + Message, MessageSource, Recipient, Status, RCPT_STATUS_CHANGED, + }, + reporting::SmtpReporting, }; impl Message { pub async fn deliver_local( &self, recipients: impl Iterator, - delivery_tx: &mpsc::Sender, + server: &Server, ) -> Status<(), Error> { // Prepare recipients list let mut total_rcpt = 0; @@ -37,54 +41,22 @@ impl Message { pending_recipients.push(rcpt); } - // Create oneshot channel - let (result_tx, result_rx) = oneshot::channel(); - - // Deliver message to JMAP server - let delivery_result = match delivery_tx - .send(DeliveryEvent::Ingest { - message: IngestMessage { - sender_address: self.return_path_lcase.clone(), - recipients: recipient_addresses, - message_blob: self.blob_hash.clone(), - message_size: self.size, - session_id: self.span_id, - }, - result_tx, + // Deliver message + let delivery_result = server + .deliver_message(IngestMessage { + sender_address: self.return_path_lcase.clone(), + recipients: recipient_addresses, + message_blob: self.blob_hash.clone(), + message_size: self.size, + session_id: self.span_id, }) - .await - { - Ok(_) => { - // Wait for result - match result_rx.await { - Ok(delivery_result) => delivery_result, - Err(_) => { - trc::event!( - Server(ServerEvent::ThreadError), - CausedBy = trc::location!(), - SpanId = self.span_id, - Reason = "Result channel closed", - ); - return Status::local_error(); - } - } - } - Err(_) => { - trc::event!( - Server(ServerEvent::ThreadError), - CausedBy = trc::location!(), - SpanId = self.span_id, - Reason = "TX channel closed", - ); - return Status::local_error(); - } - }; + .await; // Process delivery results - for (rcpt, result) in pending_recipients.into_iter().zip(delivery_result) { + for (rcpt, result) in pending_recipients.into_iter().zip(delivery_result.status) { rcpt.flags |= RCPT_STATUS_CHANGED; match result { - DeliveryResult::Success => { + LocalDeliveryStatus::Success => { rcpt.status = Status::Completed(HostResponse { hostname: "localhost".to_string(), response: Response { @@ -95,7 +67,7 @@ impl Message { }); total_completed += 1; } - DeliveryResult::TemporaryFailure { reason } => { + LocalDeliveryStatus::TemporaryFailure { reason } => { rcpt.status = Status::TemporaryFailure(HostResponse { hostname: ErrorDetails { entity: "localhost".to_string(), @@ -108,7 +80,7 @@ impl Message { }, }); } - DeliveryResult::PermanentFailure { code, reason } => { + LocalDeliveryStatus::PermanentFailure { code, reason } => { total_completed += 1; rcpt.status = Status::PermanentFailure(HostResponse { hostname: ErrorDetails { @@ -125,6 +97,56 @@ impl Message { } } + // Process autogenerated messages + for autogenerated in delivery_result.autogenerated { + let from_addr_lcase = autogenerated.sender_address.to_lowercase(); + let from_addr_domain = from_addr_lcase.domain_part().to_string(); + + let mut message = server.new_message( + autogenerated.sender_address, + from_addr_lcase, + from_addr_domain, + self.span_id, + ); + for rcpt in autogenerated.recipients { + message.add_recipient(rcpt, server).await; + } + + // Sign message + let signature = server + .sign_message( + &mut message, + &server.core.sieve.sign, + &autogenerated.message, + ) + .await; + + // Queue Message + message.size = autogenerated.message.len() + signature.as_ref().map_or(0, |s| s.len()); + if server.has_quota(&mut message).await { + message + .queue( + signature.as_deref(), + &autogenerated.message, + self.span_id, + server, + MessageSource::Autogenerated, + ) + .await; + } else { + trc::event!( + Sieve(SieveEvent::QuotaExceeded), + SpanId = self.span_id, + From = message.return_path_lcase, + To = message + .recipients + .into_iter() + .map(|r| trc::Value::from(r.address_lcase)) + .collect::>(), + ); + } + } + if total_completed == total_rcpt { Status::Completed(()) } else { diff --git a/crates/store/src/dispatch/store.rs b/crates/store/src/dispatch/store.rs index c7c14cfb..de9024cb 100644 --- a/crates/store/src/dispatch/store.rs +++ b/crates/store/src/dispatch/store.rs @@ -182,7 +182,8 @@ impl Store { result.caused_by(trc::location!()) } - pub async fn write(&self, batch: Batch) -> trc::Result { + pub async fn write(&self, batch: impl Into) -> trc::Result { + let batch = batch.into(); #[cfg(feature = "test_mode")] if std::env::var("PARANOID_WRITE").is_ok_and(|v| v == "1") { let mut account_id = u32::MAX; @@ -301,6 +302,13 @@ impl Store { result } + #[inline] + pub async fn write_expect_id(&self, batch: impl Into) -> trc::Result { + self.write(batch) + .await + .and_then(|ids| ids.last_document_id()) + } + pub async fn purge_store(&self) -> trc::Result<()> { // Delete expired reports let now = now(); @@ -925,3 +933,9 @@ impl Store { } } } + +impl From for Batch { + fn from(builder: BatchBuilder) -> Self { + builder.build() + } +} diff --git a/tests/Cargo.toml b/tests/Cargo.toml index 160214c8..7f5cca38 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -29,6 +29,7 @@ imap_proto = { path = "../crates/imap-proto" } pop3 = { path = "../crates/pop3", features = ["test_mode"] } smtp = { path = "../crates/smtp", features = ["test_mode", "enterprise"] } common = { path = "../crates/common", features = ["test_mode", "enterprise"] } +email = { path = "../crates/email", features = ["test_mode"] } spam-filter = { path = "../crates/spam-filter", features = ["test_mode", "enterprise"] } trc = { path = "../crates/trc" } managesieve = { path = "../crates/managesieve", features = ["test_mode", "enterprise"] } diff --git a/tests/src/imap/mod.rs b/tests/src/imap/mod.rs index d88a6352..4f5ed953 100644 --- a/tests/src/imap/mod.rs +++ b/tests/src/imap/mod.rs @@ -334,7 +334,7 @@ async fn init_imap_tests(store_id: &str, delete_if_exists: bool) -> IMAPTest { let cache = Caches::parse(&mut config); let store = core.storage.data.clone(); - let (ipc, mut ipc_rxs) = build_ipc(); + let (ipc, mut ipc_rxs) = build_ipc(&mut config); let inner = Arc::new(Inner { shared_core: core.into_shared(), data, diff --git a/tests/src/jmap/auth_acl.rs b/tests/src/jmap/auth_acl.rs index 53dda5a6..536a5457 100644 --- a/tests/src/jmap/auth_acl.rs +++ b/tests/src/jmap/auth_acl.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use jmap::mailbox::{INBOX_ID, TRASH_ID}; +use ::email::mailbox::{INBOX_ID, TRASH_ID}; use jmap_client::{ core::{ error::{MethodError, MethodErrorType}, diff --git a/tests/src/jmap/blob.rs b/tests/src/jmap/blob.rs index 423a8363..3236bf79 100644 --- a/tests/src/jmap/blob.rs +++ b/tests/src/jmap/blob.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use jmap::mailbox::INBOX_ID; +use email::mailbox::INBOX_ID; use jmap_proto::types::id::Id; use serde_json::Value; diff --git a/tests/src/jmap/crypto.rs b/tests/src/jmap/crypto.rs index 8ae33385..3e828d02 100644 --- a/tests/src/jmap/crypto.rs +++ b/tests/src/jmap/crypto.rs @@ -6,7 +6,7 @@ use std::path::PathBuf; -use jmap::email::crypto::{ +use email::crypto::{ try_parse_certs, Algorithm, EncryptMessage, EncryptionMethod, EncryptionParams, EncryptionType, }; use jmap_proto::types::id::Id; diff --git a/tests/src/jmap/delivery.rs b/tests/src/jmap/delivery.rs index 91481238..d8405299 100644 --- a/tests/src/jmap/delivery.rs +++ b/tests/src/jmap/delivery.rs @@ -6,10 +6,7 @@ use std::time::Duration; -use jmap::{ - mailbox::{INBOX_ID, JUNK_ID}, - JmapMethods, -}; +use email::mailbox::{INBOX_ID, JUNK_ID}; use jmap_proto::types::{collection::Collection, id::Id, property::Property}; use tokio::{ diff --git a/tests/src/jmap/email_get.rs b/tests/src/jmap/email_get.rs index 54bbfd48..8b3c1408 100644 --- a/tests/src/jmap/email_get.rs +++ b/tests/src/jmap/email_get.rs @@ -6,7 +6,7 @@ use std::{fs, path::PathBuf}; -use jmap::mailbox::INBOX_ID; +use ::email::mailbox::INBOX_ID; use jmap_client::email::{self, import::EmailImportResponse, Header, HeaderForm}; use jmap_proto::types::id::Id; use mail_parser::HeaderName; diff --git a/tests/src/jmap/email_query.rs b/tests/src/jmap/email_query.rs index 96092f37..7b1daf6e 100644 --- a/tests/src/jmap/email_query.rs +++ b/tests/src/jmap/email_query.rs @@ -10,7 +10,7 @@ use crate::{ jmap::{assert_is_empty, mailbox::destroy_all_mailboxes, wait_for_index}, store::{deflate_test_resource, query::FIELDS}, }; -use jmap::JmapMethods; + use jmap_client::{ client::Client, core::query::{Comparator, Filter}, diff --git a/tests/src/jmap/email_query_changes.rs b/tests/src/jmap/email_query_changes.rs index a40e5b48..fea2f678 100644 --- a/tests/src/jmap/email_query_changes.rs +++ b/tests/src/jmap/email_query_changes.rs @@ -4,7 +4,6 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use jmap::{changes::write::ChangeLog, JmapMethods}; use jmap_client::{ core::query::{Comparator, Filter}, email, @@ -150,7 +149,7 @@ pub async fn test(params: &mut JMAPTest) { TagValue::Id(MaybeDynamicId::Dynamic(0)), 0, ) - .custom(server.begin_changes(1).await.unwrap().with_log_move( + .custom(server.begin_changes(1).unwrap().with_log_move( Collection::Email, id, new_id, diff --git a/tests/src/jmap/email_search_snippet.rs b/tests/src/jmap/email_search_snippet.rs index 7c68d788..2fb05a7a 100644 --- a/tests/src/jmap/email_search_snippet.rs +++ b/tests/src/jmap/email_search_snippet.rs @@ -7,7 +7,8 @@ use std::{fs, path::PathBuf}; use crate::jmap::{assert_is_empty, mailbox::destroy_all_mailboxes, wait_for_index}; -use jmap::mailbox::INBOX_ID; + +use email::mailbox::INBOX_ID; use jmap_client::{core::query, email::query::Filter}; use jmap_proto::types::id::Id; use store::ahash::AHashMap; diff --git a/tests/src/jmap/email_set.rs b/tests/src/jmap/email_set.rs index 65ee1af2..24cc6324 100644 --- a/tests/src/jmap/email_set.rs +++ b/tests/src/jmap/email_set.rs @@ -8,7 +8,8 @@ use std::{fs, path::PathBuf}; use crate::jmap::{assert_is_empty, mailbox::destroy_all_mailboxes}; use ahash::AHashSet; -use jmap::mailbox::INBOX_ID; + +use ::email::mailbox::INBOX_ID; use jmap_client::{ client::Client, core::set::{SetError, SetErrorType}, diff --git a/tests/src/jmap/event_source.rs b/tests/src/jmap/event_source.rs index a74e755f..f7f8e1bf 100644 --- a/tests/src/jmap/event_source.rs +++ b/tests/src/jmap/event_source.rs @@ -13,8 +13,9 @@ use crate::{ test_account_login, }, }; +use email::mailbox::INBOX_ID; use futures::StreamExt; -use jmap::mailbox::INBOX_ID; + use jmap_client::{event_source::Changes, mailbox::Role, TypeState}; use jmap_proto::types::id::Id; use store::ahash::AHashSet; diff --git a/tests/src/jmap/mod.rs b/tests/src/jmap/mod.rs index ef7d01d0..dcacb63f 100644 --- a/tests/src/jmap/mod.rs +++ b/tests/src/jmap/mod.rs @@ -381,7 +381,7 @@ pub async fn jmap_tests() { email_query_changes::test(&mut params).await; email_copy::test(&mut params).await; thread_get::test(&mut params).await; - thread_merge::test(&mut params).await;*/ + thread_merge::test(&mut params).await; mailbox::test(&mut params).await; delivery::test(&mut params).await; auth_acl::test(&mut params).await; @@ -395,7 +395,7 @@ pub async fn jmap_tests() { websocket::test(&mut params).await; quota::test(&mut params).await; crypto::test(&mut params).await; - blob::test(&mut params).await; + blob::test(&mut params).await;*/ permissions::test(¶ms).await; purge::test(&mut params).await; enterprise::test(&mut params).await; @@ -595,7 +595,7 @@ async fn init_jmap_tests(store_id: &str, delete_if_exists: bool) -> JMAPTest { let data = Data::parse(&mut config); let cache = Caches::parse(&mut config); let store = core.storage.data.clone(); - let (ipc, mut ipc_rxs) = build_ipc(); + let (ipc, mut ipc_rxs) = build_ipc(&mut config); let inner = Arc::new(Inner { shared_core: core.into_shared(), data, diff --git a/tests/src/jmap/permissions.rs b/tests/src/jmap/permissions.rs index 30695713..a4b8015c 100644 --- a/tests/src/jmap/permissions.rs +++ b/tests/src/jmap/permissions.rs @@ -7,15 +7,12 @@ use std::sync::Arc; use ahash::AHashSet; -use common::{ - auth::{AccessToken, TenantInfo}, - ipc::{DeliveryResult, IngestMessage}, -}; +use common::auth::{AccessToken, TenantInfo}; use directory::{ backend::internal::{PrincipalField, PrincipalUpdate, PrincipalValue}, Permission, Principal, Type, }; -use jmap::{services::ingest::MailDelivery, JmapMethods}; +use email::delivery::{IngestMessage, LocalDeliveryStatus, MailDelivery}; use utils::BlobHash; use crate::jmap::assert_is_empty; @@ -615,8 +612,9 @@ pub async fn test(params: &JMAPTest) { message_size: TEST_MESSAGE.len(), session_id: 0, }) - .await, - vec![DeliveryResult::PermanentFailure { + .await + .status, + vec![LocalDeliveryStatus::PermanentFailure { code: [5, 5, 0], reason: "This account is not authorized to receive email.".into() }] @@ -652,8 +650,9 @@ pub async fn test(params: &JMAPTest) { message_size: TEST_MESSAGE.len(), session_id: 0, }) - .await, - vec![DeliveryResult::Success] + .await + .status, + vec![LocalDeliveryStatus::Success] ); // Quota for the tenant and user should be updated @@ -676,8 +675,9 @@ pub async fn test(params: &JMAPTest) { message_size: TEST_MESSAGE.len(), session_id: 0, }) - .await, - vec![DeliveryResult::TemporaryFailure { + .await + .status, + vec![LocalDeliveryStatus::TemporaryFailure { reason: "Organization over quota.".into() }] ); diff --git a/tests/src/jmap/purge.rs b/tests/src/jmap/purge.rs index c7489f64..ad628c13 100644 --- a/tests/src/jmap/purge.rs +++ b/tests/src/jmap/purge.rs @@ -7,12 +7,9 @@ use ahash::AHashSet; use common::Server; use directory::{backend::internal::manage::ManageDirectory, QueryBy}; +use email::mailbox::{INBOX_ID, JUNK_ID, TRASH_ID}; use imap_proto::ResponseType; -use jmap::{ - email::delete::EmailDeletion, - mailbox::{INBOX_ID, JUNK_ID, TRASH_ID}, - JmapMethods, -}; +use jmap::email::delete::EmailDeletion; use jmap_proto::types::{collection::Collection, id::Id, property::Property}; use store::{ write::{key::DeserializeBigEndian, TagValue}, diff --git a/tests/src/jmap/quota.rs b/tests/src/jmap/quota.rs index 7e7dffb1..e1ad9839 100644 --- a/tests/src/jmap/quota.rs +++ b/tests/src/jmap/quota.rs @@ -11,7 +11,8 @@ use crate::{ mailbox::destroy_all_mailboxes, test_account_login, }, }; -use jmap::{blob::upload::DISABLE_UPLOAD_QUOTA, mailbox::INBOX_ID, JmapMethods}; +use email::mailbox::INBOX_ID; +use jmap::blob::upload::DISABLE_UPLOAD_QUOTA; use jmap_client::{ core::set::{SetErrorType, SetObject}, email::EmailBodyPart, diff --git a/tests/src/jmap/stress_test.rs b/tests/src/jmap/stress_test.rs index 466eced5..3db76711 100644 --- a/tests/src/jmap/stress_test.rs +++ b/tests/src/jmap/stress_test.rs @@ -9,8 +9,8 @@ use std::{sync::Arc, time::Duration}; use crate::jmap::{mailbox::destroy_all_mailboxes_no_wait, wait_for_index}; use common::Server; use directory::backend::internal::manage::ManageDirectory; +use email::mailbox::UidMailbox; use futures::future::join_all; -use jmap::{mailbox::UidMailbox, JmapMethods}; use jmap_client::{ client::Client, core::set::{SetErrorType, SetObject}, diff --git a/tests/src/jmap/thread_merge.rs b/tests/src/jmap/thread_merge.rs index 62247c36..7e53d949 100644 --- a/tests/src/jmap/thread_merge.rs +++ b/tests/src/jmap/thread_merge.rs @@ -11,10 +11,8 @@ use crate::{ store::deflate_test_resource, }; use common::auth::AccessToken; -use jmap::{ - email::ingest::{EmailIngest, IngestEmail, IngestSource}, - JmapMethods, -}; + +use ::email::ingest::{EmailIngest, IngestEmail, IngestSource}; use jmap_client::{email, mailbox::Role}; use jmap_proto::types::{collection::Collection, id::Id}; use mail_parser::{mailbox::mbox::MessageIterator, MessageParser}; diff --git a/tests/src/smtp/mod.rs b/tests/src/smtp/mod.rs index 285c45ec..f3015eb1 100644 --- a/tests/src/smtp/mod.rs +++ b/tests/src/smtp/mod.rs @@ -173,7 +173,7 @@ impl TestSMTP { } pub fn inner_with_rxs(&self) -> (Arc, IpcReceivers) { - let (ipc, ipc_rxs) = build_ipc(); + let (ipc, ipc_rxs) = build_ipc(&mut Config::default()); ( Inner { @@ -191,7 +191,7 @@ impl TestSMTP { let store = core.storage.data.clone(); let blob_store = core.storage.blob.clone(); let shared_core = core.into_shared(); - let (ipc, mut ipc_rxs) = build_ipc(); + let (ipc, mut ipc_rxs) = build_ipc(&mut Config::default()); TestSMTP { queue_receiver: QueueReceiver { diff --git a/tests/src/smtp/queue/concurrent.rs b/tests/src/smtp/queue/concurrent.rs index 3ab9cdec..f0dea570 100644 --- a/tests/src/smtp/queue/concurrent.rs +++ b/tests/src/smtp/queue/concurrent.rs @@ -43,7 +43,7 @@ enable = false "#; -const NUM_MESSAGES: usize = 1000; +const NUM_MESSAGES: usize = 100; const NUM_QUEUES: usize = 10; #[tokio::test(flavor = "multi_thread", worker_threads = 18)] @@ -108,7 +108,7 @@ async fn concurrent_queue() { // Send 1000 test messages for _ in 0..(NUM_MESSAGES / 2) { session - .send_message("john@test.org", &["slow@foobar.org"], "test:no_dkim", "250") + .send_message("john@test.org", &["bill@foobar.org"], "test:no_dkim", "250") .await; } @@ -135,14 +135,14 @@ async fn concurrent_queue() { if m + e != 0 { println!("Queue still has {} messages and {} events", m, e); - for inner in &inners { + /*for inner in &inners { inner .ipc .queue_tx - .send(QueueEvent::Paused(true)) + .send(QueueEvent::Refresh) .await .unwrap(); - } + }*/ } else { break; }