From b3963412400753ccff4ff85c34445606bba23d21 Mon Sep 17 00:00:00 2001 From: mdecimus Date: Wed, 9 Apr 2025 09:56:54 +0200 Subject: [PATCH] Incremental caching --- Cargo.lock | 37 +- crates/common/Cargo.toml | 2 + crates/common/src/config/inner.rs | 32 +- crates/common/src/core.rs | 52 +- crates/common/src/lib.rs | 184 +++---- crates/common/src/sharing/document.rs | 70 +-- crates/email/Cargo.toml | 1 + crates/email/src/lib.rs | 1 - crates/email/src/mailbox/cache.rs | 279 ++++++++++ crates/email/src/mailbox/destroy.rs | 36 +- crates/email/src/mailbox/manage.rs | 183 +------ crates/email/src/mailbox/mod.rs | 1 + crates/email/src/message/cache.rs | 322 ++++++++++++ crates/email/src/message/copy.rs | 5 +- crates/email/src/message/delete.rs | 59 +-- crates/email/src/message/index.rs | 4 +- crates/email/src/message/ingest.rs | 48 +- crates/email/src/message/mod.rs | 1 + crates/email/src/sieve/ingest.rs | 45 +- crates/email/src/thread/cache.rs | 99 ---- crates/email/src/thread/mod.rs | 7 - crates/imap-proto/Cargo.toml | 1 + crates/imap-proto/src/lib.rs | 4 +- crates/imap-proto/src/parser/acl.rs | 22 +- crates/imap-proto/src/parser/append.rs | 32 +- crates/imap-proto/src/parser/authenticate.rs | 6 +- crates/imap-proto/src/parser/copy_move.rs | 8 +- crates/imap-proto/src/parser/create.rs | 16 +- crates/imap-proto/src/parser/delete.rs | 8 +- crates/imap-proto/src/parser/enable.rs | 2 +- crates/imap-proto/src/parser/fetch.rs | 44 +- crates/imap-proto/src/parser/list.rs | 82 ++- crates/imap-proto/src/parser/login.rs | 12 +- crates/imap-proto/src/parser/lsub.rs | 12 +- crates/imap-proto/src/parser/mod.rs | 7 +- crates/imap-proto/src/parser/quota.rs | 8 +- crates/imap-proto/src/parser/rename.rs | 12 +- crates/imap-proto/src/parser/search.rs | 76 +-- crates/imap-proto/src/parser/select.rs | 35 +- crates/imap-proto/src/parser/sort.rs | 14 +- crates/imap-proto/src/parser/status.rs | 4 +- crates/imap-proto/src/parser/store.rs | 8 +- crates/imap-proto/src/parser/subscribe.rs | 8 +- crates/imap-proto/src/parser/thread.rs | 6 +- crates/imap-proto/src/protocol/acl.rs | 37 +- crates/imap-proto/src/protocol/append.rs | 6 +- .../imap-proto/src/protocol/authenticate.rs | 6 +- crates/imap-proto/src/protocol/copy_move.rs | 6 +- crates/imap-proto/src/protocol/create.rs | 6 +- crates/imap-proto/src/protocol/delete.rs | 6 +- crates/imap-proto/src/protocol/enable.rs | 4 +- crates/imap-proto/src/protocol/fetch.rs | 19 +- crates/imap-proto/src/protocol/list.rs | 48 +- crates/imap-proto/src/protocol/login.rs | 8 +- crates/imap-proto/src/protocol/mod.rs | 5 +- crates/imap-proto/src/protocol/namespace.rs | 4 +- crates/imap-proto/src/protocol/quota.rs | 18 +- crates/imap-proto/src/protocol/rename.rs | 8 +- crates/imap-proto/src/protocol/search.rs | 23 +- crates/imap-proto/src/protocol/select.rs | 8 +- crates/imap-proto/src/protocol/status.rs | 17 +- crates/imap-proto/src/protocol/store.rs | 4 +- crates/imap-proto/src/protocol/subscribe.rs | 6 +- crates/imap-proto/src/protocol/thread.rs | 4 +- crates/imap-proto/src/receiver.rs | 101 ++-- crates/imap-proto/src/utf7.rs | 12 +- crates/imap/Cargo.toml | 2 +- crates/imap/src/core/mailbox.rs | 492 +++++------------- crates/imap/src/core/message.rs | 164 ++---- crates/imap/src/core/mod.rs | 97 +++- crates/imap/src/op/acl.rs | 6 +- crates/imap/src/op/append.rs | 20 +- crates/imap/src/op/authenticate.rs | 3 +- crates/imap/src/op/copy_move.rs | 11 +- crates/imap/src/op/create.rs | 115 +--- crates/imap/src/op/delete.rs | 2 +- crates/imap/src/op/expunge.rs | 38 +- crates/imap/src/op/fetch.rs | 93 ++-- crates/imap/src/op/idle.rs | 6 +- crates/imap/src/op/list.rs | 11 +- crates/imap/src/op/login.rs | 4 +- crates/imap/src/op/mod.rs | 4 +- crates/imap/src/op/namespace.rs | 2 +- crates/imap/src/op/quota.rs | 4 +- crates/imap/src/op/rename.rs | 55 +- crates/imap/src/op/search.rs | 128 +++-- crates/imap/src/op/select.rs | 58 +-- crates/imap/src/op/status.rs | 223 +++----- crates/imap/src/op/subscribe.rs | 7 +- crates/imap/src/op/thread.rs | 10 +- crates/jmap-proto/Cargo.toml | 1 + crates/jmap-proto/src/types/keyword.rs | 54 +- crates/jmap/src/blob/download.rs | 21 +- crates/jmap/src/email/copy.rs | 18 +- crates/jmap/src/email/get.rs | 21 +- crates/jmap/src/email/import.rs | 6 +- crates/jmap/src/email/query.rs | 207 ++++---- crates/jmap/src/email/set.rs | 35 +- crates/jmap/src/email/snippet.rs | 18 +- crates/jmap/src/mailbox/get.rs | 137 +++-- crates/jmap/src/mailbox/query.rs | 219 ++++---- crates/jmap/src/mailbox/set.rs | 6 +- crates/jmap/src/thread/get.rs | 10 +- crates/managesieve/Cargo.toml | 1 + crates/managesieve/src/core/mod.rs | 3 +- crates/managesieve/src/lib.rs | 20 +- crates/managesieve/src/op/authenticate.rs | 5 +- crates/pop3/src/mailbox.rs | 93 ++-- crates/trc/Cargo.toml | 1 + crates/trc/src/event/conv.rs | 7 + tests/src/imap/acl.rs | 26 +- tests/src/imap/body_structure.rs | 4 +- tests/src/imap/copy_move.rs | 2 +- tests/src/imap/mailbox.rs | 2 +- tests/src/jmap/delivery.rs | 48 +- tests/src/jmap/email_query.rs | 22 +- tests/src/jmap/email_set.rs | 2 +- tests/src/jmap/mod.rs | 2 +- tests/src/jmap/purge.rs | 58 +-- tests/src/jmap/stress_test.rs | 30 +- tests/src/store/query.rs | 5 +- 121 files changed, 2247 insertions(+), 2643 deletions(-) create mode 100644 crates/email/src/mailbox/cache.rs create mode 100644 crates/email/src/message/cache.rs delete mode 100644 crates/email/src/thread/cache.rs delete mode 100644 crates/email/src/thread/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 40986223..554515c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1037,6 +1037,15 @@ dependencies = [ "cipher 0.4.4", ] +[[package]] +name = "castaway" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0abae9be0aaf9ea96a3b1b8b1b55c602ca751eba1b1500220cea4ecbafe7c0d5" +dependencies = [ + "rustversion", +] + [[package]] name = "cbc" version = "0.1.2" @@ -1244,6 +1253,7 @@ dependencies = [ "bincode", "biscuit", "chrono", + "compact_str", "decancer", "directory", "dns-update", @@ -1294,6 +1304,7 @@ dependencies = [ "sieve-rs", "smtp-proto", "store", + "tinyvec", "tokio", "tokio-rustls 0.26.2", "trc", @@ -1305,6 +1316,22 @@ dependencies = [ "zip", ] +[[package]] +name = "compact_str" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rkyv 0.8.10", + "rustversion", + "ryu", + "serde", + "static_assertions", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -2128,6 +2155,7 @@ dependencies = [ "bincode", "cbc", "common", + "compact_str", "directory", "hashify", "jmap_proto", @@ -3466,6 +3494,7 @@ version = "0.11.8" dependencies = [ "ahash 0.8.11", "common", + "compact_str", "directory", "email", "imap_proto", @@ -3492,6 +3521,7 @@ version = "0.11.8" dependencies = [ "ahash 0.8.11", "chrono", + "compact_str", "hashify", "jmap_proto", "mail-parser", @@ -3796,6 +3826,7 @@ name = "jmap_proto" version = "0.11.8" dependencies = [ "ahash 0.8.11", + "compact_str", "fast-float", "hashify", "mail-parser", @@ -4206,6 +4237,7 @@ dependencies = [ "ahash 0.8.11", "bincode", "common", + "compact_str", "directory", "email", "imap", @@ -7341,9 +7373,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.8.1" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "022db8904dfa342efe721985167e9fcd16c29b226db4397ed752a761cfce81e8" +checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71" dependencies = [ "tinyvec_macros", ] @@ -7640,6 +7672,7 @@ dependencies = [ "ahash 0.8.11", "base64 0.22.1", "bincode", + "compact_str", "event_macro", "mail-auth", "mail-parser", diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index 264f5242..1c0e26de 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -69,6 +69,8 @@ num_cpus = "1.13.1" hashify = "0.2" rkyv = { version = "0.8.10", features = ["little_endian"] } indexmap = "2.7.1" +tinyvec = "1.9.0" +compact_str = { version = "0.9.0", features = ["rkyv", "serde"] } [target.'cfg(unix)'.dependencies] privdrop = "0.5.3" diff --git a/crates/common/src/config/inner.rs b/crates/common/src/config/inner.rs index 56c10112..1c22fa1f 100644 --- a/crates/common/src/config/inner.rs +++ b/crates/common/src/config/inner.rs @@ -23,8 +23,8 @@ use utils::{ }; use crate::{ - Account, AccountId, Caches, Data, Mailbox, MailboxId, MailboxState, NextMailboxState, Threads, - TlsConnectors, + CacheSwap, Caches, Data, DavResource, DavResources, MailboxCache, MessageItemCache, + MessageStoreCache, MessageUidCache, TlsConnectors, auth::{AccessToken, roles::RolePermissions}, config::smtp::resolver::{Policy, Tlsa}, listener::blocked::BlockedIps, @@ -107,34 +107,28 @@ impl Caches { MB_5, std::mem::size_of::() as u64, ), - account: Cache::from_config( - config, - "account", - MB_10, - (std::mem::size_of::() - + std::mem::size_of::() - + (15 * (std::mem::size_of::() + 60))) as u64, - ), - mailbox: Cache::from_config( + mailboxes: Cache::from_config( config, "mailbox", MB_10, - (std::mem::size_of::() - + std::mem::size_of::() - + std::mem::size_of::() - + (1024 * std::mem::size_of::())) as u64, + (std::mem::size_of::() + + std::mem::size_of::>>() + + (15 * (std::mem::size_of::() + 60))) as u64, ), - threads: Cache::from_config( + messages: Cache::from_config( config, - "thread", + "message", MB_10, - (std::mem::size_of::() + (500 * std::mem::size_of::())) as u64, + (std::mem::size_of::() + + std::mem::size_of::>>() + + (1024 * std::mem::size_of::())) as u64, ), dav: Cache::from_config( config, "dav", MB_10, - (std::mem::size_of::() + (500 * std::mem::size_of::())) as u64, + (std::mem::size_of::() + (500 * std::mem::size_of::())) + as u64, ), bayes: CacheWithTtl::from_config( config, diff --git a/crates/common/src/core.rs b/crates/common/src/core.rs index e001d369..46d078ab 100644 --- a/crates/common/src/core.rs +++ b/crates/common/src/core.rs @@ -18,15 +18,15 @@ use store::{ dispatch::DocumentSet, roaring::RoaringBitmap, write::{ - AlignedBytes, Archive, AssignedIds, BatchBuilder, BitmapClass, BlobOp, DirectoryClass, - QueueClass, TagValue, ValueClass, key::DeserializeBigEndian, now, + AlignedBytes, Archive, AssignedIds, BatchBuilder, BlobOp, DirectoryClass, QueueClass, + ValueClass, key::DeserializeBigEndian, now, }, }; use trc::AddContext; use utils::BlobHash; use crate::{ - ImapId, Inner, MailboxState, Server, + Inner, Server, auth::{AccessToken, ResourceToken, TenantInfo}, config::smtp::{ auth::{ArcSealer, DkimSigner, LazySignature, ResolvedSignature, build_signature}, @@ -468,35 +468,6 @@ impl Server { }) } - pub async fn get_tag( - &self, - account_id: u32, - collection: Collection, - property: impl AsRef + Sync + Send, - value: impl Into + Sync + Send, - ) -> trc::Result> { - let property = property.as_ref(); - self.core - .storage - .data - .get_bitmap(BitmapKey { - account_id, - collection: collection.into(), - class: BitmapClass::Tag { - field: property.into(), - value: value.into(), - }, - document_id: 0, - }) - .await - .add_context(|err| { - err.caused_by(trc::location!()) - .account_id(account_id) - .collection(collection) - .id(property.to_string()) - }) - } - #[inline(always)] pub fn notify_task_queue(&self) { self.inner.ipc.index_tx.notify_one(); @@ -706,20 +677,3 @@ impl BuildServer for Arc { } } } - -impl MailboxState { - pub fn map_result_id(&self, document_id: u32, is_uid: bool) -> Option<(u32, ImapId)> { - if let Some(imap_id) = self.id_to_imap.get(&document_id) { - Some((if is_uid { imap_id.uid } else { imap_id.seqnum }, *imap_id)) - } else if is_uid { - self.next_state.as_ref().and_then(|s| { - s.next_state - .id_to_imap - .get(&document_id) - .map(|imap_id| (imap_id.uid, *imap_id)) - }) - } else { - None - } - } -} diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 2a2c7540..dd7a166f 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -17,10 +17,11 @@ use std::{ use ahash::{AHashMap, AHashSet}; use arc_swap::ArcSwap; use auth::{AccessToken, oauth::config::OAuthConfig, roles::RolePermissions}; +use compact_str::CompactString; use config::{ dav::DavConfig, imap::ImapConfig, - jmap::settings::JmapConfig, + jmap::settings::{JmapConfig, SpecialUse}, network::Network, scripts::Scripting, smtp::{ @@ -32,9 +33,8 @@ use config::{ telemetry::Metrics, }; -use imap_proto::protocol::list::Attribute; -use indexmap::IndexMap; use ipc::{HousekeeperEvent, QueueEvent, ReportingEvent, StateEvent}; +use jmap_proto::types::keyword::Keyword; use listener::{asn::AsnGeoLookupData, blocked::Security, tls::AcmeProviders}; use mail_auth::{MX, Txt}; @@ -43,6 +43,7 @@ use nlp::bayes::{TokenHash, Weights}; use parking_lot::{Mutex, RwLock}; use rustls::sign::CertifiedKey; use store::roaring::RoaringBitmap; +use tinyvec::TinyVec; use tokio::sync::{Notify, Semaphore, mpsc}; use tokio_rustls::TlsConnector; use utils::{ @@ -144,9 +145,8 @@ pub struct Caches { pub http_auth: Cache, pub permissions: Cache>, - pub account: Cache>, - pub mailbox: Cache>, - pub threads: Cache>, + pub messages: Cache>>, + pub mailboxes: Cache>>, pub dav: Cache>, pub bayes: CacheWithTtl, @@ -161,6 +161,41 @@ pub struct Caches { pub dns_rbl: CacheWithTtl>>, } +#[derive(Debug, Clone)] +pub struct CacheSwap(pub Arc>); + +#[derive(Debug, Clone)] +pub struct MessageStoreCache { + pub change_id: u64, + pub items: AHashMap, + pub update_lock: Arc, + pub size: u64, +} + +#[derive(Debug, Clone)] +pub struct MessageItemCache { + pub mailboxes: TinyVec<[MessageUidCache; 2]>, + pub keywords: TinyVec<[Keyword; 2]>, + pub thread_id: u32, + pub change_id: u64, +} + +#[derive(Debug, Default, Clone, Copy)] +pub struct MessageUidCache { + pub mailbox_id: u32, + pub uid: u32, +} + +#[derive(Debug, Clone)] +pub struct MailboxCache { + pub name: CompactString, + pub path: CompactString, + pub role: SpecialUse, + pub parent_id: Option, + pub subscribers: TinyVec<[u32; 4]>, + pub uid_validity: u32, +} + #[derive(Debug, Clone, Default)] pub struct HttpAuthCache { pub account_id: u32, @@ -181,74 +216,6 @@ pub struct TlsConnectors { pub dummy_verify: TlsConnector, } -#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] -pub struct AccountId { - pub account_id: u32, - pub primary_id: u32, -} - -#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] -pub struct MailboxId { - pub account_id: u32, - pub mailbox_id: u32, -} - -#[derive(Debug, Clone, Default)] -pub struct Account { - pub account_id: u32, - pub prefix: Option, - pub mailbox_names: IndexMap, - pub mailbox_state: AHashMap, - pub state_email: Option, - pub state_mailbox: Option, - pub obj_size: u64, -} - -#[derive(Debug, Default, Clone)] -pub struct Mailbox { - pub has_children: bool, - pub is_subscribed: bool, - pub special_use: Option, - pub total_messages: Option, - pub total_unseen: Option, - pub total_deleted: Option, - pub total_deleted_storage: Option, - pub uid_validity: Option, - pub uid_next: Option, - pub size: Option, -} - -#[derive(Debug, Clone, Default)] -pub struct MailboxState { - pub uid_next: u32, - pub uid_validity: u32, - pub uid_max: u32, - pub id_to_imap: AHashMap, - pub uid_to_id: AHashMap, - pub total_messages: usize, - pub modseq: Option, - pub next_state: Option>, - pub obj_size: u64, -} - -#[derive(Debug, Clone)] -pub struct NextMailboxState { - pub next_state: MailboxState, - pub deletions: Vec, -} - -#[derive(Debug, Clone, Copy, Default)] -pub struct ImapId { - pub uid: u32, - pub seqnum: u32, -} - -#[derive(Debug, Default)] -pub struct Threads { - pub threads: AHashMap, - pub modseq: Option, -} - pub struct NameWrapper(pub String); #[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] @@ -291,39 +258,21 @@ pub struct Core { pub enterprise: Option, } -impl CacheItemWeight for AccountId { - fn weight(&self) -> u64 { - std::mem::size_of::() as u64 - } -} - -impl CacheItemWeight for MailboxId { - fn weight(&self) -> u64 { - std::mem::size_of::() as u64 - } -} - impl CacheItemWeight for DavResourceId { fn weight(&self) -> u64 { std::mem::size_of::() as u64 } } -impl CacheItemWeight for Threads { +impl CacheItemWeight for CacheSwap { fn weight(&self) -> u64 { - ((self.threads.len() + 2) * std::mem::size_of::()) as u64 + std::mem::size_of::>() as u64 + self.0.load().weight() } } -impl CacheItemWeight for MailboxState { +impl CacheItemWeight for MessageStoreCache { fn weight(&self) -> u64 { - self.obj_size - } -} - -impl CacheItemWeight for Account { - fn weight(&self) -> u64 { - self.obj_size + self.size } } @@ -339,20 +288,6 @@ impl CacheItemWeight for DavResources { } } -impl MailboxState { - pub fn calculate_weight(&self) -> u64 { - std::mem::size_of::() as u64 - + (self.id_to_imap.len() * std::mem::size_of::() + std::mem::size_of::()) - as u64 - + (self.uid_to_id.len() * std::mem::size_of::()) as u64 - + self.next_state.as_ref().map_or(0, |n| { - std::mem::size_of::() as u64 - + (n.deletions.len() * std::mem::size_of::()) as u64 - + n.next_state.calculate_weight() - }) - } -} - pub trait IntoString: Sized { fn into_string(self) -> String; } @@ -451,9 +386,8 @@ impl Default for Caches { access_tokens: Cache::new(1024, 10 * 1024 * 1024), http_auth: Cache::new(1024, 10 * 1024 * 1024), permissions: Cache::new(1024, 10 * 1024 * 1024), - account: Cache::new(1024, 10 * 1024 * 1024), - mailbox: Cache::new(1024, 10 * 1024 * 1024), - threads: Cache::new(1024, 10 * 1024 * 1024), + mailboxes: Cache::new(1024, 10 * 1024 * 1024), + messages: Cache::new(1024, 25 * 1024 * 1024), dav: Cache::new(1024, 10 * 1024 * 1024), bayes: CacheWithTtl::new(1024, 10 * 1024 * 1024), dns_rbl: CacheWithTtl::new(1024, 10 * 1024 * 1024), @@ -579,25 +513,25 @@ impl std::borrow::Borrow for DavResource { } } -impl Threads { +impl MessageStoreCache { pub fn assign_thread_id(&self, thread_name: &[u8], message_id: &[u8]) -> u32 { let mut bytes = Vec::with_capacity(thread_name.len() + message_id.len()); bytes.extend_from_slice(thread_name); bytes.extend_from_slice(message_id); let mut hash = store::gxhash::gxhash32(&bytes, 791120); - if self.threads.is_empty() { + if self.items.is_empty() { return hash; } // Naive pass, assume hash is unique let mut threads_ids = RoaringBitmap::new(); let mut is_unique_hash = true; - for &thread_id in self.threads.keys() { - if is_unique_hash && thread_id != hash { + for item in self.items.values() { + if is_unique_hash && item.thread_id != hash { is_unique_hash = false; } - threads_ids.insert(thread_id); + threads_ids.insert(item.thread_id); } if is_unique_hash { @@ -612,3 +546,17 @@ impl Threads { } } } + +impl CacheSwap { + pub fn new(value: Arc) -> Self { + Self(Arc::new(ArcSwap::new(value))) + } + + pub fn load_full(&self) -> Arc { + self.0.load_full() + } + + pub fn update(&self, value: Arc) { + self.0.store(value); + } +} diff --git a/crates/common/src/sharing/document.rs b/crates/common/src/sharing/document.rs index 68f5e5d5..a2073416 100644 --- a/crates/common/src/sharing/document.rs +++ b/crates/common/src/sharing/document.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use jmap_proto::types::{acl::Acl, collection::Collection, property::Property}; +use jmap_proto::types::{acl::Acl, collection::Collection}; use store::{ValueKey, query::acl::AclQuery, roaring::RoaringBitmap, write::ValueClass}; use trc::AddContext; use utils::map::bitmap::Bitmap; @@ -50,46 +50,6 @@ impl Server { Ok(document_ids) } - pub async fn shared_items( - &self, - access_token: &AccessToken, - to_account_id: u32, - to_container_collection: Collection, - to_item_collection: Collection, - property: Property, - check_acls: impl Into>, - ) -> trc::Result { - let check_acls = check_acls.into(); - let shared_containers = self - .shared_containers( - access_token, - to_account_id, - to_container_collection, - check_acls, - ) - .await?; - if shared_containers.is_empty() { - return Ok(shared_containers); - } - let todo = "maybe cache?"; - let mut shared_items = RoaringBitmap::new(); - for document_id in shared_containers { - if let Some(documents_in_folder) = self - .get_tag( - to_account_id, - to_item_collection, - property.clone(), - document_id, - ) - .await? - { - shared_items |= documents_in_folder; - } - } - - Ok(shared_items) - } - pub async fn owned_or_shared_containers( &self, access_token: &AccessToken, @@ -110,34 +70,6 @@ impl Server { Ok(document_ids) } - pub async fn owned_or_shared_items( - &self, - access_token: &AccessToken, - account_id: u32, - container_collection: Collection, - item_collection: Collection, - property: Property, - check_acls: impl Into>, - ) -> trc::Result { - let mut document_ids = self - .get_document_ids(account_id, item_collection) - .await? - .unwrap_or_default(); - if !document_ids.is_empty() && !access_token.is_member(account_id) { - document_ids &= self - .shared_items( - access_token, - account_id, - container_collection, - item_collection, - property, - check_acls, - ) - .await?; - } - Ok(document_ids) - } - pub async fn has_access_to_document( &self, access_token: &AccessToken, diff --git a/crates/email/Cargo.toml b/crates/email/Cargo.toml index c749f108..15519ab8 100644 --- a/crates/email/Cargo.toml +++ b/crates/email/Cargo.toml @@ -33,6 +33,7 @@ rand = "0.8" sequoia-openpgp = { version = "1.16", default-features = false, features = ["crypto-rust", "allow-experimental-crypto", "allow-variable-time-crypto"] } hashify = "0.2" rkyv = { version = "0.8.10", features = ["little_endian"] } +compact_str = "0.9.0" [features] test_mode = [] diff --git a/crates/email/src/lib.rs b/crates/email/src/lib.rs index 4d22e499..3dfdb23c 100644 --- a/crates/email/src/lib.rs +++ b/crates/email/src/lib.rs @@ -10,4 +10,3 @@ pub mod message; pub mod push; pub mod sieve; pub mod submission; -pub mod thread; diff --git a/crates/email/src/mailbox/cache.rs b/crates/email/src/mailbox/cache.rs new file mode 100644 index 00000000..5907fed7 --- /dev/null +++ b/crates/email/src/mailbox/cache.rs @@ -0,0 +1,279 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::sync::Arc; + +use common::{ + CacheSwap, MailboxCache, MessageStoreCache, Server, config::jmap::settings::SpecialUse, +}; +use compact_str::CompactString; +use jmap_proto::types::collection::Collection; +use std::future::Future; +use store::{ + ahash::AHashMap, + query::log::{Change, Query}, +}; +use tokio::sync::Semaphore; +use trc::AddContext; +use utils::topological::TopologicalSort; + +use super::{ArchivedMailbox, Mailbox, manage::MailboxFnc}; + +pub trait MessageMailboxCache: Sync + Send { + fn get_cached_mailboxes( + &self, + account_id: u32, + ) -> impl Future>>> + Send; +} + +impl MessageMailboxCache for Server { + async fn get_cached_mailboxes( + &self, + account_id: u32, + ) -> trc::Result>> { + let cache_ = match self + .inner + .cache + .mailboxes + .get_value_or_guard_async(&account_id) + .await + { + Ok(cache) => cache, + Err(guard) => { + let cache = full_cache_build(self, account_id, Arc::new(Semaphore::new(1))).await?; + + if guard.insert(CacheSwap::new(cache.clone())).is_err() { + self.inner + .cache + .mailboxes + .insert(account_id, CacheSwap::new(cache.clone())); + } + + return Ok(cache); + } + }; + + // Perform full refresh on stale ids + let cache = cache_.load_full(); + if cache.change_id > 0 + && self + .core + .jmap + .changes_max_history + .and_then(|history| self.inner.data.jmap_id_gen.past_id(history)) + .is_some_and(|last_change_id| cache.change_id < last_change_id) + { + let cache = full_cache_build(self, account_id, cache.update_lock.clone()).await?; + cache_.update(cache.clone()); + return Ok(cache); + } + + // Obtain current state + let changes = self + .core + .storage + .data + .changes( + account_id, + Collection::Mailbox, + Query::Since(cache.change_id), + ) + .await + .caused_by(trc::location!())?; + + // Verify changes + if changes.changes.is_empty() { + return Ok(cache); + } + + // Lock for updates + let _permit = cache.update_lock.acquire().await; + let cache = cache_.load_full(); + if cache.change_id >= changes.to_change_id { + return Ok(cache); + } + + let mut has_changes = false; + let mut cache = cache.as_ref().clone(); + cache.change_id = changes.to_change_id; + + for change in changes.changes { + match change { + Change::Insert(id) | Change::Update(id) => { + let document_id = id as u32; + if let Some(archive) = self + .get_archive(account_id, Collection::Mailbox, document_id) + .await + .caused_by(trc::location!())? + { + insert_item(&mut cache, document_id, archive.unarchive::()?); + has_changes = true; + } + } + Change::Delete(id) => { + if cache.items.remove(&(id as u32)).is_some() { + has_changes = true; + } + } + Change::ChildUpdate(_) => {} + } + } + + if has_changes { + build_tree(&mut cache); + } + + let cache = Arc::new(cache); + cache_.update(cache.clone()); + + Ok(cache) + } +} + +async fn full_cache_build( + server: &Server, + account_id: u32, + update_lock: Arc, +) -> trc::Result>> { + // Build cache + let mut cache = MessageStoreCache { + items: AHashMap::with_capacity(16), + size: 0, + change_id: 0, + update_lock, + }; + + server + .get_archives( + account_id, + Collection::Mailbox, + &(), + |document_id, archive| { + insert_item(&mut cache, document_id, archive.unarchive::()?); + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; + + if cache.items.is_empty() { + server + .create_system_folders(account_id) + .await + .caused_by(trc::location!())?; + server + .get_archives( + account_id, + Collection::Mailbox, + &(), + |document_id, archive| { + insert_item(&mut cache, document_id, archive.unarchive::()?); + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; + } + + build_tree(&mut cache); + + Ok(Arc::new(cache)) +} + +fn insert_item( + cache: &mut MessageStoreCache, + document_id: u32, + mailbox: &ArchivedMailbox, +) { + let parent_id = mailbox.parent_id.to_native(); + let item = MailboxCache { + name: mailbox.name.as_str().into(), + path: "".into(), + role: (&mailbox.role).into(), + parent_id: if parent_id > 0 { + Some(parent_id - 1) + } else { + None + }, + subscribers: mailbox.subscribers.iter().map(|s| s.to_native()).collect(), + uid_validity: mailbox.uid_validity.to_native(), + }; + + cache.items.insert(document_id, item); +} + +fn build_tree(cache: &mut MessageStoreCache) { + cache.size = 0; + let mut topological_sort = TopologicalSort::with_capacity(cache.items.len()); + + for (idx, (&document_id, mailbox)) in cache.items.iter_mut().enumerate() { + topological_sort.insert( + mailbox.parent_id.map(|id| id + 1).unwrap_or(0), + document_id + 1, + ); + mailbox.path = if matches!(mailbox.role, SpecialUse::Inbox) { + "INBOX".into() + } else if mailbox.parent_id.is_none() && mailbox.name.as_str().eq_ignore_ascii_case("inbox") + { + format!("INBOX {}", idx + 1).into() + } else { + mailbox.name.clone() + }; + + cache.size += (std::mem::size_of::() + + std::mem::size_of::() + + mailbox.name.len() + + mailbox.path.len()) as u64; + } + + for folder_id in topological_sort.into_iterator() { + if folder_id != 0 { + let folder_id = folder_id - 1; + if let Some((path, parent_path)) = cache + .items + .get(&folder_id) + .and_then(|folder| folder.parent_id.map(|parent_id| (&folder.path, parent_id))) + .and_then(|(path, parent_id)| { + cache + .items + .get(&parent_id) + .map(|folder| (path, &folder.path)) + }) + { + let mut new_path = CompactString::with_capacity(parent_path.len() + path.len() + 1); + new_path.push_str(parent_path.as_str()); + new_path.push('/'); + new_path.push_str(path.as_str()); + let folder = cache.items.get_mut(&folder_id).unwrap(); + folder.path = new_path; + } + } + } +} + +pub trait MailboxCacheAccess { + fn by_name(&self, name: &str) -> Option<(&u32, &MailboxCache)>; + fn by_path(&self, name: &str) -> Option<(&u32, &MailboxCache)>; + fn by_role(&self, role: &SpecialUse) -> Option<(&u32, &MailboxCache)>; +} + +impl MailboxCacheAccess for MessageStoreCache { + fn by_name(&self, name: &str) -> Option<(&u32, &MailboxCache)> { + self.items + .iter() + .find(|(_, m)| m.name.eq_ignore_ascii_case(name)) + } + + fn by_path(&self, path: &str) -> Option<(&u32, &MailboxCache)> { + self.items + .iter() + .find(|(_, m)| m.path.eq_ignore_ascii_case(path)) + } + + fn by_role(&self, role: &SpecialUse) -> Option<(&u32, &MailboxCache)> { + self.items.iter().find(|(_, m)| &m.role == role) + } +} diff --git a/crates/email/src/mailbox/destroy.rs b/crates/email/src/mailbox/destroy.rs index 77256ca9..bdb1534f 100644 --- a/crates/email/src/mailbox/destroy.rs +++ b/crates/email/src/mailbox/destroy.rs @@ -15,7 +15,11 @@ use jmap_proto::{ use store::{SerializeInfallible, query::Filter, roaring::RoaringBitmap, write::BatchBuilder}; use trc::AddContext; -use crate::message::{delete::EmailDeletion, metadata::MessageData}; +use crate::message::{ + cache::{MessageCache, MessageCacheAccess}, + delete::EmailDeletion, + metadata::MessageData, +}; use super::*; @@ -80,18 +84,18 @@ impl MailboxDestroy for Server { batch.with_account_id(account_id); - if let Some(message_ids) = self - .get_tag( - account_id, - Collection::Email, - Property::MailboxIds, - document_id, - ) - .await? - { - if remove_emails { - // If the message is in multiple mailboxes, untag it from the current mailbox, - // otherwise delete it. + if remove_emails { + // If the message is in multiple mailboxes, untag it from the current mailbox, + // otherwise delete it. + let message_ids = RoaringBitmap::from_iter( + self.get_cached_messages(account_id) + .await + .caused_by(trc::location!())? + .in_mailbox(document_id) + .map(|(id, _)| id), + ); + + if !message_ids.is_empty() { let mut destroy_ids = RoaringBitmap::new(); self.get_archives( @@ -148,10 +152,10 @@ impl MailboxDestroy for Server { self.emails_tombstone(account_id, &mut batch, destroy_ids) .await?; } - } else { - return Ok(Err(SetError::new(SetErrorType::MailboxHasEmail) - .with_description("Mailbox is not empty."))); } + } else { + return Ok(Err(SetError::new(SetErrorType::MailboxHasEmail) + .with_description("Mailbox is not empty."))); } // Obtain mailbox diff --git a/crates/email/src/mailbox/manage.rs b/crates/email/src/mailbox/manage.rs index 45cff535..84b96b5f 100644 --- a/crates/email/src/mailbox/manage.rs +++ b/crates/email/src/mailbox/manage.rs @@ -6,72 +6,30 @@ use std::future::Future; +use super::{cache::MessageMailboxCache, *}; use common::{Server, config::jmap::settings::SpecialUse, storage::index::ObjectIndexBuilder}; -use jmap_proto::types::{collection::Collection, keyword::Keyword, property::Property}; -use store::{ - SerializeInfallible, - ahash::{AHashMap, AHashSet}, - query::Filter, - roaring::RoaringBitmap, - write::BatchBuilder, -}; +use jmap_proto::types::collection::Collection; +use store::write::BatchBuilder; use trc::AddContext; -use crate::thread::cache::ThreadCache; - -use super::*; - pub trait MailboxFnc: Sync + Send { - fn mailbox_get_or_create( + fn create_system_folders( &self, account_id: u32, - ) -> impl Future> + Send; + ) -> impl Future> + Send; fn mailbox_create_path( &self, account_id: u32, path: &str, ) -> impl Future>> + Send; - - fn mailbox_count_threads( - &self, - account_id: u32, - document_ids: Option, - ) -> impl Future> + Send; - - fn mailbox_unread_tags( - &self, - account_id: u32, - document_id: u32, - message_ids: &Option, - ) -> impl Future>> + Send; - - fn mailbox_get_by_name( - &self, - account_id: u32, - path: &str, - ) -> impl Future>> + Send; - - fn mailbox_get_by_role( - &self, - account_id: u32, - role: SpecialUse, - ) -> impl Future>> + Send; } impl MailboxFnc for Server { - async fn mailbox_get_or_create(&self, account_id: u32) -> trc::Result { - let mut mailbox_ids = self - .get_document_ids(account_id, Collection::Mailbox) - .await? - .unwrap_or_default(); - if !mailbox_ids.is_empty() { - return Ok(mailbox_ids); - } - + async fn create_system_folders(&self, account_id: u32) -> trc::Result<()> { #[cfg(feature = "test_mode")] - if mailbox_ids.is_empty() && account_id == 0 { - return Ok(mailbox_ids); + if account_id == 0 { + return Ok(()); } let mut batch = BatchBuilder::new(); @@ -104,7 +62,6 @@ impl MailboxFnc for Server { .create_document(document_id) .custom(ObjectIndexBuilder::<(), _>::new().with_changes(object)) .caused_by(trc::location!())?; - mailbox_ids.insert(document_id); } self.store() .assign_document_ids(account_id, Collection::Mailbox, (ARCHIVE_ID + 1) as u64) @@ -118,24 +75,14 @@ impl MailboxFnc for Server { .await .caused_by(trc::location!())?; - Ok(mailbox_ids) + Ok(()) } async fn mailbox_create_path(&self, account_id: u32, path: &str) -> trc::Result> { let folders = self - .fetch_folders::(account_id, Collection::Mailbox) + .get_cached_mailboxes(account_id) .await - .caused_by(trc::location!())? - .format(|f| { - f.name = if f.document_id == INBOX_ID { - "inbox".to_string() - } else { - f.name.to_lowercase() - }; - }) - .into_iterator() - .map(|e| (e.name, e.document_id)) - .collect::>(); + .caused_by(trc::location!())?; let mut next_parent_id = 0; let mut create_paths = Vec::with_capacity(2); @@ -154,7 +101,11 @@ impl MailboxFnc for Server { } } - if let Some(document_id) = folders.get(&found_path) { + if let Some((document_id, _)) = folders + .items + .iter() + .find(|(_, item)| item.path == found_path) + { next_parent_id = *document_id + 1; } else { create_paths.push(name.to_string()); @@ -199,106 +150,4 @@ impl MailboxFnc for Server { Ok(Some(next_parent_id - 1)) } - - async fn mailbox_count_threads( - &self, - account_id: u32, - document_ids: Option, - ) -> trc::Result { - if let Some(document_ids) = document_ids { - let thread_ids = self - .get_cached_thread_ids(account_id) - .await - .caused_by(trc::location!())? - .threads - .iter() - .filter_map(|(document_id, thread_id)| { - if document_ids.contains(*document_id) { - Some(*thread_id) - } else { - None - } - }) - .collect::>(); - Ok(thread_ids.len()) - } else { - Ok(0) - } - } - - async fn mailbox_unread_tags( - &self, - account_id: u32, - document_id: u32, - message_ids: &Option, - ) -> trc::Result> { - if let (Some(message_ids), Some(mailbox_message_ids)) = ( - message_ids, - self.get_tag( - account_id, - Collection::Email, - Property::MailboxIds, - document_id, - ) - .await?, - ) { - if let Some(mut seen) = self - .get_tag( - account_id, - Collection::Email, - Property::Keywords, - Keyword::Seen, - ) - .await? - { - seen ^= message_ids; - seen &= &mailbox_message_ids; - if !seen.is_empty() { - Ok(Some(seen)) - } else { - Ok(None) - } - } else { - Ok(mailbox_message_ids.into()) - } - } else { - Ok(None) - } - } - - async fn mailbox_get_by_name(&self, account_id: u32, path: &str) -> trc::Result> { - self.fetch_folders::(account_id, Collection::Mailbox) - .await - .map(|folders| { - folders - .format(|f| { - if f.document_id == INBOX_ID { - f.name = "INBOX".to_string(); - } - }) - .into_iterator() - .find(|e| e.name.eq_ignore_ascii_case(path)) - .map(|e| e.document_id) - }) - } - - async fn mailbox_get_by_role( - &self, - account_id: u32, - role: SpecialUse, - ) -> trc::Result> { - if let Some(role) = role.as_str() { - self.store() - .filter( - account_id, - Collection::Mailbox, - vec![Filter::eq(Property::Role, role.serialize())], - ) - .await - .caused_by(trc::location!()) - .map(|r| r.results.min()) - } else { - Ok(None) - } - } } diff --git a/crates/email/src/mailbox/mod.rs b/crates/email/src/mailbox/mod.rs index 24decdd1..d7d6e64f 100644 --- a/crates/email/src/mailbox/mod.rs +++ b/crates/email/src/mailbox/mod.rs @@ -8,6 +8,7 @@ use common::config::jmap::settings::SpecialUse; use jmap_proto::types::value::AclGrant; use store::{SERIALIZE_OBJ_04_V1, SerializedVersion}; +pub mod cache; pub mod destroy; pub mod index; pub mod manage; diff --git a/crates/email/src/message/cache.rs b/crates/email/src/message/cache.rs new file mode 100644 index 00000000..eec15d13 --- /dev/null +++ b/crates/email/src/message/cache.rs @@ -0,0 +1,322 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::sync::Arc; + +use common::{ + CacheSwap, MessageItemCache, MessageStoreCache, MessageUidCache, Server, auth::AccessToken, +}; +use jmap_proto::types::{acl::Acl, collection::Collection, keyword::Keyword}; +use std::future::Future; +use store::{ + ahash::{AHashMap, AHashSet}, + query::log::{Change, Query}, + roaring::RoaringBitmap, +}; +use tokio::sync::Semaphore; +use trc::AddContext; +use utils::map::bitmap::Bitmap; + +use super::metadata::{ArchivedMessageData, MessageData}; + +pub trait MessageCache: Sync + Send { + fn get_cached_messages( + &self, + account_id: u32, + ) -> impl Future>>> + Send; + + fn shared_messages( + &self, + access_token: &AccessToken, + to_account_id: u32, + check_acls: impl Into> + Sync + Send, + ) -> impl Future> + Send; + + fn owned_or_shared_messages( + &self, + access_token: &AccessToken, + account_id: u32, + check_acls: impl Into> + Sync + Send, + ) -> impl Future> + Send; +} + +impl MessageCache for Server { + async fn get_cached_messages( + &self, + account_id: u32, + ) -> trc::Result>> { + let cache_ = match self + .inner + .cache + .messages + .get_value_or_guard_async(&account_id) + .await + { + Ok(cache) => cache, + Err(guard) => { + let cache = full_cache_build(self, account_id, Arc::new(Semaphore::new(1))).await?; + if guard.insert(CacheSwap::new(cache.clone())).is_err() { + self.inner + .cache + .messages + .insert(account_id, CacheSwap::new(cache.clone())); + } + return Ok(cache); + } + }; + + // Perform full refresh on stale ids + let cache = cache_.load_full(); + if cache.change_id > 0 + && self + .core + .jmap + .changes_max_history + .and_then(|history| self.inner.data.jmap_id_gen.past_id(history)) + .is_some_and(|last_change_id| cache.change_id < last_change_id) + { + let cache = full_cache_build(self, account_id, cache.update_lock.clone()).await?; + cache_.update(cache.clone()); + return Ok(cache); + } + + // Obtain current state + let changes = self + .core + .storage + .data + .changes(account_id, Collection::Email, Query::Since(cache.change_id)) + .await + .caused_by(trc::location!())?; + + // Verify changes + if changes.changes.is_empty() { + return Ok(cache); + } + + // Lock for updates + let _permit = cache.update_lock.acquire().await; + let cache = cache_.load_full(); + if cache.change_id >= changes.to_change_id { + return Ok(cache); + } + + let mut cache = cache.as_ref().clone(); + cache.change_id = changes.to_change_id; + let mut delete = AHashSet::with_capacity(changes.changes.len() / 2); + let mut update = AHashMap::with_capacity(changes.changes.len()); + + for change in changes.changes { + match change { + Change::Insert(id) => { + if let Some(item) = cache.items.get_mut(&(id as u32)) { + item.thread_id = (id >> 32) as u32; + } + update.insert(id as u32, true); + } + Change::Update(id) | Change::ChildUpdate(id) => { + update.insert(id as u32, false); + } + Change::Delete(id) => { + delete.insert(id as u32); + } + } + } + + for document_id in delete { + if update.remove(&document_id).is_none() { + if let Some(item) = cache.items.remove(&document_id) { + cache.size -= (std::mem::size_of::() + + std::mem::size_of::() + + (item.mailboxes.len() * std::mem::size_of::())) + as u64; + } + } + } + + for (document_id, is_insert) in update { + if let Some(archive) = self + .get_archive(account_id, Collection::Email, document_id) + .await + .caused_by(trc::location!())? + { + let message = archive.unarchive::()?; + insert_item(&mut cache, document_id, message, is_insert); + } + } + + let cache = Arc::new(cache); + cache_.update(cache.clone()); + + Ok(cache) + } + + async fn shared_messages( + &self, + access_token: &AccessToken, + to_account_id: u32, + check_acls: impl Into> + Sync + Send, + ) -> trc::Result { + let check_acls = check_acls.into(); + let shared_containers = self + .shared_containers(access_token, to_account_id, Collection::Mailbox, check_acls) + .await?; + if shared_containers.is_empty() { + return Ok(shared_containers); + } + let mut shared_messages = RoaringBitmap::new(); + for document_id in shared_containers { + shared_messages.extend( + self.get_cached_messages(to_account_id) + .await? + .in_mailbox(document_id) + .map(|(id, _)| *id), + ); + } + + Ok(shared_messages) + } + + async fn owned_or_shared_messages( + &self, + access_token: &AccessToken, + account_id: u32, + check_acls: impl Into> + Sync + Send, + ) -> trc::Result { + let mut document_ids = self + .get_document_ids(account_id, Collection::Email) + .await? + .unwrap_or_default(); + if !document_ids.is_empty() && !access_token.is_member(account_id) { + document_ids &= self + .shared_messages(access_token, account_id, check_acls) + .await?; + } + Ok(document_ids) + } +} + +async fn full_cache_build( + server: &Server, + account_id: u32, + update_lock: Arc, +) -> trc::Result>> { + // Build cache + let mut cache = MessageStoreCache { + items: AHashMap::with_capacity(16), + size: 0, + change_id: 0, + update_lock, + }; + + server + .get_archives( + account_id, + Collection::Email, + &(), + |document_id, archive| { + let message = archive.unarchive::()?; + cache.change_id = std::cmp::max(cache.change_id, message.change_id.to_native()); + + insert_item(&mut cache, document_id, message, true); + + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; + + Ok(Arc::new(cache)) +} + +fn insert_item( + cache: &mut MessageStoreCache, + document_id: u32, + message: &ArchivedMessageData, + update_size: bool, +) { + let item = MessageItemCache { + mailboxes: message + .mailboxes + .iter() + .map(|m| MessageUidCache { + mailbox_id: m.mailbox_id.to_native(), + uid: m.uid.to_native(), + }) + .collect(), + keywords: message.keywords.iter().map(Into::into).collect(), + thread_id: message.thread_id.to_native(), + change_id: message.change_id.to_native(), + }; + + if update_size { + cache.size += (std::mem::size_of::() + + std::mem::size_of::() + + (item.mailboxes.len() * std::mem::size_of::())) + as u64; + } + cache.items.insert(document_id, item); +} + +pub trait MessageCacheAccess { + fn in_mailbox(&self, mailbox_id: u32) -> impl Iterator; + + fn in_thread(&self, thread_id: u32) -> impl Iterator; + + fn with_keyword(&self, keyword: &Keyword) -> impl Iterator; + + fn in_mailbox_with_keyword( + &self, + mailbox_id: u32, + keyword: &Keyword, + ) -> impl Iterator; + + fn in_mailbox_without_keyword( + &self, + mailbox_id: u32, + keyword: &Keyword, + ) -> impl Iterator; +} + +impl MessageCacheAccess for MessageStoreCache { + fn in_mailbox(&self, mailbox_id: u32) -> impl Iterator { + self.items + .iter() + .filter(move |(_, m)| m.mailboxes.iter().any(|m| m.mailbox_id == mailbox_id)) + } + + fn in_thread(&self, thread_id: u32) -> impl Iterator { + self.items + .iter() + .filter(move |(_, m)| m.thread_id == thread_id) + } + + fn with_keyword(&self, keyword: &Keyword) -> impl Iterator { + self.items + .iter() + .filter(move |(_, m)| m.keywords.contains(keyword)) + } + + fn in_mailbox_with_keyword( + &self, + mailbox_id: u32, + keyword: &Keyword, + ) -> impl Iterator { + self.items.iter().filter(move |(_, m)| { + m.mailboxes.iter().any(|m| m.mailbox_id == mailbox_id) && m.keywords.contains(keyword) + }) + } + + fn in_mailbox_without_keyword( + &self, + mailbox_id: u32, + keyword: &Keyword, + ) -> impl Iterator { + self.items.iter().filter(move |(_, m)| { + m.mailboxes.iter().any(|m| m.mailbox_id == mailbox_id) && !m.keywords.contains(keyword) + }) + } +} diff --git a/crates/email/src/message/copy.rs b/crates/email/src/message/copy.rs index 85e3335d..589d13e3 100644 --- a/crates/email/src/message/copy.rs +++ b/crates/email/src/message/copy.rs @@ -19,9 +19,10 @@ use store::{ }; use trc::AddContext; -use crate::{mailbox::UidMailbox, thread::cache::ThreadCache}; +use crate::mailbox::UidMailbox; use super::{ + cache::MessageCache, index::{MAX_ID_LENGTH, MAX_SORT_FIELD_LENGTH, TrimTextValue}, ingest::{EmailIngest, IngestedEmail, ThreadResult}, metadata::{HeaderName, HeaderValue, MessageData, MessageMetadata}, @@ -141,7 +142,7 @@ impl EmailCopy for Server { ThreadResult::Id(thread_id) => (false, thread_id), ThreadResult::Create => ( true, - self.get_cached_thread_ids(account_id) + self.get_cached_messages(account_id) .await .caused_by(trc::location!())? .assign_thread_id(subject.as_bytes(), message_id.as_bytes()), diff --git a/crates/email/src/message/delete.rs b/crates/email/src/message/delete.rs index 34dc4bd4..b1a3fbd1 100644 --- a/crates/email/src/message/delete.rs +++ b/crates/email/src/message/delete.rs @@ -19,7 +19,10 @@ use utils::BlobHash; use std::future::Future; use store::rand::prelude::SliceRandom; -use crate::{mailbox::*, message::metadata::MessageMetadata}; +use crate::{ + mailbox::*, + message::{cache::MessageCache, metadata::MessageMetadata}, +}; use super::metadata::MessageData; @@ -171,51 +174,27 @@ impl EmailDeletion for Server { } async fn emails_auto_expunge(&self, account_id: u32, period: Duration) -> trc::Result<()> { - let deletion_candidates = self - .get_tag( - account_id, - Collection::Email, - Property::MailboxIds, - TagValue::Id(TRASH_ID), - ) - .await? - .unwrap_or_default() - | self - .get_tag( - account_id, - Collection::Email, - Property::MailboxIds, - TagValue::Id(JUNK_ID), - ) - .await? - .unwrap_or_default(); - - if deletion_candidates.is_empty() { - return Ok(()); - } let reference_cid = self.inner.data.jmap_id_gen.past_id(period).ok_or_else(|| { trc::StoreEvent::UnexpectedError .into_err() .caused_by(trc::location!()) .ctx(trc::Key::Reason, "Failed to generate reference cid.") })?; - - // Find messages to destroy - let mut destroy_ids = RoaringBitmap::new(); - self.get_archives( - account_id, - Collection::Email, - &deletion_candidates, - |document_id, data| { - if data.unarchive::()?.change_id < reference_cid { - destroy_ids.insert(document_id); - } - - Ok(true) - }, - ) - .await?; - + let destroy_ids = RoaringBitmap::from_iter( + self.get_cached_messages(account_id) + .await + .caused_by(trc::location!())? + .items + .iter() + .filter(|(_, item)| { + item.change_id < reference_cid + && item + .mailboxes + .iter() + .any(|id| id.mailbox_id == TRASH_ID || id.mailbox_id == JUNK_ID) + }) + .map(|(id, _)| id), + ); if destroy_ids.is_empty() { return Ok(()); } diff --git a/crates/email/src/message/index.rs b/crates/email/src/message/index.rs index 68320e02..05141525 100644 --- a/crates/email/src/message/index.rs +++ b/crates/email/src/message/index.rs @@ -592,7 +592,7 @@ impl IndexableObject for MessageData { .iter() .map(|k| match k.id() { Ok(id) => TagValue::Id(id), - Err(string) => TagValue::Text(string.into_bytes()), + Err(string) => TagValue::Text(string.as_bytes().to_vec()), }) .collect(), }, @@ -634,7 +634,7 @@ impl IndexableObject for &ArchivedMessageData { .iter() .map(|k| match k.id() { Ok(id) => TagValue::Id(id), - Err(string) => TagValue::Text(string.into_bytes()), + Err(string) => TagValue::Text(string.as_bytes().to_vec()), }) .collect(), }, diff --git a/crates/email/src/message/ingest.rs b/crates/email/src/message/ingest.rs index 42e7f0c7..0bac4be0 100644 --- a/crates/email/src/message/ingest.rs +++ b/crates/email/src/message/ingest.rs @@ -45,14 +45,15 @@ use trc::{AddContext, MessageIngestEvent}; use crate::{ mailbox::{INBOX_ID, JUNK_ID, UidMailbox}, message::{ + cache::MessageCache, crypto::EncryptionParams, index::{IndexMessage, MAX_ID_LENGTH, VisitValues}, metadata::MessageData, }, - thread::cache::ThreadCache, }; use super::{ + cache::MessageCacheAccess, crypto::{EncryptMessage, EncryptMessageError}, index::{MAX_SORT_FIELD_LENGTH, TrimTextValue}, }; @@ -334,7 +335,7 @@ impl EmailIngest for Server { ThreadResult::Id(thread_id) => thread_id, ThreadResult::Create => { log_thread_create = true; - self.get_cached_thread_ids(account_id) + self.get_cached_messages(account_id) .await .caused_by(trc::location!())? .assign_thread_id( @@ -722,39 +723,32 @@ impl EmailIngest for Server { return Ok(ThreadResult::Create); } + // Fetch cached messages + let cache = self + .get_cached_messages(account_id) + .await + .caused_by(trc::location!())?; + // Skip duplicate messages - if !found_message_id.is_empty() { - if let Some(ids) = self - .get_tag( - account_id, - Collection::Email, - Property::MailboxIds, - skip_duplicate.unwrap().1, - ) - .await - .caused_by(trc::location!())? - { - if found_message_id.iter().any(|id| ids.contains(*id)) { - return Ok(ThreadResult::Skip); - } - } + if !found_message_id.is_empty() + && cache + .in_mailbox(skip_duplicate.unwrap().1) + .any(|(id, _)| found_message_id.contains(id)) + { + return Ok(ThreadResult::Skip); } // Find the most common threadId let mut thread_counts = AHashMap::::with_capacity(16); let mut thread_id = u32::MAX; let mut thread_count = 0; - let thread_cache = self - .get_cached_thread_ids(account_id) - .await - .caused_by(trc::location!())?; - for (document_id, thread_id_) in thread_cache.threads.iter() { + for (document_id, item) in &cache.items { if results.contains(*document_id) { - let tc = thread_counts.entry(*thread_id_).or_default(); + let tc = thread_counts.entry(item.thread_id).or_default(); *tc += 1; if *tc > thread_count { thread_count = *tc; - thread_id = *thread_id_; + thread_id = item.thread_id; } } } @@ -779,8 +773,8 @@ impl EmailIngest for Server { // Move messages to the new threadId batch.with_collection(Collection::Email); - for (&document_id, &old_thread_id) in &thread_cache.threads { - if thread_id == old_thread_id || !thread_counts.contains_key(&old_thread_id) { + for (&document_id, item) in &cache.items { + if thread_id == item.thread_id || !thread_counts.contains_key(&item.thread_id) { continue; } if let Some(data_) = self @@ -791,7 +785,7 @@ impl EmailIngest for Server { let data = data_ .to_unarchived::() .caused_by(trc::location!())?; - if data.inner.thread_id != old_thread_id { + if data.inner.thread_id != item.thread_id { continue; } let mut new_data = data.deserialize().caused_by(trc::location!())?; diff --git a/crates/email/src/message/mod.rs b/crates/email/src/message/mod.rs index e0ee70c5..14b634ca 100644 --- a/crates/email/src/message/mod.rs +++ b/crates/email/src/message/mod.rs @@ -5,6 +5,7 @@ */ pub mod bayes; +pub mod cache; pub mod copy; pub mod crypto; pub mod delete; diff --git a/crates/email/src/sieve/ingest.rs b/crates/email/src/sieve/ingest.rs index 6cc7e224..9165a34a 100644 --- a/crates/email/src/sieve/ingest.rs +++ b/crates/email/src/sieve/ingest.rs @@ -7,7 +7,11 @@ use std::{borrow::Cow, sync::Arc}; use crate::{ - mailbox::{INBOX_ID, TRASH_ID, manage::MailboxFnc}, + mailbox::{ + INBOX_ID, TRASH_ID, + cache::{MailboxCacheAccess, MessageMailboxCache}, + manage::MailboxFnc, + }, message::{ delivery::AutogeneratedMessage, ingest::{EmailIngest, IngestEmail, IngestSource, IngestedEmail}, @@ -96,8 +100,8 @@ impl SieveScriptIngest for Server { // Obtain mailboxIds let account_id = access_token.primary_id; - let mailbox_ids = self - .mailbox_get_or_create(account_id) + let mailbox_cache = self + .get_cached_mailboxes(account_id) .await .caused_by(trc::location!())?; @@ -185,12 +189,9 @@ impl SieveScriptIngest for Server { } else { let mut mailbox_id = u32::MAX; if let Ok(role) = SpecialUse::parse_value(&role) { - if let Some(mailbox_id_) = self - .mailbox_get_by_role(account_id, role) - .await - .caused_by(trc::location!())? + if let Some((mailbox_id_, _)) = mailbox_cache.by_role(&role) { - mailbox_id = mailbox_id_; + mailbox_id = *mailbox_id_; } } @@ -203,9 +204,9 @@ impl SieveScriptIngest for Server { match mailbox { Mailbox::Name(name) => { if !matches!( - self.mailbox_get_by_name(account_id, &name).await.caused_by(trc::location!())?, - Some(document_id) if special_use_ids.is_empty() || - special_use_ids.contains(&document_id) + mailbox_cache.by_name(&name), + Some((document_id, _)) if special_use_ids.is_empty() || + special_use_ids.contains(document_id) ) { result = false; break; @@ -213,7 +214,7 @@ impl SieveScriptIngest for Server { } Mailbox::Id(id) => { if !matches!(Id::from_bytes(id.as_bytes()), Some(id) if - mailbox_ids.contains(id.document_id()) && + mailbox_cache.items.contains_key(&id.document_id()) && (special_use_ids.is_empty() || special_use_ids.contains(&id.document_id()))) { @@ -233,11 +234,7 @@ impl SieveScriptIngest for Server { { let role = SpecialUse::parse_value(&role); if role.is_err() - || !matches!( - self.mailbox_get_by_role(account_id, role.unwrap()) - .await, - Ok(Some(_)) - ) + || mailbox_cache.by_role(&role.unwrap()).is_some() { result = false; break; @@ -313,7 +310,7 @@ impl SieveScriptIngest for Server { mailbox_id.and_then(|m| Id::from_bytes(m.as_bytes())) { let mailbox_id = mailbox_id.document_id(); - if mailbox_ids.contains(mailbox_id) { + if mailbox_cache.items.contains_key(&mailbox_id) { target_id = mailbox_id; } } @@ -326,10 +323,8 @@ impl SieveScriptIngest for Server { } else if special_use.eq_ignore_ascii_case("trash") { target_id = TRASH_ID; } else if let Ok(role) = SpecialUse::parse_value(&special_use) { - if let Ok(Some(mailbox_id_)) = - self.mailbox_get_by_role(account_id, role).await - { - target_id = mailbox_id_; + if let Some((mailbox_id_, _)) = mailbox_cache.by_role(&role) { + target_id = *mailbox_id_; } } } @@ -338,10 +333,8 @@ impl SieveScriptIngest for Server { // Find mailbox by name if target_id == u32::MAX { if !create { - if let Ok(Some(document_id)) = - self.mailbox_get_by_name(account_id, &folder).await - { - target_id = document_id; + if let Some((document_id, _)) = mailbox_cache.by_name(&folder) { + target_id = *document_id; } } else if let Ok(Some(document_id)) = self.mailbox_create_path(account_id, &folder).await diff --git a/crates/email/src/thread/cache.rs b/crates/email/src/thread/cache.rs deleted file mode 100644 index 7f5c4b64..00000000 --- a/crates/email/src/thread/cache.rs +++ /dev/null @@ -1,99 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use std::sync::Arc; - -use common::{Server, Threads}; -use jmap_proto::types::{collection::Collection, property::Property}; -use std::future::Future; -use store::{ - BitmapKey, IterateParams, U32_LEN, - ahash::AHashMap, - write::{BitmapClass, TagValue, key::DeserializeBigEndian}, -}; -use trc::AddContext; -use utils::codec::leb128::Leb128Reader; - -pub trait ThreadCache: Sync + Send { - fn get_cached_thread_ids( - &self, - account_id: u32, - ) -> impl Future>> + Send; -} - -impl ThreadCache for Server { - async fn get_cached_thread_ids(&self, account_id: u32) -> trc::Result> { - // Obtain current state - let modseq = self - .core - .storage - .data - .get_last_change_id(account_id, Collection::Thread) - .await - .caused_by(trc::location!())?; - - // Lock the cache - if let Some(thread_cache) = self.inner.cache.threads.get(&account_id).and_then(|t| { - if t.modseq.unwrap_or(0) >= modseq.unwrap_or(0) { - Some(t) - } else { - None - } - }) { - Ok(thread_cache) - } else { - let mut threads = AHashMap::new(); - self.core - .storage - .data - .iterate( - IterateParams::new( - BitmapKey { - account_id, - collection: Collection::Email.into(), - class: BitmapClass::Tag { - field: Property::ThreadId.into(), - value: TagValue::Id(0), - }, - document_id: 0, - }, - BitmapKey { - account_id, - collection: Collection::Email.into(), - class: BitmapClass::Tag { - field: Property::ThreadId.into(), - value: TagValue::Id(u32::MAX), - }, - document_id: u32::MAX, - }, - ) - .no_values(), - |key, _| { - let (thread_id, _) = key - .get(U32_LEN + 2..) - .and_then(|bytes| bytes.read_leb128::()) - .ok_or_else(|| { - trc::Error::corrupted_key(key, None, trc::location!()) - })?; - let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; - - threads.insert(document_id, thread_id); - - Ok(true) - }, - ) - .await - .caused_by(trc::location!())?; - - let thread_cache = Arc::new(Threads { threads, modseq }); - self.inner - .cache - .threads - .insert(account_id, thread_cache.clone()); - Ok(thread_cache) - } - } -} diff --git a/crates/email/src/thread/mod.rs b/crates/email/src/thread/mod.rs deleted file mode 100644 index e785c447..00000000 --- a/crates/email/src/thread/mod.rs +++ /dev/null @@ -1,7 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -pub mod cache; diff --git a/crates/imap-proto/Cargo.toml b/crates/imap-proto/Cargo.toml index f8145bcf..c5dac71d 100644 --- a/crates/imap-proto/Cargo.toml +++ b/crates/imap-proto/Cargo.toml @@ -12,6 +12,7 @@ ahash = { version = "0.8" } chrono = { version = "0.4"} trc = { path = "../trc" } hashify = { version = "0.2" } +compact_str = "0.9.0" [dev-dependencies] tokio = { version = "1.23", features = ["full"] } diff --git a/crates/imap-proto/src/lib.rs b/crates/imap-proto/src/lib.rs index 1e4e4a31..dfc04407 100644 --- a/crates/imap-proto/src/lib.rs +++ b/crates/imap-proto/src/lib.rs @@ -174,9 +174,9 @@ pub enum ResponseType { } impl ResponseCode { - pub fn highest_modseq(modseq: Option) -> Self { + pub fn highest_modseq(modseq: u64) -> Self { ResponseCode::HighestModseq { - modseq: modseq.map(|id| id + 1).unwrap_or(0), + modseq: if modseq > 0 { modseq + 1 } else { 0 }, } } } diff --git a/crates/imap-proto/src/parser/acl.rs b/crates/imap-proto/src/parser/acl.rs index de4c24bf..3c420fc2 100644 --- a/crates/imap-proto/src/parser/acl.rs +++ b/crates/imap-proto/src/parser/acl.rs @@ -143,9 +143,9 @@ mod tests { ( "A003 Setacl INBOX/Drafts Byron lrswikda\r\n", acl::Arguments { - tag: "A003".to_string(), - mailbox_name: "INBOX/Drafts".to_string(), - identifier: "Byron".to_string().into(), + tag: "A003".into(), + mailbox_name: "INBOX/Drafts".into(), + identifier: Some("Byron".into()), mod_rights: ModRights { op: ModRightsOp::Replace, rights: vec![ @@ -165,9 +165,9 @@ mod tests { ( "A002 SETACL INBOX/Drafts Chris +cda\r\n", acl::Arguments { - tag: "A002".to_string(), - mailbox_name: "INBOX/Drafts".to_string(), - identifier: "Chris".to_string().into(), + tag: "A002".into(), + mailbox_name: "INBOX/Drafts".into(), + identifier: Some("Chris".into()), mod_rights: ModRights { op: ModRightsOp::Add, rights: vec![ @@ -182,9 +182,9 @@ mod tests { ( "A036 SETACL INBOX/Drafts John -lrswicda\r\n", acl::Arguments { - tag: "A036".to_string(), - mailbox_name: "INBOX/Drafts".to_string(), - identifier: "John".to_string().into(), + tag: "A036".into(), + mailbox_name: "INBOX/Drafts".into(), + identifier: Some("John".into()), mod_rights: ModRights { op: ModRightsOp::Remove, rights: vec![ @@ -204,8 +204,8 @@ mod tests { ( "A001 GETACL INBOX/Drafts\r\n", acl::Arguments { - tag: "A001".to_string(), - mailbox_name: "INBOX/Drafts".to_string(), + tag: "A001".into(), + mailbox_name: "INBOX/Drafts".into(), identifier: None, mod_rights: None, }, diff --git a/crates/imap-proto/src/parser/append.rs b/crates/imap-proto/src/parser/append.rs index 5b2aeb2d..0a59de4b 100644 --- a/crates/imap-proto/src/parser/append.rs +++ b/crates/imap-proto/src/parser/append.rs @@ -161,8 +161,8 @@ mod tests { ( "A003 APPEND saved-messages (\\Seen) {1+}\r\na\r\n", append::Arguments { - tag: "A003".to_string(), - mailbox_name: "saved-messages".to_string(), + tag: "A003".into(), + mailbox_name: "saved-messages".into(), messages: vec![Message { message: vec![b'a'], flags: vec![Flag::Seen], @@ -173,8 +173,8 @@ mod tests { ( "A003 APPEND \"hello world\" (\\Seen \\Draft $MDNSent) {1+}\r\na\r\n", append::Arguments { - tag: "A003".to_string(), - mailbox_name: "hello world".to_string(), + tag: "A003".into(), + mailbox_name: "hello world".into(), messages: vec![Message { message: vec![b'a'], flags: vec![Flag::Seen, Flag::Draft, Flag::MDNSent], @@ -185,8 +185,8 @@ mod tests { ( "A003 APPEND \"hi\" ($Junk) \"7-Feb-1994 22:43:04 -0800\" {1+}\r\na\r\n", append::Arguments { - tag: "A003".to_string(), - mailbox_name: "hi".to_string(), + tag: "A003".into(), + mailbox_name: "hi".into(), messages: vec![Message { message: vec![b'a'], flags: vec![Flag::Junk], @@ -197,8 +197,8 @@ mod tests { ( "A003 APPEND \"hi\" \"20-Nov-2022 23:59:59 +0300\" {1+}\r\na\r\n", append::Arguments { - tag: "A003".to_string(), - mailbox_name: "hi".to_string(), + tag: "A003".into(), + mailbox_name: "hi".into(), messages: vec![Message { message: vec![b'a'], flags: vec![], @@ -209,8 +209,8 @@ mod tests { ( "A003 APPEND \"hi\" \"20-Nov-2022 23:59:59 +0300\" ~{1+}\r\na\r\n", append::Arguments { - tag: "A003".to_string(), - mailbox_name: "hi".to_string(), + tag: "A003".into(), + mailbox_name: "hi".into(), messages: vec![Message { message: vec![b'a'], flags: vec![], @@ -221,8 +221,8 @@ mod tests { ( "42 APPEND \"Drafts\" (\\Draft) UTF8 (~{5+}\r\nhello)\r\n", append::Arguments { - tag: "42".to_string(), - mailbox_name: "Drafts".to_string(), + tag: "42".into(), + mailbox_name: "Drafts".into(), messages: vec![Message { message: vec![b'h', b'e', b'l', b'l', b'o'], flags: vec![Flag::Draft], @@ -233,8 +233,8 @@ mod tests { ( "42 APPEND \"Drafts\" (\\Draft) \"20-Nov-2022 23:59:59 +0300\" UTF8 (~{5+}\r\nhello)\r\n", append::Arguments { - tag: "42".to_string(), - mailbox_name: "Drafts".to_string(), + tag: "42".into(), + mailbox_name: "Drafts".into(), messages: vec![Message { message: vec![b'h', b'e', b'l', b'l', b'o'], flags: vec![Flag::Draft], @@ -282,8 +282,8 @@ mod tests { assert_eq!( request.parse_append(ProtocolVersion::Rev1).unwrap(), append::Arguments { - tag: "A003".to_string(), - mailbox_name: "saved-messages".to_string(), + tag: "A003".into(), + mailbox_name: "saved-messages".into(), messages: vec![ Message { message: concat!( diff --git a/crates/imap-proto/src/parser/authenticate.rs b/crates/imap-proto/src/parser/authenticate.rs index 870a6126..b4ba120a 100644 --- a/crates/imap-proto/src/parser/authenticate.rs +++ b/crates/imap-proto/src/parser/authenticate.rs @@ -69,15 +69,15 @@ mod tests { ( "a002 AUTHENTICATE \"EXTERNAL\" {16+}\r\nfred@example.com\r\n", authenticate::Arguments { - tag: "a002".to_string(), + tag: "a002".into(), mechanism: Mechanism::External, - params: vec!["fred@example.com".to_string()], + params: vec!["fred@example.com".into()], }, ), ( "A01 AUTHENTICATE PLAIN\r\n", authenticate::Arguments { - tag: "A01".to_string(), + tag: "A01".into(), mechanism: Mechanism::Plain, params: vec![], }, diff --git a/crates/imap-proto/src/parser/copy_move.rs b/crates/imap-proto/src/parser/copy_move.rs index 63dc6208..5ffa5c09 100644 --- a/crates/imap-proto/src/parser/copy_move.rs +++ b/crates/imap-proto/src/parser/copy_move.rs @@ -64,8 +64,8 @@ mod tests { start: 2.into(), end: 4.into(), }, - mailbox_name: "MEETING".to_string(), - tag: "A003".to_string(), + mailbox_name: "MEETING".into(), + tag: "A003".into(), } ); assert_eq!( @@ -79,8 +79,8 @@ mod tests { start: 2.into(), end: 4.into(), }, - mailbox_name: "You & Me".to_string(), - tag: "A003".to_string(), + mailbox_name: "You & Me".into(), + tag: "A003".into(), } ); } diff --git a/crates/imap-proto/src/parser/create.rs b/crates/imap-proto/src/parser/create.rs index e72a1d0d..80879219 100644 --- a/crates/imap-proto/src/parser/create.rs +++ b/crates/imap-proto/src/parser/create.rs @@ -102,32 +102,32 @@ mod tests { ( "A142 CREATE 12345\r\n", create::Arguments { - tag: "A142".to_string(), - mailbox_name: "12345".to_string(), + tag: "A142".into(), + mailbox_name: "12345".into(), mailbox_role: None, }, ), ( "A142 CREATE \"my funky mailbox\"\r\n", create::Arguments { - tag: "A142".to_string(), - mailbox_name: "my funky mailbox".to_string(), + tag: "A142".into(), + mailbox_name: "my funky mailbox".into(), mailbox_role: None, }, ), ( "t1 CREATE \"Important Messages\" (USE (\\Important))\r\n", create::Arguments { - tag: "t1".to_string(), - mailbox_name: "Important Messages".to_string(), + tag: "t1".into(), + mailbox_name: "Important Messages".into(), mailbox_role: Some(Attribute::Important), }, ), ( "A142 CREATE \"Test-ąęć-Test\"\r\n", create::Arguments { - tag: "A142".to_string(), - mailbox_name: "Test-ąęć-Test".to_string(), + tag: "A142".into(), + mailbox_name: "Test-ąęć-Test".into(), mailbox_role: None, }, ), diff --git a/crates/imap-proto/src/parser/delete.rs b/crates/imap-proto/src/parser/delete.rs index e9669d2c..180c5cd4 100644 --- a/crates/imap-proto/src/parser/delete.rs +++ b/crates/imap-proto/src/parser/delete.rs @@ -47,15 +47,15 @@ mod tests { ( "A142 DELETE INBOX\r\n", delete::Arguments { - mailbox_name: "INBOX".to_string(), - tag: "A142".to_string(), + mailbox_name: "INBOX".into(), + tag: "A142".into(), }, ), ( "A142 DELETE \"my funky mailbox\"\r\n", delete::Arguments { - mailbox_name: "my funky mailbox".to_string(), - tag: "A142".to_string(), + mailbox_name: "my funky mailbox".into(), + tag: "A142".into(), }, ), ] { diff --git a/crates/imap-proto/src/parser/enable.rs b/crates/imap-proto/src/parser/enable.rs index 124bc6e2..9f5ebe43 100644 --- a/crates/imap-proto/src/parser/enable.rs +++ b/crates/imap-proto/src/parser/enable.rs @@ -69,7 +69,7 @@ mod tests { .parse_enable() .unwrap(), enable::Arguments { - tag: "t2".to_string(), + tag: "t2".into(), capabilities: vec![Capability::IMAP4rev2, Capability::CondStore], } ); diff --git a/crates/imap-proto/src/parser/fetch.rs b/crates/imap-proto/src/parser/fetch.rs index c0d3f77e..5a288f7c 100644 --- a/crates/imap-proto/src/parser/fetch.rs +++ b/crates/imap-proto/src/parser/fetch.rs @@ -8,6 +8,8 @@ use std::borrow::Cow; use std::iter::Peekable; use std::vec::IntoIter; +use compact_str::CompactString; + use crate::{ Command, protocol::fetch::{self, Attribute, Section}, @@ -192,7 +194,7 @@ impl Request { match token { Token::ParenthesisClose => break, Token::Argument(value) => { - fields.push(String::from_utf8(value).map_err( + fields.push(CompactString::from_utf8(value).map_err( |_| bad(self.tag.clone(), "Invalid UTF-8 in header field name."), )?); } @@ -397,7 +399,7 @@ impl Request { _ => { return Err(bad( self.tag.clone(), - format!("Unsupported parameter '{}'.", token), + format!("Unsupported parameter '{}'.", token.to_string()), )); } } @@ -513,7 +515,7 @@ mod tests { ( "A654 FETCH 2:4 (FLAGS BODY[HEADER.FIELDS (DATE FROM)])\r\n", fetch::Arguments { - tag: "A654".to_string(), + tag: "A654".into(), sequence_set: Sequence::range(2.into(), 4.into()), attributes: vec![ Attribute::Flags, @@ -521,7 +523,7 @@ mod tests { peek: false, sections: vec![Section::HeaderFields { not: false, - fields: vec!["DATE".to_string(), "FROM".to_string()], + fields: vec!["DATE".into(), "FROM".into()], }], partial: None, }, @@ -533,7 +535,7 @@ mod tests { ( "A001 FETCH 1 BODY[]\r\n", fetch::Arguments { - tag: "A001".to_string(), + tag: "A001".into(), sequence_set: Sequence::number(1), attributes: vec![Attribute::BodySection { peek: false, @@ -547,7 +549,7 @@ mod tests { ( "A001 FETCH 1 (BODY[HEADER])\r\n", fetch::Arguments { - tag: "A001".to_string(), + tag: "A001".into(), sequence_set: Sequence::number(1), attributes: vec![Attribute::BodySection { peek: false, @@ -561,14 +563,14 @@ mod tests { ( "A001 FETCH 1 (BODY.PEEK[HEADER.FIELDS (X-MAILER)] PREVIEW(LAZY))\r\n", fetch::Arguments { - tag: "A001".to_string(), + tag: "A001".into(), sequence_set: Sequence::number(1), attributes: vec![ Attribute::BodySection { peek: true, sections: vec![Section::HeaderFields { not: false, - fields: vec!["X-MAILER".to_string()], + fields: vec!["X-MAILER".into()], }], partial: None, }, @@ -581,17 +583,13 @@ mod tests { ( "A001 FETCH 1 (BODY[HEADER.FIELDS.NOT (FROM TO SUBJECT)])\r\n", fetch::Arguments { - tag: "A001".to_string(), + tag: "A001".into(), sequence_set: Sequence::number(1), attributes: vec![Attribute::BodySection { peek: false, sections: vec![Section::HeaderFields { not: true, - fields: vec![ - "FROM".to_string(), - "TO".to_string(), - "SUBJECT".to_string(), - ], + fields: vec!["FROM".into(), "TO".into(), "SUBJECT".into()], }], partial: None, }], @@ -602,7 +600,7 @@ mod tests { ( "A001 FETCH 1 (BODY[MIME] BODY[TEXT] PREVIEW)\r\n", fetch::Arguments { - tag: "A001".to_string(), + tag: "A001".into(), sequence_set: Sequence::number(1), attributes: vec![ Attribute::BodySection { @@ -624,7 +622,7 @@ mod tests { ( "A001 FETCH 1 (BODYSTRUCTURE ENVELOPE FLAGS INTERNALDATE UID)\r\n", fetch::Arguments { - tag: "A001".to_string(), + tag: "A001".into(), sequence_set: Sequence::number(1), attributes: vec![ Attribute::BodyStructure, @@ -640,7 +638,7 @@ mod tests { ( "A001 FETCH 1 (RFC822 RFC822.HEADER RFC822.SIZE RFC822.TEXT)\r\n", fetch::Arguments { - tag: "A001".to_string(), + tag: "A001".into(), sequence_set: Sequence::number(1), attributes: vec![ Attribute::Rfc822, @@ -666,7 +664,7 @@ mod tests { ")\r\n" ), fetch::Arguments { - tag: "A001".to_string(), + tag: "A001".into(), sequence_set: Sequence::number(1), attributes: vec![ Attribute::BodySection { @@ -728,7 +726,7 @@ mod tests { ( "A001 FETCH 1 ALL\r\n", fetch::Arguments { - tag: "A001".to_string(), + tag: "A001".into(), sequence_set: Sequence::number(1), attributes: vec![ Attribute::Flags, @@ -743,7 +741,7 @@ mod tests { ( "A001 FETCH 1 FULL\r\n", fetch::Arguments { - tag: "A001".to_string(), + tag: "A001".into(), sequence_set: Sequence::number(1), attributes: vec![ Attribute::Flags, @@ -759,7 +757,7 @@ mod tests { ( "A001 FETCH 1 FAST\r\n", fetch::Arguments { - tag: "A001".to_string(), + tag: "A001".into(), sequence_set: Sequence::number(1), attributes: vec![ Attribute::Flags, @@ -773,7 +771,7 @@ mod tests { ( "s100 UID FETCH 1:* (FLAGS MODSEQ) (CHANGEDSINCE 12345 VANISHED)\r\n", fetch::Arguments { - tag: "s100".to_string(), + tag: "s100".into(), sequence_set: Sequence::range(1.into(), None), attributes: vec![Attribute::Flags, Attribute::ModSeq], changed_since: 12345.into(), @@ -783,7 +781,7 @@ mod tests { ( "9 UID FETCH 1:* UID (VANISHED CHANGEDSINCE 1)\r\n", fetch::Arguments { - tag: "9".to_string(), + tag: "9".into(), sequence_set: Sequence::range(1.into(), None), attributes: vec![Attribute::Uid], changed_since: 1.into(), diff --git a/crates/imap-proto/src/parser/list.rs b/crates/imap-proto/src/parser/list.rs index 6a39bfa2..a3bf1bf7 100644 --- a/crates/imap-proto/src/parser/list.rs +++ b/crates/imap-proto/src/parser/list.rs @@ -225,17 +225,17 @@ mod tests { ( "A682 LIST \"\" *\r\n", list::Arguments::Basic { - tag: "A682".to_string(), - reference_name: "".to_string(), - mailbox_name: "*".to_string(), + tag: "A682".into(), + reference_name: "".into(), + mailbox_name: "*".into(), }, ), ( "A02 LIST (SUBSCRIBED) \"\" \"*\"\r\n", list::Arguments::Extended { - tag: "A02".to_string(), - reference_name: "".to_string(), - mailbox_name: vec!["*".to_string()], + tag: "A02".into(), + reference_name: "".into(), + mailbox_name: vec!["*".into()], selection_options: vec![SelectionOption::Subscribed], return_options: vec![], }, @@ -243,9 +243,9 @@ mod tests { ( "A03 LIST () \"\" \"%\" RETURN (CHILDREN)\r\n", list::Arguments::Extended { - tag: "A03".to_string(), - reference_name: "".to_string(), - mailbox_name: vec!["%".to_string()], + tag: "A03".into(), + reference_name: "".into(), + mailbox_name: vec!["%".into()], selection_options: vec![], return_options: vec![ReturnOption::Children], }, @@ -253,9 +253,9 @@ mod tests { ( "A04 LIST (REMOTE) \"\" \"%\" RETURN (CHILDREN)\r\n", list::Arguments::Extended { - tag: "A04".to_string(), - reference_name: "".to_string(), - mailbox_name: vec!["%".to_string()], + tag: "A04".into(), + reference_name: "".into(), + mailbox_name: vec!["%".into()], selection_options: vec![SelectionOption::Remote], return_options: vec![ReturnOption::Children], }, @@ -263,9 +263,9 @@ mod tests { ( "A05 LIST (REMOTE SUBSCRIBED) \"\" \"*\"\r\n", list::Arguments::Extended { - tag: "A05".to_string(), - reference_name: "".to_string(), - mailbox_name: vec!["*".to_string()], + tag: "A05".into(), + reference_name: "".into(), + mailbox_name: vec!["*".into()], selection_options: vec![SelectionOption::Remote, SelectionOption::Subscribed], return_options: vec![], }, @@ -273,9 +273,9 @@ mod tests { ( "A06 LIST (REMOTE) \"\" \"*\" RETURN (SUBSCRIBED)\r\n", list::Arguments::Extended { - tag: "A06".to_string(), - reference_name: "".to_string(), - mailbox_name: vec!["*".to_string()], + tag: "A06".into(), + reference_name: "".into(), + mailbox_name: vec!["*".into()], selection_options: vec![SelectionOption::Remote], return_options: vec![ReturnOption::Subscribed], }, @@ -283,9 +283,9 @@ mod tests { ( "C04 LIST (SUBSCRIBED RECURSIVEMATCH) \"\" \"%\"\r\n", list::Arguments::Extended { - tag: "C04".to_string(), - reference_name: "".to_string(), - mailbox_name: vec!["%".to_string()], + tag: "C04".into(), + reference_name: "".into(), + mailbox_name: vec!["%".into()], selection_options: vec![ SelectionOption::Subscribed, SelectionOption::RecursiveMatch, @@ -296,9 +296,9 @@ mod tests { ( "C04 LIST (SUBSCRIBED RECURSIVEMATCH) \"\" \"%\" RETURN (CHILDREN)\r\n", list::Arguments::Extended { - tag: "C04".to_string(), - reference_name: "".to_string(), - mailbox_name: vec!["%".to_string()], + tag: "C04".into(), + reference_name: "".into(), + mailbox_name: vec!["%".into()], selection_options: vec![ SelectionOption::Subscribed, SelectionOption::RecursiveMatch, @@ -309,9 +309,9 @@ mod tests { ( "a1 LIST \"\" (\"foo\")\r\n", list::Arguments::Extended { - tag: "a1".to_string(), - reference_name: "".to_string(), - mailbox_name: vec!["foo".to_string()], + tag: "a1".into(), + reference_name: "".into(), + mailbox_name: vec!["foo".into()], selection_options: vec![], return_options: vec![], }, @@ -319,9 +319,9 @@ mod tests { ( "a3.1 LIST \"\" (% music/rock)\r\n", list::Arguments::Extended { - tag: "a3.1".to_string(), - reference_name: "".to_string(), - mailbox_name: vec!["%".to_string(), "music/rock".to_string()], + tag: "a3.1".into(), + reference_name: "".into(), + mailbox_name: vec!["%".into(), "music/rock".into()], selection_options: vec![], return_options: vec![], }, @@ -329,13 +329,9 @@ mod tests { ( "BBB LIST \"\" (\"INBOX\" \"Drafts\" \"Sent/%\")\r\n", list::Arguments::Extended { - tag: "BBB".to_string(), - reference_name: "".to_string(), - mailbox_name: vec![ - "INBOX".to_string(), - "Drafts".to_string(), - "Sent/%".to_string(), - ], + tag: "BBB".into(), + reference_name: "".into(), + mailbox_name: vec!["INBOX".into(), "Drafts".into(), "Sent/%".into()], selection_options: vec![], return_options: vec![], }, @@ -343,9 +339,9 @@ mod tests { ( "A01 LIST \"\" % RETURN (STATUS (MESSAGES UNSEEN))\r\n", list::Arguments::Extended { - tag: "A01".to_string(), - reference_name: "".to_string(), - mailbox_name: vec!["%".to_string()], + tag: "A01".into(), + reference_name: "".into(), + mailbox_name: vec!["%".into()], selection_options: vec![], return_options: vec![ReturnOption::Status(vec![ Status::Messages, @@ -359,9 +355,9 @@ mod tests { "% RETURN (CHILDREN STATUS (MESSAGES))\r\n" ), list::Arguments::Extended { - tag: "A02".to_string(), - reference_name: "".to_string(), - mailbox_name: vec!["%".to_string()], + tag: "A02".into(), + reference_name: "".into(), + mailbox_name: vec!["%".into()], selection_options: vec![ SelectionOption::Subscribed, SelectionOption::RecursiveMatch, diff --git a/crates/imap-proto/src/parser/login.rs b/crates/imap-proto/src/parser/login.rs index 04ccc99c..5c4b82fd 100644 --- a/crates/imap-proto/src/parser/login.rs +++ b/crates/imap-proto/src/parser/login.rs @@ -47,17 +47,17 @@ mod tests { ( "a001 LOGIN SMITH SESAME\r\n", login::Arguments { - tag: "a001".to_string(), - username: "SMITH".to_string(), - password: "SESAME".to_string(), + tag: "a001".into(), + username: "SMITH".into(), + password: "SESAME".into(), }, ), ( "A001 LOGIN {11+}\r\nFRED FOOBAR {7+}\r\nfat man\r\n", login::Arguments { - tag: "A001".to_string(), - username: "FRED FOOBAR".to_string(), - password: "fat man".to_string(), + tag: "A001".into(), + username: "FRED FOOBAR".into(), + password: "fat man".into(), }, ), ] { diff --git a/crates/imap-proto/src/parser/lsub.rs b/crates/imap-proto/src/parser/lsub.rs index ebc566ce..529611bb 100644 --- a/crates/imap-proto/src/parser/lsub.rs +++ b/crates/imap-proto/src/parser/lsub.rs @@ -58,9 +58,9 @@ mod tests { ( "A002 LSUB \"#news.\" \"comp.mail.*\"\r\n", list::Arguments::Extended { - tag: "A002".to_string(), - reference_name: "#news.".to_string(), - mailbox_name: vec!["comp.mail.*".to_string()], + tag: "A002".into(), + reference_name: "#news.".into(), + mailbox_name: vec!["comp.mail.*".into()], selection_options: vec![SelectionOption::Subscribed], return_options: vec![], }, @@ -68,9 +68,9 @@ mod tests { ( "A002 LSUB \"#news.\" \"comp.%\"\r\n", list::Arguments::Extended { - tag: "A002".to_string(), - reference_name: "#news.".to_string(), - mailbox_name: vec!["comp.%".to_string()], + tag: "A002".into(), + reference_name: "#news.".into(), + mailbox_name: vec!["comp.%".into()], selection_options: vec![SelectionOption::Subscribed], return_options: vec![], }, diff --git a/crates/imap-proto/src/parser/mod.rs b/crates/imap-proto/src/parser/mod.rs index 0c6be3a6..66adb86e 100644 --- a/crates/imap-proto/src/parser/mod.rs +++ b/crates/imap-proto/src/parser/mod.rs @@ -28,6 +28,7 @@ pub mod thread; use std::{borrow::Cow, str::FromStr}; use chrono::{DateTime, NaiveDate}; +use compact_str::CompactString; use crate::{ Command, @@ -111,7 +112,7 @@ impl Flag { if let Some(flag) = flag { Ok(flag) } else { - String::from_utf8(value) + CompactString::from_utf8(value) .map_err(|_| Cow::from("Invalid UTF-8.")) .map(Flag::Keyword) } @@ -136,9 +137,9 @@ impl Flag { "$forwarded" => Flag::Forwarded, "$mdnsent" => Flag::MDNSent, ) - .unwrap_or_else(|| Flag::Keyword(value)) + .unwrap_or_else(|| Flag::Keyword(value.into())) } else { - let mut keyword = String::with_capacity(value.len()); + let mut keyword = CompactString::with_capacity(value.len()); for c in value.chars() { if c.is_ascii_alphanumeric() { keyword.push(c); diff --git a/crates/imap-proto/src/parser/quota.rs b/crates/imap-proto/src/parser/quota.rs index a469fcd5..2b0c0e07 100644 --- a/crates/imap-proto/src/parser/quota.rs +++ b/crates/imap-proto/src/parser/quota.rs @@ -63,8 +63,8 @@ mod tests { let (command, arguments) = ( "A142 GETQUOTAROOT INBOX\r\n", quota::Arguments { - name: "INBOX".to_string(), - tag: "A142".to_string(), + name: "INBOX".into(), + tag: "A142".into(), }, ); assert_eq!( @@ -79,8 +79,8 @@ mod tests { let (command, arguments) = ( "A142 GETQUOTA \"my funky mailbox\"\r\n", quota::Arguments { - name: "my funky mailbox".to_string(), - tag: "A142".to_string(), + name: "my funky mailbox".into(), + tag: "A142".into(), }, ); assert_eq!( diff --git a/crates/imap-proto/src/parser/rename.rs b/crates/imap-proto/src/parser/rename.rs index 67eb5eb6..2f0fc6f6 100644 --- a/crates/imap-proto/src/parser/rename.rs +++ b/crates/imap-proto/src/parser/rename.rs @@ -58,17 +58,17 @@ mod tests { ( "A142 RENAME \"my funky mailbox\" Private\r\n", rename::Arguments { - mailbox_name: "my funky mailbox".to_string(), - new_mailbox_name: "Private".to_string(), - tag: "A142".to_string(), + mailbox_name: "my funky mailbox".into(), + new_mailbox_name: "Private".into(), + tag: "A142".into(), }, ), ( "A142 RENAME {1+}\r\na {1+}\r\nb\r\n", rename::Arguments { - mailbox_name: "a".to_string(), - new_mailbox_name: "b".to_string(), - tag: "A142".to_string(), + mailbox_name: "a".into(), + new_mailbox_name: "b".into(), + tag: "A142".into(), }, ), ] { diff --git a/crates/imap-proto/src/parser/search.rs b/crates/imap-proto/src/parser/search.rs index 532b5b52..5023bd94 100644 --- a/crates/imap-proto/src/parser/search.rs +++ b/crates/imap-proto/src/parser/search.rs @@ -8,6 +8,7 @@ use std::borrow::Cow; use std::iter::Peekable; use std::vec::IntoIter; +use compact_str::CompactString; use mail_parser::decoders::charsets::DecoderFnc; use mail_parser::decoders::charsets::map::charset_decoder; @@ -373,7 +374,7 @@ pub fn parse_filters( } Some(token) => { return Err( - format!("Unsupported MODSEQ parameter '{}'.", token).into() + format!("Unsupported MODSEQ parameter '{}'.", token.to_string()).into() ); } None => { @@ -499,17 +500,16 @@ pub fn parse_filters( pub fn decode_argument( tokens: &mut Peekable>, decoder: Option, -) -> super::Result { +) -> super::Result { let argument = tokens .next() .ok_or_else(|| Cow::from("Expected string."))? .unwrap_bytes(); if let Some(decoder) = decoder { - Ok(decoder(&argument)) + Ok(decoder(&argument).into()) } else { - Ok(String::from_utf8(argument.to_vec()) - .map_err(|_| Cow::from("Invalid UTF-8 argument."))?) + Ok(CompactString::from_utf8(argument).map_err(|_| Cow::from("Invalid UTF-8 argument."))?) } } @@ -553,13 +553,13 @@ mod tests { b"A282 SEARCH RETURN (MIN COUNT) FLAGGED SINCE 1-Feb-1994 NOT FROM \"Smith\"\r\n" .to_vec(), search::Arguments { - tag: "A282".to_string(), + tag: "A282".into(), result_options: vec![ResultOption::Min, ResultOption::Count], filter: vec![ Filter::Flagged, Filter::Since(760060800), Filter::Not, - Filter::From("Smith".to_string()), + Filter::From("Smith".into()), Filter::End, ], is_esearch: true, @@ -569,13 +569,13 @@ mod tests { ( b"A283 SEARCH RETURN () FLAGGED SINCE 1-Feb-1994 NOT FROM \"Smith\"\r\n".to_vec(), search::Arguments { - tag: "A283".to_string(), + tag: "A283".into(), result_options: vec![], filter: vec![ Filter::Flagged, Filter::Since(760060800), Filter::Not, - Filter::From("Smith".to_string()), + Filter::From("Smith".into()), Filter::End, ], is_esearch: true, @@ -585,7 +585,7 @@ mod tests { ( b"A301 SEARCH $ SMALLER 4096\r\n".to_vec(), search::Arguments { - tag: "A301".to_string(), + tag: "A301".into(), result_options: vec![], filter: vec![Filter::seq_saved_search(), Filter::Smaller(4096)], is_esearch: true, @@ -597,7 +597,7 @@ mod tests { .as_bytes() .to_vec(), search::Arguments { - tag: "P283".to_string(), + tag: "P283".into(), result_options: vec![], filter: vec![ Filter::Or, @@ -612,7 +612,7 @@ mod tests { false, ), Filter::End, - Filter::Text("мать".to_string()), + Filter::Text("мать".into()), ], is_esearch: true, sort: None, @@ -621,7 +621,7 @@ mod tests { ( b"F282 SEARCH RETURN (SAVE) KEYWORD $Junk\r\n".to_vec(), search::Arguments { - tag: "F282".to_string(), + tag: "F282".into(), result_options: vec![ResultOption::Save], filter: vec![Filter::Keyword(Flag::Junk)], is_esearch: true, @@ -636,17 +636,17 @@ mod tests { ] .concat(), search::Arguments { - tag: "F282".to_string(), + tag: "F282".into(), result_options: vec![], filter: vec![ Filter::Or, Filter::Or, - Filter::From("hello@world.com".to_string()), - Filter::To("test@example.com".to_string()), + Filter::From("hello@world.com".into()), + Filter::To("test@example.com".into()), Filter::End, Filter::Or, - Filter::Bcc("jane@foobar.com".to_string()), - Filter::Cc("john@doe.com".to_string()), + Filter::Bcc("jane@foobar.com".into()), + Filter::Cc("john@doe.com".into()), Filter::End, Filter::End, ], @@ -662,14 +662,14 @@ mod tests { ] .concat(), search::Arguments { - tag: "abc".to_string(), + tag: "abc".into(), result_options: vec![], filter: vec![ Filter::Or, Filter::Smaller(10000), Filter::Or, - Filter::Header("Subject".to_string(), "ravioli festival".to_string()), - Filter::Header("From".to_string(), "dr. ravioli".to_string()), + Filter::Header("Subject".into(), "ravioli festival".into()), + Filter::Header("From".into(), "dr. ravioli".into()), Filter::End, Filter::End, ], @@ -685,16 +685,16 @@ mod tests { ] .concat(), search::Arguments { - tag: "abc".to_string(), + tag: "abc".into(), result_options: vec![], filter: vec![ Filter::Deleted, Filter::Seen, Filter::Answered, Filter::Not, - Filter::From("john".to_string()), - Filter::To("jane".to_string()), - Filter::Bcc("bill".to_string()), + Filter::From("john".into()), + Filter::To("jane".into()), + Filter::Bcc("bill".into()), Filter::End, Filter::Sequence( Sequence::List { @@ -727,7 +727,7 @@ mod tests { ] .concat(), search::Arguments { - tag: "abc".to_string(), + tag: "abc".into(), result_options: vec![], filter: vec![ Filter::seq_range(None, None), @@ -761,14 +761,14 @@ mod tests { ] .concat(), search::Arguments { - tag: "abc".to_string(), + tag: "abc".into(), result_options: vec![], filter: vec![ Filter::Not, - Filter::From("john".to_string()), + Filter::From("john".into()), Filter::Or, - Filter::To("jane".to_string()), - Filter::Cc("bill".to_string()), + Filter::To("jane".into()), + Filter::Cc("bill".into()), Filter::End, Filter::End, Filter::Or, @@ -783,7 +783,7 @@ mod tests { Filter::End, Filter::End, Filter::End, - Filter::Keyword(Flag::Keyword("tps report".to_string())), + Filter::Keyword(Flag::Keyword("tps report".into())), ], is_esearch: true, sort: None, @@ -796,9 +796,9 @@ mod tests { ] .concat(), search::Arguments { - tag: "B283".to_string(), + tag: "B283".into(), result_options: vec![ResultOption::Save, ResultOption::Min, ResultOption::Max], - filter: vec![Filter::Text("Привет, мир".to_string())], + filter: vec![Filter::Text("Привет, мир".into())], is_esearch: true, sort: None, }, @@ -806,9 +806,9 @@ mod tests { ( b"B283 SEARCH CHARSET BIG5 FROM \"\xa7A\xa6n\xa1A\xa5@\xac\xc9\"\r\n".to_vec(), search::Arguments { - tag: "B283".to_string(), + tag: "B283".into(), result_options: vec![], - filter: vec![Filter::From("你好，世界".to_string())], + filter: vec![Filter::From("你好，世界".into())], is_esearch: true, sort: None, }, @@ -816,7 +816,7 @@ mod tests { ( b"a SEARCH MODSEQ \"/flags/\\draft\" all 620162338\r\n".to_vec(), search::Arguments { - tag: "a".to_string(), + tag: "a".into(), result_options: vec![], filter: vec![Filter::ModSeq((620162338, ModSeqEntry::All(Flag::Draft)))], is_esearch: true, @@ -826,7 +826,7 @@ mod tests { ( b"t SEARCH OR NOT MODSEQ 720162338 LARGER 50000\r\n".to_vec(), search::Arguments { - tag: "t".to_string(), + tag: "t".into(), result_options: vec![], filter: vec![ Filter::Or, @@ -843,7 +843,7 @@ mod tests { ( b"5 UID SEARCH BEFORE 1-Dec-2023\r\n".to_vec(), search::Arguments { - tag: "5".to_string(), + tag: "5".into(), result_options: vec![], filter: vec![Filter::Before(1701388800)], is_esearch: true, diff --git a/crates/imap-proto/src/parser/select.rs b/crates/imap-proto/src/parser/select.rs index 186040c1..1004dafa 100644 --- a/crates/imap-proto/src/parser/select.rs +++ b/crates/imap-proto/src/parser/select.rs @@ -152,14 +152,17 @@ impl Request { _ => { return Err(bad( self.tag, - format!("Unexpected value '{}'.", token), + format!("Unexpected value '{}'.", token.to_string()), )); } } } } Some(token) => { - return Err(bad(self.tag, format!("Unexpected value '{}'.", token))); + return Err(bad( + self.tag, + format!("Unexpected value '{}'.", token.to_string()), + )); } None => (), } @@ -194,8 +197,8 @@ mod tests { ( "A142 SELECT INBOX\r\n", select::Arguments { - mailbox_name: "INBOX".to_string(), - tag: "A142".to_string(), + mailbox_name: "INBOX".into(), + tag: "A142".into(), condstore: false, qresync: None, }, @@ -203,8 +206,8 @@ mod tests { ( "A142 SELECT \"my funky mailbox\"\r\n", select::Arguments { - mailbox_name: "my funky mailbox".to_string(), - tag: "A142".to_string(), + mailbox_name: "my funky mailbox".into(), + tag: "A142".into(), condstore: false, qresync: None, }, @@ -212,8 +215,8 @@ mod tests { ( "A142 SELECT INBOX (CONDSTORE)\r\n", select::Arguments { - mailbox_name: "INBOX".to_string(), - tag: "A142".to_string(), + mailbox_name: "INBOX".into(), + tag: "A142".into(), condstore: true, qresync: None, }, @@ -221,8 +224,8 @@ mod tests { ( "A142 SELECT INBOX (QRESYNC (3857529045 20010715194032001 1:198))\r\n", select::Arguments { - mailbox_name: "INBOX".to_string(), - tag: "A142".to_string(), + mailbox_name: "INBOX".into(), + tag: "A142".into(), condstore: false, qresync: QResync { uid_validity: 3857529045, @@ -242,8 +245,8 @@ mod tests { "41:211,214:541) CONDSTORE)\r\n" ), select::Arguments { - mailbox_name: "INBOX".to_string(), - tag: "A03".to_string(), + mailbox_name: "INBOX".into(), + tag: "A03".into(), condstore: true, qresync: QResync { uid_validity: 67890007, @@ -273,8 +276,8 @@ mod tests { "29994,29997)))\r\n" ), select::Arguments { - mailbox_name: "INBOX".to_string(), - tag: "B04".to_string(), + mailbox_name: "INBOX".into(), + tag: "B04".into(), condstore: false, qresync: QResync { uid_validity: 67890007, @@ -320,8 +323,8 @@ mod tests { ( "A12 SELECT \"INBOX\" (QRESYNC (1693237464 16582))\r\n", select::Arguments { - mailbox_name: "INBOX".to_string(), - tag: "A12".to_string(), + mailbox_name: "INBOX".into(), + tag: "A12".into(), condstore: false, qresync: QResync { uid_validity: 1693237464, diff --git a/crates/imap-proto/src/parser/sort.rs b/crates/imap-proto/src/parser/sort.rs index a8696de2..16d9bceb 100644 --- a/crates/imap-proto/src/parser/sort.rs +++ b/crates/imap-proto/src/parser/sort.rs @@ -134,7 +134,7 @@ mod tests { filter: vec![Filter::Since(760060800)], result_options: Vec::new(), is_esearch: false, - tag: "A282".to_string(), + tag: "A282".into(), }, ), ( @@ -154,7 +154,7 @@ mod tests { filter: vec![Filter::All], result_options: Vec::new(), is_esearch: false, - tag: "A283".to_string(), + tag: "A283".into(), }, ), ( @@ -165,10 +165,10 @@ mod tests { ascending: true, }] .into(), - filter: vec![Filter::Text("not in mailbox".to_string())], + filter: vec![Filter::Text("not in mailbox".into())], result_options: Vec::new(), is_esearch: false, - tag: "A284".to_string(), + tag: "A284".into(), }, ), ( @@ -189,10 +189,10 @@ mod tests { }, ] .into(), - filter: vec![Filter::Subject("مرحبا بالعالم".to_string())], + filter: vec![Filter::Subject("مرحبا بالعالم".into())], result_options: Vec::new(), is_esearch: false, - tag: "A284".to_string(), + tag: "A284".into(), }, ), ( @@ -210,7 +210,7 @@ mod tests { filter: vec![Filter::Undeleted, Filter::Unkeyword(Flag::Junk)], result_options: vec![ResultOption::Count], is_esearch: true, - tag: "E01".to_string(), + tag: "E01".into(), }, ), ] { diff --git a/crates/imap-proto/src/parser/status.rs b/crates/imap-proto/src/parser/status.rs index 98e874ac..714a27f9 100644 --- a/crates/imap-proto/src/parser/status.rs +++ b/crates/imap-proto/src/parser/status.rs @@ -114,8 +114,8 @@ mod tests { .parse_status(ProtocolVersion::Rev2) .unwrap(), status::Arguments { - tag: "A042".to_string(), - mailbox_name: "blurdybloop".to_string(), + tag: "A042".into(), + mailbox_name: "blurdybloop".into(), items: vec![status::Status::UidNext, status::Status::Messages], } ); diff --git a/crates/imap-proto/src/parser/store.rs b/crates/imap-proto/src/parser/store.rs index 6afc6fd2..767c14b4 100644 --- a/crates/imap-proto/src/parser/store.rs +++ b/crates/imap-proto/src/parser/store.rs @@ -52,7 +52,7 @@ impl Request { _ => { return Err(bad( self.tag.to_string(), - format!("Unsupported parameter '{}'.", token), + format!("Unsupported parameter '{}'.", token.to_string()), )); } } @@ -154,7 +154,7 @@ mod tests { is_silent: false, operation: Operation::Add, keywords: vec![Flag::Deleted], - tag: "A003".to_string(), + tag: "A003".into(), unchanged_since: None, }, ), @@ -168,7 +168,7 @@ mod tests { is_silent: true, operation: Operation::Clear, keywords: vec![Flag::Phishing, Flag::Junk], - tag: "A004".to_string(), + tag: "A004".into(), unchanged_since: None, }, ), @@ -185,7 +185,7 @@ mod tests { is_silent: true, operation: Operation::Add, keywords: vec![Flag::Deleted], - tag: "d105".to_string(), + tag: "d105".into(), unchanged_since: Some(320162338), }, ), diff --git a/crates/imap-proto/src/parser/subscribe.rs b/crates/imap-proto/src/parser/subscribe.rs index fb0a7d83..24f9bd54 100644 --- a/crates/imap-proto/src/parser/subscribe.rs +++ b/crates/imap-proto/src/parser/subscribe.rs @@ -47,15 +47,15 @@ mod tests { ( "A142 SUBSCRIBE #news.comp.mail.mime\r\n", subscribe::Arguments { - mailbox_name: "#news.comp.mail.mime".to_string(), - tag: "A142".to_string(), + mailbox_name: "#news.comp.mail.mime".into(), + tag: "A142".into(), }, ), ( "A142 SUBSCRIBE \"#news.comp.mail.mime\"\r\n", subscribe::Arguments { - mailbox_name: "#news.comp.mail.mime".to_string(), - tag: "A142".to_string(), + mailbox_name: "#news.comp.mail.mime".into(), + tag: "A142".into(), }, ), ] { diff --git a/crates/imap-proto/src/parser/thread.rs b/crates/imap-proto/src/parser/thread.rs index 25140a81..28e439cd 100644 --- a/crates/imap-proto/src/parser/thread.rs +++ b/crates/imap-proto/src/parser/thread.rs @@ -87,15 +87,15 @@ mod tests { thread::Arguments { algorithm: Algorithm::OrderedSubject, filter: vec![Filter::Since(952214400)], - tag: "A283".to_string(), + tag: "A283".into(), }, ), ( b"A284 THREAD REFERENCES US-ASCII TEXT \"gewp\"\r\n".to_vec(), thread::Arguments { algorithm: Algorithm::References, - filter: vec![Filter::Text("gewp".to_string())], - tag: "A284".to_string(), + filter: vec![Filter::Text("gewp".into())], + tag: "A284".into(), }, ), ] { diff --git a/crates/imap-proto/src/protocol/acl.rs b/crates/imap-proto/src/protocol/acl.rs index 2fbff2a3..8dddabc2 100644 --- a/crates/imap-proto/src/protocol/acl.rs +++ b/crates/imap-proto/src/protocol/acl.rs @@ -35,6 +35,7 @@ use std::fmt::Display; +use compact_str::CompactString; use jmap_proto::types::acl::Acl; use crate::utf7::utf7_encode; @@ -71,28 +72,28 @@ pub enum ModRightsOp { #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { - pub tag: String, - pub mailbox_name: String, - pub identifier: Option, + pub tag: CompactString, + pub mailbox_name: CompactString, + pub identifier: Option, pub mod_rights: Option, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct GetAclResponse { - pub mailbox_name: String, - pub permissions: Vec<(String, Vec)>, + pub mailbox_name: CompactString, + pub permissions: Vec<(CompactString, Vec)>, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct ListRightsResponse { - pub mailbox_name: String, - pub identifier: String, + pub mailbox_name: CompactString, + pub identifier: CompactString, pub permissions: Vec>, } #[derive(Debug, Clone, PartialEq, Eq)] pub struct MyRightsResponse { - pub mailbox_name: String, + pub mailbox_name: CompactString, pub rights: Vec, } @@ -248,17 +249,19 @@ impl From for Acl { #[cfg(test)] mod tests { + use compact_str::CompactString; + use crate::protocol::acl::{GetAclResponse, ListRightsResponse, MyRightsResponse, Rights}; #[test] fn serialize_acl() { assert_eq!( - String::from_utf8( + CompactString::from_utf8( GetAclResponse { - mailbox_name: "INBOX".to_string(), + mailbox_name: "INBOX".into(), permissions: vec![ ( - "Fred".to_string(), + "Fred".into(), vec![ Rights::Lookup, Rights::Read, @@ -271,7 +274,7 @@ mod tests { ] ), ( - "David".to_string(), + "David".into(), vec![ Rights::CreateMailbox, Rights::DeleteMessages, @@ -287,10 +290,10 @@ mod tests { ); assert_eq!( - String::from_utf8( + CompactString::from_utf8( ListRightsResponse { - mailbox_name: "Deleted Items".to_string(), - identifier: "Fred".to_string(), + mailbox_name: "Deleted Items".into(), + identifier: "Fred".into(), permissions: vec![ vec![Rights::Lookup, Rights::Read], vec![Rights::Administer], @@ -304,9 +307,9 @@ mod tests { ); assert_eq!( - String::from_utf8( + CompactString::from_utf8( MyRightsResponse { - mailbox_name: "Important".to_string(), + mailbox_name: "Important".into(), rights: vec![Rights::Lookup, Rights::Read, Rights::DeleteMailbox] } .into_bytes(true) diff --git a/crates/imap-proto/src/protocol/append.rs b/crates/imap-proto/src/protocol/append.rs index 24f13bdf..f42671d3 100644 --- a/crates/imap-proto/src/protocol/append.rs +++ b/crates/imap-proto/src/protocol/append.rs @@ -4,12 +4,14 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use compact_str::CompactString; + use super::Flag; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { - pub tag: String, - pub mailbox_name: String, + pub tag: CompactString, + pub mailbox_name: CompactString, pub messages: Vec, } diff --git a/crates/imap-proto/src/protocol/authenticate.rs b/crates/imap-proto/src/protocol/authenticate.rs index 8be74653..f165b717 100644 --- a/crates/imap-proto/src/protocol/authenticate.rs +++ b/crates/imap-proto/src/protocol/authenticate.rs @@ -4,11 +4,13 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use compact_str::CompactString; + #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { - pub tag: String, + pub tag: CompactString, pub mechanism: Mechanism, - pub params: Vec, + pub params: Vec, } #[derive(Debug, Clone, PartialEq, Eq)] diff --git a/crates/imap-proto/src/protocol/copy_move.rs b/crates/imap-proto/src/protocol/copy_move.rs index 669af0ac..42ca7192 100644 --- a/crates/imap-proto/src/protocol/copy_move.rs +++ b/crates/imap-proto/src/protocol/copy_move.rs @@ -4,11 +4,13 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use compact_str::CompactString; + use super::Sequence; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { - pub tag: String, + pub tag: CompactString, pub sequence_set: Sequence, - pub mailbox_name: String, + pub mailbox_name: CompactString, } diff --git a/crates/imap-proto/src/protocol/create.rs b/crates/imap-proto/src/protocol/create.rs index 32962541..cbc41da4 100644 --- a/crates/imap-proto/src/protocol/create.rs +++ b/crates/imap-proto/src/protocol/create.rs @@ -4,11 +4,13 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use compact_str::CompactString; + use super::list::Attribute; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { - pub tag: String, - pub mailbox_name: String, + pub tag: CompactString, + pub mailbox_name: CompactString, pub mailbox_role: Option, } diff --git a/crates/imap-proto/src/protocol/delete.rs b/crates/imap-proto/src/protocol/delete.rs index 8cbd6e32..e6a78de5 100644 --- a/crates/imap-proto/src/protocol/delete.rs +++ b/crates/imap-proto/src/protocol/delete.rs @@ -4,8 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use compact_str::CompactString; + #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { - pub tag: String, - pub mailbox_name: String, + pub tag: CompactString, + pub mailbox_name: CompactString, } diff --git a/crates/imap-proto/src/protocol/enable.rs b/crates/imap-proto/src/protocol/enable.rs index eec6b1b0..740ca972 100644 --- a/crates/imap-proto/src/protocol/enable.rs +++ b/crates/imap-proto/src/protocol/enable.rs @@ -4,11 +4,13 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use compact_str::CompactString; + use super::{ImapResponse, capability::Capability}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { - pub tag: String, + pub tag: CompactString, pub capabilities: Vec, } diff --git a/crates/imap-proto/src/protocol/fetch.rs b/crates/imap-proto/src/protocol/fetch.rs index 09d6a687..c8a51edb 100644 --- a/crates/imap-proto/src/protocol/fetch.rs +++ b/crates/imap-proto/src/protocol/fetch.rs @@ -6,6 +6,7 @@ use std::borrow::Cow; +use compact_str::CompactString; use mail_parser::DateTime; use super::{ @@ -15,7 +16,7 @@ use super::{ #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { - pub tag: String, + pub tag: CompactString, pub sequence_set: Sequence, pub attributes: Vec, pub changed_since: Option, @@ -68,9 +69,14 @@ pub enum Attribute { #[derive(Debug, Clone, PartialEq, Eq)] pub enum Section { - Part { num: u32 }, + Part { + num: u32, + }, Header, - HeaderFields { not: bool, fields: Vec }, + HeaderFields { + not: bool, + fields: Vec, + }, Text, Mime, } @@ -128,10 +134,10 @@ pub enum DataItem<'x> { modseq: u64, }, EmailId { - email_id: String, + email_id: CompactString, }, ThreadId { - thread_id: String, + thread_id: CompactString, }, } @@ -916,6 +922,7 @@ impl ImapResponse for Response<'_> { #[cfg(test)] mod tests { + use compact_str::CompactString; use mail_parser::DateTime; use crate::protocol::{Flag, ImapResponse}; @@ -1363,7 +1370,7 @@ mod tests { item.serialize(&mut buf); - assert_eq!(String::from_utf8(buf).unwrap(), expected_response); + assert_eq!(CompactString::from_utf8(buf).unwrap(), expected_response); } } diff --git a/crates/imap-proto/src/protocol/list.rs b/crates/imap-proto/src/protocol/list.rs index 9dede9f7..ca8820f8 100644 --- a/crates/imap-proto/src/protocol/list.rs +++ b/crates/imap-proto/src/protocol/list.rs @@ -4,6 +4,8 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use compact_str::CompactString; + use crate::utf7::utf7_encode; use super::{ @@ -14,14 +16,14 @@ use super::{ #[derive(Debug, Clone, PartialEq, Eq)] pub enum Arguments { Basic { - tag: String, - reference_name: String, - mailbox_name: String, + tag: CompactString, + reference_name: CompactString, + mailbox_name: CompactString, }, Extended { - tag: String, - reference_name: String, - mailbox_name: Vec, + tag: CompactString, + reference_name: CompactString, + mailbox_name: Vec, selection_options: Vec, return_options: Vec, }, @@ -80,12 +82,12 @@ pub enum ChildInfo { #[derive(Debug, Clone, PartialEq, Eq)] pub enum Tag { ChildInfo(Vec), - OldName(String), + OldName(CompactString), } #[derive(Debug, Clone, PartialEq, Eq)] pub struct ListItem { - pub mailbox_name: String, + pub mailbox_name: CompactString, pub attributes: Vec, pub tags: Vec, } @@ -106,7 +108,7 @@ impl Arguments { } } - pub fn unwrap_tag(self) -> String { + pub fn unwrap_tag(self) -> CompactString { match self { Arguments::Basic { tag, .. } => tag, Arguments::Extended { tag, .. } => tag, @@ -194,7 +196,7 @@ impl Tag { } impl ListItem { - pub fn new(name: impl Into) -> Self { + pub fn new(name: impl Into) -> Self { ListItem { mailbox_name: name.into(), attributes: Vec::new(), @@ -275,6 +277,8 @@ impl ImapResponse for Response { #[cfg(test)] mod tests { + use compact_str::CompactString; + use crate::protocol::{ ImapResponse, status::{Status, StatusItem, StatusItemType}, @@ -287,7 +291,7 @@ mod tests { for (response, expected_v2, expected_v1) in [ ( super::ListItem { - mailbox_name: "".to_string(), + mailbox_name: "".into(), attributes: vec![], tags: vec![], }, @@ -296,7 +300,7 @@ mod tests { ), ( super::ListItem { - mailbox_name: "中國書店".to_string(), + mailbox_name: "中國書店".into(), attributes: vec![Attribute::NoInferiors, Attribute::Drafts], tags: vec![], }, @@ -308,7 +312,7 @@ mod tests { ), ( super::ListItem { - mailbox_name: "☺".to_string(), + mailbox_name: "☺".into(), attributes: vec![Attribute::Subscribed, Attribute::Remote], tags: vec![Tag::ChildInfo(vec![ChildInfo::Subscribed])], }, @@ -323,7 +327,7 @@ mod tests { ), ( super::ListItem { - mailbox_name: "foo".to_string(), + mailbox_name: "foo".into(), attributes: vec![Attribute::HasNoChildren], tags: vec![Tag::ChildInfo(vec![ChildInfo::Subscribed])], }, @@ -337,8 +341,8 @@ mod tests { response.serialize(&mut buf_1, false, false); response.serialize(&mut buf_2, true, false); - let response_v1 = String::from_utf8(buf_1).unwrap(); - let response_v2 = String::from_utf8(buf_2).unwrap(); + let response_v1 = CompactString::from_utf8(buf_1).unwrap(); + let response_v2 = CompactString::from_utf8(buf_2).unwrap(); assert_eq!(response_v2, expected_v2); assert_eq!(response_v1, expected_v1); @@ -350,23 +354,23 @@ mod tests { let mut response = super::Response { list_items: vec![ ListItem { - mailbox_name: "INBOX".to_string(), + mailbox_name: "INBOX".into(), attributes: vec![Attribute::Subscribed], tags: vec![], }, ListItem { - mailbox_name: "foo".to_string(), + mailbox_name: "foo".into(), attributes: vec![], tags: vec![Tag::ChildInfo(vec![ChildInfo::Subscribed])], }, ], status_items: vec![ StatusItem { - mailbox_name: "INBOX".to_string(), + mailbox_name: "INBOX".into(), items: vec![(Status::Messages, StatusItemType::Number(17))], }, StatusItem { - mailbox_name: "foo".to_string(), + mailbox_name: "foo".into(), items: vec![ (Status::Messages, StatusItemType::Number(30)), (Status::Unseen, StatusItemType::Number(29)), @@ -387,11 +391,11 @@ mod tests { "* LSUB () \"/\" \"foo\" (\"CHILDINFO\" (\"SUBSCRIBED\"))\r\n", ); - let response_v2 = String::from_utf8(response.clone().serialize()).unwrap(); + let response_v2 = CompactString::from_utf8(response.clone().serialize()).unwrap(); response.is_rev2 = false; response.is_lsub = true; response.status_items.clear(); - let response_v1 = String::from_utf8(response.serialize()).unwrap(); + let response_v1 = CompactString::from_utf8(response.serialize()).unwrap(); assert_eq!(response_v2, expected_v2); assert_eq!(response_v1, expected_v1); diff --git a/crates/imap-proto/src/protocol/login.rs b/crates/imap-proto/src/protocol/login.rs index 84c92c1a..7597a000 100644 --- a/crates/imap-proto/src/protocol/login.rs +++ b/crates/imap-proto/src/protocol/login.rs @@ -4,9 +4,11 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use compact_str::CompactString; + #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { - pub tag: String, - pub username: String, - pub password: String, + pub tag: CompactString, + pub username: CompactString, + pub password: CompactString, } diff --git a/crates/imap-proto/src/protocol/mod.rs b/crates/imap-proto/src/protocol/mod.rs index 7ffb0ffc..fcdab80a 100644 --- a/crates/imap-proto/src/protocol/mod.rs +++ b/crates/imap-proto/src/protocol/mod.rs @@ -8,6 +8,7 @@ use std::{cmp::Ordering, fmt::Display}; use ahash::AHashSet; use chrono::{DateTime, Utc}; +use compact_str::CompactString; use jmap_proto::types::keyword::{ArchivedKeyword, Keyword}; use crate::{Command, ResponseCode, ResponseType, StatusResponse}; @@ -239,7 +240,7 @@ pub enum Flag { Deleted, Forwarded, MDNSent, - Keyword(String), + Keyword(CompactString), } impl Flag { @@ -297,7 +298,7 @@ impl From<&ArchivedKeyword> for Flag { ArchivedKeyword::Deleted => Flag::Deleted, ArchivedKeyword::Forwarded => Flag::Forwarded, ArchivedKeyword::MdnSent => Flag::MDNSent, - ArchivedKeyword::Other(value) => Flag::Keyword(value.to_string()), + ArchivedKeyword::Other(value) => Flag::Keyword(value.as_str().into()), } } } diff --git a/crates/imap-proto/src/protocol/namespace.rs b/crates/imap-proto/src/protocol/namespace.rs index 25d76bfb..9b0d4d18 100644 --- a/crates/imap-proto/src/protocol/namespace.rs +++ b/crates/imap-proto/src/protocol/namespace.rs @@ -4,10 +4,12 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use compact_str::CompactString; + use super::{ImapResponse, quoted_string}; pub struct Response { - pub shared_prefix: Option, + pub shared_prefix: Option, } impl ImapResponse for Response { diff --git a/crates/imap-proto/src/protocol/quota.rs b/crates/imap-proto/src/protocol/quota.rs index c0fceec6..366bd911 100644 --- a/crates/imap-proto/src/protocol/quota.rs +++ b/crates/imap-proto/src/protocol/quota.rs @@ -4,16 +4,18 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use compact_str::CompactString; + use super::{ImapResponse, capability::QuotaResourceName, quoted_string}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { - pub tag: String, - pub name: String, + pub tag: CompactString, + pub name: CompactString, } pub struct QuotaItem { - pub name: String, + pub name: CompactString, pub resources: Vec, } @@ -24,7 +26,7 @@ pub struct QuotaResource { } pub struct Response { - pub quota_root_items: Vec, + pub quota_root_items: Vec, pub quota_items: Vec, } @@ -88,7 +90,7 @@ mod tests { for (response, expected) in [ ( super::Response { - quota_root_items: vec!["INBOX".to_string(), "#test".to_string()], + quota_root_items: vec!["INBOX".into(), "#test".into()], quota_items: vec![], }, "* QUOTAROOT \"INBOX\" \"#test\"\r\n", @@ -97,7 +99,7 @@ mod tests { super::Response { quota_root_items: vec![], quota_items: vec![QuotaItem { - name: "INBOX".to_string(), + name: "INBOX".into(), resources: vec![QuotaResource { resource: QuotaResourceName::Storage, total: 1073741824, @@ -109,9 +111,9 @@ mod tests { ), ( super::Response { - quota_root_items: vec!["my mailbox".to_string(), "".to_string()], + quota_root_items: vec!["my mailbox".into(), "".into()], quota_items: vec![QuotaItem { - name: "INBOX".to_string(), + name: "INBOX".into(), resources: vec![ QuotaResource { resource: QuotaResourceName::Storage, diff --git a/crates/imap-proto/src/protocol/rename.rs b/crates/imap-proto/src/protocol/rename.rs index 26e139ea..87e8e804 100644 --- a/crates/imap-proto/src/protocol/rename.rs +++ b/crates/imap-proto/src/protocol/rename.rs @@ -4,9 +4,11 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use compact_str::CompactString; + #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { - pub tag: String, - pub mailbox_name: String, - pub new_mailbox_name: String, + pub tag: CompactString, + pub mailbox_name: CompactString, + pub new_mailbox_name: CompactString, } diff --git a/crates/imap-proto/src/protocol/search.rs b/crates/imap-proto/src/protocol/search.rs index 7658c4f5..fef3f4ce 100644 --- a/crates/imap-proto/src/protocol/search.rs +++ b/crates/imap-proto/src/protocol/search.rs @@ -4,13 +4,14 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use compact_str::CompactString; use store::fts::{FilterItem, FilterType}; use super::{Flag, Sequence, quoted_string, serialize_sequence}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { - pub tag: String, + pub tag: CompactString, pub is_esearch: bool, pub sort: Option>, pub result_options: Vec, @@ -63,15 +64,15 @@ pub enum Filter { Sequence(Sequence, bool), All, Answered, - Bcc(String), + Bcc(CompactString), Before(i64), - Body(String), - Cc(String), + Body(CompactString), + Cc(CompactString), Deleted, Draft, Flagged, - From(String), - Header(String, String), + From(CompactString), + Header(CompactString, CompactString), Keyword(Flag), Larger(u32), On(i64), @@ -81,9 +82,9 @@ pub enum Filter { SentSince(i64), Since(i64), Smaller(u32), - Subject(String), - Text(String), - To(String), + Subject(CompactString), + Text(CompactString), + To(CompactString), Unanswered, Undeleted, Undraft, @@ -110,8 +111,8 @@ pub enum Filter { ModSeq((u64, ModSeqEntry)), // RFC 8474 - ObjectID - EmailId(String), - ThreadId(String), + EmailId(CompactString), + ThreadId(CompactString), } impl FilterItem for Filter { diff --git a/crates/imap-proto/src/protocol/select.rs b/crates/imap-proto/src/protocol/select.rs index 9a91abad..e9fa55e4 100644 --- a/crates/imap-proto/src/protocol/select.rs +++ b/crates/imap-proto/src/protocol/select.rs @@ -4,14 +4,16 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use compact_str::CompactString; + use crate::{ResponseCode, StatusResponse}; use super::{ImapResponse, Sequence, list::ListItem}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { - pub tag: String, - pub mailbox_name: String, + pub tag: CompactString, + pub mailbox_name: CompactString, pub condstore: bool, pub qresync: Option, } @@ -38,7 +40,7 @@ pub struct Response { pub is_rev2: bool, pub closed_previous: bool, pub highest_modseq: Option, - pub mailbox_id: String, + pub mailbox_id: CompactString, } #[derive(Debug, Clone)] diff --git a/crates/imap-proto/src/protocol/status.rs b/crates/imap-proto/src/protocol/status.rs index 35f1c29f..259618b4 100644 --- a/crates/imap-proto/src/protocol/status.rs +++ b/crates/imap-proto/src/protocol/status.rs @@ -4,14 +4,16 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use compact_str::CompactString; + use crate::utf7::utf7_encode; use super::quoted_string; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { - pub tag: String, - pub mailbox_name: String, + pub tag: CompactString, + pub mailbox_name: CompactString, pub items: Vec, } @@ -31,14 +33,14 @@ pub enum Status { #[derive(Debug, Clone, PartialEq, Eq)] pub struct StatusItem { - pub mailbox_name: String, + pub mailbox_name: CompactString, pub items: Vec<(Status, StatusItemType)>, } #[derive(Debug, Clone, PartialEq, Eq)] pub enum StatusItemType { Number(u64), - String(String), + String(CompactString), } impl StatusItem { @@ -91,14 +93,11 @@ mod tests { fn serialize_status() { let mut buf = Vec::new(); StatusItem { - mailbox_name: "blurdybloop".to_string(), + mailbox_name: "blurdybloop".into(), items: vec![ (Status::Messages, StatusItemType::Number(231)), (Status::UidNext, StatusItemType::Number(44292)), - ( - Status::MailboxId, - StatusItemType::String("abc-123".to_string()), - ), + (Status::MailboxId, StatusItemType::String("abc-123".into())), ], } .serialize(&mut buf, true); diff --git a/crates/imap-proto/src/protocol/store.rs b/crates/imap-proto/src/protocol/store.rs index eddc4458..c636b41b 100644 --- a/crates/imap-proto/src/protocol/store.rs +++ b/crates/imap-proto/src/protocol/store.rs @@ -4,11 +4,13 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use compact_str::CompactString; + use super::{Flag, ImapResponse, Sequence, fetch::FetchItem}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { - pub tag: String, + pub tag: CompactString, pub sequence_set: Sequence, pub operation: Operation, pub is_silent: bool, diff --git a/crates/imap-proto/src/protocol/subscribe.rs b/crates/imap-proto/src/protocol/subscribe.rs index 8cbd6e32..e6a78de5 100644 --- a/crates/imap-proto/src/protocol/subscribe.rs +++ b/crates/imap-proto/src/protocol/subscribe.rs @@ -4,8 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use compact_str::CompactString; + #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { - pub tag: String, - pub mailbox_name: String, + pub tag: CompactString, + pub mailbox_name: CompactString, } diff --git a/crates/imap-proto/src/protocol/thread.rs b/crates/imap-proto/src/protocol/thread.rs index 33c36197..eede8b59 100644 --- a/crates/imap-proto/src/protocol/thread.rs +++ b/crates/imap-proto/src/protocol/thread.rs @@ -4,11 +4,13 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use compact_str::CompactString; + use super::{ImapResponse, search::Filter}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Arguments { - pub tag: String, + pub tag: CompactString, pub filter: Vec, pub algorithm: Algorithm, } diff --git a/crates/imap-proto/src/receiver.rs b/crates/imap-proto/src/receiver.rs index 568a6115..b2662bae 100644 --- a/crates/imap-proto/src/receiver.rs +++ b/crates/imap-proto/src/receiver.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::fmt::Display; +use compact_str::CompactString; use super::{ResponseCode, ResponseType}; @@ -17,7 +17,7 @@ pub enum Error { #[derive(Debug, Clone, PartialEq, Eq)] pub struct Request { - pub tag: String, + pub tag: CompactString, pub command: T, pub tokens: Vec, } @@ -43,7 +43,7 @@ pub enum Token { impl Default for Request { fn default() -> Self { Self { - tag: String::with_capacity(0), + tag: CompactString::new(""), command: T::default(), tokens: Vec::new(), } @@ -150,7 +150,7 @@ impl Receiver { State::Tag => match ch { b' ' => { if !self.buf.is_empty() { - self.request.tag = String::from_utf8(std::mem::replace( + self.request.tag = CompactString::from_utf8(std::mem::replace( &mut self.buf, Vec::with_capacity(10), )) @@ -386,10 +386,10 @@ impl Receiver { } impl Token { - pub fn unwrap_string(self) -> crate::parser::Result { + pub fn unwrap_string(self) -> crate::parser::Result { match self { Token::Argument(value) => { - String::from_utf8(value).map_err(|_| "Invalid UTF-8 in argument.".into()) + CompactString::from_utf8(value).map_err(|_| "Invalid UTF-8 in argument.".into()) } other => Ok(other.to_string()), } @@ -398,7 +398,7 @@ impl Token { pub fn unwrap_bytes(self) -> Vec { match self { Token::Argument(value) => value, - other => other.to_string().into_bytes(), + other => other.as_bytes().to_vec(), } } @@ -445,24 +445,27 @@ impl Token { } } -impl Display for Token { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { +impl Token { + pub fn as_bytes(&self) -> &[u8] { match self { - Token::Argument(value) => write!(f, "{}", String::from_utf8_lossy(value)), - Token::ParenthesisOpen => write!(f, "("), - Token::ParenthesisClose => write!(f, ")"), - Token::BracketOpen => write!(f, "["), - Token::BracketClose => write!(f, "]"), - Token::Gt => write!(f, ">"), - Token::Lt => write!(f, "<"), - Token::Dot => write!(f, "."), - Token::Nil => write!(f, ""), + Token::Argument(value) => value, + Token::ParenthesisOpen => b"(", + Token::ParenthesisClose => b")", + Token::BracketOpen => b"[", + Token::BracketClose => b"]", + Token::Gt => b">", + Token::Lt => b"<", + Token::Dot => b".", + Token::Nil => b"", } } + pub fn to_string(&self) -> CompactString { + CompactString::from_utf8_lossy(self.as_bytes()) + } } impl Error { - pub fn err(tag: Option, message: impl Into) -> Self { + pub fn err(tag: Option, message: impl Into) -> Self { Error::Error { response: trc::ImapEvent::Error .ctx(trc::Key::Details, message) @@ -553,7 +556,7 @@ mod tests { ( vec!["abcd CAPABILITY\r\n"], vec![Request { - tag: "abcd".to_string(), + tag: "abcd".into(), command: Command::Capability, tokens: vec![], }], @@ -561,7 +564,7 @@ mod tests { ( vec!["A023 LO", "GOUT\r\n"], vec![Request { - tag: "A023".to_string(), + tag: "A023".into(), command: Command::Logout, tokens: vec![], }], @@ -569,7 +572,7 @@ mod tests { ( vec![" A001 AUTHENTICATE GSSAPI \r\n"], vec![Request { - tag: "A001".to_string(), + tag: "A001".into(), command: Command::Authenticate, tokens: vec![Token::Argument(b"GSSAPI".to_vec())], }], @@ -577,7 +580,7 @@ mod tests { ( vec!["A03 AUTHENTICATE ", "PLAIN dGVzdAB0ZXN", "0AHRlc3Q=\r\n"], vec![Request { - tag: "A03".to_string(), + tag: "A03".into(), command: Command::Authenticate, tokens: vec![ Token::Argument(b"PLAIN".to_vec()), @@ -588,7 +591,7 @@ mod tests { ( vec!["A003 CREATE owatagusiam/\r\n"], vec![Request { - tag: "A003".to_string(), + tag: "A003".into(), command: Command::Create, tokens: vec![Token::Argument(b"owatagusiam/".to_vec())], }], @@ -596,7 +599,7 @@ mod tests { ( vec!["A682 LIST \"\" *\r\n"], vec![Request { - tag: "A682".to_string(), + tag: "A682".into(), command: Command::List, tokens: vec![Token::Nil, Token::Argument(b"*".to_vec())], }], @@ -604,7 +607,7 @@ mod tests { ( vec!["A03 LIST () \"\" \"%\" RETURN (CHILDREN)\r\n"], vec![Request { - tag: "A03".to_string(), + tag: "A03".into(), command: Command::List, tokens: vec![ Token::ParenthesisOpen, @@ -621,7 +624,7 @@ mod tests { ( vec!["A05 LIST (REMOTE SUBSCRIBED) \"\" \"*\"\r\n"], vec![Request { - tag: "A05".to_string(), + tag: "A05".into(), command: Command::List, tokens: vec![ Token::ParenthesisOpen, @@ -636,7 +639,7 @@ mod tests { ( vec!["a1 list \"\" (\"foo\")\r\n"], vec![Request { - tag: "a1".to_string(), + tag: "a1".into(), command: Command::List, tokens: vec![ Token::Nil, @@ -649,7 +652,7 @@ mod tests { ( vec!["a3.1 LIST \"\" (% music/rock)\r\n"], vec![Request { - tag: "a3.1".to_string(), + tag: "a3.1".into(), command: Command::List, tokens: vec![ Token::Nil, @@ -663,7 +666,7 @@ mod tests { ( vec!["A01 LIST \"\" % RETURN (STATUS (MESSAGES UNSEEN))\r\n"], vec![Request { - tag: "A01".to_string(), + tag: "A01".into(), command: Command::List, tokens: vec![ Token::Nil, @@ -682,7 +685,7 @@ mod tests { ( vec![" A01 LiSt \"\" % RETURN ( STATUS ( MESSAGES UNSEEN ) ) \r\n"], vec![Request { - tag: "A01".to_string(), + tag: "A01".into(), command: Command::List, tokens: vec![ Token::Nil, @@ -701,7 +704,7 @@ mod tests { ( vec!["A02 LIST (SUBSCRIBED RECURSIVEMATCH) \"\" % RETURN (STATUS (MESSAGES))\r\n"], vec![Request { - tag: "A02".to_string(), + tag: "A02".into(), command: Command::List, tokens: vec![ Token::ParenthesisOpen, @@ -723,7 +726,7 @@ mod tests { ( vec!["A002 CREATE \"INBOX.Sent Mail\"\r\n"], vec![Request { - tag: "A002".to_string(), + tag: "A002".into(), command: Command::Create, tokens: vec![Token::Argument(b"INBOX.Sent Mail".to_vec())], }], @@ -731,7 +734,7 @@ mod tests { ( vec!["A002 CREATE \"Maibox \\\"quo\\\\ted\\\" \"\r\n"], vec![Request { - tag: "A002".to_string(), + tag: "A002".into(), command: Command::Create, tokens: vec![Token::Argument(b"Maibox \"quo\\ted\" ".to_vec())], }], @@ -739,7 +742,7 @@ mod tests { ( vec!["A004 COPY 2:4 meeting\r\n"], vec![Request { - tag: "A004".to_string(), + tag: "A004".into(), command: Command::Copy(false), tokens: vec![ Token::Argument(b"2:4".to_vec()), @@ -754,7 +757,7 @@ mod tests { "NOT FROM \"Smith\"\r\n", ], vec![Request { - tag: "A282".to_string(), + tag: "A282".into(), command: Command::Search(false), tokens: vec![ Token::Argument(b"RETURN".to_vec()), @@ -774,7 +777,7 @@ mod tests { ( vec!["F284 UID STORE $ +FLAGS.Silent (\\Deleted)\r\n"], vec![Request { - tag: "F284".to_string(), + tag: "F284".into(), command: Command::Store(true), tokens: vec![ Token::Argument(b"$".to_vec()), @@ -788,7 +791,7 @@ mod tests { ( vec!["A654 FETCH 2:4 (FLAGS BODY[HEADER.FIELDS (DATE FROM)])\r\n"], vec![Request { - tag: "A654".to_string(), + tag: "A654".into(), command: Command::Fetch(false), tokens: vec![ Token::Argument(b"2:4".to_vec()), @@ -814,7 +817,7 @@ mod tests { "KOI8-R (OR $ 1,3000:3021) TEXT \"hello world\"\r\n", ], vec![Request { - tag: "B283".to_string(), + tag: "B283".into(), command: Command::Search(true), tokens: vec![ Token::Argument(b"RETURN".to_vec()), @@ -839,7 +842,7 @@ mod tests { "TEXT {8+}\r\nмать\r\n", ], vec![Request { - tag: "P283".to_string(), + tag: "P283".into(), command: Command::Search(false), tokens: vec![ Token::Argument(b"CHARSET".to_vec()), @@ -857,7 +860,7 @@ mod tests { ( vec!["A001 LOGIN {11}\r\n", "FRED FOOBAR {7}\r\n", "fat man\r\n"], vec![Request { - tag: "A001".to_string(), + tag: "A001".into(), command: Command::Login, tokens: vec![ Token::Argument(b"FRED FOOBAR".to_vec()), @@ -868,7 +871,7 @@ mod tests { ( vec!["TAG3 CREATE \"Test-ąęć-Test\"\r\n"], vec![Request { - tag: "TAG3".to_string(), + tag: "TAG3".into(), command: Command::Create, tokens: vec![Token::Argument("Test-ąęć-Test".as_bytes().to_vec())], }], @@ -876,7 +879,7 @@ mod tests { ( vec!["abc LOGIN {0}\r\n", "\r\n"], vec![Request { - tag: "abc".to_string(), + tag: "abc".into(), command: Command::Login, tokens: vec![Token::Nil], }], @@ -884,7 +887,7 @@ mod tests { ( vec!["abc LOGIN {0+}\r\n\r\n"], vec![Request { - tag: "abc".to_string(), + tag: "abc".into(), command: Command::Login, tokens: vec![Token::Nil], }], @@ -903,7 +906,7 @@ mod tests { "Hello Joe, do you think we can meet at 3:30 tomorrow?\r\n\r\n", ], vec![Request { - tag: "A003".to_string(), + tag: "A003".into(), command: Command::Append, tokens: vec![ Token::Argument(b"saved-messages".to_vec()), @@ -942,7 +945,7 @@ mod tests { "Hello Joe, do you think we can meet at 3:30 tomorrow?\r\n\r\n", ], vec![Request { - tag: "A003".to_string(), + tag: "A003".into(), command: Command::Append, tokens: vec![ Token::Argument(b"saved-messages".to_vec()), @@ -971,17 +974,17 @@ mod tests { vec!["001 NOOP\r\n002 CAPABILITY\r\nabc LOGIN hello world\r\n"], vec![ Request { - tag: "001".to_string(), + tag: "001".into(), command: Command::Noop, tokens: vec![], }, Request { - tag: "002".to_string(), + tag: "002".into(), command: Command::Capability, tokens: vec![], }, Request { - tag: "abc".to_string(), + tag: "abc".into(), command: Command::Login, tokens: vec![ Token::Argument(b"hello".to_vec()), diff --git a/crates/imap-proto/src/utf7.rs b/crates/imap-proto/src/utf7.rs index 4af7cdfa..6298843f 100644 --- a/crates/imap-proto/src/utf7.rs +++ b/crates/imap-proto/src/utf7.rs @@ -7,6 +7,8 @@ // Ported from https://github.com/jstedfast/MailKit/blob/master/MailKit/Net/Imap/ImapEncoding.cs // Author: Jeffrey Stedfast +use compact_str::CompactString; + use crate::protocol::ProtocolVersion; static UTF_7_RANK: &[u8] = &[ @@ -20,7 +22,7 @@ static UTF_7_RANK: &[u8] = &[ static UTF_7_MAP: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+,"; -pub fn utf7_decode(text: &str) -> Option { +pub fn utf7_decode(text: &str) -> Option { let mut bytes: Vec = Vec::with_capacity(text.len()); let mut bits = 0; let mut v: u32 = 0; @@ -68,11 +70,11 @@ pub fn utf7_decode(text: &str) -> Option { } } - String::from_utf16(&bytes).ok() + CompactString::from_utf16(&bytes).ok() } -pub fn utf7_encode(text: &str) -> String { - let mut result = String::with_capacity(text.len()); +pub fn utf7_encode(text: &str) -> CompactString { + let mut result = CompactString::with_capacity(text.len()); let mut shifted = false; let mut bits = 0; let mut u: u32 = 0; @@ -122,7 +124,7 @@ pub fn utf7_encode(text: &str) -> String { } #[inline(always)] -pub fn utf7_maybe_decode(text: String, version: ProtocolVersion) -> String { +pub fn utf7_maybe_decode(text: CompactString, version: ProtocolVersion) -> CompactString { if version.is_rev2() { text } else { diff --git a/crates/imap/Cargo.toml b/crates/imap/Cargo.toml index c760e4db..e26d5035 100644 --- a/crates/imap/Cargo.toml +++ b/crates/imap/Cargo.toml @@ -25,7 +25,7 @@ ahash = { version = "0.8" } md5 = "0.7.0" rand = "0.9.0" indexmap = "2.7.1" - +compact_str = "0.9.0" [features] test_mode = [] diff --git a/crates/imap/src/core/mailbox.rs b/crates/imap/src/core/mailbox.rs index 8e809341..9bb39f49 100644 --- a/crates/imap/src/core/mailbox.rs +++ b/crates/imap/src/core/mailbox.rs @@ -4,26 +4,29 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use super::{Account, AccountState, MailboxId, MailboxSync, Session, SessionData}; +use crate::core::Mailbox; use ahash::AHashMap; use common::{ - AccountId, Mailbox, auth::AccessToken, - config::jmap::settings::{ArchivedSpecialUse, SpecialUse}, + config::jmap::settings::SpecialUse, listener::{SessionStream, limiter::InFlight}, sharing::EffectiveAcl, }; +use compact_str::CompactString; use directory::{QueryBy, backend::internal::PrincipalField}; -use email::mailbox::{INBOX_ID, manage::MailboxFnc}; +use email::{ + mailbox::{INBOX_ID, cache::MessageMailboxCache}, + message::cache::{MessageCache, MessageCacheAccess}, +}; use imap_proto::protocol::list::Attribute; -use indexmap::IndexMap; -use jmap_proto::types::{acl::Acl, collection::Collection, id::Id, property::Property}; +use jmap_proto::types::{acl::Acl, collection::Collection, id::Id, keyword::Keyword}; use parking_lot::Mutex; -use std::sync::{Arc, atomic::Ordering}; -use store::query::log::{Change, Query}; +use std::{ + collections::BTreeMap, + sync::{Arc, atomic::Ordering}, +}; use trc::AddContext; -use utils::topological::TopologicalSort; - -use super::{Account, MailboxId, MailboxSync, Session, SessionData}; impl SessionData { pub async fn new( @@ -46,36 +49,35 @@ impl SessionData { // Fetch mailboxes for the main account let mut mailboxes = vec![ session - .fetch_account_mailboxes(session.account_id, None, &access_token) + .fetch_account_mailboxes(session.account_id, None, &access_token, None) .await - .caused_by(trc::location!())?, + .caused_by(trc::location!())? + .unwrap(), ]; // Fetch shared mailboxes for &account_id in access_token.shared_accounts(Collection::Mailbox) { + let prefix: CompactString = format!( + "{}/{}", + session.server.core.jmap.shared_folder, + session + .server + .core + .storage + .directory + .query(QueryBy::Id(account_id), false) + .await + .unwrap_or_default() + .and_then(|mut p| p.take_str(PrincipalField::Name)) + .unwrap_or_else(|| Id::from(account_id).to_string()) + ) + .into(); mailboxes.push( session - .fetch_account_mailboxes( - account_id, - format!( - "{}/{}", - session.server.core.jmap.shared_folder, - session - .server - .core - .storage - .directory - .query(QueryBy::Id(account_id), false) - .await - .unwrap_or_default() - .and_then(|mut p| p.take_str(PrincipalField::Name)) - .unwrap_or_else(|| Id::from(account_id).to_string()) - ) - .into(), - &access_token, - ) + .fetch_account_mailboxes(account_id, prefix.into(), &access_token, None) .await - .caused_by(trc::location!())?, + .caused_by(trc::location!())? + .unwrap(), ); } @@ -87,212 +89,80 @@ impl SessionData { async fn fetch_account_mailboxes( &self, account_id: u32, - mailbox_prefix: Option, + mailbox_prefix: Option, access_token: &AccessToken, - ) -> trc::Result { - let state_mailbox = self + current_state: Option, + ) -> trc::Result> { + let cached_mailboxes = self .server - .core - .storage - .data - .get_last_change_id(account_id, Collection::Mailbox) + .get_cached_mailboxes(account_id) .await .caused_by(trc::location!())?; - let state_email = self + let cached_messages = self .server - .core - .storage - .data - .get_last_change_id(account_id, Collection::Email) + .get_cached_messages(account_id) .await .caused_by(trc::location!())?; - let cached_account_id = AccountId { - account_id, - primary_id: access_token.primary_id(), - }; - if let Some(cached_account) = self - .server - .inner - .cache - .account - .get(&cached_account_id) - .and_then(|cached_account| { - if cached_account.state_mailbox == state_mailbox - && cached_account.state_email == state_email - { - Some(cached_account) - } else { - None - } - }) - { - return Ok(cached_account.as_ref().clone()); + if current_state.is_some_and(|state| { + state.email == cached_messages.change_id && state.mailbox == cached_mailboxes.change_id + }) { + return Ok(None); } - let mailbox_ids = if access_token.is_primary_id(account_id) + let shared_mailbox_ids = if access_token.is_primary_id(account_id) || access_token.member_of.contains(&account_id) { - self.server - .mailbox_get_or_create(account_id) - .await - .caused_by(trc::location!())? + None } else { self.server .shared_containers(access_token, account_id, Collection::Mailbox, Acl::Read) .await .caused_by(trc::location!())? + .into() }; - // Fetch mailboxes - struct MailboxData { - mailbox_id: u32, - parent_id: u32, - role: SpecialUse, - name: String, - is_subscribed: bool, + // Build special uses + let mut special_uses = AHashMap::new(); + for (&mailbox_id, mailbox) in &cached_mailboxes.items { + if shared_mailbox_ids + .as_ref() + .is_none_or(|ids| ids.contains(mailbox_id)) + && !matches!(mailbox.role, SpecialUse::None) + { + special_uses.insert(mailbox.role, mailbox_id); + } } - let mut mailboxes = AHashMap::with_capacity(10); - let mut special_uses = AHashMap::new(); - let mut mailbox_topology = TopologicalSort::with_capacity(10); - - self.server - .get_archives( - account_id, - Collection::Mailbox, - &mailbox_ids, - |mailbox_id, mailbox_| { - let mailbox = mailbox_ - .unarchive::() - .caused_by(trc::location!())?; - // Map special uses - let role = SpecialUse::from(&mailbox.role); - if !matches!(mailbox.role, ArchivedSpecialUse::None) { - special_uses.insert(role, mailbox_id); - } - - // Build mailbox data - let mailbox = MailboxData { - mailbox_id, - parent_id: u32::from(mailbox.parent_id), - role, - name: mailbox.name.to_string(), - is_subscribed: mailbox.is_subscribed(access_token.primary_id()), - }; - mailbox_topology.insert(mailbox.parent_id, mailbox.mailbox_id + 1); - - // Add mailbox id - mailboxes.insert(mailbox.mailbox_id, mailbox); - - Ok(true) - }, - ) - .await - .caused_by(trc::location!())?; - // Build account - let message_ids = self - .server - .get_document_ids(account_id, Collection::Email) - .await - .caused_by(trc::location!())?; let mut account = Account { account_id, prefix: mailbox_prefix, - mailbox_names: IndexMap::with_capacity(mailboxes.len()), - mailbox_state: AHashMap::with_capacity(mailboxes.len()), - state_mailbox, - state_email, - obj_size: 0, + mailbox_names: BTreeMap::new(), + mailbox_state: AHashMap::with_capacity(cached_mailboxes.items.len()), + state: AccountState { + email: cached_messages.change_id, + mailbox: cached_mailboxes.change_id, + }, }; - account.obj_size = (std::mem::size_of::() - + account.prefix.as_ref().map_or(0, |p| p.len()) - + account - .mailbox_names - .keys() - .map(|k| k.len() + std::mem::size_of::()) - .sum::() - + (account.mailbox_state.len() - * (std::mem::size_of::() + std::mem::size_of::()))) - as u64; - // Build mailbox state - for mailbox in mailboxes.values() { - account.mailbox_state.insert( - mailbox.mailbox_id, - Mailbox { - has_children: mailboxes - .values() - .any(|child| child.parent_id == mailbox.mailbox_id + 1), - is_subscribed: mailbox.is_subscribed, - special_use: match mailbox.role { - SpecialUse::Trash => Some(Attribute::Trash), - SpecialUse::Junk => Some(Attribute::Junk), - SpecialUse::Drafts => Some(Attribute::Drafts), - SpecialUse::Archive => Some(Attribute::Archive), - SpecialUse::Sent => Some(Attribute::Sent), - SpecialUse::Important => Some(Attribute::Important), - _ => None, - }, - total_messages: self - .server - .get_tag( - account_id, - Collection::Email, - Property::MailboxIds, - mailbox.mailbox_id, - ) - .await - .caused_by(trc::location!())? - .map(|v| v.len()) - .unwrap_or(0) - .into(), - total_unseen: self - .server - .mailbox_unread_tags(account_id, mailbox.mailbox_id, &message_ids) - .await - .caused_by(trc::location!())? - .map(|v| v.len()) - .unwrap_or(0) - .into(), - ..Default::default() - }, - ); - } - - // Build mailbox tree - for mailbox_id in mailbox_topology.into_iterator() { - if mailbox_id == 0 { + for (&mailbox_id, mailbox) in &cached_mailboxes.items { + if shared_mailbox_ids + .as_ref() + .is_some_and(|ids| !ids.contains(mailbox_id)) + { continue; } - let mailbox_id = mailbox_id - 1; - let (mailbox_name, parent_id) = mailboxes - .get(&mailbox_id) - .map(|m| { - ( - m.name.as_str(), - if m.parent_id == 0 { - None - } else { - Some(m.parent_id - 1) - }, - ) - }) - .unwrap(); - // Obtain folder name - let (mailbox_name, did_rename) = if mailbox_id != INBOX_ID || account.prefix.is_some() { - // If there is another mailbox called Inbox, rename it to avoid conflicts - if parent_id.is_none() || !mailbox_name.eq_ignore_ascii_case("inbox") { - (mailbox_name, false) - } else { - ("INBOX 2", true) - } + // Build mailbox path and map it to its effective id + let mailbox_name = if let Some(prefix) = &account.prefix { + let mut name = CompactString::with_capacity(prefix.len() + mailbox.path.len() + 1); + name.push_str(prefix.as_str()); + name.push('/'); + name.push_str(mailbox.path.as_str()); + name } else { - ("INBOX", true) + mailbox.path.clone() }; - - // Map special use folder aliases to their internal ids let effective_mailbox_id = self .server .core @@ -303,45 +173,49 @@ impl SessionData { .and_then(|f| special_uses.get(&f.special_use)) .copied() .unwrap_or(mailbox_id); - - // Update mailbox name - let full_name = if let Some(parent_id) = parent_id { - let full_name = format!( - "{}/{}", - mailboxes - .get(&parent_id) - .map(|m| m.name.as_str()) - .unwrap_or_default(), - mailbox_name - ); - mailboxes.get_mut(&mailbox_id).unwrap().name = full_name.clone(); - full_name - } else if let Some(prefix) = &account.prefix { - let full_name = format!("{prefix}/{mailbox_name}"); - mailboxes.get_mut(&mailbox_id).unwrap().name = full_name.clone(); - full_name - } else if did_rename { - let full_name = mailbox_name.to_string(); - mailboxes.get_mut(&mailbox_id).unwrap().name = full_name.clone(); - full_name - } else { - mailbox_name.to_string() - }; - - // Insert mailbox account .mailbox_names - .insert(full_name, effective_mailbox_id); + .insert(mailbox_name, effective_mailbox_id); + account.mailbox_state.insert( + mailbox_id, + Mailbox { + has_children: cached_mailboxes.items.values().any(|child| { + child + .parent_id + .is_some_and(|parent_id| parent_id == mailbox_id) + }), + is_subscribed: mailbox.subscribers.contains(&access_token.primary_id()), + special_use: match mailbox.role { + SpecialUse::Trash => Some(Attribute::Trash), + SpecialUse::Junk => Some(Attribute::Junk), + SpecialUse::Drafts => Some(Attribute::Drafts), + SpecialUse::Archive => Some(Attribute::Archive), + SpecialUse::Sent => Some(Attribute::Sent), + SpecialUse::Important => Some(Attribute::Important), + _ => None, + }, + total_messages: cached_messages.in_mailbox(mailbox_id).count() as u64, + total_unseen: cached_messages + .in_mailbox_without_keyword(mailbox_id, &Keyword::Seen) + .count() as u64, + total_deleted: cached_messages + .in_mailbox_with_keyword(mailbox_id, &Keyword::Deleted) + .count() as u64, + uid_validity: mailbox.uid_validity as u64, + uid_next: self + .get_uid_next(&MailboxId { + account_id, + mailbox_id, + }) + .await + .caused_by(trc::location!())? as u64, + total_deleted_storage: None, + size: None, + }, + ); } - // Update cache - self.server - .inner - .cache - .account - .insert(cached_account_id, Arc::new(account.clone())); - - Ok(account) + Ok(account.into()) } pub async fn synchronize_mailboxes( @@ -404,7 +278,7 @@ impl SessionData { // Fetch mailboxes for each new shared account for account_id in added_account_ids { - let prefix = format!( + let prefix: CompactString = format!( "{}/{}", self.server.core.jmap.shared_folder, self.server @@ -416,10 +290,12 @@ impl SessionData { .caused_by(trc::location!())? .and_then(|mut p| p.take_str(PrincipalField::Name)) .unwrap_or_else(|| Id::from(account_id).to_string()) - ); + ) + .into(); added_accounts.push( - self.fetch_account_mailboxes(account_id, prefix.into(), &access_token) - .await?, + self.fetch_account_mailboxes(account_id, prefix.into(), &access_token, None) + .await? + .unwrap(), ); } @@ -433,109 +309,15 @@ impl SessionData { .mailboxes .lock() .iter() - .map(|m| (m.account_id, m.state_mailbox)) + .map(|m| (m.account_id, m.prefix.clone(), m.state)) .collect::>(); - for (account_id, last_state) in account_states { - let changelog = self - .server - .store() - .changes( - account_id, - Collection::Mailbox, - last_state.map(Query::Since).unwrap_or(Query::All), - ) - .await?; - if !changelog.changes.is_empty() { - let mut has_changes = false; - let mut has_child_changes = false; - - for change in changelog.changes { - match change { - Change::Insert(_) | Change::Update(_) | Change::Delete(_) => { - has_changes = true - } - Change::ChildUpdate(_) => has_child_changes = true, - } - } - - if has_child_changes && !has_changes && changes.is_none() { - // Only child changes, no need to re-fetch mailboxes - let state_email = self - .server - .core - .storage - .data - .get_last_change_id(account_id, Collection::Email) - .await - .caused_by(trc::location!())?; - let state_mailbox = Some(changelog.to_change_id); - for account in self.mailboxes.lock().iter_mut() { - if account.account_id == account_id { - account.mailbox_state.values_mut().for_each(|v| { - v.total_deleted = None; - v.total_unseen = None; - v.total_messages = None; - v.size = None; - v.uid_next = None; - }); - account.state_mailbox = state_mailbox; - account.state_email = state_email; - break; - } - } - - // Update cache - let ac_id = AccountId { - account_id, - primary_id: access_token.primary_id(), - }; - if let Some(cached_account_) = self.server.inner.cache.account.get(&ac_id) { - if cached_account_.state_mailbox != state_mailbox - || cached_account_.state_email != state_email - { - let mut cached_account = cached_account_.as_ref().clone(); - cached_account.mailbox_state.values_mut().for_each(|v| { - v.total_deleted = None; - v.total_unseen = None; - v.total_messages = None; - v.size = None; - v.uid_next = None; - }); - cached_account.state_mailbox = state_mailbox; - cached_account.state_email = state_email; - self.server - .inner - .cache - .account - .insert(ac_id, Arc::new(cached_account)); - } - } - } else { - // Refresh mailboxes for changed account - let mailbox_prefix = if !access_token.is_primary_id(account_id) { - format!( - "{}/{}", - self.server.core.jmap.shared_folder, - self.server - .core - .storage - .directory - .query(QueryBy::Id(account_id), false) - .await - .caused_by(trc::location!())? - .and_then(|mut p| p.take_str(PrincipalField::Name)) - .unwrap_or_else(|| Id::from(account_id).to_string()) - ) - .into() - } else { - None - }; - - changed_accounts.push( - self.fetch_account_mailboxes(account_id, mailbox_prefix, &access_token) - .await?, - ); - } + for (account_id, prefix, last_state) in account_states { + if let Some(changed_account) = self + .fetch_account_mailboxes(account_id, prefix, &access_token, last_state.into()) + .await + .caused_by(trc::location!())? + { + changed_accounts.push(changed_account); } } @@ -557,23 +339,21 @@ impl SessionData { for (mailbox_name, mailbox_id) in new_account.mailbox_names.iter() { if let Some(old_mailbox) = old_account.mailbox_state.get(mailbox_id) { if let Some(mailbox) = new_account.mailbox_state.get(mailbox_id) { - if mailbox.total_messages.unwrap_or(0) - != old_mailbox.total_messages.unwrap_or(0) - || mailbox.total_unseen.unwrap_or(0) - != old_mailbox.total_unseen.unwrap_or(0) + if mailbox.total_messages != old_mailbox.total_messages + || mailbox.total_unseen != old_mailbox.total_unseen { - changes.changed.push(mailbox_name.to_string()); + changes.changed.push(mailbox_name.clone()); } } } else { - changes.added.push(mailbox_name.to_string()); + changes.added.push(mailbox_name.clone()); } } // Add deleted mailboxes for (mailbox_name, mailbox_id) in &old_account.mailbox_names { if !new_account.mailbox_state.contains_key(mailbox_id) { - changes.deleted.push(mailbox_name.to_string()); + changes.deleted.push(mailbox_name.clone()); } } } @@ -613,7 +393,7 @@ impl SessionData { if account .prefix .as_ref() - .is_none_or(|p| mailbox_name.starts_with(p)) + .is_none_or(|p| mailbox_name.starts_with(p.as_str())) { for (mailbox_name_, mailbox_id_) in account.mailbox_names.iter() { if (!is_inbox && mailbox_name_ == mailbox_name) diff --git a/crates/imap/src/core/message.rs b/crates/imap/src/core/message.rs index e6e66b08..368c4a00 100644 --- a/crates/imap/src/core/message.rs +++ b/crates/imap/src/core/message.rs @@ -4,85 +4,51 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{collections::BTreeMap, sync::Arc}; - use ahash::AHashMap; -use common::{NextMailboxState, listener::SessionStream}; -use email::message::metadata::MessageData; +use common::listener::SessionStream; +use email::message::cache::MessageCache; use imap_proto::protocol::{Sequence, expunge, select::Exists}; use jmap_proto::types::{collection::Collection, property::Property}; +use std::collections::BTreeMap; use store::{ValueKey, write::ValueClass}; use trc::AddContext; use crate::core::ImapId; -use super::{ImapUidToId, MailboxId, MailboxState, SelectedMailbox, SessionData}; +use super::{ + ImapUidToId, Mailbox, MailboxId, MailboxState, NextMailboxState, SelectedMailbox, SessionData, +}; impl SessionData { - pub async fn fetch_messages(&self, mailbox: &MailboxId) -> trc::Result { - // Obtain message ids - let message_ids = self + pub async fn fetch_messages( + &self, + mailbox: &MailboxId, + current_state: Option, + ) -> trc::Result> { + let cached_messages = self .server - .get_tag( - mailbox.account_id, - Collection::Email, - Property::MailboxIds, - mailbox.mailbox_id, - ) - .await? - .unwrap_or_default(); - - // Obtain UID validity and UID next - let uid_validity = self.get_uid_validity(mailbox).await?; - let uid_next = self.get_uid_next(mailbox).await?; - - // Obtain current state - let modseq = self - .server - .core - .storage - .data - .get_last_change_id(mailbox.account_id, Collection::Email) + .get_cached_messages(mailbox.account_id) .await - .add_context(|e| e.caused_by(trc::location!()).account_id(mailbox.account_id))?; + .caused_by(trc::location!())?; - // Obtain all message ids - let mut uid_map = BTreeMap::new(); - self.server - .get_archives( - mailbox.account_id, - Collection::Email, - &message_ids, - |message_id, message_data_| { - let message_data = message_data_ - .unarchive::() - .caused_by(trc::location!())?; - // Make sure the message is still in this mailbox - if let Some(item) = message_data - .mailboxes - .iter() - .find(|item| item.mailbox_id == mailbox.mailbox_id) - { - debug_assert!(item.uid != 0, "UID is zero for message {item:?}"); - if uid_map.insert(u32::from(item.uid), message_id).is_some() { - trc::event!( - Store(trc::StoreEvent::UnexpectedError), - AccountId = mailbox.account_id, - Collection = Collection::Mailbox, - MailboxId = mailbox.mailbox_id, - MessageId = message_id, - SpanId = self.session_id, - Details = "Duplicate IMAP UID" - ); - } - } - - Ok(true) - }, - ) - .await?; + if current_state.is_some_and(|state| state == cached_messages.change_id) { + return Ok(None); + } // Obtain UID next and assign UIDs + let uid_map = cached_messages + .items + .iter() + .filter_map(|(document_id, item)| { + item.mailboxes.iter().find_map(|m| { + if m.mailbox_id == mailbox.mailbox_id { + Some((m.uid, *document_id)) + } else { + None + } + }) + }) + .collect::>(); let mut uid_max = 0; let mut id_to_imap = AHashMap::with_capacity(uid_map.len()); let mut uid_to_id = AHashMap::with_capacity(uid_map.len()); @@ -101,32 +67,26 @@ impl SessionData { uid_to_id.insert(uid, message_id); } - let mut state = MailboxState { - uid_next, - uid_validity, + Ok(Some(MailboxState { total_messages: id_to_imap.len(), id_to_imap, uid_to_id, uid_max, - modseq, + modseq: cached_messages.change_id, next_state: None, - obj_size: 0, - }; - state.obj_size = state.calculate_weight(); - - Ok(state) + })) } - pub async fn synchronize_messages( - &self, - mailbox: &SelectedMailbox, - ) -> trc::Result> { + pub async fn synchronize_messages(&self, mailbox: &SelectedMailbox) -> trc::Result { // Obtain current modseq - let modseq = self.get_modseq(mailbox.id.account_id).await?; - if mailbox.state.lock().modseq != modseq { + let mut current_modseq = mailbox.state.lock().modseq; + if let Some(new_state) = self + .fetch_messages(&mailbox.id, current_modseq.into()) + .await? + { // Synchronize messages - let new_state = self.fetch_messages(&mailbox.id).await?; let mut current_state = mailbox.state.lock(); + current_modseq = new_state.modseq; // Add missing uids let mut deletions = current_state @@ -148,13 +108,6 @@ impl SessionData { } current_state.id_to_imap = id_to_imap; - // Update cache - self.server - .inner - .cache - .mailbox - .insert(mailbox.id, Arc::new(new_state.clone())); - // Update state current_state.modseq = new_state.modseq; current_state.next_state = Some(Box::new(NextMailboxState { @@ -163,14 +116,14 @@ impl SessionData { })); } - Ok(modseq) + Ok(current_modseq) } pub async fn write_mailbox_changes( &self, mailbox: &SelectedMailbox, is_qresync: bool, - ) -> trc::Result> { + ) -> trc::Result { // Resync mailbox let modseq = self.synchronize_messages(mailbox).await?; let mut buf = Vec::new(); @@ -223,24 +176,6 @@ impl SessionData { }) } - pub async fn get_uid_validity(&self, mailbox: &MailboxId) -> trc::Result { - self.server - .get_archive(mailbox.account_id, Collection::Mailbox, mailbox.mailbox_id) - .await? - .ok_or_else(|| { - trc::ImapEvent::Error - .caused_by(trc::location!()) - .details("Mailbox unavailable") - .account_id(mailbox.account_id) - .collection(Collection::Mailbox) - .document_id(mailbox.mailbox_id) - }) - .and_then(|m| { - m.unarchive::() - .map(|m| u32::from(m.uid_validity)) - }) - } - pub async fn get_uid_next(&self, mailbox: &MailboxId) -> trc::Result { self.server .core @@ -255,6 +190,15 @@ impl SessionData { .await .map(|v| (v + 1) as u32) } + + pub fn mailbox_state(&self, mailbox: &MailboxId) -> Option { + self.mailboxes + .lock() + .iter() + .find(|m| m.account_id == mailbox.account_id) + .and_then(|m| m.mailbox_state.get(&mailbox.mailbox_id)) + .cloned() + } } impl SelectedMailbox { @@ -337,9 +281,9 @@ impl SelectedMailbox { deleted_ids } - pub fn append_messages(&self, ids: Vec, modseq: Option) -> u32 { + pub fn append_messages(&self, ids: Vec, modseq: Option) { let mut mailbox = self.state.lock(); - if modseq.unwrap_or(0) > mailbox.modseq.unwrap_or(0) { + if modseq.unwrap_or(0) > mailbox.modseq { let mut uid_max = 0; for id in ids { mailbox.total_messages += 1; @@ -355,8 +299,6 @@ impl SelectedMailbox { uid_max = id.uid; } mailbox.uid_max = uid_max; - mailbox.uid_next = uid_max + 1; } - mailbox.uid_validity } } diff --git a/crates/imap/src/core/mod.rs b/crates/imap/src/core/mod.rs index 54163390..4e234fc8 100644 --- a/crates/imap/src/core/mod.rs +++ b/crates/imap/src/core/mod.rs @@ -5,16 +5,23 @@ */ use std::{ + collections::BTreeMap, net::IpAddr, sync::{Arc, atomic::AtomicU32}, }; +use ahash::AHashMap; use common::{ - Account, ImapId, Inner, MailboxId, MailboxState, Server, + Inner, Server, auth::AccessToken, listener::{ServerInstance, SessionStream, limiter::InFlight}, }; -use imap_proto::{Command, protocol::ProtocolVersion, receiver::Receiver}; +use compact_str::CompactString; +use imap_proto::{ + Command, + protocol::{ProtocolVersion, list::Attribute}, + receiver::Receiver, +}; use tokio::{ io::{ReadHalf, WriteHalf}, sync::watch, @@ -72,11 +79,74 @@ pub struct SelectedMailbox { pub is_condstore: bool, } +#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] +pub struct AccountId { + pub account_id: u32, + pub primary_id: u32, +} + +#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] +pub struct MailboxId { + pub account_id: u32, + pub mailbox_id: u32, +} + +#[derive(Debug, Clone, Default)] +pub struct Account { + pub account_id: u32, + pub prefix: Option, + pub mailbox_names: BTreeMap, + pub mailbox_state: AHashMap, + pub state: AccountState, +} + +#[derive(Debug, Clone, Copy, Default)] +pub struct AccountState { + pub email: u64, + pub mailbox: u64, +} + +#[derive(Debug, Default, Clone)] +pub struct Mailbox { + pub has_children: bool, + pub is_subscribed: bool, + pub special_use: Option, + pub total_messages: u64, + pub total_unseen: u64, + pub total_deleted: u64, + pub total_deleted_storage: Option, + pub uid_validity: u64, + pub uid_next: u64, + pub size: Option, +} + +#[derive(Debug, Clone, Default)] +pub struct MailboxState { + pub uid_max: u32, + pub id_to_imap: AHashMap, + pub uid_to_id: AHashMap, + pub total_messages: usize, + pub modseq: u64, + pub next_state: Option>, +} + +#[derive(Debug, Clone)] +pub struct NextMailboxState { + pub next_state: MailboxState, + pub deletions: Vec, +} + +#[derive(Debug, Clone, Copy, Default)] +pub struct ImapId { + pub uid: u32, + pub seqnum: u32, +} + #[derive(Debug, Default)] pub struct MailboxSync { - pub added: Vec, - pub changed: Vec, - pub deleted: Vec, + pub added: Vec, + pub changed: Vec, + pub deleted: Vec, } pub enum SavedSearch { @@ -156,3 +226,20 @@ impl SessionData { } } } + +impl MailboxState { + pub fn map_result_id(&self, document_id: u32, is_uid: bool) -> Option<(u32, ImapId)> { + if let Some(imap_id) = self.id_to_imap.get(&document_id) { + Some((if is_uid { imap_id.uid } else { imap_id.seqnum }, *imap_id)) + } else if is_uid { + self.next_state.as_ref().and_then(|s| { + s.next_state + .id_to_imap + .get(&document_id) + .map(|imap_id| (imap_id.uid, *imap_id)) + }) + } else { + None + } + } +} diff --git a/crates/imap/src/op/acl.rs b/crates/imap/src/op/acl.rs index c46d59bd..40004a69 100644 --- a/crates/imap/src/op/acl.rs +++ b/crates/imap/src/op/acl.rs @@ -7,7 +7,7 @@ use std::{sync::Arc, time::Instant}; use common::{ - MailboxId, auth::AccessToken, listener::SessionStream, sharing::EffectiveAcl, + auth::AccessToken, listener::SessionStream, sharing::EffectiveAcl, storage::index::ObjectIndexBuilder, }; use directory::{ @@ -28,7 +28,7 @@ use trc::AddContext; use utils::map::bitmap::Bitmap; use crate::{ - core::{Session, SessionData, State}, + core::{MailboxId, Session, SessionData, State}, op::ImapContext, spawn_op, }; @@ -104,7 +104,7 @@ impl Session { } } - permissions.push((account_name, rights)); + permissions.push((account_name.into(), rights)); } } diff --git a/crates/imap/src/op/append.rs b/crates/imap/src/op/append.rs index 924173cb..b3d46582 100644 --- a/crates/imap/src/op/append.rs +++ b/crates/imap/src/op/append.rs @@ -15,10 +15,10 @@ use imap_proto::{ }; use crate::{ - core::{ImapUidToId, SelectedMailbox, Session, SessionData}, + core::{ImapUidToId, MailboxId, SelectedMailbox, Session, SessionData}, spawn_op, }; -use common::{MailboxId, listener::SessionStream}; +use common::listener::SessionStream; use jmap_proto::types::{acl::Acl, keyword::Keyword, state::StateChange, type_state::DataType}; use mail_parser::MessageParser; @@ -168,23 +168,25 @@ impl SessionData { if !created_ids.is_empty() { let uids = created_ids.iter().map(|id| id.uid).collect(); - let uid_validity = match selected_mailbox { + match selected_mailbox { Some(selected_mailbox) if selected_mailbox.id == mailbox => { // Write updated modseq if is_qresync { self.write_bytes( - HighestModSeq::new(last_change_id.to_modseq()).into_bytes(), + HighestModSeq::new(last_change_id.unwrap_or_default().to_modseq()) + .into_bytes(), ) .await?; } - selected_mailbox.append_messages(created_ids, last_change_id) + selected_mailbox.append_messages(created_ids, last_change_id); } - _ => self - .get_uid_validity(&mailbox) - .await - .imap_ctx(&arguments.tag, trc::location!())?, + _ => {} }; + let uid_validity = self + .mailbox_state(&mailbox) + .map(|m| m.uid_validity as u32) + .unwrap_or_default(); response = response.with_code(ResponseCode::AppendUid { uid_validity, uids }); } diff --git a/crates/imap/src/op/authenticate.rs b/crates/imap/src/op/authenticate.rs index 0d069fd2..26f4f8e3 100644 --- a/crates/imap/src/op/authenticate.rs +++ b/crates/imap/src/op/authenticate.rs @@ -11,6 +11,7 @@ use common::{ }, listener::{SessionStream, limiter::LimiterResult}, }; +use compact_str::CompactString; use directory::Permission; use imap_proto::{ Command, ResponseCode, StatusResponse, @@ -73,7 +74,7 @@ impl Session { pub async fn authenticate( &mut self, credentials: Credentials, - tag: String, + tag: CompactString, ) -> trc::Result<()> { // Authenticate let access_token = self diff --git a/crates/imap/src/op/copy_move.rs b/crates/imap/src/op/copy_move.rs index 313df1e9..1f4b0780 100644 --- a/crates/imap/src/op/copy_move.rs +++ b/crates/imap/src/op/copy_move.rs @@ -19,10 +19,10 @@ use imap_proto::{ }; use crate::{ - core::{SelectedMailbox, Session, SessionData}, + core::{MailboxId, SelectedMailbox, Session, SessionData}, spawn_op, }; -use common::{MailboxId, listener::SessionStream, storage::index::ObjectIndexBuilder}; +use common::{listener::SessionStream, storage::index::ObjectIndexBuilder}; use jmap_proto::{ error::set::SetErrorType, types::{acl::Acl, collection::Collection, state::StateChange, type_state::DataType}, @@ -396,9 +396,10 @@ impl SessionData { // Prepare response let uid_validity = self - .get_uid_validity(&dest_mailbox) - .await - .imap_ctx(&arguments.tag, trc::location!())?; + .mailbox_state(&dest_mailbox) + .map(|m| m.uid_validity as u32) + .unwrap_or_default(); + let mut src_uids = Vec::with_capacity(copied_ids.len()); let mut dest_uids = Vec::with_capacity(copied_ids.len()); for (src_uid, dest_uid) in copied_ids { diff --git a/crates/imap/src/op/create.rs b/crates/imap/src/op/create.rs index e04077f9..25c5e840 100644 --- a/crates/imap/src/op/create.rs +++ b/crates/imap/src/op/create.rs @@ -12,9 +12,9 @@ use crate::{ spawn_op, }; use common::{ - Account, Mailbox, config::jmap::settings::SpecialUse, listener::SessionStream, - storage::index::ObjectIndexBuilder, + config::jmap::settings::SpecialUse, listener::SessionStream, storage::index::ObjectIndexBuilder, }; +use compact_str::CompactString; use directory::Permission; use imap_proto::{ Command, ResponseCode, StatusResponse, @@ -63,7 +63,7 @@ impl SessionData { .imap_ctx(&arguments.tag, trc::location!())?; // Validate mailbox name - let mut params = self + let params = self .validate_mailbox_create(&arguments.mailbox_name, arguments.mailbox_role) .await .imap_ctx(&arguments.tag, trc::location!())?; @@ -72,7 +72,6 @@ impl SessionData { // Build batch let mut parent_id = params.parent_mailbox_id.map(|id| id + 1).unwrap_or(0); let mut create_ids = Vec::with_capacity(params.path.len()); - let mut change_id = 0; let mut next_document_id = self .server .store() @@ -94,7 +93,6 @@ impl SessionData { } let mailbox_id = next_document_id; next_document_id -= 1; - change_id = batch.change_id(); batch .with_account_id(params.account_id) .with_collection(Collection::Mailbox) @@ -123,12 +121,6 @@ impl SessionData { Elapsed = op_start.elapsed() ); - // Add created mailboxes to session - std::mem::drop( - self.add_created_mailboxes(&mut params, change_id, create_ids) - .imap_ctx(&arguments.tag, trc::location!())?, - ); - // Build response Ok(StatusResponse::ok("Mailbox created.") .with_code(ResponseCode::MailboxId { @@ -137,93 +129,6 @@ impl SessionData { .with_tag(arguments.tag)) } - pub fn add_created_mailboxes( - &self, - params: &mut CreateParams<'_>, - new_state: u64, - mailbox_ids: Vec, - ) -> trc::Result>> { - // Lock mailboxes - let mut mailboxes = self.mailboxes.lock(); - let account = if let Some(account) = mailboxes - .iter_mut() - .find(|account| account.account_id == params.account_id) - { - account - } else { - return Err(trc::ImapEvent::Error - .into_err() - .details("Account no longer available.") - .caused_by(trc::location!())); - }; - - // Update state - account.state_mailbox = new_state.into(); - - // Add mailboxes - let mut mailbox_name = if let Some(parent_mailbox_name) = params.parent_mailbox_name.take() - { - if let Some(parent_mailbox) = account - .mailbox_state - .get_mut(params.parent_mailbox_id.as_ref().unwrap()) - { - parent_mailbox.has_children = true; - } - parent_mailbox_name - } else if let Some(account_prefix) = account.prefix.as_ref() { - account_prefix.to_string() - } else { - "".to_string() - }; - - for (pos, (mailbox_id, path_item)) in - mailbox_ids.into_iter().zip(params.path.iter()).enumerate() - { - mailbox_name = if !mailbox_name.is_empty() { - format!("{}/{}", mailbox_name, path_item) - } else { - path_item.to_string() - }; - - let effective_id = self - .server - .core - .jmap - .default_folders - .iter() - .find(|f| f.aliases.iter().any(|a| a == &mailbox_name)) - .and_then(|f| account.mailbox_names.get(&f.name)) - .copied() - .unwrap_or(mailbox_id); - - account - .mailbox_names - .insert(mailbox_name.clone(), effective_id); - - account.mailbox_state.insert( - mailbox_id, - Mailbox { - has_children: pos < params.path.len() - 1 || params.is_rename, - is_subscribed: false, - total_messages: 0.into(), - total_unseen: 0.into(), - total_deleted: 0.into(), - total_deleted_storage: 0.into(), - uid_validity: None, - uid_next: None, - size: 0.into(), - special_use: if pos == params.path.len() - 1 { - params.special_use - } else { - None - }, - }, - ); - } - - Ok(mailboxes) - } - pub async fn validate_mailbox_create<'x>( &self, mailbox_name: &'x str, @@ -271,7 +176,7 @@ impl SessionData { } // Validate special folders - let full_path = path.join("/"); + let full_path: CompactString = path.join("/").into(); let mut parent_mailbox_id = None; let mut parent_mailbox_name = None; let (account_id, path) = { @@ -284,7 +189,11 @@ impl SessionData { .details("Mailboxes under root shared folders are not allowed.") .code(ResponseCode::Cannot)); } - let prefix = Some(format!("{}/{}", path.remove(0), path.remove(0))); + let prefix = Some(CompactString::from(format!( + "{}/{}", + path.remove(0), + path.remove(0) + ))); // Locate account if let Some(account) = mailboxes @@ -322,7 +231,7 @@ impl SessionData { if path.len() > 1 { let mut create_path = Vec::with_capacity(path.len()); while !path.is_empty() { - let mailbox_name = path.join("/"); + let mailbox_name: CompactString = path.join("/").into(); if let Some(&mailbox_id) = account.mailbox_names.get(&mailbox_name) { parent_mailbox_id = mailbox_id.into(); parent_mailbox_name = mailbox_name.into(); @@ -403,9 +312,9 @@ impl SessionData { pub struct CreateParams<'x> { pub account_id: u32, pub path: Vec<&'x str>, - pub full_path: String, + pub full_path: CompactString, pub parent_mailbox_id: Option, - pub parent_mailbox_name: Option, + pub parent_mailbox_name: Option, pub special_use: Option, pub is_rename: bool, } diff --git a/crates/imap/src/op/delete.rs b/crates/imap/src/op/delete.rs index 318a385c..dd7ff846 100644 --- a/crates/imap/src/op/delete.rs +++ b/crates/imap/src/op/delete.rs @@ -90,7 +90,7 @@ impl SessionData { // Update mailbox cache for account in self.mailboxes.lock().iter_mut() { if account.account_id == account_id { - account.mailbox_names.shift_remove(&arguments.mailbox_name); + account.mailbox_names.remove(&arguments.mailbox_name); account.mailbox_state.remove(&mailbox_id); break; } diff --git a/crates/imap/src/op/expunge.rs b/crates/imap/src/op/expunge.rs index 7a7fcb90..34ac5d5a 100644 --- a/crates/imap/src/op/expunge.rs +++ b/crates/imap/src/op/expunge.rs @@ -8,7 +8,11 @@ use std::{sync::Arc, time::Instant}; use ahash::AHashMap; use directory::Permission; -use email::message::{delete::EmailDeletion, metadata::MessageData}; +use email::message::{ + cache::{MessageCache, MessageCacheAccess}, + delete::EmailDeletion, + metadata::MessageData, +}; use imap_proto::{ Command, ResponseCode, ResponseType, StatusResponse, parser::parse_sequence_set, @@ -16,9 +20,9 @@ use imap_proto::{ }; use trc::AddContext; -use crate::core::{SavedSearch, SelectedMailbox, Session, SessionData}; -use common::{ImapId, listener::SessionStream, storage::index::ObjectIndexBuilder}; -use jmap_proto::types::{acl::Acl, collection::Collection, keyword::Keyword, property::Property}; +use crate::core::{ImapId, SavedSearch, SelectedMailbox, Session, SessionData}; +use common::{listener::SessionStream, storage::index::ObjectIndexBuilder}; +use jmap_proto::types::{acl::Acl, collection::Collection, keyword::Keyword}; use store::{roaring::RoaringBitmap, write::BatchBuilder}; use super::{ImapContext, ToModSeq}; @@ -111,28 +115,14 @@ impl SessionData { ) -> trc::Result<()> { // Obtain message ids let account_id = mailbox.id.account_id; - let mut deleted_ids = self - .server - .get_tag( - account_id, - Collection::Email, - Property::MailboxIds, - mailbox.id.mailbox_id, - ) - .await - .caused_by(trc::location!())? - .unwrap_or_default() - & self - .server - .get_tag( - account_id, - Collection::Email, - Property::Keywords, - Keyword::Deleted, - ) + let mut deleted_ids = RoaringBitmap::from_iter( + self.server + .get_cached_messages(account_id) .await .caused_by(trc::location!())? - .unwrap_or_default(); + .in_mailbox_with_keyword(mailbox.id.mailbox_id, &Keyword::Deleted) + .map(|(id, _)| id), + ); // Filter by sequence if let Some(sequence) = &sequence { diff --git a/crates/imap/src/op/fetch.rs b/crates/imap/src/op/fetch.rs index 10fb2cf9..7cb94453 100644 --- a/crates/imap/src/op/fetch.rs +++ b/crates/imap/src/op/fetch.rs @@ -13,10 +13,13 @@ use crate::{ use ahash::AHashMap; use common::{listener::SessionStream, storage::index::ObjectIndexBuilder}; use directory::Permission; -use email::message::metadata::{ - ArchivedAddress, ArchivedGetHeader, ArchivedHeaderName, ArchivedHeaderValue, - ArchivedMessageMetadata, ArchivedMessageMetadataContents, ArchivedMetadataPartType, - DecodedParts, MessageData, MessageMetadata, +use email::message::{ + cache::MessageCache, + metadata::{ + ArchivedAddress, ArchivedGetHeader, ArchivedHeaderName, ArchivedHeaderValue, + ArchivedMessageMetadata, ArchivedMessageMetadataContents, ArchivedMetadataPartType, + DecodedParts, MessageData, MessageMetadata, + }, }; use imap_proto::{ Command, ResponseCode, ResponseType, StatusResponse, @@ -297,10 +300,15 @@ impl SessionData { .iter() .map(|id| trc::Value::from(id.2)) .collect::>(); + let message_cache = self + .server + .get_cached_messages(account_id) + .await + .imap_ctx(&arguments.tag, trc::location!())?; for (seqnum, uid, id) in ids { // Obtain attributes and keywords - let (metadata_, data_) = if let (Some(email), Some(keywords)) = ( + let (metadata_, data) = if let (Some(email), Some(data)) = ( self.server .get_archive_by_property( account_id, @@ -310,12 +318,9 @@ impl SessionData { ) .await .imap_ctx(&arguments.tag, trc::location!())?, - self.server - .get_archive(account_id, Collection::Email, id) - .await - .imap_ctx(&arguments.tag, trc::location!())?, + message_cache.items.get(&id), ) { - (email, keywords) + (email, data) } else { trc::event!( Store(trc::StoreEvent::NotFound), @@ -330,9 +335,6 @@ impl SessionData { let metadata = metadata_ .unarchive::() .imap_ctx(&arguments.tag, trc::location!())?; - let data = data_ - .to_unarchived::() - .imap_ctx(&arguments.tag, trc::location!())?; // Fetch and parse blob let raw_message: Cow<[u8]> = if needs_blobs { @@ -367,12 +369,8 @@ impl SessionData { // Build response let mut items = Vec::with_capacity(arguments.attributes.len()); - let set_seen_flag = set_seen_flags - && !data - .inner - .keywords - .iter() - .any(|k| k == &ArchivedKeyword::Seen); + let set_seen_flag = + set_seen_flags && !data.keywords.iter().any(|k| k == &ArchivedKeyword::Seen); for attribute in &arguments.attributes { match attribute { @@ -383,9 +381,9 @@ impl SessionData { } Attribute::Flags => { let mut flags = data - .inner .keywords .iter() + .cloned() .map(Flag::from) .collect::>(); if set_seen_flag { @@ -499,18 +497,19 @@ impl SessionData { } Attribute::ModSeq => { items.push(DataItem::ModSeq { - modseq: u64::from(data.inner.change_id) + 1, + modseq: data.change_id + 1, }); } Attribute::EmailId => { items.push(DataItem::EmailId { - email_id: Id::from_parts(account_id, id).to_string(), + email_id: Id::from_parts(account_id, id).to_string().into(), }); } Attribute::ThreadId => { items.push(DataItem::ThreadId { - thread_id: Id::from_parts(account_id, u32::from(data.inner.thread_id)) - .to_string(), + thread_id: Id::from_parts(account_id, data.thread_id) + .to_string() + .into(), }); } } @@ -519,9 +518,9 @@ impl SessionData { // Add flags to the response if the message was unseen if set_seen_flag && !arguments.attributes.contains(&Attribute::Flags) { let mut flags = data - .inner .keywords .iter() + .cloned() .map(Flag::from) .collect::>(); flags.push(Flag::Seen); @@ -535,23 +534,33 @@ impl SessionData { // Add to set flags if set_seen_flag { - let mut new_data = data - .deserialize() - .imap_ctx(&arguments.tag, trc::location!())?; - new_data.keywords.push(Keyword::Seen); - new_data.change_id = batch.change_id(); - - batch - .with_account_id(account_id) - .with_collection(Collection::Email) - .update_document(id) - .custom( - ObjectIndexBuilder::new() - .with_current(data) - .with_changes(new_data), - ) + if let Some(data_) = self + .server + .get_archive(account_id, Collection::Email, id) + .await .imap_ctx(&arguments.tag, trc::location!())? - .commit_point(); + { + let data = data_ + .to_unarchived::() + .imap_ctx(&arguments.tag, trc::location!())?; + let mut new_data = data + .deserialize() + .imap_ctx(&arguments.tag, trc::location!())?; + new_data.keywords.push(Keyword::Seen); + new_data.change_id = batch.change_id(); + + batch + .with_account_id(account_id) + .with_collection(Collection::Email) + .update_document(id) + .custom( + ObjectIndexBuilder::new() + .with_current(data) + .with_changes(new_data), + ) + .imap_ctx(&arguments.tag, trc::location!())? + .commit_point(); + } } } @@ -565,7 +574,7 @@ impl SessionData { .imap_ctx(&arguments.tag, trc::location!()) { Ok(_) => { - modseq = change_id.into(); + modseq = change_id; } Err(err) => { if !err.is_assertion_failure() { diff --git a/crates/imap/src/op/idle.rs b/crates/imap/src/op/idle.rs index d206a0a8..cf35d80e 100644 --- a/crates/imap/src/op/idle.rs +++ b/crates/imap/src/op/idle.rs @@ -156,7 +156,7 @@ impl SessionData { // List added mailboxes for mailbox_name in changes.added { ListItem { - mailbox_name: mailbox_name.to_string(), + mailbox_name, attributes: vec![], tags: vec![], } @@ -206,7 +206,7 @@ impl SessionData { .changes( mailbox.id.account_id, Collection::Email, - modseq.map(Query::Since).unwrap_or(Query::All), + Query::Since(modseq), ) .await .caused_by(trc::location!())?; @@ -229,7 +229,7 @@ impl SessionData { return self .fetch( fetch::Arguments { - tag: String::new(), + tag: "".into(), sequence_set: Sequence::List { items: changed_ids .into_iter() diff --git a/crates/imap/src/op/list.rs b/crates/imap/src/op/list.rs index 4231ec1b..93ac6188 100644 --- a/crates/imap/src/op/list.rs +++ b/crates/imap/src/op/list.rs @@ -11,6 +11,7 @@ use crate::{ spawn_op, }; use common::listener::SessionStream; +use compact_str::CompactString; use directory::Permission; use imap_proto::{ Command, StatusResponse, @@ -57,7 +58,7 @@ impl Session { is_rev2: self.version.is_rev2(), is_lsub, list_items: vec![ListItem { - mailbox_name: String::new(), + mailbox_name: "".into(), attributes: vec![Attribute::NoSelect], tags: vec![], }], @@ -162,7 +163,7 @@ impl SessionData { // Append reference name if !patterns.is_empty() && !reference_name.is_empty() { patterns.iter_mut().for_each(|item| { - *item = format!("{}{}", reference_name, item); + *item = format!("{}{}", reference_name, item).into(); }) } @@ -177,7 +178,7 @@ impl SessionData { && matches_pattern(&patterns, &self.server.core.jmap.shared_folder) { list_items.push(ListItem { - mailbox_name: self.server.core.jmap.shared_folder.clone(), + mailbox_name: self.server.core.jmap.shared_folder.as_str().into(), attributes: if include_children { vec![Attribute::HasChildren, Attribute::NoSelect] } else { @@ -269,7 +270,7 @@ impl SessionData { if let Some(include_status) = include_status { for list_item in &list_items { match self - .status(list_item.mailbox_name.to_string(), include_status) + .status(list_item.mailbox_name.clone(), include_status) .await .imap_ctx(&tag, trc::location!()) { @@ -320,7 +321,7 @@ impl SessionData { } #[allow(clippy::while_let_on_iterator)] -pub fn matches_pattern(patterns: &[String], mailbox_name: &str) -> bool { +pub fn matches_pattern(patterns: &[CompactString], mailbox_name: &str) -> bool { if patterns.is_empty() { return true; } diff --git a/crates/imap/src/op/login.rs b/crates/imap/src/op/login.rs index 2312a56b..046e6ea1 100644 --- a/crates/imap/src/op/login.rs +++ b/crates/imap/src/op/login.rs @@ -16,8 +16,8 @@ impl Session { self.authenticate( Credentials::Plain { - username: arguments.username, - secret: arguments.password, + username: arguments.username.to_string(), + secret: arguments.password.to_string(), }, arguments.tag, ) diff --git a/crates/imap/src/op/mod.rs b/crates/imap/src/op/mod.rs index 6036f18c..64343026 100644 --- a/crates/imap/src/op/mod.rs +++ b/crates/imap/src/op/mod.rs @@ -51,9 +51,9 @@ impl FromModSeq for Query { } } -impl ToModSeq for Option { +impl ToModSeq for u64 { fn to_modseq(&self) -> u64 { - self.map(|modseq| modseq + 1).unwrap_or(0) + if *self > 0 { *self + 1 } else { 0 } } } diff --git a/crates/imap/src/op/namespace.rs b/crates/imap/src/op/namespace.rs index 465c4990..3d5b37c6 100644 --- a/crates/imap/src/op/namespace.rs +++ b/crates/imap/src/op/namespace.rs @@ -30,7 +30,7 @@ impl Session { .serialize( Response { shared_prefix: if self.state.session_data().mailboxes.lock().len() > 1 { - self.server.core.jmap.shared_folder.clone().into() + Some(self.server.core.jmap.shared_folder.as_str().into()) } else { None }, diff --git a/crates/imap/src/op/quota.rs b/crates/imap/src/op/quota.rs index af2187b5..82605a7c 100644 --- a/crates/imap/src/op/quota.rs +++ b/crates/imap/src/op/quota.rs @@ -185,9 +185,9 @@ impl SessionData { // Build response let response = Response { - quota_root_items: vec![arguments.name, format!("#{account_id}")], + quota_root_items: vec![arguments.name, format!("#{account_id}").into()], quota_items: vec![QuotaItem { - name: format!("#{account_id}"), + name: format!("#{account_id}").into(), resources: vec![QuotaResource { resource: QuotaResourceName::Storage, total: access_token.quota, diff --git a/crates/imap/src/op/rename.rs b/crates/imap/src/op/rename.rs index 0b87697c..691e8751 100644 --- a/crates/imap/src/op/rename.rs +++ b/crates/imap/src/op/rename.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{collections::BTreeMap, time::Instant}; +use std::time::Instant; use crate::{ core::{Session, SessionData}, @@ -169,64 +169,11 @@ impl SessionData { .with_changes(new_mailbox), ) .imap_ctx(&arguments.tag, trc::location!())?; - let change_id = batch.change_id(); self.server .commit_batch(batch) .await .imap_ctx(&arguments.tag, trc::location!())?; - let mut mailboxes = if !create_ids.is_empty() { - self.add_created_mailboxes(&mut params, change_id, create_ids) - .add_context(|err| err.id(arguments.tag.clone()))? - } else { - self.mailboxes.lock() - }; - - // Rename mailbox cache - for account in mailboxes.iter_mut() { - if account.account_id == params.account_id { - // Update state - account.state_mailbox = change_id.into(); - - // Update parents - if arguments.mailbox_name.contains('/') { - let mut parent_path = arguments.mailbox_name.split('/').collect::>(); - parent_path.pop(); - let parent_path = parent_path.join("/"); - if let Some(old_parent_id) = account.mailbox_names.get(&parent_path) { - if let Some(old_parent) = account.mailbox_state.get_mut(old_parent_id) { - let prefix = format!("{}/", parent_path); - old_parent.has_children = account.mailbox_names.keys().any(|name| { - name != &arguments.mailbox_name && name.starts_with(&prefix) - }); - } - } - } - if let Some(parent_mailbox) = params - .parent_mailbox_id - .and_then(|id| account.mailbox_state.get_mut(&id)) - { - parent_mailbox.has_children = true; - } - - let prefix = format!("{}/", arguments.mailbox_name); - let mut new_mailbox_names = BTreeMap::new(); - for (mailbox_name, mailbox_id) in std::mem::take(&mut account.mailbox_names) { - if mailbox_name != arguments.mailbox_name { - if let Some(child_name) = mailbox_name.strip_prefix(&prefix) { - new_mailbox_names - .insert(format!("{}/{}", params.full_path, child_name), mailbox_id); - } else { - new_mailbox_names.insert(mailbox_name, mailbox_id); - } - } - } - new_mailbox_names.insert(params.full_path, mailbox_id); - account.mailbox_names = new_mailbox_names.into_iter().collect(); - break; - } - } - trc::event!( Imap(trc::ImapEvent::RenameMailbox), SpanId = self.session_id, diff --git a/crates/imap/src/op/search.rs b/crates/imap/src/op/search.rs index b88394ba..9624044a 100644 --- a/crates/imap/src/op/search.rs +++ b/crates/imap/src/op/search.rs @@ -6,8 +6,9 @@ use std::{sync::Arc, time::Instant}; -use common::{ImapId, listener::SessionStream}; +use common::listener::SessionStream; use directory::Permission; +use email::message::cache::{MessageCache, MessageCacheAccess}; use imap_proto::{ Command, StatusResponse, protocol::{ @@ -30,7 +31,7 @@ use tokio::sync::watch; use trc::AddContext; use crate::{ - core::{SavedSearch, SelectedMailbox, Session, SessionData}, + core::{ImapId, SavedSearch, SelectedMailbox, Session, SessionData}, spawn_op, }; @@ -260,16 +261,14 @@ impl SessionData { ) -> trc::Result<(ResultSet, bool)> { // Obtain message ids let mut filters = Vec::with_capacity(imap_filter.len() + 1); - let message_ids = self + let cache = self .server - .get_tag( - mailbox.id.account_id, - Collection::Email, - Property::MailboxIds, - mailbox.id.mailbox_id, - ) - .await? - .unwrap_or_default(); + .get_cached_messages(mailbox.id.account_id) + .await + .caused_by(trc::location!())?; + let message_ids = + RoaringBitmap::from_iter(cache.in_mailbox(mailbox.id.mailbox_id).map(|(id, _)| id)); + filters.push(query::Filter::is_in_set(message_ids.clone())); // Convert query @@ -358,32 +357,32 @@ impl SessionData { fts_filters.push(FtsFilter::Or); fts_filters.push(FtsFilter::has_text( Field::Header(HeaderName::From), - &text, + text.as_str(), Language::None, )); fts_filters.push(FtsFilter::has_text( Field::Header(HeaderName::To), - &text, + text.as_str(), Language::None, )); fts_filters.push(FtsFilter::has_text( Field::Header(HeaderName::Cc), - &text, + text.as_str(), Language::None, )); fts_filters.push(FtsFilter::has_text( Field::Header(HeaderName::Bcc), - &text, + text.as_str(), Language::None, )); fts_filters.push(FtsFilter::has_text_detect( Field::Header(HeaderName::Subject), - &text, + text.as_str(), self.server.core.jmap.default_language, )); fts_filters.push(FtsFilter::has_text_detect( Field::Body, - &text, + text.as_str(), self.server.core.jmap.default_language, )); fts_filters.push(FtsFilter::has_text_detect( @@ -452,10 +451,9 @@ impl SessionData { filters.push(query::Filter::is_in_set(message_ids.clone())); } search::Filter::Answered => { - filters.push(query::Filter::is_in_bitmap( - Property::Keywords, - Keyword::Answered, - )); + filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( + cache.with_keyword(&Keyword::Answered).map(|(id, _)| id), + ))); } search::Filter::Before(date) => { filters.push(query::Filter::lt( @@ -464,28 +462,26 @@ impl SessionData { )); } search::Filter::Deleted => { - filters.push(query::Filter::is_in_bitmap( - Property::Keywords, - Keyword::Deleted, - )); + filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( + cache.with_keyword(&Keyword::Deleted).map(|(id, _)| id), + ))); } search::Filter::Draft => { - filters.push(query::Filter::is_in_bitmap( - Property::Keywords, - Keyword::Draft, - )); + filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( + cache.with_keyword(&Keyword::Draft).map(|(id, _)| id), + ))); } search::Filter::Flagged => { - filters.push(query::Filter::is_in_bitmap( - Property::Keywords, - Keyword::Flagged, - )); + filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( + cache.with_keyword(&Keyword::Flagged).map(|(id, _)| id), + ))); } search::Filter::Keyword(keyword) => { - filters.push(query::Filter::is_in_bitmap( - Property::Keywords, - Keyword::from(keyword), - )); + filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( + cache + .with_keyword(&Keyword::from(keyword)) + .map(|(id, _)| id), + ))); } search::Filter::Larger(size) => { filters.push(query::Filter::gt(Property::Size, size.serialize())); @@ -503,10 +499,9 @@ impl SessionData { filters.push(query::Filter::End); } search::Filter::Seen => { - filters.push(query::Filter::is_in_bitmap( - Property::Keywords, - Keyword::Seen, - )); + filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( + cache.with_keyword(&Keyword::Seen).map(|(id, _)| id), + ))); } search::Filter::SentBefore(date) => { filters.push(query::Filter::lt( @@ -543,50 +538,46 @@ impl SessionData { } search::Filter::Unanswered => { filters.push(query::Filter::Not); - filters.push(query::Filter::is_in_bitmap( - Property::Keywords, - Keyword::Answered, - )); + filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( + cache.with_keyword(&Keyword::Answered).map(|(id, _)| id), + ))); filters.push(query::Filter::End); } search::Filter::Undeleted => { filters.push(query::Filter::Not); - filters.push(query::Filter::is_in_bitmap( - Property::Keywords, - Keyword::Deleted, - )); + filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( + cache.with_keyword(&Keyword::Deleted).map(|(id, _)| id), + ))); filters.push(query::Filter::End); } search::Filter::Undraft => { filters.push(query::Filter::Not); - filters.push(query::Filter::is_in_bitmap( - Property::Keywords, - Keyword::Draft, - )); + filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( + cache.with_keyword(&Keyword::Draft).map(|(id, _)| id), + ))); filters.push(query::Filter::End); } search::Filter::Unflagged => { filters.push(query::Filter::Not); - filters.push(query::Filter::is_in_bitmap( - Property::Keywords, - Keyword::Flagged, - )); + filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( + cache.with_keyword(&Keyword::Flagged).map(|(id, _)| id), + ))); filters.push(query::Filter::End); } search::Filter::Unkeyword(keyword) => { filters.push(query::Filter::Not); - filters.push(query::Filter::is_in_bitmap( - Property::Keywords, - Keyword::from(keyword), - )); + filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( + cache + .with_keyword(&Keyword::from(keyword)) + .map(|(id, _)| id), + ))); filters.push(query::Filter::End); } search::Filter::Unseen => { filters.push(query::Filter::Not); - filters.push(query::Filter::is_in_bitmap( - Property::Keywords, - Keyword::Seen, - )); + filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( + cache.with_keyword(&Keyword::Seen).map(|(id, _)| id), + ))); filters.push(query::Filter::End); } search::Filter::And => { @@ -666,10 +657,9 @@ impl SessionData { } search::Filter::ThreadId(id) => { if let Some(id) = Id::from_bytes(id.as_bytes()) { - filters.push(query::Filter::is_in_bitmap( - Property::ThreadId, - id.document_id(), - )); + filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( + cache.in_thread(id.document_id()).map(|(id, _)| id), + ))); } else { return Err(trc::ImapEvent::Error .into_err() diff --git a/crates/imap/src/op/select.rs b/crates/imap/src/op/select.rs index ff468691..150cf498 100644 --- a/crates/imap/src/op/select.rs +++ b/crates/imap/src/op/select.rs @@ -45,41 +45,11 @@ impl Session { if let Some(mailbox) = data.get_mailbox_by_name(&arguments.mailbox_name) { // Try obtaining the mailbox from the cache - let state = { - let modseq = data - .get_modseq(mailbox.account_id) - .await - .imap_ctx(&arguments.tag, trc::location!())?; - - if let Some(cached_state) = - self.server - .inner - .cache - .mailbox - .get(&mailbox) - .and_then(|cached_state| { - if cached_state.modseq.unwrap_or(0) >= modseq.unwrap_or(0) { - Some(cached_state) - } else { - None - } - }) - { - cached_state.as_ref().clone() - } else { - let new_state = Arc::new( - data.fetch_messages(&mailbox) - .await - .imap_ctx(&arguments.tag, trc::location!())?, - ); - self.server - .inner - .cache - .mailbox - .insert(mailbox, new_state.clone()); - new_state.as_ref().clone() - } - }; + let state = data + .fetch_messages(&mailbox, None) + .await + .imap_ctx(&arguments.tag, trc::location!())? + .unwrap(); // Synchronize messages let closed_previous = self.state.close_mailbox(); @@ -87,8 +57,7 @@ impl Session { // Build new state let is_rev2 = self.version.is_rev2(); - let uid_validity = state.uid_validity; - let uid_next = state.uid_next; + let mailbox_state = data.mailbox_state(&mailbox).unwrap(); let total_messages = state.total_messages; let highest_modseq = if is_condstore { HighestModSeq::new(state.modseq.to_modseq()).into() @@ -111,11 +80,11 @@ impl Session { .details("QRESYNC is not enabled.") .id(arguments.tag)); } - if qresync.uid_validity == uid_validity { + if qresync.uid_validity == mailbox_state.uid_validity as u32 { // Send flags for changed messages data.fetch( fetch::Arguments { - tag: String::new(), + tag: "".into(), sequence_set: qresync .known_uids .or_else(|| qresync.seq_match.map(|(_, s)| s)) @@ -146,8 +115,8 @@ impl Session { AccountId = mailbox.id.account_id, MailboxId = mailbox.id.mailbox_id, Total = total_messages, - UidNext = uid_next, - UidValidity = uid_validity, + UidNext = mailbox_state.uid_next, + UidValidity = mailbox_state.uid_validity, Elapsed = op_start.elapsed() ); @@ -157,13 +126,14 @@ impl Session { total_messages, recent_messages: 0, unseen_seq: 0, - uid_validity, - uid_next, + uid_validity: mailbox_state.uid_validity as u32, + uid_next: mailbox_state.uid_next as u32, closed_previous, is_rev2, highest_modseq, mailbox_id: Id::from_parts(mailbox.id.account_id, mailbox.id.mailbox_id) - .to_string(), + .to_string() + .into(), }; // Update state diff --git a/crates/imap/src/op/status.rs b/crates/imap/src/op/status.rs index 926ef69a..9eb131d4 100644 --- a/crates/imap/src/op/status.rs +++ b/crates/imap/src/op/status.rs @@ -4,15 +4,17 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{sync::Arc, time::Instant}; +use std::time::Instant; use crate::{ - core::{Session, SessionData}, + core::{Mailbox, Session, SessionData}, op::ImapContext, spawn_op, }; -use common::{Mailbox, listener::SessionStream}; +use common::listener::SessionStream; +use compact_str::CompactString; use directory::Permission; +use email::message::cache::{MessageCache, MessageCacheAccess}; use imap_proto::{ Command, ResponseCode, StatusResponse, parser::PushUnique, @@ -88,7 +90,11 @@ impl Session { } impl SessionData { - pub async fn status(&self, mailbox_name: String, items: &[Status]) -> trc::Result { + pub async fn status( + &self, + mailbox_name: CompactString, + items: &[Status], + ) -> trc::Result { // Get mailbox id let mailbox = if let Some(mailbox) = self.get_mailbox_by_name(&mailbox_name) { mailbox @@ -119,7 +125,7 @@ impl SessionData { Status::UidNext | Status::UidValidity => { StatusItemType::Number(1) } - Status::MailboxId => StatusItemType::String("none".to_string()), + Status::MailboxId => StatusItemType::String("none".into()), }, ) }) @@ -139,47 +145,35 @@ impl SessionData { for account in self.mailboxes.lock().iter_mut() { if account.account_id == mailbox.account_id { - let mailbox_state = account - .mailbox_state - .entry(mailbox.mailbox_id) - .or_insert_with(Mailbox::default); - let update_recent = mailbox_state.total_messages.is_none(); + let mailbox_state = + if let Some(mailbox_state) = account.mailbox_state.get(&mailbox.mailbox_id) { + mailbox_state + } else { + continue; + }; for item in items { match item { Status::Messages => { - if let Some(value) = mailbox_state.total_messages { - items_response.push((*item, StatusItemType::Number(value))); - } else { - items_update.push_unique(*item); - } + items_response.push(( + *item, + StatusItemType::Number(mailbox_state.total_messages), + )); } Status::UidNext => { - if let Some(value) = mailbox_state.uid_next { - items_response.push((*item, StatusItemType::Number(value))); - } else { - items_update.push_unique(*item); - } + items_response + .push((*item, StatusItemType::Number(mailbox_state.uid_next))); } Status::UidValidity => { - if let Some(value) = mailbox_state.uid_validity { - items_response.push((*item, StatusItemType::Number(value))); - } else { - items_update.push_unique(*item); - } + items_response + .push((*item, StatusItemType::Number(mailbox_state.uid_validity))); } Status::Unseen => { - if let Some(value) = mailbox_state.total_unseen { - items_response.push((*item, StatusItemType::Number(value))); - } else { - items_update.push_unique(*item); - } + items_response + .push((*item, StatusItemType::Number(mailbox_state.total_unseen))); } Status::Deleted => { - if let Some(value) = mailbox_state.total_deleted { - items_response.push((*item, StatusItemType::Number(value))); - } else { - items_update.push_unique(*item); - } + items_response + .push((*item, StatusItemType::Number(mailbox_state.total_deleted))); } Status::DeletedStorage => { if let Some(value) = mailbox_state.total_deleted_storage { @@ -198,7 +192,7 @@ impl SessionData { Status::HighestModSeq => { items_response.push(( *item, - StatusItemType::Number(account.state_email.to_modseq()), + StatusItemType::Number(account.state.email.to_modseq()), )); } Status::MailboxId => { @@ -206,16 +200,13 @@ impl SessionData { *item, StatusItemType::String( Id::from_parts(mailbox.account_id, mailbox.mailbox_id) - .to_string(), + .to_string() + .into(), ), )); } Status::Recent => { - if !update_recent { - items_response.push((*item, StatusItemType::Number(0))); - } else { - items_update.push_unique(*item); - } + items_response.push((*item, StatusItemType::Number(0))); } } } @@ -226,130 +217,37 @@ impl SessionData { if !items_update.is_empty() { // Retrieve latest values let mut values_update = Vec::with_capacity(items_update.len()); - let mailbox_message_ids = self + + let cache = self .server - .get_tag( - mailbox.account_id, - Collection::Email, - Property::MailboxIds, - mailbox.mailbox_id, - ) - .await - .caused_by(trc::location!())? - .map(Arc::new); - let message_ids = self - .server - .get_document_ids(mailbox.account_id, Collection::Email) + .get_cached_messages(mailbox.account_id) .await .caused_by(trc::location!())?; for item in items_update { let result = match item { - Status::Messages => mailbox_message_ids.as_ref().map(|v| v.len()).unwrap_or(0), - Status::UidNext => self - .get_uid_next(&mailbox) + Status::DeletedStorage => self + .calculate_mailbox_size( + mailbox.account_id, + &RoaringBitmap::from_iter( + cache + .in_mailbox_with_keyword(mailbox.mailbox_id, &Keyword::Deleted) + .map(|x| x.0), + ), + ) .await - .caused_by(trc::location!())? as u64, - Status::UidValidity => u32::from( - self.server - .get_archive( - mailbox.account_id, - Collection::Mailbox, - mailbox.mailbox_id, - ) - .await? - .ok_or_else(|| { - trc::StoreEvent::UnexpectedError - .into_err() - .details("Mailbox unavailable") - .ctx(trc::Key::Reason, "Failed to obtain uid validity") - .caused_by(trc::location!()) - .account_id(mailbox.account_id) - .document_id(mailbox.mailbox_id) - })? - .unarchive::() - .caused_by(trc::location!())? - .uid_validity, - ) as u64, - Status::Unseen => { - if let (Some(message_ids), Some(mailbox_message_ids)) = - (&message_ids, &mailbox_message_ids) - { - if let Some(mut seen) = self - .server - .get_tag( - mailbox.account_id, - Collection::Email, - Property::Keywords, - Keyword::Seen, - ) - .await - .caused_by(trc::location!())? - { - seen ^= message_ids; - seen &= mailbox_message_ids.as_ref(); - seen.len() - } else { - mailbox_message_ids.len() - } - } else { - 0 - } - } - Status::Deleted => { - if let (Some(mailbox_message_ids), Some(mut deleted)) = ( - &mailbox_message_ids, - self.server - .get_tag( - mailbox.account_id, - Collection::Email, - Property::Keywords, - Keyword::Deleted, - ) - .await - .caused_by(trc::location!())?, - ) { - deleted &= mailbox_message_ids.as_ref(); - deleted.len() - } else { - 0 - } - } - Status::DeletedStorage => { - if let (Some(mailbox_message_ids), Some(mut deleted)) = ( - &mailbox_message_ids, - self.server - .get_tag( - mailbox.account_id, - Collection::Email, - Property::Keywords, - Keyword::Deleted, - ) - .await - .caused_by(trc::location!())?, - ) { - deleted &= mailbox_message_ids.as_ref(); - self.calculate_mailbox_size(mailbox.account_id, &deleted) - .await - .caused_by(trc::location!())? - } else { - 0 - } - } - Status::Size => { - if let Some(mailbox_message_ids) = &mailbox_message_ids { - self.calculate_mailbox_size(mailbox.account_id, mailbox_message_ids) - .await - .caused_by(trc::location!())? - } else { - 0 - } - } - Status::Recent => { - self.fetch_messages(&mailbox).await?; - 0 - } - Status::HighestModSeq | Status::MailboxId => { + .caused_by(trc::location!())?, + Status::Size => self + .calculate_mailbox_size( + mailbox.account_id, + &RoaringBitmap::from_iter( + cache.in_mailbox(mailbox.mailbox_id).map(|x| x.0), + ), + ) + .await + .caused_by(trc::location!())?, + + _ => { unreachable!() } }; @@ -368,11 +266,6 @@ impl SessionData { for (item, value) in values_update { match item { - Status::Messages => mailbox_state.total_messages = value.into(), - Status::UidNext => mailbox_state.uid_next = value.into(), - Status::UidValidity => mailbox_state.uid_validity = value.into(), - Status::Unseen => mailbox_state.total_unseen = value.into(), - Status::Deleted => mailbox_state.total_deleted = value.into(), Status::DeletedStorage => { mailbox_state.total_deleted_storage = value.into() } @@ -384,7 +277,7 @@ impl SessionData { .unwrap() .1 = StatusItemType::Number(0); } - Status::HighestModSeq | Status::MailboxId => { + _ => { unreachable!() } } diff --git a/crates/imap/src/op/subscribe.rs b/crates/imap/src/op/subscribe.rs index bab49207..cf3dc66c 100644 --- a/crates/imap/src/op/subscribe.rs +++ b/crates/imap/src/op/subscribe.rs @@ -11,6 +11,7 @@ use crate::{ spawn_op, }; use common::{listener::SessionStream, storage::index::ObjectIndexBuilder}; +use compact_str::CompactString; use directory::Permission; use imap_proto::{Command, ResponseCode, StatusResponse, receiver::Request}; use jmap_proto::types::collection::Collection; @@ -49,8 +50,8 @@ impl Session { impl SessionData { pub async fn subscribe_folder( &self, - tag: String, - mailbox_name: String, + tag: CompactString, + mailbox_name: CompactString, subscribe: bool, op_start: Instant, ) -> trc::Result { @@ -130,7 +131,6 @@ impl SessionData { .with_changes(new_mailbox), ) .imap_ctx(&tag, trc::location!())?; - let change_id = batch.change_id(); self.server .commit_batch(batch) .await @@ -139,7 +139,6 @@ impl SessionData { // Update mailbox cache for account in self.mailboxes.lock().iter_mut() { if account.account_id == account_id { - account.state_mailbox = change_id.into(); if let Some(mailbox) = account.mailbox_state.get_mut(&mailbox_id) { mailbox.is_subscribed = subscribe; } diff --git a/crates/imap/src/op/thread.rs b/crates/imap/src/op/thread.rs index 52d94da7..a86f6212 100644 --- a/crates/imap/src/op/thread.rs +++ b/crates/imap/src/op/thread.rs @@ -13,7 +13,7 @@ use crate::{ use ahash::AHashMap; use common::listener::SessionStream; use directory::Permission; -use email::thread::cache::ThreadCache; +use email::message::cache::MessageCache; use imap_proto::{ Command, StatusResponse, protocol::{ @@ -80,19 +80,19 @@ impl SessionData { } // Lock the cache - let thread_cache = self + let cache = self .server - .get_cached_thread_ids(mailbox.id.account_id) + .get_cached_messages(mailbox.id.account_id) .await .caused_by(trc::location!())?; // Group messages by thread let mut threads: AHashMap> = AHashMap::new(); let state = mailbox.state.lock(); - for (document_id, thread_id) in &thread_cache.threads { + for (document_id, item) in &cache.items { if result_set.results.contains(*document_id) { if let Some((imap_id, _)) = state.map_result_id(*document_id, is_uid) { - threads.entry(*thread_id).or_default().push(imap_id); + threads.entry(item.thread_id).or_default().push(imap_id); } } } diff --git a/crates/jmap-proto/Cargo.toml b/crates/jmap-proto/Cargo.toml index 01951df5..7e358cc7 100644 --- a/crates/jmap-proto/Cargo.toml +++ b/crates/jmap-proto/Cargo.toml @@ -15,6 +15,7 @@ ahash = { version = "0.8.2", features = ["serde"] } serde_json = { version = "1.0", features = ["raw_value"] } hashify = "0.2" rkyv = { version = "0.8.10", features = ["little_endian"] } +compact_str = { version = "0.9.0", features = ["rkyv", "serde"] } [dev-dependencies] tokio = { version = "1.23", features = ["full"] } diff --git a/crates/jmap-proto/src/types/keyword.rs b/crates/jmap-proto/src/types/keyword.rs index 3bfe6dbe..693a651d 100644 --- a/crates/jmap-proto/src/types/keyword.rs +++ b/crates/jmap-proto/src/types/keyword.rs @@ -6,6 +6,7 @@ use std::fmt::Display; +use compact_str::CompactString; use store::{Serialize, write::TagValue}; use crate::parser::{JsonObjectParser, json::Parser}; @@ -33,6 +34,7 @@ pub const OTHER: usize = 12; PartialEq, Eq, Hash, + Default, serde::Serialize, )] #[serde(untagged)] @@ -46,6 +48,7 @@ pub enum Keyword { Flagged, #[serde(rename(serialize = "$answered"))] Answered, + #[default] #[serde(rename(serialize = "$recent"))] Recent, #[serde(rename(serialize = "$important"))] @@ -62,7 +65,7 @@ pub enum Keyword { Forwarded, #[serde(rename(serialize = "$mdnsent"))] MdnSent, - Other(String), + Other(CompactString), } impl JsonObjectParser for Keyword { @@ -106,17 +109,18 @@ impl JsonObjectParser for Keyword { } if parser.is_eof || parser.skip_string() { - Ok(Keyword::Other( - String::from_utf8_lossy(parser.bytes[pos..parser.pos - 1].as_ref()).into_owned(), - )) + Ok(Keyword::Other(CompactString::from_utf8_lossy( + parser.bytes[pos..parser.pos - 1].as_ref(), + ))) } else { Err(parser.error_unterminated()) } } } -impl From for Keyword { - fn from(value: String) -> Self { +impl> From for Keyword { + fn from(value: T) -> Self { + let value = value.as_ref(); if value .as_bytes() .first() @@ -151,7 +155,7 @@ impl From for Keyword { } } - Keyword::Other(value) + Keyword::Other(CompactString::from(value)) } } @@ -216,7 +220,7 @@ impl Serialize for Keyword { } impl Keyword { - pub fn id(&self) -> Result { + pub fn id(&self) -> Result { match self { Keyword::Seen => Ok(SEEN as u32), Keyword::Draft => Ok(DRAFT as u32), @@ -230,11 +234,11 @@ impl Keyword { Keyword::Deleted => Ok(DELETED as u32), Keyword::Forwarded => Ok(FORWARDED as u32), Keyword::MdnSent => Ok(MDN_SENT as u32), - Keyword::Other(string) => Err(string.clone()), + Keyword::Other(string) => Err(string.as_str()), } } - pub fn into_id(self) -> Result { + pub fn into_id(self) -> Result { match self { Keyword::Seen => Ok(SEEN as u32), Keyword::Draft => Ok(DRAFT as u32), @@ -254,7 +258,7 @@ impl Keyword { } impl ArchivedKeyword { - pub fn id(&self) -> Result { + pub fn id(&self) -> Result { match self { ArchivedKeyword::Seen => Ok(SEEN as u32), ArchivedKeyword::Draft => Ok(DRAFT as u32), @@ -268,7 +272,7 @@ impl ArchivedKeyword { ArchivedKeyword::Deleted => Ok(DELETED as u32), ArchivedKeyword::Forwarded => Ok(FORWARDED as u32), ArchivedKeyword::MdnSent => Ok(MDN_SENT as u32), - ArchivedKeyword::Other(string) => Err(string.to_string()), + ArchivedKeyword::Other(string) => Err(string.as_str()), } } } @@ -277,7 +281,7 @@ impl From for TagValue { fn from(value: Keyword) -> Self { match value.into_id() { Ok(id) => TagValue::Id(id), - Err(string) => TagValue::Text(string.into_bytes()), + Err(string) => TagValue::Text(string.as_bytes().to_vec()), } } } @@ -286,7 +290,7 @@ impl From<&Keyword> for TagValue { fn from(value: &Keyword) -> Self { match value.id() { Ok(id) => TagValue::Id(id), - Err(string) => TagValue::Text(string.into_bytes()), + Err(string) => TagValue::Text(string.as_bytes().to_vec()), } } } @@ -295,7 +299,27 @@ impl From<&ArchivedKeyword> for TagValue { fn from(value: &ArchivedKeyword) -> Self { match value.id() { Ok(id) => TagValue::Id(id), - Err(string) => TagValue::Text(string.into_bytes()), + Err(string) => TagValue::Text(string.as_bytes().to_vec()), + } + } +} + +impl From<&ArchivedKeyword> for Keyword { + fn from(value: &ArchivedKeyword) -> Self { + match value { + ArchivedKeyword::Seen => Keyword::Seen, + ArchivedKeyword::Draft => Keyword::Draft, + ArchivedKeyword::Flagged => Keyword::Flagged, + ArchivedKeyword::Answered => Keyword::Answered, + ArchivedKeyword::Recent => Keyword::Recent, + ArchivedKeyword::Important => Keyword::Important, + ArchivedKeyword::Phishing => Keyword::Phishing, + ArchivedKeyword::Junk => Keyword::Junk, + ArchivedKeyword::NotJunk => Keyword::NotJunk, + ArchivedKeyword::Deleted => Keyword::Deleted, + ArchivedKeyword::Forwarded => Keyword::Forwarded, + ArchivedKeyword::MdnSent => Keyword::MdnSent, + ArchivedKeyword::Other(string) => Keyword::Other(string.as_str().into()), } } } diff --git a/crates/jmap/src/blob/download.rs b/crates/jmap/src/blob/download.rs index 5db30967..9ced3683 100644 --- a/crates/jmap/src/blob/download.rs +++ b/crates/jmap/src/blob/download.rs @@ -7,7 +7,8 @@ use std::ops::Range; use common::{Server, auth::AccessToken}; -use jmap_proto::types::{acl::Acl, blob::BlobId, collection::Collection, property::Property}; +use email::message::cache::MessageCache; +use jmap_proto::types::{acl::Acl, blob::BlobId, collection::Collection}; use std::future::Future; use store::BlobClass; use trc::AddContext; @@ -60,14 +61,7 @@ impl BlobDownload for Server { } => { if Collection::from(*collection) == Collection::Email { match self - .shared_items( - access_token, - *account_id, - Collection::Mailbox, - Collection::Email, - Property::MailboxIds, - Acl::ReadItems, - ) + .shared_messages(access_token, *account_id, Acl::ReadItems) .await { Ok(shared_messages) if shared_messages.contains(*document_id) => (), @@ -133,14 +127,7 @@ impl BlobDownload for Server { if Collection::from(*collection) == Collection::Email { access_token.is_member(*account_id) || self - .shared_items( - access_token, - *account_id, - Collection::Mailbox, - Collection::Email, - Property::MailboxIds, - Acl::ReadItems, - ) + .shared_messages(access_token, *account_id, Acl::ReadItems) .await? .contains(*document_id) } else { diff --git a/crates/jmap/src/email/copy.rs b/crates/jmap/src/email/copy.rs index f7733530..27bacd72 100644 --- a/crates/jmap/src/email/copy.rs +++ b/crates/jmap/src/email/copy.rs @@ -6,7 +6,10 @@ use common::{Server, auth::AccessToken}; -use email::{mailbox::manage::MailboxFnc, message::copy::EmailCopy}; +use email::{ + mailbox::cache::MessageMailboxCache, + message::{cache::MessageCache, copy::EmailCopy}, +}; use http_proto::HttpSessionData; use jmap_proto::{ error::set::SetError, @@ -74,16 +77,9 @@ impl JmapEmailCopy for Server { }; let from_message_ids = self - .owned_or_shared_items( - access_token, - from_account_id, - Collection::Mailbox, - Collection::Email, - Property::MailboxIds, - Acl::ReadItems, - ) + .owned_or_shared_messages(access_token, from_account_id, Acl::ReadItems) .await?; - let mailbox_ids = self.mailbox_get_or_create(account_id).await?; + let mailbox_ids = self.get_cached_mailboxes(account_id).await?; let can_add_mailbox_ids = if access_token.is_shared(account_id) { self.shared_containers(access_token, account_id, Collection::Mailbox, Acl::AddItems) .await? @@ -193,7 +189,7 @@ impl JmapEmailCopy for Server { // Verify that the mailboxIds are valid for mailbox_id in &mailboxes { - if !mailbox_ids.contains(*mailbox_id) { + if !mailbox_ids.items.contains_key(mailbox_id) { response.not_created.append( id, SetError::invalid_properties() diff --git a/crates/jmap/src/email/get.rs b/crates/jmap/src/email/get.rs index e82ed853..f774f812 100644 --- a/crates/jmap/src/email/get.rs +++ b/crates/jmap/src/email/get.rs @@ -6,12 +6,12 @@ use common::{Server, auth::AccessToken}; -use email::{ - message::metadata::{ +use email::message::{ + cache::MessageCache, + metadata::{ ArchivedGetHeader, ArchivedHeaderName, ArchivedMetadataPartType, MessageData, MessageMetadata, }, - thread::cache::ThreadCache, }; use jmap_proto::{ method::get::{GetRequest, GetResponse}, @@ -103,25 +103,18 @@ impl EmailGet for Server { let account_id = request.account_id.document_id(); let message_ids = self - .owned_or_shared_items( - access_token, - account_id, - Collection::Mailbox, - Collection::Email, - Property::MailboxIds, - Acl::ReadItems, - ) + .owned_or_shared_messages(access_token, account_id, Acl::ReadItems) .await?; let ids = if let Some(ids) = ids { ids } else { - self.get_cached_thread_ids(account_id) + self.get_cached_messages(account_id) .await .caused_by(trc::location!())? - .threads + .items .iter() .take(self.core.jmap.get_max_objects) - .map(|(document_id, thread_id)| Id::from_parts(*thread_id, *document_id)) + .map(|(document_id, item)| Id::from_parts(item.thread_id, *document_id)) .collect() }; let mut response = GetResponse { diff --git a/crates/jmap/src/email/import.rs b/crates/jmap/src/email/import.rs index bc567ead..899b0a2d 100644 --- a/crates/jmap/src/email/import.rs +++ b/crates/jmap/src/email/import.rs @@ -6,7 +6,7 @@ use common::{Server, auth::AccessToken}; use email::{ - mailbox::manage::MailboxFnc, + mailbox::cache::MessageMailboxCache, message::ingest::{EmailIngest, IngestEmail, IngestSource}, }; use http_proto::HttpSessionData; @@ -51,7 +51,7 @@ impl EmailImport for Server { .assert_state(account_id, Collection::Email, &request.if_in_state) .await?; - let valid_mailbox_ids = self.mailbox_get_or_create(account_id).await?; + let valid_mailbox_ids = self.get_cached_mailboxes(account_id).await?; let can_add_mailbox_ids = if access_token.is_shared(account_id) { self.shared_containers(access_token, account_id, Collection::Mailbox, Acl::AddItems) .await? @@ -91,7 +91,7 @@ impl EmailImport for Server { continue; } for mailbox_id in &mailbox_ids { - if !valid_mailbox_ids.contains(*mailbox_id) { + if !valid_mailbox_ids.items.contains_key(mailbox_id) { response.not_created.append( id, SetError::invalid_properties() diff --git a/crates/jmap/src/email/query.rs b/crates/jmap/src/email/query.rs index e1dc5641..91df3a96 100644 --- a/crates/jmap/src/email/query.rs +++ b/crates/jmap/src/email/query.rs @@ -4,8 +4,8 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{Server, auth::AccessToken}; -use email::thread::cache::ThreadCache; +use common::{MessageItemCache, MessageStoreCache, Server, auth::AccessToken}; +use email::message::cache::{MessageCache, MessageCacheAccess}; use jmap_proto::{ method::query::{Comparator, Filter, QueryRequest, QueryResponse, SortProperty}, object::email::QueryArguments, @@ -16,6 +16,7 @@ use nlp::language::Language; use std::future::Future; use store::{ SerializeInfallible, + ahash::AHashMap, fts::{Field, FilterGroup, FtsFilter, IntoFilterGroup}, query::{self}, roaring::RoaringBitmap, @@ -29,13 +30,6 @@ pub trait EmailQuery: Sync + Send { request: QueryRequest, access_token: &AccessToken, ) -> impl Future> + Send; - - fn thread_keywords( - &self, - account_id: u32, - keyword: Keyword, - match_all: bool, - ) -> impl Future> + Send; } impl EmailQuery for Server { @@ -46,6 +40,7 @@ impl EmailQuery for Server { ) -> trc::Result { let account_id = request.account_id.document_id(); let mut filters = Vec::with_capacity(request.filter.len()); + let cache = self.get_cached_messages(account_id).await?; for cond_group in std::mem::take(&mut request.filter).into_filter_group() { match cond_group { @@ -185,18 +180,18 @@ impl EmailQuery for Server { } FilterGroup::Store(cond) => { match cond { - Filter::InMailbox(mailbox) => filters.push(query::Filter::is_in_bitmap( - Property::MailboxIds, - mailbox.document_id(), - )), + Filter::InMailbox(mailbox) => { + filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( + cache.in_mailbox(mailbox.document_id()).map(|(id, _)| *id), + ))) + } Filter::InMailboxOtherThan(mailboxes) => { filters.push(query::Filter::Not); filters.push(query::Filter::Or); for mailbox in mailboxes { - filters.push(query::Filter::is_in_bitmap( - Property::MailboxIds, - mailbox.document_id(), - )); + filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( + cache.in_mailbox(mailbox.document_id()).map(|(id, _)| *id), + ))); } filters.push(query::Filter::End); filters.push(query::Filter::End); @@ -213,29 +208,29 @@ impl EmailQuery for Server { Filter::MaxSize(size) => { filters.push(query::Filter::lt(Property::Size, size.serialize())) } - Filter::AllInThreadHaveKeyword(keyword) => { - filters.push(query::Filter::is_in_set( - self.thread_keywords(account_id, keyword, true).await?, - )) - } - Filter::SomeInThreadHaveKeyword(keyword) => { - filters.push(query::Filter::is_in_set( - self.thread_keywords(account_id, keyword, false).await?, - )) - } + Filter::AllInThreadHaveKeyword(keyword) => filters.push( + query::Filter::is_in_set(thread_keywords(&cache, keyword, true)), + ), + Filter::SomeInThreadHaveKeyword(keyword) => filters.push( + query::Filter::is_in_set(thread_keywords(&cache, keyword, false)), + ), Filter::NoneInThreadHaveKeyword(keyword) => { filters.push(query::Filter::Not); - filters.push(query::Filter::is_in_set( - self.thread_keywords(account_id, keyword, false).await?, - )); + filters.push(query::Filter::is_in_set(thread_keywords( + &cache, keyword, false, + ))); filters.push(query::Filter::End); } Filter::HasKeyword(keyword) => { - filters.push(query::Filter::is_in_bitmap(Property::Keywords, keyword)) + filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( + cache.with_keyword(&keyword).map(|(id, _)| *id), + ))); } Filter::NotKeyword(keyword) => { filters.push(query::Filter::Not); - filters.push(query::Filter::is_in_bitmap(Property::Keywords, keyword)); + filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( + cache.with_keyword(&keyword).map(|(id, _)| *id), + ))); filters.push(query::Filter::End); } Filter::HasAttachment(has_attach) => { @@ -262,10 +257,11 @@ impl EmailQuery for Server { Filter::SentAfter(date) => { filters.push(query::Filter::gt(Property::SentAt, date.serialize())) } - Filter::InThread(id) => filters.push(query::Filter::is_in_bitmap( - Property::ThreadId, - id.document_id(), - )), + Filter::InThread(id) => { + filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( + cache.in_thread(id.document_id()).map(|(id, _)| *id), + ))) + } Filter::And | Filter::Or | Filter::Not | Filter::Close => { filters.push(cond.into()); } @@ -283,15 +279,8 @@ impl EmailQuery for Server { let mut result_set = self.filter(account_id, Collection::Email, filters).await?; if access_token.is_shared(account_id) { result_set.apply_mask( - self.shared_items( - access_token, - account_id, - Collection::Mailbox, - Collection::Email, - Property::MailboxIds, - Acl::ReadItems, - ) - .await?, + self.shared_messages(access_token, account_id, Acl::ReadItems) + .await?, ); } let (response, paginate) = self.build_query_response(&result_set, &request).await?; @@ -324,32 +313,19 @@ impl EmailQuery for Server { query::Comparator::field(Property::SentAt, comparator.is_ascending) } SortProperty::HasKeyword => query::Comparator::set( - self.get_tag( - account_id, - Collection::Email, - Property::Keywords, - comparator.keyword.unwrap_or(Keyword::Seen), - ) - .await? - .unwrap_or_default(), + RoaringBitmap::from_iter( + cache + .with_keyword(&comparator.keyword.unwrap_or(Keyword::Seen)) + .map(|(id, _)| *id), + ), comparator.is_ascending, ), SortProperty::AllInThreadHaveKeyword => query::Comparator::set( - self.thread_keywords( - account_id, - comparator.keyword.unwrap_or(Keyword::Seen), - true, - ) - .await?, + thread_keywords(&cache, comparator.keyword.unwrap_or(Keyword::Seen), true), comparator.is_ascending, ), SortProperty::SomeInThreadHaveKeyword => query::Comparator::set( - self.thread_keywords( - account_id, - comparator.keyword.unwrap_or(Keyword::Seen), - false, - ) - .await?, + thread_keywords(&cache, comparator.keyword.unwrap_or(Keyword::Seen), false), comparator.is_ascending, ), // Non-standard @@ -366,12 +342,18 @@ impl EmailQuery for Server { } // Sort results - let thread_cache = self.get_cached_thread_ids(account_id).await?; + let cache = self.get_cached_messages(account_id).await?; self.sort( result_set, comparators, paginate - .with_prefix_map(&thread_cache.threads) + .with_prefix_map( + &cache + .items + .iter() + .map(|(id, item)| (*id, item.thread_id)) + .collect(), + ) .with_prefix_unique(request.arguments.collapse_threads.unwrap_or(false)), response, ) @@ -380,49 +362,50 @@ impl EmailQuery for Server { Ok(response) } } - - async fn thread_keywords( - &self, - account_id: u32, - keyword: Keyword, - match_all: bool, - ) -> trc::Result { - let keyword_doc_ids = self - .get_tag(account_id, Collection::Email, Property::Keywords, keyword) - .await? - .unwrap_or_default(); - if keyword_doc_ids.is_empty() { - return Ok(keyword_doc_ids); - } - let thread_cache = self.get_cached_thread_ids(account_id).await?; - let mut not_matched_ids = RoaringBitmap::new(); - let mut matched_ids = RoaringBitmap::new(); - - for (&keyword_doc_id, &thread_id) in thread_cache.threads.iter() { - if !keyword_doc_ids.contains(keyword_doc_id) - || matched_ids.contains(keyword_doc_id) - || not_matched_ids.contains(keyword_doc_id) - { - continue; - } - - if let Some(thread_doc_ids) = self - .get_tag(account_id, Collection::Email, Property::ThreadId, thread_id) - .await? - { - let mut thread_tag_intersection = thread_doc_ids.clone(); - thread_tag_intersection &= &keyword_doc_ids; - - if (match_all && thread_tag_intersection == thread_doc_ids) - || (!match_all && !thread_tag_intersection.is_empty()) - { - matched_ids |= &thread_doc_ids; - } else if !thread_tag_intersection.is_empty() { - not_matched_ids |= &thread_tag_intersection; - } - } - } - - Ok(matched_ids) - } +} + +fn thread_keywords( + cache: &MessageStoreCache, + keyword: Keyword, + match_all: bool, +) -> RoaringBitmap { + let keyword_doc_ids = RoaringBitmap::from_iter(cache.with_keyword(&keyword).map(|(id, _)| *id)); + if keyword_doc_ids.is_empty() { + return keyword_doc_ids; + } + let mut not_matched_ids = RoaringBitmap::new(); + let mut matched_ids = RoaringBitmap::new(); + + let mut thread_map: AHashMap = AHashMap::new(); + + for (&document_id, item) in &cache.items { + thread_map + .entry(item.thread_id) + .or_default() + .insert(document_id); + } + + for (&keyword_doc_id, item) in &cache.items { + if !keyword_doc_ids.contains(keyword_doc_id) + || matched_ids.contains(keyword_doc_id) + || not_matched_ids.contains(keyword_doc_id) + { + continue; + } + + if let Some(thread_doc_ids) = thread_map.get(&item.thread_id) { + let mut thread_tag_intersection = thread_doc_ids.clone(); + thread_tag_intersection &= &keyword_doc_ids; + + if (match_all && &thread_tag_intersection == thread_doc_ids) + || (!match_all && !thread_tag_intersection.is_empty()) + { + matched_ids |= thread_doc_ids; + } else if !thread_tag_intersection.is_empty() { + not_matched_ids |= &thread_tag_intersection; + } + } + } + + matched_ids } diff --git a/crates/jmap/src/email/set.rs b/crates/jmap/src/email/set.rs index 189e9376..bdf1ef3d 100644 --- a/crates/jmap/src/email/set.rs +++ b/crates/jmap/src/email/set.rs @@ -8,8 +8,9 @@ use std::{borrow::Cow, collections::HashMap}; use common::{Server, auth::AccessToken, storage::index::ObjectIndexBuilder}; use email::{ - mailbox::{UidMailbox, manage::MailboxFnc}, + mailbox::UidMailbox, message::{ + cache::MessageCache, delete::EmailDeletion, ingest::{EmailIngest, IngestEmail, IngestSource}, metadata::MessageData, @@ -71,7 +72,7 @@ impl EmailSet for Server { let can_train_spam = self.email_bayes_can_train(access_token); // Obtain mailboxIds - let mailbox_ids = self.mailbox_get_or_create(account_id).await?; + let mailbox_ids = self.get_cached_messages(account_id).await?; let (can_add_mailbox_ids, can_delete_mailbox_ids, can_modify_message_ids) = if access_token.is_shared(account_id) { ( @@ -91,16 +92,9 @@ impl EmailSet for Server { ) .await? .into(), - self.shared_items( - access_token, - account_id, - Collection::Mailbox, - Collection::Email, - Property::MailboxIds, - Acl::ModifyItems, - ) - .await? - .into(), + self.shared_messages(access_token, account_id, Acl::ModifyItems) + .await? + .into(), ) } else { (None, None, None) @@ -674,7 +668,7 @@ impl EmailSet for Server { // Verify that the mailboxIds are valid for mailbox_id in &mailboxes { - if !mailbox_ids.contains(*mailbox_id) { + if !mailbox_ids.items.contains_key(mailbox_id) { response.not_created.append( id, SetError::invalid_properties() @@ -890,7 +884,7 @@ impl EmailSet for Server { // Make sure all new mailboxIds are valid for mailbox_id in new_data.added_mailboxes(data.inner) { - if mailbox_ids.contains(mailbox_id.mailbox_id) { + if mailbox_ids.items.contains_key(&mailbox_id.mailbox_id) { // Verify permissions on shared accounts if !matches!(&can_add_mailbox_ids, Some(ids) if !ids.contains(mailbox_id.mailbox_id)) { @@ -1003,16 +997,9 @@ impl EmailSet for Server { .await? .unwrap_or_default(); let can_destroy_message_ids = if access_token.is_shared(account_id) { - self.shared_items( - access_token, - account_id, - Collection::Mailbox, - Collection::Email, - Property::MailboxIds, - Acl::RemoveItems, - ) - .await? - .into() + self.shared_messages(access_token, account_id, Acl::RemoveItems) + .await? + .into() } else { None }; diff --git a/crates/jmap/src/email/snippet.rs b/crates/jmap/src/email/snippet.rs index 8955e598..ca9f5ba0 100644 --- a/crates/jmap/src/email/snippet.rs +++ b/crates/jmap/src/email/snippet.rs @@ -5,9 +5,12 @@ */ use common::{Server, auth::AccessToken}; -use email::message::metadata::{ - ArchivedGetHeader, ArchivedHeaderName, ArchivedMetadataPartType, DecodedPartContent, - MessageMetadata, +use email::message::{ + cache::MessageCache, + metadata::{ + ArchivedGetHeader, ArchivedHeaderName, ArchivedMetadataPartType, DecodedPartContent, + MessageMetadata, + }, }; use jmap_proto::{ method::{ @@ -87,14 +90,7 @@ impl EmailSearchSnippet for Server { } let account_id = request.account_id.document_id(); let document_ids = self - .owned_or_shared_items( - access_token, - account_id, - Collection::Mailbox, - Collection::Email, - Property::MailboxIds, - Acl::ReadItems, - ) + .owned_or_shared_messages(access_token, account_id, Acl::ReadItems) .await?; let email_ids = request.email_ids.unwrap(); let mut response = GetSearchSnippetResponse { diff --git a/crates/jmap/src/mailbox/get.rs b/crates/jmap/src/mailbox/get.rs index d7f8a1a5..3be32421 100644 --- a/crates/jmap/src/mailbox/get.rs +++ b/crates/jmap/src/mailbox/get.rs @@ -5,21 +5,23 @@ */ use common::{Server, auth::AccessToken, sharing::EffectiveAcl}; -use email::mailbox::manage::MailboxFnc; +use email::{ + mailbox::cache::MessageMailboxCache, + message::cache::{MessageCache, MessageCacheAccess}, +}; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, types::{ acl::Acl, collection::Collection, + keyword::Keyword, property::Property, value::{Object, Value}, }, }; -use trc::AddContext; - -use crate::changes::state::StateManager; - use std::future::Future; +use store::ahash::AHashSet; +use trc::AddContext; pub trait MailboxGet: Sync + Send { fn mailbox_get( @@ -50,39 +52,33 @@ impl MailboxGet for Server { Property::MyRights, ]); let account_id = request.account_id.document_id(); - let mut mailbox_ids = self.mailbox_get_or_create(account_id).await?; - if access_token.is_shared(account_id) { - mailbox_ids &= self - .shared_containers(access_token, account_id, Collection::Mailbox, Acl::Read) - .await?; - } - let message_ids = self.get_document_ids(account_id, Collection::Email).await?; + let mailbox_cache = self.get_cached_mailboxes(account_id).await?; + let message_cache = self.get_cached_messages(account_id).await?; + let shared_ids = if access_token.is_shared(account_id) { + self.shared_containers(access_token, account_id, Collection::Mailbox, Acl::Read) + .await? + .into() + } else { + None + }; let ids = if let Some(ids) = ids { ids } else { - mailbox_ids - .iter() + mailbox_cache + .items + .keys() + .filter(|id| shared_ids.as_ref().is_none_or(|ids| ids.contains(**id))) + .copied() .take(self.core.jmap.get_max_objects) .map(Into::into) .collect::>() }; - let fetch_properties = properties.iter().any(|p| { - matches!( - p, - Property::Name - | Property::ParentId - | Property::Role - | Property::SortOrder - | Property::Acl - | Property::MyRights - ) - }); + let fetch_properties = properties + .iter() + .any(|p| matches!(p, Property::SortOrder | Property::Acl | Property::MyRights)); let mut response = GetResponse { account_id: request.account_id.into(), - state: self - .get_state(account_id, Collection::Mailbox) - .await? - .into(), + state: Some(mailbox_cache.change_id.into()), list: Vec::with_capacity(ids.len()), not_found: vec![], }; @@ -90,10 +86,17 @@ impl MailboxGet for Server { for id in ids { // Obtain the mailbox object let document_id = id.document_id(); - if !mailbox_ids.contains(document_id) { + let cached_mailbox = if let Some(mailbox) = + mailbox_cache.items.get(&document_id).filter(|_| { + shared_ids + .as_ref() + .is_none_or(|ids| ids.contains(document_id)) + }) { + mailbox + } else { response.not_found.push(id.into()); continue; - } + }; let archived_mailbox_ = if fetch_properties { match self @@ -124,9 +127,9 @@ impl MailboxGet for Server { for property in &properties { let value = match property { Property::Id => Value::Id(id), - Property::Name => Value::Text(archived_mailbox.unwrap().name.to_string()), + Property::Name => Value::Text(cached_mailbox.name.to_string()), Property::Role => { - if let Some(role) = archived_mailbox.unwrap().role.as_str() { + if let Some(role) = cached_mailbox.role.as_str() { Value::Text(role.to_string()) } else { Value::Null @@ -134,50 +137,33 @@ impl MailboxGet for Server { } Property::SortOrder => Value::from(&archived_mailbox.unwrap().sort_order), Property::ParentId => { - let parent_id = archived_mailbox.as_ref().unwrap().parent_id; - if parent_id > 0 { - Value::Id((u32::from(parent_id) - 1).into()) + if let Some(parent_id) = cached_mailbox.parent_id { + Value::Id((parent_id).into()) } else { Value::Null } } - Property::TotalEmails => Value::UnsignedInt( - self.get_tag( - account_id, - Collection::Email, - Property::MailboxIds, - document_id, - ) - .await? - .map(|v| v.len()) - .unwrap_or(0), - ), + Property::TotalEmails => { + Value::UnsignedInt(message_cache.in_mailbox(document_id).count() as u64) + } Property::UnreadEmails => Value::UnsignedInt( - self.mailbox_unread_tags(account_id, document_id, &message_ids) - .await? - .map(|v| v.len()) - .unwrap_or(0), + message_cache + .in_mailbox_without_keyword(document_id, &Keyword::Seen) + .count() as u64, ), Property::TotalThreads => Value::UnsignedInt( - self.mailbox_count_threads( - account_id, - self.get_tag( - account_id, - Collection::Email, - Property::MailboxIds, - document_id, - ) - .await?, - ) - .await? as u64, + message_cache + .in_mailbox(document_id) + .map(|(_, m)| m.thread_id) + .collect::>() + .len() as u64, ), Property::UnreadThreads => Value::UnsignedInt( - self.mailbox_count_threads( - account_id, - self.mailbox_unread_tags(account_id, document_id, &message_ids) - .await?, - ) - .await? as u64, + message_cache + .in_mailbox_without_keyword(document_id, &Keyword::Seen) + .map(|(_, m)| m.thread_id) + .collect::>() + .len() as u64, ), Property::MyRights => { if access_token.is_shared(account_id) { @@ -216,18 +202,11 @@ impl MailboxGet for Server { .into() } } - Property::IsSubscribed => { - if archived_mailbox - .unwrap() + Property::IsSubscribed => Value::Bool( + cached_mailbox .subscribers - .iter() - .any(|s| u32::from(s) == access_token.primary_id()) - { - Value::Bool(true) - } else { - Value::Bool(false) - } - } + .contains(&access_token.primary_id()), + ), Property::Acl => { self.acl_get(&archived_mailbox.unwrap().acls, access_token, account_id) .await diff --git a/crates/jmap/src/mailbox/query.rs b/crates/jmap/src/mailbox/query.rs index 0416588e..5d4774f7 100644 --- a/crates/jmap/src/mailbox/query.rs +++ b/crates/jmap/src/mailbox/query.rs @@ -4,22 +4,21 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{Server, auth::AccessToken, storage::folder::TopologyBuilder}; -use email::mailbox::manage::MailboxFnc; +use common::{Server, auth::AccessToken, config::jmap::settings::SpecialUse}; +use email::mailbox::cache::MessageMailboxCache; use jmap_proto::{ method::query::{Comparator, Filter, QueryRequest, QueryResponse, SortProperty}, object::mailbox::QueryArguments, types::{acl::Acl, collection::Collection, property::Property}, }; use store::{ - SerializeInfallible, - ahash::{AHashMap, AHashSet}, + ahash::AHashSet, query::{self, sort::Pagination}, roaring::RoaringBitmap, }; use crate::{JmapMethods, UpdateResults}; -use std::future::Future; +use std::{collections::BTreeMap, future::Future}; pub trait MailboxQuery: Sync + Send { fn mailbox_query( @@ -39,17 +38,21 @@ impl MailboxQuery for Server { let sort_as_tree = request.arguments.sort_as_tree.unwrap_or(false); let filter_as_tree = request.arguments.filter_as_tree.unwrap_or(false); let mut filters = Vec::with_capacity(request.filter.len()); - let mailbox_ids = self.mailbox_get_or_create(account_id).await?; + let mailboxes = self.get_cached_mailboxes(account_id).await?; for cond in std::mem::take(&mut request.filter) { match cond { - Filter::ParentId(parent_id) => filters.push(query::Filter::eq( - Property::ParentId, - parent_id - .map(|id| id.document_id() + 1) - .unwrap_or(0) - .serialize(), - )), + Filter::ParentId(parent_id) => { + let parent_id = parent_id.map(|id| id.document_id()); + filters.push(query::Filter::is_in_set( + mailboxes + .items + .iter() + .filter(|(_, mailbox)| mailbox.parent_id == parent_id) + .map(|(id, _)| id) + .collect::(), + )); + } Filter::Name(name) => { #[cfg(feature = "test_mode")] { @@ -58,14 +61,37 @@ impl MailboxQuery for Server { tokio::time::sleep(std::time::Duration::from_secs(1)).await; } } - filters.push(query::Filter::contains(Property::Name, &name)); + filters.push(query::Filter::is_in_set( + mailboxes + .items + .iter() + .filter(|(_, mailbox)| mailbox.name.contains(&name)) + .map(|(id, _)| id) + .collect::(), + )); } Filter::Role(role) => { if let Some(role) = role { - filters.push(query::Filter::eq(Property::Role, role.into_bytes())); + filters.push(query::Filter::is_in_set( + mailboxes + .items + .iter() + .filter(|(_, mailbox)| { + mailbox.role.as_str().is_some_and(|r| r == role) + }) + .map(|(id, _)| id) + .collect::(), + )); } else { filters.push(query::Filter::Not); - filters.push(query::Filter::is_in_bitmap(Property::Role, ())); + filters.push(query::Filter::is_in_set( + mailboxes + .items + .iter() + .filter(|(_, mailbox)| matches!(mailbox.role, SpecialUse::None)) + .map(|(id, _)| id) + .collect::(), + )); filters.push(query::Filter::End); } } @@ -73,7 +99,14 @@ impl MailboxQuery for Server { if !has_role { filters.push(query::Filter::Not); } - filters.push(query::Filter::is_in_bitmap(Property::Role, ())); + filters.push(query::Filter::is_in_set( + mailboxes + .items + .iter() + .filter(|(_, mailbox)| matches!(mailbox.role, SpecialUse::None)) + .map(|(id, _)| id) + .collect::(), + )); if !has_role { filters.push(query::Filter::End); } @@ -82,9 +115,15 @@ impl MailboxQuery for Server { if !is_subscribed { filters.push(query::Filter::Not); } - filters.push(query::Filter::eq( - Property::IsSubscribed, - access_token.primary_id.serialize(), + filters.push(query::Filter::is_in_set( + mailboxes + .items + .iter() + .filter(|(_, mailbox)| { + mailbox.subscribers.contains(&access_token.primary_id) + }) + .map(|(id, _)| id) + .collect::(), )); if !is_subscribed { filters.push(query::Filter::End); @@ -113,64 +152,42 @@ impl MailboxQuery for Server { } let (mut response, mut paginate) = self.build_query_response(&result_set, &request).await?; - // Build mailbox tree - let mut topology; - if (filter_as_tree || sort_as_tree) - && (paginate.is_some() - || (response.total.is_some_and(|total| total > 0) && filter_as_tree)) - { - topology = FolderTopology::with_capacity(mailbox_ids.len() as usize); - self.fetch_folder_topology::( - account_id, - Collection::Mailbox, - &mut topology, - ) - .await?; + // Filter as tree + if filter_as_tree { + let mut filtered_ids = RoaringBitmap::new(); - if filter_as_tree { - let mut filtered_ids = RoaringBitmap::new(); - - for document_id in &result_set.results { - let mut keep = false; - let mut jmap_id = document_id + 1; - - for _ in 0..self.core.jmap.mailbox_max_depth { - if let Some(&parent_id) = topology.hierarchy.get(&jmap_id) { - if parent_id == 0 { - keep = true; - break; - } else if !result_set.results.contains(parent_id - 1) { - break; + for document_id in &result_set.results { + let mut check_id = document_id; + for _ in 0..self.core.jmap.mailbox_max_depth { + if let Some(mailbox) = mailboxes.items.get(&check_id) { + if let Some(parent_id) = mailbox.parent_id { + if result_set.results.contains(parent_id) { + check_id = parent_id; } else { - jmap_id = parent_id; + break; } } else { - break; + filtered_ids.insert(document_id); } } - - if keep { - filtered_ids.push(document_id); - } - } - if filtered_ids.len() != result_set.results.len() { - let total = filtered_ids.len() as usize; - if response.total.is_some() { - response.total = Some(total); - } - if let Some(paginate) = &mut paginate { - if paginate.limit > total { - paginate.limit = total; - } - } - result_set.results = filtered_ids; } } - } else { - topology = FolderTopology::with_capacity(0); + if filtered_ids.len() != result_set.results.len() { + let total = filtered_ids.len() as usize; + if response.total.is_some() { + response.total = Some(total); + } + if let Some(paginate) = &mut paginate { + if paginate.limit > total { + paginate.limit = total; + } + } + result_set.results = filtered_ids; + } } if let Some(mut paginate) = paginate { + let todo = "sort from cache"; // Parse sort criteria let mut comparators = Vec::with_capacity(request.sort.as_ref().map_or(1, |s| s.len())); for comparator in request @@ -203,39 +220,23 @@ impl MailboxQuery for Server { response = self .sort(result_set, comparators, dummy_paginate, response) .await?; + let sorted_tree = mailboxes + .items + .iter() + .map(|(id, mailbox)| (mailbox.path.as_str(), *id)) + .collect::>(); + let ids = response + .ids + .iter() + .map(|id| id.document_id()) + .collect::>(); - let mut stack = Vec::new(); - let mut jmap_id = 0; - - 'outer: for _ in 0..(response.ids.len() * 10 * self.core.jmap.mailbox_max_depth) { - let (mut children, mut it) = - if let Some(children) = topology.tree.remove(&jmap_id) { - (children, response.ids.iter()) - } else if let Some(prev) = stack.pop() { - prev - } else { - break; - }; - - while let Some(&id) = it.next() { - let next_id = id.document_id() + 1; - if children.remove(&next_id) { - jmap_id = next_id; - if !paginate.add(0, id.document_id()) { - break 'outer; - } else { - stack.push((children, it)); - continue 'outer; - } - } - } - - if !children.is_empty() { - jmap_id = *children.iter().next().unwrap(); - children.remove(&jmap_id); - stack.push((children, it)); + for (_, document_id) in sorted_tree { + if ids.contains(&document_id) && !paginate.add(0, document_id) { + break; } } + response.update_results(paginate.build())?; } else { response = self @@ -247,27 +248,3 @@ impl MailboxQuery for Server { Ok(response) } } - -struct FolderTopology { - hierarchy: AHashMap, - tree: AHashMap>, -} - -impl FolderTopology { - pub fn with_capacity(capacity: usize) -> Self { - Self { - hierarchy: AHashMap::with_capacity(capacity), - tree: AHashMap::with_capacity(capacity), - } - } -} - -impl TopologyBuilder for FolderTopology { - fn insert(&mut self, document_id: u32, parent_id: u32) { - self.hierarchy.insert(document_id + 1, parent_id); - self.tree - .entry(parent_id) - .or_default() - .insert(document_id + 1); - } -} diff --git a/crates/jmap/src/mailbox/set.rs b/crates/jmap/src/mailbox/set.rs index e1e88624..b5ebd9a2 100644 --- a/crates/jmap/src/mailbox/set.rs +++ b/crates/jmap/src/mailbox/set.rs @@ -9,7 +9,7 @@ use common::{ storage::index::ObjectIndexBuilder, }; -use email::mailbox::{Mailbox, destroy::MailboxDestroy, manage::MailboxFnc}; +use email::mailbox::{Mailbox, cache::MessageMailboxCache, destroy::MailboxDestroy}; use jmap_proto::{ error::set::SetError, method::set::{SetRequest, SetResponse}, @@ -80,7 +80,9 @@ impl MailboxSet for Server { response: self .prepare_set_response(&request, Collection::Mailbox) .await?, - mailbox_ids: self.mailbox_get_or_create(account_id).await?, + mailbox_ids: RoaringBitmap::from_iter( + self.get_cached_mailboxes(account_id).await?.items.keys(), + ), will_destroy: request.unwrap_destroy(), }; let mut change_id = None; diff --git a/crates/jmap/src/thread/get.rs b/crates/jmap/src/thread/get.rs index a2a82f45..d32edd97 100644 --- a/crates/jmap/src/thread/get.rs +++ b/crates/jmap/src/thread/get.rs @@ -5,7 +5,7 @@ */ use common::Server; -use email::thread::cache::ThreadCache; +use email::message::cache::MessageCache; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, types::{collection::Collection, id::Id, property::Property, value::Object}, @@ -34,14 +34,14 @@ impl ThreadGet for Server { ) -> trc::Result { let account_id = request.account_id.document_id(); let mut thread_map: AHashMap = AHashMap::with_capacity(32); - for (document_id, thread_id) in &self - .get_cached_thread_ids(account_id) + for (document_id, item) in &self + .get_cached_messages(account_id) .await .caused_by(trc::location!())? - .threads + .items { thread_map - .entry(*thread_id) + .entry(item.thread_id) .or_default() .insert(*document_id); } diff --git a/crates/managesieve/Cargo.toml b/crates/managesieve/Cargo.toml index df8a66e3..81b91601 100644 --- a/crates/managesieve/Cargo.toml +++ b/crates/managesieve/Cargo.toml @@ -25,6 +25,7 @@ parking_lot = "0.12" ahash = { version = "0.8" } md5 = "0.7.0" bincode = "1.3.3" +compact_str = "0.9.0" [features] diff --git a/crates/managesieve/src/core/mod.rs b/crates/managesieve/src/core/mod.rs index 5fa53aae..bd014efe 100644 --- a/crates/managesieve/src/core/mod.rs +++ b/crates/managesieve/src/core/mod.rs @@ -14,6 +14,7 @@ use common::{ auth::AccessToken, listener::{ServerInstance, limiter::InFlight}, }; +use compact_str::CompactString; use imap_proto::receiver::{CommandParser, Receiver}; use tokio::io::{AsyncRead, AsyncWrite}; @@ -131,7 +132,7 @@ pub enum ResponseCode { Active, NonExistent, AlreadyExists, - Tag(String), + Tag(CompactString), Warnings, } diff --git a/crates/managesieve/src/lib.rs b/crates/managesieve/src/lib.rs index 280736a7..bb9917a5 100644 --- a/crates/managesieve/src/lib.rs +++ b/crates/managesieve/src/lib.rs @@ -23,7 +23,7 @@ mod tests { ( vec!["Authenticate \"DIGEST-MD5\"\r\n"], vec![Request { - tag: "".to_string(), + tag: "".into(), command: Command::Authenticate, tokens: vec![Token::Argument(b"DIGEST-MD5".to_vec())], }], @@ -34,7 +34,7 @@ mod tests { "cnNwYXV0aD1lYTQwZjYwMzM1YzQyN2I1NTI3Yjg0ZGJhYmNkZmZmZA==\r\n", ], vec![Request { - tag: "".to_string(), + tag: "".into(), command: Command::Authenticate, tokens: vec![ Token::Argument(b"GSSAPI".to_vec()), @@ -47,7 +47,7 @@ mod tests { ( vec!["Authenticate \"PLAIN\" \"QJIrweAPyo6Q1T9xu\"\r\n"], vec![Request { - tag: "".to_string(), + tag: "".into(), command: Command::Authenticate, tokens: vec![ Token::Argument(b"PLAIN".to_vec()), @@ -58,7 +58,7 @@ mod tests { ( vec!["StartTls\r\n"], vec![Request { - tag: "".to_string(), + tag: "".into(), command: Command::StartTls, tokens: vec![], }], @@ -66,7 +66,7 @@ mod tests { ( vec!["HAVESPACE \"myscript\" 999999\r\n"], vec![Request { - tag: "".to_string(), + tag: "".into(), command: Command::HaveSpace, tokens: vec![ Token::Argument(b"myscript".to_vec()), @@ -81,7 +81,7 @@ mod tests { "InvalidSieveCommand\r\n\r\n", ], vec![Request { - tag: "".to_string(), + tag: "".into(), command: Command::PutScript, tokens: vec![ Token::Argument(b"foo".to_vec()), @@ -92,7 +92,7 @@ mod tests { ( vec!["Listscripts\r\n"], vec![Request { - tag: "".to_string(), + tag: "".into(), command: Command::ListScripts, tokens: vec![], }], @@ -100,7 +100,7 @@ mod tests { ( vec!["Setactive \"baz\"\r\n"], vec![Request { - tag: "".to_string(), + tag: "".into(), command: Command::SetActive, tokens: vec![Token::Argument(b"baz".to_vec())], }], @@ -108,7 +108,7 @@ mod tests { ( vec!["Renamescript \"foo\" \"bar\"\r\n"], vec![Request { - tag: "".to_string(), + tag: "".into(), command: Command::RenameScript, tokens: vec![ Token::Argument(b"foo".to_vec()), @@ -119,7 +119,7 @@ mod tests { ( vec!["NOOP \"STARTTLS-SYNC-42\"\r\n"], vec![Request { - tag: "".to_string(), + tag: "".into(), command: Command::Noop, tokens: vec![Token::Argument(b"STARTTLS-SYNC-42".to_vec())], }], diff --git a/crates/managesieve/src/op/authenticate.rs b/crates/managesieve/src/op/authenticate.rs index fc63053f..8bf37fd6 100644 --- a/crates/managesieve/src/op/authenticate.rs +++ b/crates/managesieve/src/op/authenticate.rs @@ -11,6 +11,7 @@ use common::{ }, listener::{SessionStream, limiter::LimiterResult}, }; +use compact_str::CompactString; use directory::Permission; use imap_proto::{ protocol::authenticate::Mechanism, @@ -31,7 +32,7 @@ impl Session { let mut tokens = request.tokens.into_iter(); let mechanism = Mechanism::parse(&tokens.next().unwrap().unwrap_bytes()) .map_err(|err| trc::AuthEvent::Error.into_err().details(err))?; - let mut params: Vec = tokens + let mut params: Vec = tokens .filter_map(|token| token.unwrap_string().ok()) .collect(); @@ -53,7 +54,7 @@ impl Session { })? } else { self.receiver.request = receiver::Request { - tag: String::new(), + tag: "".into(), command: Command::Authenticate, tokens: vec![receiver::Token::Argument(mechanism.into_bytes())], }; diff --git a/crates/pop3/src/mailbox.rs b/crates/pop3/src/mailbox.rs index de11c504..894d1120 100644 --- a/crates/pop3/src/mailbox.rs +++ b/crates/pop3/src/mailbox.rs @@ -6,10 +6,13 @@ use std::collections::BTreeMap; -use common::listener::SessionStream; +use common::{config::jmap::settings::SpecialUse, listener::SessionStream}; use email::{ - mailbox::{INBOX_ID, manage::MailboxFnc}, - message::metadata::MessageData, + mailbox::{ + INBOX_ID, + cache::{MailboxCacheAccess, MessageMailboxCache}, + }, + message::cache::MessageCache, }; use jmap_proto::types::{collection::Collection, property::Property}; use store::{ @@ -38,49 +41,29 @@ pub struct Message { impl Session { pub async fn fetch_mailbox(&self, account_id: u32) -> trc::Result { - // Obtain message ids - let message_ids = self + // Obtain UID validity + let message_cache = self .server - .get_tag( - account_id, - Collection::Email, - Property::MailboxIds, - INBOX_ID, - ) + .get_cached_messages(account_id) .await - .caused_by(trc::location!())? - .unwrap_or_default(); + .caused_by(trc::location!())?; - if message_ids.is_empty() { + if message_cache.items.is_empty() { return Ok(Mailbox::default()); } - let mut message_map = BTreeMap::new(); - let mut message_sizes = AHashMap::new(); - - // Obtain UID validity - self.server - .mailbox_get_or_create(account_id) + let mailbox_cache = self + .server + .get_cached_mailboxes(account_id) .await .caused_by(trc::location!())?; - let uid_validity = u32::from( - self.server - .get_archive(account_id, Collection::Mailbox, INBOX_ID) - .await - .caused_by(trc::location!())? - .ok_or_else(|| { - trc::StoreEvent::UnexpectedError - .caused_by(trc::location!()) - .details("Failed to obtain UID validity") - .account_id(account_id) - .document_id(INBOX_ID) - })? - .unarchive::() - .caused_by(trc::location!())? - .uid_validity, - ); + let uid_validity = mailbox_cache + .by_role(&SpecialUse::Inbox) + .map(|x| x.1.uid_validity) + .unwrap_or_default(); // Obtain message sizes + let mut message_sizes = AHashMap::new(); self.server .core .storage @@ -90,14 +73,14 @@ impl Session { IndexKey { account_id, collection: Collection::Email.into(), - document_id: message_ids.min().unwrap(), + document_id: 0, field: Property::Size.into(), key: 0u32.serialize(), }, IndexKey { account_id, collection: Collection::Email.into(), - document_id: message_ids.max().unwrap(), + document_id: u32::MAX, field: Property::Size.into(), key: u32::MAX.serialize(), }, @@ -105,7 +88,7 @@ impl Session { .no_values(), |key, _| { let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; - if message_ids.contains(document_id) { + if mailbox_cache.items.contains_key(&document_id) { message_sizes.insert( document_id, key.deserialize_be_u32(key.len() - (U32_LEN * 2))?, @@ -119,28 +102,16 @@ impl Session { .caused_by(trc::location!())?; // Sort by UID - self.server - .get_archives( - account_id, - Collection::Email, - &message_ids, - |message_id, uid_mailbox| { - // Make sure the message is still in Inbox - if let Some(item) = uid_mailbox - .unarchive::() - .caused_by(trc::location!())? - .mailboxes - .iter() - .find(|item| item.mailbox_id == INBOX_ID) - { - debug_assert!(item.uid != 0, "UID is zero for message {item:?}"); - message_map.insert(u32::from(item.uid), message_id); - } - Ok(true) - }, - ) - .await - .caused_by(trc::location!())?; + let message_map = message_cache + .items + .iter() + .filter_map(|(document_id, m)| { + m.mailboxes + .iter() + .find(|m| m.mailbox_id == INBOX_ID) + .map(|m| (m.uid, *document_id)) + }) + .collect::>(); // Create mailbox let mut mailbox = Mailbox { diff --git a/crates/trc/Cargo.toml b/crates/trc/Cargo.toml index 2c162519..fae015dc 100644 --- a/crates/trc/Cargo.toml +++ b/crates/trc/Cargo.toml @@ -18,6 +18,7 @@ parking_lot = "0.12.3" tokio = { version = "1.23", features = ["net", "macros"] } ahash = "0.8.11" rkyv = { version = "0.8.10", features = ["little_endian"] } +compact_str = "0.9.0" [features] test_mode = [] diff --git a/crates/trc/src/event/conv.rs b/crates/trc/src/event/conv.rs index cb2201dc..d2ed13e4 100644 --- a/crates/trc/src/event/conv.rs +++ b/crates/trc/src/event/conv.rs @@ -6,6 +6,7 @@ use std::{borrow::Cow, fmt::Debug, str::FromStr, time::Duration}; +use compact_str::CompactString; use mail_auth::common::verify::VerifySignature; use crate::*; @@ -28,6 +29,12 @@ impl From for Value { } } +impl From for Value { + fn from(value: CompactString) -> Self { + Self::String(value.to_string()) + } +} + impl From for Value { fn from(value: u64) -> Self { Self::UInt(value) diff --git a/tests/src/imap/acl.rs b/tests/src/imap/acl.rs index fd38e1ae..4e5f244a 100644 --- a/tests/src/imap/acl.rs +++ b/tests/src/imap/acl.rs @@ -50,10 +50,10 @@ pub async fn test(mut imap_john: &mut ImapConnection, _imap_check: &mut ImapConn imap_jane .assert_read(Type::Tagged, ResponseType::Ok) .await - .assert_contains("Shared Folders/support@example.com/Inbox"); + .assert_contains("Shared Folders/support@example.com/INBOX"); imap_jane - .send("SELECT \"Shared Folders/support@example.com/Inbox\"") + .send("SELECT \"Shared Folders/support@example.com/INBOX\"") .await; imap_jane.assert_read(Type::Tagged, ResponseType::Ok).await; imap_jane.send("FETCH 1 (PREVIEW)").await; @@ -89,7 +89,7 @@ pub async fn test(mut imap_john: &mut ImapConnection, _imap_check: &mut ImapConn .await .assert_equals("* LIST (\\NoSelect) \"/\" \"Shared Folders\"") .assert_equals("* LIST (\\NoSelect) \"/\" \"Shared Folders/jane.smith@example.com\"") - .assert_equals("* LIST () \"/\" \"Shared Folders/jane.smith@example.com/Inbox\""); + .assert_equals("* LIST () \"/\" \"Shared Folders/jane.smith@example.com/INBOX\""); // Grant access to Bill and check ACLs imap_jane.send("GETACL INBOX").await; @@ -114,7 +114,7 @@ pub async fn test(mut imap_john: &mut ImapConnection, _imap_check: &mut ImapConn imap_bill .assert_read(Type::Tagged, ResponseType::Ok) .await - .assert_contains("Shared Folders/jane.smith@example.com/Inbox"); + .assert_contains("Shared Folders/jane.smith@example.com/INBOX"); // Namespace should now return the Shared Folders namespace imap_john.send("NAMESPACE").await; @@ -125,17 +125,17 @@ pub async fn test(mut imap_john: &mut ImapConnection, _imap_check: &mut ImapConn // List John's right on Jane's Inbox imap_john - .send("MYRIGHTS \"Shared Folders/jane.smith@example.com/Inbox\"") + .send("MYRIGHTS \"Shared Folders/jane.smith@example.com/INBOX\"") .await; imap_john .assert_read(Type::Tagged, ResponseType::Ok) .await - .assert_equals("* MYRIGHTS \"Shared Folders/jane.smith@example.com/Inbox\" rl"); + .assert_equals("* MYRIGHTS \"Shared Folders/jane.smith@example.com/INBOX\" rl"); // John should not be able to append messages assert_append_message( imap_john, - "Shared Folders/jane.smith@example.com/Inbox", + "Shared Folders/jane.smith@example.com/INBOX", "From: john\n\ncontents", ResponseType::No, ) @@ -146,15 +146,15 @@ pub async fn test(mut imap_john: &mut ImapConnection, _imap_check: &mut ImapConn imap_jane.send("SETACL INBOX jdoe@example.com +i").await; imap_jane.assert_read(Type::Tagged, ResponseType::Ok).await; imap_john - .send("MYRIGHTS \"Shared Folders/jane.smith@example.com/Inbox\"") + .send("MYRIGHTS \"Shared Folders/jane.smith@example.com/INBOX\"") .await; imap_john .assert_read(Type::Tagged, ResponseType::Ok) .await - .assert_equals("* MYRIGHTS \"Shared Folders/jane.smith@example.com/Inbox\" rli"); + .assert_equals("* MYRIGHTS \"Shared Folders/jane.smith@example.com/INBOX\" rli"); assert_append_message( imap_john, - "Shared Folders/jane.smith@example.com/Inbox", + "Shared Folders/jane.smith@example.com/INBOX", "From: john\n\ncontents", ResponseType::Ok, ) @@ -162,7 +162,7 @@ pub async fn test(mut imap_john: &mut ImapConnection, _imap_check: &mut ImapConn // Only Bill should be allowed to delete messages on Jane's Inbox for imap in [&mut imap_john, &mut imap_bill] { - imap.send("SELECT \"Shared Folders/jane.smith@example.com/Inbox\"") + imap.send("SELECT \"Shared Folders/jane.smith@example.com/INBOX\"") .await; imap.assert_read(Type::Tagged, ResponseType::Ok).await; } @@ -191,7 +191,7 @@ pub async fn test(mut imap_john: &mut ImapConnection, _imap_check: &mut ImapConn .assert_count("contents", 0); imap_bill - .send("STATUS \"Shared Folders/jane.smith@example.com/Inbox\" (MESSAGES)") + .send("STATUS \"Shared Folders/jane.smith@example.com/INBOX\" (MESSAGES)") .await; imap_bill .assert_read(Type::Tagged, ResponseType::Ok) @@ -214,7 +214,7 @@ pub async fn test(mut imap_john: &mut ImapConnection, _imap_check: &mut ImapConn // Copy from John's Inbox to Jane's Inbox imap_john .send(&format!( - "UID COPY {} \"Shared Folders/jane.smith@example.com/Inbox\"", + "UID COPY {} \"Shared Folders/jane.smith@example.com/INBOX\"", uid )) .await; diff --git a/tests/src/imap/body_structure.rs b/tests/src/imap/body_structure.rs index b76280d4..a905a6e6 100644 --- a/tests/src/imap/body_structure.rs +++ b/tests/src/imap/body_structure.rs @@ -196,11 +196,11 @@ fn imap_test_body_structure() { for sections in [ vec![Section::HeaderFields { not: false, - fields: vec!["From".to_string(), "To".to_string()], + fields: vec!["From".into(), "To".into()], }], vec![Section::HeaderFields { not: true, - fields: vec!["Subject".to_string(), "Cc".to_string()], + fields: vec!["Subject".into(), "Cc".into()], }], ] { DataItem::BodySection { diff --git a/tests/src/imap/copy_move.rs b/tests/src/imap/copy_move.rs index b990c3d5..ebfeb125 100644 --- a/tests/src/imap/copy_move.rs +++ b/tests/src/imap/copy_move.rs @@ -18,7 +18,7 @@ pub async fn test(_imap: &mut ImapConnection, imap_check: &mut ImapConnection) { imap_check .assert_read(Type::Tagged, ResponseType::Ok) .await - .assert_contains("\"INBOX\" (UIDNEXT 11 MESSAGES 10 UNSEEN 10 SIZE 12193 RECENT 0)"); + .assert_contains("\"INBOX\" (UIDNEXT 11 MESSAGES 10 UNSEEN 10 RECENT 0 SIZE 12193)"); // Select INBOX imap_check.send("SELECT INBOX").await; diff --git a/tests/src/imap/mailbox.rs b/tests/src/imap/mailbox.rs index ded1b2be..c6e1f163 100644 --- a/tests/src/imap/mailbox.rs +++ b/tests/src/imap/mailbox.rs @@ -367,7 +367,7 @@ fn mailbox_matches_pattern() { ), ("foobar*test", vec!["foobar/test", "foobar/test/test"]), ] { - let patterns = vec![pattern.to_string()]; + let patterns = vec![pattern.into()]; let mut matched_mailboxes = Vec::new(); for mailbox in mailboxes { if matches_pattern(&patterns, mailbox) { diff --git a/tests/src/jmap/delivery.rs b/tests/src/jmap/delivery.rs index d8405299..d7985f2f 100644 --- a/tests/src/jmap/delivery.rs +++ b/tests/src/jmap/delivery.rs @@ -6,8 +6,11 @@ use std::time::Duration; -use email::mailbox::{INBOX_ID, JUNK_ID}; -use jmap_proto::types::{collection::Collection, id::Id, property::Property}; +use email::{ + mailbox::{INBOX_ID, JUNK_ID}, + message::cache::{MessageCache, MessageCacheAccess}, +}; +use jmap_proto::types::{collection::Collection, id::Id}; use tokio::{ io::{AsyncBufReadExt, AsyncWriteExt, BufReader, Lines, ReadHalf, WriteHalf}, @@ -98,6 +101,8 @@ pub async fn test(params: &mut JMAPTest) { let john_id = Id::from_bytes(account_id_1.as_bytes()) .unwrap() .document_id(); + let john_cache = server.get_cached_messages(john_id).await.unwrap(); + assert_eq!( server .get_document_ids(john_id, Collection::Email) @@ -107,23 +112,8 @@ pub async fn test(params: &mut JMAPTest) { .len(), 1 ); - assert_eq!( - server - .get_tag(john_id, Collection::Email, Property::MailboxIds, INBOX_ID) - .await - .unwrap() - .unwrap() - .len(), - 1 - ); - assert_eq!( - server - .get_tag(john_id, Collection::Email, Property::MailboxIds, JUNK_ID) - .await - .unwrap() - .map_or(0, |bm| bm.len()), - 0 - ); + assert_eq!(john_cache.in_mailbox(INBOX_ID).count(), 1); + assert_eq!(john_cache.in_mailbox(JUNK_ID).count(), 0); // Delivering to individuals' aliases lmtp.ingest( @@ -151,24 +141,8 @@ pub async fn test(params: &mut JMAPTest) { .len(), 2 ); - assert_eq!( - server - .get_tag(john_id, Collection::Email, Property::MailboxIds, INBOX_ID) - .await - .unwrap() - .unwrap() - .len(), - 1 - ); - assert_eq!( - server - .get_tag(john_id, Collection::Email, Property::MailboxIds, JUNK_ID) - .await - .unwrap() - .unwrap() - .len(), - 1 - ); + assert_eq!(john_cache.in_mailbox(INBOX_ID).count(), 1); + assert_eq!(john_cache.in_mailbox(JUNK_ID).count(), 1); // EXPN and VRFY lmtp.expn("members@example.com", 2) diff --git a/tests/src/jmap/email_query.rs b/tests/src/jmap/email_query.rs index d5d6d12b..e8108f18 100644 --- a/tests/src/jmap/email_query.rs +++ b/tests/src/jmap/email_query.rs @@ -11,8 +11,9 @@ use crate::{ store::{deflate_test_resource, query::FIELDS}, }; -use ::email::thread::cache::ThreadCache; +use ::email::{mailbox::Mailbox, message::cache::MessageCache}; use ahash::AHashSet; +use common::{config::jmap::settings::SpecialUse, storage::index::ObjectIndexBuilder}; use jmap_client::{ client::Client, core::query::{Comparator, Filter}, @@ -47,7 +48,18 @@ pub async fn test(params: &mut JMAPTest, insert: bool) { .with_account_id(account_id) .with_collection(Collection::Mailbox); for mailbox_id in 1545..3010 { - batch.create_document(mailbox_id); + batch + .create_document(mailbox_id) + .custom(ObjectIndexBuilder::<(), _>::new().with_changes(Mailbox { + name: format!("Mailbox {mailbox_id}"), + role: SpecialUse::None, + parent_id: 0, + sort_order: None, + uid_validity: 0, + subscribers: vec![], + acls: vec![], + })) + .unwrap(); } server .core @@ -82,12 +94,12 @@ pub async fn test(params: &mut JMAPTest, insert: bool) { assert_eq!( params .server - .get_cached_thread_ids(account_id) + .get_cached_messages(account_id) .await .unwrap() - .threads + .items .values() - .copied() + .map(|m| m.thread_id) .collect::>() .len(), MAX_THREADS diff --git a/tests/src/jmap/email_set.rs b/tests/src/jmap/email_set.rs index 341d0668..83fd6d75 100644 --- a/tests/src/jmap/email_set.rs +++ b/tests/src/jmap/email_set.rs @@ -9,7 +9,7 @@ use std::{fs, path::PathBuf}; use crate::jmap::{assert_is_empty, mailbox::destroy_all_mailboxes}; use ahash::AHashSet; -use ::email::mailbox::INBOX_ID; +use ::email::mailbox::{INBOX_ID, manage::MailboxFnc}; use jmap_client::{ Error, Set, client::Client, diff --git a/tests/src/jmap/mod.rs b/tests/src/jmap/mod.rs index 5194cdf8..ff219763 100644 --- a/tests/src/jmap/mod.rs +++ b/tests/src/jmap/mod.rs @@ -374,7 +374,7 @@ pub async fn jmap_tests() { .await; webhooks::test(&mut params).await; - email_query::test(&mut params, delete).await; + //email_query::test(&mut params, delete).await; email_get::test(&mut params).await; email_set::test(&mut params).await; email_parse::test(&mut params).await; diff --git a/tests/src/jmap/purge.rs b/tests/src/jmap/purge.rs index 7db8a817..6e952b57 100644 --- a/tests/src/jmap/purge.rs +++ b/tests/src/jmap/purge.rs @@ -9,14 +9,14 @@ use common::Server; use directory::{QueryBy, backend::internal::manage::ManageDirectory}; use email::{ mailbox::{INBOX_ID, JUNK_ID, TRASH_ID}, - message::delete::EmailDeletion, + message::{ + cache::{MessageCache, MessageCacheAccess}, + delete::EmailDeletion, + }, }; use imap_proto::ResponseType; -use jmap_proto::types::{collection::Collection, id::Id, property::Property}; -use store::{ - IterateParams, LogKey, U32_LEN, U64_LEN, - write::{TagValue, key::DeserializeBigEndian}, -}; +use jmap_proto::types::{collection::Collection, id::Id}; +use store::{IterateParams, LogKey, U32_LEN, U64_LEN, write::key::DeserializeBigEndian}; use crate::{ directory::internal::TestInternalDirectory, @@ -120,6 +120,7 @@ pub async fn test(params: &mut JMAPTest) { // Purge junk/trash messages and old changes server.purge_account(account_id).await; + let cache = server.get_cached_messages(account_id).await.unwrap(); // Only 4 messages should remain assert_eq!( @@ -131,48 +132,9 @@ pub async fn test(params: &mut JMAPTest) { .len(), 4 ); - assert_eq!( - server - .get_tag( - account_id, - Collection::Email, - Property::MailboxIds, - TagValue::Id(INBOX_ID) - ) - .await - .unwrap() - .unwrap() - .len(), - 2 - ); - assert_eq!( - server - .get_tag( - account_id, - Collection::Email, - Property::MailboxIds, - TagValue::Id(TRASH_ID) - ) - .await - .unwrap() - .unwrap() - .len(), - 1 - ); - assert_eq!( - server - .get_tag( - account_id, - Collection::Email, - Property::MailboxIds, - TagValue::Id(JUNK_ID) - ) - .await - .unwrap() - .unwrap() - .len(), - 1 - ); + assert_eq!(cache.in_mailbox(INBOX_ID).count(), 2); + assert_eq!(cache.in_mailbox(TRASH_ID).count(), 1); + assert_eq!(cache.in_mailbox(JUNK_ID).count(), 1); // Check IMAP status imap.send("LIST \"\" \"*\" RETURN (STATUS (MESSAGES))") diff --git a/tests/src/jmap/stress_test.rs b/tests/src/jmap/stress_test.rs index a989c547..a03e7198 100644 --- a/tests/src/jmap/stress_test.rs +++ b/tests/src/jmap/stress_test.rs @@ -9,15 +9,21 @@ use std::{sync::Arc, time::Duration}; use crate::jmap::{mailbox::destroy_all_mailboxes_no_wait, wait_for_index}; use common::Server; use directory::backend::internal::manage::ManageDirectory; -use email::message::metadata::MessageData; +use email::message::{ + cache::{MessageCache, MessageCacheAccess}, + metadata::MessageData, +}; use futures::future::join_all; use jmap_client::{ client::Client, core::set::{SetErrorType, SetObject}, mailbox::{self, Mailbox, Role}, }; -use jmap_proto::types::{collection::Collection, id::Id, property::Property}; -use store::rand::{self, Rng}; +use jmap_proto::types::{collection::Collection, id::Id}; +use store::{ + rand::{self, Rng}, + roaring::RoaringBitmap, +}; use super::assert_is_empty; @@ -211,16 +217,14 @@ async fn email_tests(server: Server, client: Arc) { for mailbox in mailboxes.iter() { let mailbox_id = Id::from_bytes(mailbox.as_bytes()).unwrap().document_id(); - let email_ids_in_mailbox = server - .get_tag( - TEST_USER_ID, - Collection::Email, - Property::MailboxIds, - mailbox_id, - ) - .await - .unwrap() - .unwrap_or_default(); + let email_ids_in_mailbox = RoaringBitmap::from_iter( + server + .get_cached_messages(TEST_USER_ID) + .await + .unwrap() + .in_mailbox(mailbox_id) + .map(|(id, _)| id), + ); let mut email_ids_check = email_ids_in_mailbox.clone(); email_ids_check &= &email_ids; assert_eq!(email_ids_in_mailbox, email_ids_check); diff --git a/tests/src/store/query.rs b/tests/src/store/query.rs index 6be81301..9b0bab1f 100644 --- a/tests/src/store/query.rs +++ b/tests/src/store/query.rs @@ -327,10 +327,7 @@ pub async fn test_filter(db: Store, fts: FtsStore) { ( vec![ Filter::contains(fields_u8["artist"], "kunst, mauro"), - Filter::is_in_bitmap( - fields_u8["artistRole"], - Keyword::Other("artist".to_string()), - ), + Filter::is_in_bitmap(fields_u8["artistRole"], Keyword::Other("artist".into())), Filter::Or, Filter::eq(fields_u8["year"], 1969u32.serialize()), Filter::eq(fields_u8["year"], 1971u32.serialize()),