diff --git a/Cargo.lock b/Cargo.lock index c9c1b02a..f45e55e8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3445,6 +3445,7 @@ dependencies = [ "tokio", "trc", "types", + "utils", ] [[package]] diff --git a/crates/common/src/core.rs b/crates/common/src/core.rs index 228421be..7debf547 100644 --- a/crates/common/src/core.rs +++ b/crates/common/src/core.rs @@ -30,7 +30,7 @@ use store::{ roaring::RoaringBitmap, write::{ AlignedBytes, AnyClass, Archive, AssignedIds, BatchBuilder, BlobOp, DirectoryClass, - IndexPropertyClass, QueueClass, ValueClass, key::DeserializeBigEndian, now, + QueueClass, ValueClass, key::DeserializeBigEndian, now, }, }; use trc::AddContext; @@ -38,7 +38,7 @@ use types::{ blob::{BlobClass, BlobId}, blob_hash::BlobHash, collection::{Collection, SyncCollection}, - field::{EmailField, Field}, + field::Field, type_state::{DataType, StateChange}, }; use utils::{map::bitmap::Bitmap, snowflake::SnowflakeIdGenerator}; @@ -343,52 +343,6 @@ impl Server { .add_context(|err| err.caused_by(trc::location!()).account_id(account_id)) } - pub async fn recalculate_quota(&self, account_id: u32) -> trc::Result<()> { - let mut quota = 0i64; - - self.store() - .iterate( - IterateParams::new( - ValueKey { - account_id, - collection: Collection::Email.into(), - document_id: 0, - class: ValueClass::IndexProperty(IndexPropertyClass::Integer { - property: EmailField::ReceivedToSize.into(), - value: 0, - }), - }, - ValueKey { - account_id, - collection: Collection::Email.into(), - document_id: u32::MAX, - class: ValueClass::IndexProperty(IndexPropertyClass::Integer { - property: EmailField::ReceivedToSize.into(), - value: u64::MAX, - }), - }, - ) - .ascending(), - |_, value| { - quota += value.deserialize_be_u32(0)? as i64; - - Ok(true) - }, - ) - .await - .caused_by(trc::location!())?; - - let mut batch = BatchBuilder::new(); - batch - .clear(DirectoryClass::UsedQuota(account_id)) - .add(DirectoryClass::UsedQuota(account_id), quota); - self.store() - .write(batch.build_all()) - .await - .caused_by(trc::location!()) - .map(|_| ()) - } - pub async fn has_available_quota( &self, quotas: &ResourceToken, diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index b96925aa..efa7ecd1 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -196,16 +196,16 @@ pub struct MessageStoreCache { pub struct MailboxesCache { pub change_id: u64, pub index: AHashMap, - pub items: Vec, + pub items: Box<[MailboxCache]>, pub size: u64, } #[derive(Debug, Clone)] pub struct MessagesCache { pub change_id: u64, - pub items: Vec, + pub items: Box<[MessageCache]>, pub index: AHashMap, - pub keywords: Vec, + pub keywords: Box<[Box]>, pub size: u64, } @@ -216,6 +216,7 @@ pub struct MessageCache { pub keywords: u128, pub thread_id: u32, pub change_id: u64, + pub size: u32, } #[derive(Debug, Default, Clone, Copy)] diff --git a/crates/common/src/storage/index.rs b/crates/common/src/storage/index.rs index 399dfaf2..3ea2b625 100644 --- a/crates/common/src/storage/index.rs +++ b/crates/common/src/storage/index.rs @@ -296,6 +296,11 @@ impl ObjectIndexBuilder) -> Self { + self.tenant_id = tenant_id; + self + } } impl IntoOperations diff --git a/crates/email/src/cache/email.rs b/crates/email/src/cache/email.rs index 40163dc2..3a28729e 100644 --- a/crates/email/src/cache/email.rs +++ b/crates/email/src/cache/email.rs @@ -18,18 +18,26 @@ use types::{ }; use utils::map::bitmap::Bitmap; +struct MessagesCacheBuilder { + pub change_id: u64, + pub items: Vec, + pub index: AHashMap, + pub keywords: Vec>, + pub size: u64, +} + pub(crate) async fn update_email_cache( server: &Server, account_id: u32, changed_ids: &AHashMap, store_cache: &MessageStoreCache, ) -> trc::Result { - let mut new_cache = MessagesCache { + let mut new_cache = MessagesCacheBuilder { index: AHashMap::with_capacity(store_cache.emails.items.len()), items: Vec::with_capacity(store_cache.emails.items.len()), size: 0, change_id: 0, - keywords: store_cache.emails.keywords.clone(), + keywords: store_cache.emails.keywords.to_vec(), }; for (document_id, is_update) in changed_ids { @@ -53,15 +61,7 @@ pub(crate) async fn update_email_cache( } } - if store_cache.emails.items.len() > new_cache.items.len() { - new_cache.items.shrink_to_fit(); - new_cache.index.shrink_to_fit(); - } - if store_cache.emails.keywords.len() > new_cache.keywords.len() { - new_cache.keywords.shrink_to_fit(); - } - - Ok(new_cache) + Ok(new_cache.build()) } pub(crate) async fn full_email_cache_build( @@ -69,7 +69,7 @@ pub(crate) async fn full_email_cache_build( account_id: u32, ) -> trc::Result { // Build cache - let mut cache = MessagesCache { + let mut cache = MessagesCacheBuilder { items: Vec::with_capacity(16), index: AHashMap::with_capacity(16), keywords: Vec::new(), @@ -94,14 +94,11 @@ pub(crate) async fn full_email_cache_build( .await .caused_by(trc::location!())?; - cache.items.shrink_to_fit(); - cache.index.shrink_to_fit(); - - Ok(cache) + Ok(cache.build()) } fn insert_item( - cache: &mut MessagesCache, + cache: &mut MessagesCacheBuilder, document_id: u32, archive: Archive<&ArchivedMessageData>, ) { @@ -119,6 +116,7 @@ fn insert_item( thread_id: message.thread_id.to_native(), change_id: archive.version.change_id().unwrap_or_default(), document_id, + size: message.size.to_native(), }; for keyword in message.keywords.iter() { match keyword.id() { @@ -126,10 +124,10 @@ fn insert_item( item.keywords |= 1 << id; } Err(custom) => { - if let Some(idx) = cache.keywords.iter().position(|k| k == custom) { + if let Some(idx) = cache.keywords.iter().position(|k| **k == *custom) { item.keywords |= 1 << (OTHER + idx); } else if cache.keywords.len() < (128 - OTHER) { - cache.keywords.push(String::from(custom)); + cache.keywords.push(custom.into()); item.keywords |= 1 << (OTHER + cache.keywords.len() - 1); } } @@ -139,6 +137,19 @@ fn insert_item( email_insert(cache, item); } +impl MessagesCacheBuilder { + pub fn build(mut self) -> MessagesCache { + self.index.shrink_to_fit(); + MessagesCache { + change_id: self.change_id, + items: self.items.into_boxed_slice(), + index: self.index, + keywords: self.keywords.into_boxed_slice(), + size: self.size, + } + } +} + pub trait MessageCacheAccess { fn email_by_id(&self, id: &u32) -> Option<&MessageCache>; @@ -282,7 +293,7 @@ impl MessageCacheAccess for MessageStoreCache { } } -fn email_insert(cache: &mut MessagesCache, item: MessageCache) { +fn email_insert(cache: &mut MessagesCacheBuilder, item: MessageCache) { let id = item.document_id; if let Some(idx) = cache.index.get(&id) { cache.items[*idx as usize] = item; @@ -306,7 +317,7 @@ fn keyword_to_id(cache: &MessageStoreCache, keyword: &Keyword) -> Option { .emails .keywords .iter() - .position(|k| k == name) + .position(|k| **k == *name) .map(|idx| (OTHER + idx) as u32), } } diff --git a/crates/email/src/cache/mailbox.rs b/crates/email/src/cache/mailbox.rs index ccb0fb45..99387283 100644 --- a/crates/email/src/cache/mailbox.rs +++ b/crates/email/src/cache/mailbox.rs @@ -18,13 +18,20 @@ use types::{ }; use utils::{map::bitmap::Bitmap, topological::TopologicalSort}; +struct MailboxesCacheBuilder { + pub change_id: u64, + pub index: AHashMap, + pub items: Vec, + pub size: u64, +} + pub(crate) async fn update_mailbox_cache( server: &Server, account_id: u32, changed_ids: &AHashMap, store_cache: &MessageStoreCache, ) -> trc::Result { - let mut new_cache = MailboxesCache { + let mut new_cache = MailboxesCacheBuilder { items: Vec::with_capacity(store_cache.mailboxes.items.len()), index: AHashMap::with_capacity(store_cache.mailboxes.items.len()), size: 0, @@ -54,12 +61,7 @@ pub(crate) async fn update_mailbox_cache( build_tree(&mut new_cache); - if store_cache.mailboxes.items.len() > new_cache.items.len() { - new_cache.items.shrink_to_fit(); - new_cache.index.shrink_to_fit(); - } - - Ok(new_cache) + Ok(new_cache.build()) } pub(crate) async fn full_mailbox_cache_build( @@ -67,7 +69,7 @@ pub(crate) async fn full_mailbox_cache_build( account_id: u32, ) -> trc::Result { // Build cache - let mut cache = MailboxesCache { + let mut cache = MailboxesCacheBuilder { items: Default::default(), index: Default::default(), size: 0, @@ -108,10 +110,10 @@ pub(crate) async fn full_mailbox_cache_build( build_tree(&mut cache); - Ok(cache) + Ok(cache.build()) } -fn insert_item(cache: &mut MailboxesCache, document_id: u32, mailbox: &ArchivedMailbox) { +fn insert_item(cache: &mut MailboxesCacheBuilder, document_id: u32, mailbox: &ArchivedMailbox) { let parent_id = mailbox.parent_id.to_native(); let item = MailboxCache { document_id, @@ -143,7 +145,7 @@ fn insert_item(cache: &mut MailboxesCache, document_id: u32, mailbox: &ArchivedM mailbox_insert(cache, item); } -fn build_tree(cache: &mut MailboxesCache) { +fn build_tree(cache: &mut MailboxesCacheBuilder) { cache.size = 0; let mut topological_sort = TopologicalSort::with_capacity(cache.items.len()); @@ -191,6 +193,18 @@ fn build_tree(cache: &mut MailboxesCache) { } } +impl MailboxesCacheBuilder { + fn build(mut self) -> MailboxesCache { + self.index.shrink_to_fit(); + MailboxesCache { + change_id: self.change_id, + index: self.index, + items: self.items.into_boxed_slice(), + size: self.size, + } + } +} + pub trait MailboxCacheAccess { fn mailbox_by_id(&self, id: &u32) -> Option<&MailboxCache>; fn mailbox_by_name(&self, name: &str) -> Option<&MailboxCache>; @@ -257,7 +271,7 @@ impl MailboxCacheAccess for MessageStoreCache { } #[inline(always)] -fn by_id<'x>(cache: &'x MailboxesCache, id: &u32) -> Option<&'x MailboxCache> { +fn by_id<'x>(cache: &'x MailboxesCacheBuilder, id: &u32) -> Option<&'x MailboxCache> { cache .index .get(id) @@ -265,14 +279,14 @@ fn by_id<'x>(cache: &'x MailboxesCache, id: &u32) -> Option<&'x MailboxCache> { } #[inline(always)] -fn by_id_mut<'x>(cache: &'x mut MailboxesCache, id: &u32) -> Option<&'x mut MailboxCache> { +fn by_id_mut<'x>(cache: &'x mut MailboxesCacheBuilder, id: &u32) -> Option<&'x mut MailboxCache> { cache .index .get(id) .and_then(|idx| cache.items.get_mut(*idx as usize)) } -fn mailbox_insert(cache: &mut MailboxesCache, item: MailboxCache) { +fn mailbox_insert(cache: &mut MailboxesCacheBuilder, item: MailboxCache) { let id = item.document_id; if let Some(idx) = cache.index.get(&id) { cache.items[*idx as usize] = item; diff --git a/crates/email/src/mailbox/destroy.rs b/crates/email/src/mailbox/destroy.rs index 0e294794..4ea74ba1 100644 --- a/crates/email/src/mailbox/destroy.rs +++ b/crates/email/src/mailbox/destroy.rs @@ -7,14 +7,22 @@ use super::*; use crate::{ cache::{MessageCacheFetch, email::MessageCacheAccess}, - message::{delete::EmailDeletion, metadata::MessageData}, + message::metadata::MessageData, }; use common::{ Server, auth::AccessToken, sharing::EffectiveAcl, storage::index::ObjectIndexBuilder, }; -use store::{roaring::RoaringBitmap, write::BatchBuilder}; +use store::{ + SerializeInfallible, + roaring::RoaringBitmap, + write::{BatchBuilder, SearchIndex, TaskEpoch, TaskQueueClass, ValueClass}, +}; use trc::AddContext; -use types::{acl::Acl, collection::Collection, field::MailboxField}; +use types::{ + acl::Acl, + collection::{Collection, VanishedCollection}, + field::MailboxField, +}; pub trait MailboxDestroy: Sync + Send { fn mailbox_destroy( @@ -76,8 +84,6 @@ impl MailboxDestroy for Server { // If the message is in multiple mailboxes, untag it from the current mailbox, // otherwise delete it. - let mut destroy_ids = RoaringBitmap::new(); - self.archives( account_id, Collection::Email, @@ -98,40 +104,68 @@ impl MailboxDestroy for Server { if prev_message_data.inner.mailboxes.len() == 1 { // Delete message - destroy_ids.insert(message_id); - return Ok(true); + for mailbox in prev_message_data.inner.mailboxes.iter() { + batch.log_vanished_item( + VanishedCollection::Email, + (mailbox.mailbox_id.to_native(), mailbox.uid.to_native()), + ); + } + batch + .with_collection(Collection::Email) + .with_document(message_id) + .custom( + ObjectIndexBuilder::<_, ()>::new() + .with_access_token(access_token) + .with_current(prev_message_data), + ) + .caused_by(trc::location!())? + .set( + ValueClass::TaskQueue(TaskQueueClass::UpdateIndex { + index: SearchIndex::Email, + due: TaskEpoch::now(), + is_insert: false, + }), + 0u64.serialize(), + ) + .commit_point(); + } else { + let new_message_data = MessageData { + mailboxes: prev_message_data + .inner + .mailboxes + .iter() + .filter(|m| m.mailbox_id != document_id) + .map(|m| m.to_native()) + .collect(), + keywords: prev_message_data + .inner + .keywords + .iter() + .map(|k| k.to_native()) + .collect(), + thread_id: prev_message_data.inner.thread_id.to_native(), + size: prev_message_data.inner.size.to_native(), + }; + + // Untag message from mailbox + batch + .with_collection(Collection::Email) + .with_document(message_id) + .custom( + ObjectIndexBuilder::new() + .with_access_token(access_token) + .with_changes(new_message_data) + .with_current(prev_message_data), + ) + .caused_by(trc::location!())? + .commit_point(); } - let mut new_message_data = prev_message_data - .deserialize() - .caused_by(trc::location!())?; - - new_message_data - .mailboxes - .retain(|id| id.mailbox_id != document_id); - - // Untag message from mailbox - batch - .with_collection(Collection::Email) - .with_document(message_id) - .custom( - ObjectIndexBuilder::new() - .with_changes(new_message_data) - .with_current(prev_message_data), - ) - .caused_by(trc::location!())? - .commit_point(); Ok(true) }, ) .await .caused_by(trc::location!())?; - - // Bulk delete messages - if !destroy_ids.is_empty() { - self.emails_delete(account_id, &mut batch, destroy_ids) - .await?; - } } else { return Ok(Err(MailboxDestroyError::HasEmails)); } diff --git a/crates/email/src/message/copy.rs b/crates/email/src/message/copy.rs index 285747f6..156f26d4 100644 --- a/crates/email/src/message/copy.rs +++ b/crates/email/src/message/copy.rs @@ -11,12 +11,15 @@ use super::{ use crate::{ mailbox::UidMailbox, message::{ - index::extractors::VisitText, + index::extractors::VisitTextArchived, ingest::{MergeThreadIds, ThreadInfo}, + metadata::{ + MESSAGE_HAS_ATTACHMENT, MESSAGE_RECEIVED_MASK, MetadataHeaderName, MetadataHeaderValue, + }, }, }; use common::{Server, auth::ResourceToken, storage::index::ObjectIndexBuilder}; -use mail_parser::{HeaderName, HeaderValue, parsers::fields::thread::thread_name}; +use mail_parser::parsers::fields::thread::thread_name; use store::write::{ BatchBuilder, IndexPropertyClass, SearchIndex, TaskEpoch, TaskQueueClass, ValueClass, }; @@ -79,10 +82,8 @@ impl EmailCopy for Server { }; // Check quota - match self - .has_available_quota(resource_token, metadata.size as u64) - .await - { + let size = metadata.root_part().offset_end; + match self.has_available_quota(resource_token, size as u64).await { Ok(_) => (), Err(err) => { if err.matches(trc::EventType::Limit(trc::LimitEvent::Quota)) @@ -98,7 +99,8 @@ impl EmailCopy for Server { // Set receivedAt if let Some(received_at) = received_at { - metadata.received_at = received_at; + metadata.rcvd_attach = (metadata.rcvd_attach & MESSAGE_HAS_ATTACHMENT) + | (received_at & MESSAGE_RECEIVED_MASK); } // Obtain threadId @@ -106,24 +108,26 @@ impl EmailCopy for Server { let mut subject = ""; for header in &metadata.contents[0].parts[0].headers { match &header.name { - HeaderName::MessageId => { + MetadataHeaderName::MessageId => { header.value.visit_text(|id| { if !id.is_empty() { message_ids.push(CheekyHash::new(id.as_bytes())); } }); } - HeaderName::InReplyTo | HeaderName::References | HeaderName::ResentMessageId => { + MetadataHeaderName::InReplyTo + | MetadataHeaderName::References + | MetadataHeaderName::ResentMessageId => { header.value.visit_text(|id| { if !id.is_empty() { message_ids.push(CheekyHash::new(id.as_bytes())); } }); } - HeaderName::Subject if subject.is_empty() => { + MetadataHeaderName::Subject if subject.is_empty() => { subject = thread_name(match &header.value { - HeaderValue::Text(text) => text.as_ref(), - HeaderValue::TextList(list) if !list.is_empty() => { + MetadataHeaderValue::Text(text) => text.as_ref(), + MetadataHeaderValue::TextList(list) if !list.is_empty() => { list.first().unwrap().as_ref() } _ => "", @@ -141,7 +145,7 @@ impl EmailCopy for Server { // Assign id let mut email = IngestedEmail { - size: metadata.size as usize, + size: size as usize, ..Default::default() }; let blob_hash = metadata.blob_hash.clone(); @@ -183,11 +187,14 @@ impl EmailCopy for Server { .with_collection(Collection::Email) .with_document(document_id) .custom( - ObjectIndexBuilder::<(), _>::new().with_changes(MessageData { - mailboxes: mailbox_ids, - keywords, - thread_id, - }), + ObjectIndexBuilder::<(), _>::new() + .with_tenant_id(resource_token.tenant.map(|t| t.id)) + .with_changes(MessageData { + mailboxes: mailbox_ids.into_boxed_slice(), + keywords: keywords.into_boxed_slice(), + thread_id, + size, + }), ) .caused_by(trc::location!())? .set( @@ -217,12 +224,7 @@ impl EmailCopy for Server { } metadata - .index( - &mut batch, - account_id, - resource_token.tenant.map(|t| t.id), - true, - ) + .index(&mut batch, true) .caused_by(trc::location!())?; // Insert and obtain ids diff --git a/crates/email/src/message/delete.rs b/crates/email/src/message/delete.rs index bdd1108f..b10ea2b2 100644 --- a/crates/email/src/message/delete.rs +++ b/crates/email/src/message/delete.rs @@ -20,12 +20,13 @@ use store::{ }; use trc::AddContext; use types::collection::{Collection, VanishedCollection}; -use types::field::{EmailField, EmailSubmissionField}; +use types::field::EmailSubmissionField; pub trait EmailDeletion: Sync + Send { fn emails_delete( &self, account_id: u32, + tenant_id: Option, batch: &mut BatchBuilder, document_ids: RoaringBitmap, ) -> impl Future> + Send; @@ -51,6 +52,7 @@ impl EmailDeletion for Server { async fn emails_delete( &self, account_id: u32, + tenant_id: Option, batch: &mut BatchBuilder, document_ids: RoaringBitmap, ) -> trc::Result { @@ -75,7 +77,11 @@ impl EmailDeletion for Server { } batch .with_document(document_id) - .custom(ObjectIndexBuilder::<_, ()>::new().with_current(metadata)) + .custom( + ObjectIndexBuilder::<_, ()>::new() + .with_tenant_id(tenant_id) + .with_current(metadata), + ) .caused_by(trc::location!())? .set( ValueClass::TaskQueue(TaskQueueClass::UpdateIndex { @@ -226,7 +232,8 @@ impl EmailDeletion for Server { } // Filter messages by received date - let mut destroy_ids = RoaringBitmap::new(); + let todo = "fix"; + /*let mut destroy_ids = RoaringBitmap::new(); self.store() .iterate( IterateParams::new( @@ -282,7 +289,7 @@ impl EmailDeletion for Server { self.emails_delete(account_id, &mut batch, destroy_ids) .await?; self.commit_batch(batch).await?; - self.notify_task_queue(); + self.notify_task_queue();*/ Ok(()) } diff --git a/crates/email/src/message/delivery.rs b/crates/email/src/message/delivery.rs index 2d48b874..b2717b79 100644 --- a/crates/email/src/message/delivery.rs +++ b/crates/email/src/message/delivery.rs @@ -158,6 +158,7 @@ impl MailDelivery for Server { // Ingest message self.email_ingest(IngestEmail { raw_message: &raw_message, + blob_hash: Some(&message.message_blob), message: MessageParser::new().parse(&raw_message), access_token: &access_token, mailbox_ids: vec![INBOX_ID], @@ -177,6 +178,7 @@ impl MailDelivery for Server { Ok(Some(active_script)) => { self.sieve_script_ingest( &access_token, + &message.message_blob, &raw_message, &message.sender_address, message.sender_authenticated, diff --git a/crates/email/src/message/index/extractors.rs b/crates/email/src/message/index/extractors.rs index b621cdd9..8b969da6 100644 --- a/crates/email/src/message/index/extractors.rs +++ b/crates/email/src/message/index/extractors.rs @@ -4,11 +4,11 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::message::metadata::{ArchivedMessageMetadataContents, ArchivedMessageMetadataPart}; -use mail_parser::{ - Addr, Address, ArchivedAddress, ArchivedHeaderName, ArchivedHeaderValue, Group, HeaderValue, - core::rkyv::ArchivedGetHeader, +use crate::message::metadata::{ + ArchivedMessageMetadataContents, ArchivedMessageMetadataPart, ArchivedMetadataHeaderValue, + MetadataHeaderName, MetadataHeaderValue, }; +use mail_parser::{Addr, Address, Group, HeaderValue}; use nlp::language::Language; use rkyv::option::ArchivedOption; use std::borrow::Cow; @@ -25,18 +25,11 @@ impl ArchivedMessageMetadataContents { impl ArchivedMessageMetadataPart { pub fn language(&self) -> Option { - self.headers - .header_value(&ArchivedHeaderName::ContentLanguage) + self.header_value(&MetadataHeaderName::ContentLanguage) .and_then(|v| { - Language::from_iso_639(match v { - ArchivedHeaderValue::Text(v) => v.as_ref(), - ArchivedHeaderValue::TextList(v) => v.first()?, - _ => { - return None; - } - }) - .unwrap_or(Language::Unknown) - .into() + Language::from_iso_639(v.as_text()?) + .unwrap_or(Language::Unknown) + .into() }) } } @@ -121,10 +114,58 @@ pub trait VisitTextArchived { fn visit_text(&self, visitor: impl FnMut(&str)); } -impl VisitTextArchived for ArchivedHeaderValue<'static> { +impl VisitTextArchived for MetadataHeaderValue { fn visit_addresses(&self, mut visitor: impl FnMut(AddressElement, &str)) { match self { - ArchivedHeaderValue::Address(ArchivedAddress::List(addr_list)) => { + MetadataHeaderValue::AddressList(addr_list) => { + for addr in addr_list.iter() { + if let Some(name) = &addr.name { + visitor(AddressElement::Name, name); + } + if let Some(addr) = &addr.address { + visitor(AddressElement::Address, addr); + } + } + } + MetadataHeaderValue::AddressGroup(groups) => { + for group in groups.iter() { + if let Some(name) = &group.name { + visitor(AddressElement::GroupName, name); + } + + for addr in group.addresses.iter() { + if let Some(name) = &addr.name { + visitor(AddressElement::Name, name); + } + if let Some(addr) = &addr.address { + visitor(AddressElement::Address, addr); + } + } + } + } + _ => (), + } + } + + fn visit_text(&self, mut visitor: impl FnMut(&str)) { + match &self { + MetadataHeaderValue::Text(text) => { + visitor(text.as_ref()); + } + MetadataHeaderValue::TextList(texts) => { + for text in texts.iter() { + visitor(text.as_ref()); + } + } + _ => (), + } + } +} + +impl VisitTextArchived for ArchivedMetadataHeaderValue { + fn visit_addresses(&self, mut visitor: impl FnMut(AddressElement, &str)) { + match self { + ArchivedMetadataHeaderValue::AddressList(addr_list) => { for addr in addr_list.iter() { if let ArchivedOption::Some(name) = &addr.name { visitor(AddressElement::Name, name); @@ -134,7 +175,7 @@ impl VisitTextArchived for ArchivedHeaderValue<'static> { } } } - ArchivedHeaderValue::Address(ArchivedAddress::Group(groups)) => { + ArchivedMetadataHeaderValue::AddressGroup(groups) => { for group in groups.iter() { if let ArchivedOption::Some(name) = &group.name { visitor(AddressElement::GroupName, name); @@ -156,10 +197,10 @@ impl VisitTextArchived for ArchivedHeaderValue<'static> { fn visit_text(&self, mut visitor: impl FnMut(&str)) { match &self { - ArchivedHeaderValue::Text(text) => { + ArchivedMetadataHeaderValue::Text(text) => { visitor(text.as_ref()); } - ArchivedHeaderValue::TextList(texts) => { + ArchivedMetadataHeaderValue::TextList(texts) => { for text in texts.iter() { visitor(text.as_ref()); } diff --git a/crates/email/src/message/index/metadata.rs b/crates/email/src/message/index/metadata.rs index f5fff5a1..34390242 100644 --- a/crates/email/src/message/index/metadata.rs +++ b/crates/email/src/message/index/metadata.rs @@ -7,19 +7,20 @@ use crate::message::{ index::{IndexMessage, MAX_MESSAGE_PARTS, PREVIEW_LENGTH}, metadata::{ - ArchivedMessageMetadata, ArchivedMessageMetadataPart, MessageData, MessageMetadata, - MessageMetadataPart, + ArchivedMessageMetadata, ArchivedMessageMetadataPart, ArchivedMetadataHeaderName, + MESSAGE_HAS_ATTACHMENT, MESSAGE_RECEIVED_MASK, MessageData, MessageMetadata, + MessageMetadataPart, build_metadata_contents, }, }; use common::storage::index::ObjectIndexBuilder; use mail_parser::{ - ArchivedHeaderName, + PartType, decoders::html::html_to_text, parsers::{fields::thread::thread_name, preview::preview_text}, }; use store::{ - Serialize, SerializeInfallible, - write::{Archiver, BatchBuilder, BlobOp, DirectoryClass, IndexPropertyClass, ValueClass}, + Serialize, + write::{Archiver, BatchBuilder, BlobOp, IndexPropertyClass, ValueClass}, }; use trc::AddContext; use types::{blob_hash::BlobHash, field::EmailField}; @@ -31,59 +32,22 @@ impl MessageMetadata { &self.contents[0].parts[0] } - pub fn index( - self, - batch: &mut BatchBuilder, - account_id: u32, - tenant_id: Option, - set: bool, - ) -> trc::Result<()> { + pub fn index(self, batch: &mut BatchBuilder, set: bool) -> trc::Result<()> { if set { - // Serialize metadata - batch.set( - ValueClass::IndexProperty(IndexPropertyClass::Integer { - property: EmailField::ReceivedToSize.into(), - value: self.received_at, - }), - self.size.serialize(), - ); - } else { - // Delete metadata batch - .clear(EmailField::Metadata) - .clear(ValueClass::IndexProperty(IndexPropertyClass::Integer { - property: EmailField::ReceivedToSize.into(), - value: self.received_at, - })); - } - - // Index properties - let quota = if set { - self.size as i64 + .set( + BlobOp::Link { + hash: self.blob_hash.clone(), + }, + Vec::new(), + ) + .set(EmailField::Metadata, Archiver::new(self).serialize()?); } else { - -(self.size as i64) - }; - batch.add(DirectoryClass::UsedQuota(account_id), quota); - if let Some(tenant_id) = tenant_id { - batch.add(DirectoryClass::UsedQuota(tenant_id), quota); - } - - // Link blob - if set { - batch.set( - BlobOp::Link { + batch + .clear(BlobOp::Link { hash: self.blob_hash.clone(), - }, - Vec::new(), - ); - } else { - batch.clear(BlobOp::Link { - hash: self.blob_hash.clone(), - }); - } - - if set { - batch.set(EmailField::Metadata, Archiver::new(self).serialize()?); + }) + .clear(EmailField::Metadata); } Ok(()) @@ -96,7 +60,7 @@ impl ArchivedMessageMetadata { &self.contents[0].parts[0] } - pub fn unindex(&self, batch: &mut BatchBuilder, account_id: u32, tenant_id: Option) { + pub fn unindex(&self, batch: &mut BatchBuilder) { // Delete metadata let thread_name = self .contents @@ -104,7 +68,7 @@ impl ArchivedMessageMetadata { .and_then(|c| c.parts.first()) .and_then(|p| { p.headers.iter().rev().find_map(|h| { - if let ArchivedHeaderName::Subject = &h.name { + if let ArchivedMetadataHeaderName::Subject = &h.name { h.value.as_text() } else { None @@ -113,12 +77,9 @@ impl ArchivedMessageMetadata { }) .map(thread_name) .unwrap_or_default(); + batch .clear(EmailField::Metadata) - .clear(ValueClass::IndexProperty(IndexPropertyClass::Integer { - property: EmailField::ReceivedToSize.into(), - value: self.received_at.to_native(), - })) .clear(ValueClass::IndexProperty(IndexPropertyClass::Hash { property: EmailField::Threading.into(), hash: CheekyHash::new(if !thread_name.is_empty() { @@ -126,51 +87,24 @@ impl ArchivedMessageMetadata { } else { "!" }), - })); - - // Index properties - let quota = -(u32::from(self.size) as i64); - batch.add(DirectoryClass::UsedQuota(account_id), quota); - if let Some(tenant_id) = tenant_id { - batch.add(DirectoryClass::UsedQuota(tenant_id), quota); - } - - // Unlink blob - batch.clear(BlobOp::Link { - hash: BlobHash::from(&self.blob_hash), - }); + })) + .clear(BlobOp::Link { + hash: BlobHash::from(&self.blob_hash), + }); } } impl IndexMessage for BatchBuilder { - fn index_message( + fn index_message<'x>( &mut self, - account_id: u32, tenant_id: Option, - message: mail_parser::Message<'_>, + mut message: mail_parser::Message<'x>, + extra_headers: Vec, + mut extra_headers_parsed: Vec>, blob_hash: BlobHash, data: MessageData, received_at: u64, ) -> trc::Result<&mut Self> { - // Index size - self.set( - ValueClass::IndexProperty(IndexPropertyClass::Integer { - property: EmailField::ReceivedToSize.into(), - value: received_at, - }), - (message.raw_message.len() as u32).serialize(), - ) - .add( - DirectoryClass::UsedQuota(account_id), - message.raw_message.len() as i64, - ); - if let Some(tenant_id) = tenant_id { - self.add( - DirectoryClass::UsedQuota(tenant_id), - message.raw_message.len() as i64, - ); - } - let mut has_attachments = false; let mut preview = None; let preview_part_id = message @@ -217,38 +151,91 @@ impl IndexMessage for BatchBuilder { } } - // Build metadata + // Build raw headers let root_part = message.root_part(); - let metadata = MessageMetadata { - preview: preview.unwrap_or_default().into_owned(), - size: message.raw_message.len() as u32, - raw_headers: message + let mut raw_headers = Vec::with_capacity( + (root_part.offset_body - root_part.offset_header) as usize + extra_headers.len(), + ); + raw_headers.extend_from_slice(&extra_headers); + raw_headers.extend_from_slice( + message .raw_message .as_ref() .get(root_part.offset_header as usize..root_part.offset_body as usize) - .unwrap_or_default() - .to_vec(), - contents: vec![], - received_at, - has_attachments, - blob_hash, - } - .with_contents(message); + .unwrap_or_default(), + ); + + // Add additional headers to message + let blob_body_offset = if !extra_headers.is_empty() { + // Add extra headers to root part + let offset_start = extra_headers.len() as u32; + let mut part_iter_stack = Vec::new(); + let mut part_iter = message.parts.iter_mut(); + + loop { + if let Some(part) = part_iter.next() { + // Increment header offsets + for header in part.headers.iter_mut() { + header.offset_field += offset_start; + header.offset_start += offset_start; + header.offset_end += offset_start; + } + + // Adjust part offsets + part.offset_body += offset_start; + part.offset_end += offset_start; + part.offset_header += offset_start; + + if let PartType::Message(sub_message) = &mut part.body + && sub_message.root_part().offset_header != 0 + { + part_iter_stack.push(part_iter); + part_iter = sub_message.parts.iter_mut(); + } + } else if let Some(iter) = part_iter_stack.pop() { + part_iter = iter; + } else { + break; + } + } + + // Add extra headers to root part + let root_part = &mut message.parts[0]; + extra_headers_parsed.append(&mut root_part.headers); + root_part.offset_header = 0; + root_part.headers = extra_headers_parsed; + root_part.offset_body - offset_start + } else { + message.root_part().offset_body + }; + + // Build metadata + let metadata = MessageMetadata { + preview: preview.unwrap_or_default().into_owned().into_boxed_str(), + raw_headers: raw_headers.into_boxed_slice(), + contents: build_metadata_contents(message), + blob_hash, + blob_body_offset, + rcvd_attach: (if has_attachments { + MESSAGE_HAS_ATTACHMENT + } else { + 0 + }) | (received_at & MESSAGE_RECEIVED_MASK), + }; - // Link blob self.set( BlobOp::Link { hash: metadata.blob_hash.clone(), }, Vec::new(), - ); - - // Store message data - self.custom(ObjectIndexBuilder::<(), _>::new().with_changes(data)) - .caused_by(trc::location!())?; - - // Store message metadata - self.set( + ) + .custom( + ObjectIndexBuilder::<(), _>::new() + .with_tenant_id(tenant_id) + .with_changes(data), + ) + .caused_by(trc::location!())? + .set( EmailField::Metadata, Archiver::new(metadata) .serialize() diff --git a/crates/email/src/message/index/mod.rs b/crates/email/src/message/index/mod.rs index 8d275a7e..85e31ed1 100644 --- a/crates/email/src/message/index/mod.rs +++ b/crates/email/src/message/index/mod.rs @@ -18,6 +18,7 @@ pub const PREVIEW_LENGTH: usize = 256; impl IndexableObject for MessageData { fn index_values(&self) -> impl Iterator> { [ + IndexValue::Quota { used: self.size }, IndexValue::LogItem { sync_collection: SyncCollection::Email, prefix: self.thread_id.into(), @@ -38,6 +39,9 @@ impl IndexableObject for MessageData { impl IndexableObject for &ArchivedMessageData { fn index_values(&self) -> impl Iterator> { [ + IndexValue::Quota { + used: self.size.to_native(), + }, IndexValue::LogItem { sync_collection: SyncCollection::Email, prefix: self.thread_id.to_native().into(), @@ -61,11 +65,12 @@ impl IndexableObject for &ArchivedMessageData { pub(super) trait IndexMessage { #[allow(clippy::too_many_arguments)] - fn index_message( + fn index_message<'x>( &mut self, - account_id: u32, tenant_id: Option, - message: mail_parser::Message<'_>, + message: mail_parser::Message<'x>, + extra_headers: Vec, + extra_headers_parsed: Vec>, blob_hash: BlobHash, data: MessageData, received_at: u64, diff --git a/crates/email/src/message/index/search.rs b/crates/email/src/message/index/search.rs index 39da649e..dce77e37 100644 --- a/crates/email/src/message/index/search.rs +++ b/crates/email/src/message/index/search.rs @@ -6,12 +6,12 @@ use crate::message::{ index::{MAX_MESSAGE_PARTS, extractors::VisitTextArchived}, - metadata::{ArchivedMessageMetadata, ArchivedMetadataPartType, DecodedPartContent}, -}; -use mail_parser::{ - ArchivedHeaderName, ArchivedHeaderValue, DateTime, core::rkyv::ArchivedGetHeader, - decoders::html::html_to_text, parsers::fields::thread::thread_name, + metadata::{ + ArchivedMessageMetadata, ArchivedMetadataHeaderName, ArchivedMetadataHeaderValue, + ArchivedMetadataPartType, DecodedPartContent, MESSAGE_RECEIVED_MASK, MetadataHeaderName, + }, }; +use mail_parser::{DateTime, decoders::html::html_to_text, parsers::fields::thread::thread_name}; use nlp::{ language::{ Language, @@ -25,6 +25,7 @@ use store::{ search::{EmailSearchField, IndexDocument, SearchField}, write::SearchIndex, }; +use utils::chained_bytes::ChainedBytes; impl ArchivedMessageMetadata { pub fn index_document( @@ -42,12 +43,18 @@ impl ArchivedMessageMetadata { .with_account_id(account_id) .with_document_id(document_id); + let raw_message = ChainedBytes::new(self.raw_headers.as_ref()).with_last( + raw_message + .get(self.blob_body_offset.to_native() as usize..) + .unwrap_or_default(), + ); + if index_fields.is_empty() || index_fields.contains(&SearchField::Email(EmailSearchField::ReceivedAt)) { document.index_unsigned( SearchField::Email(EmailSearchField::ReceivedAt), - self.received_at.to_native(), + self.rcvd_attach.to_native() & MESSAGE_RECEIVED_MASK, ); } if index_fields.is_empty() @@ -55,7 +62,7 @@ impl ArchivedMessageMetadata { { document.index_unsigned( SearchField::Email(EmailSearchField::Size), - self.size.to_native(), + raw_message.len() as u32, ); } @@ -71,7 +78,7 @@ impl ArchivedMessageMetadata { for header in part.headers.iter().rev() { match &header.name { - ArchivedHeaderName::From => { + ArchivedMetadataHeaderName::From => { if index_fields.is_empty() || index_fields .contains(&SearchField::Email(EmailSearchField::From)) @@ -85,7 +92,7 @@ impl ArchivedMessageMetadata { }); } } - ArchivedHeaderName::To => { + ArchivedMetadataHeaderName::To => { if index_fields.is_empty() || index_fields.contains(&SearchField::Email(EmailSearchField::To)) { @@ -98,7 +105,7 @@ impl ArchivedMessageMetadata { }); } } - ArchivedHeaderName::Cc => { + ArchivedMetadataHeaderName::Cc => { if index_fields.is_empty() || index_fields.contains(&SearchField::Email(EmailSearchField::Cc)) { @@ -111,7 +118,7 @@ impl ArchivedMessageMetadata { }); } } - ArchivedHeaderName::Bcc => { + ArchivedMetadataHeaderName::Bcc => { if index_fields.is_empty() || index_fields.contains(&SearchField::Email(EmailSearchField::Bcc)) { @@ -124,7 +131,7 @@ impl ArchivedMessageMetadata { }); } } - ArchivedHeaderName::Subject => { + ArchivedMetadataHeaderName::Subject => { if (index_fields.is_empty() || index_fields .contains(&SearchField::Email(EmailSearchField::Subject))) @@ -143,7 +150,7 @@ impl ArchivedMessageMetadata { ); } } - ArchivedHeaderName::Date => { + ArchivedMetadataHeaderName::Date => { if (index_fields.is_empty() || index_fields .contains(&SearchField::Email(EmailSearchField::SentAt))) @@ -166,7 +173,8 @@ impl ArchivedMessageMetadata { if index_headers { let mut value = String::new(); match &header.value { - ArchivedHeaderValue::Address(_) => { + ArchivedMetadataHeaderValue::AddressList(_) + | ArchivedMetadataHeaderValue::AddressGroup(_) => { header.value.visit_addresses(|_, addr| { if !value.is_empty() { value.push(' '); @@ -174,8 +182,8 @@ impl ArchivedMessageMetadata { value.push_str(addr); }); } - ArchivedHeaderValue::Text(_) - | ArchivedHeaderValue::TextList(_) => { + ArchivedMetadataHeaderValue::Text(_) + | ArchivedMetadataHeaderValue::TextList(_) => { header.value.visit_text(|text| { if !value.is_empty() { value.push(' '); @@ -183,15 +191,19 @@ impl ArchivedMessageMetadata { value.push_str(text); }); } - ArchivedHeaderValue::ContentType(_) - | ArchivedHeaderValue::Received(_) => { - if let Some(header) = raw_message - .get( - header.offset_start.to_native() as usize - ..header.offset_end.to_native() as usize, - ) - .and_then(|bytes| std::str::from_utf8(bytes).ok()) + _ => { + if (matches!( + header.value, + ArchivedMetadataHeaderValue::ContentType(_) + ) || matches!( + header.name, + ArchivedMetadataHeaderName::Received + )) && let Some(header) = + raw_message.get(header.value_range()) { + let header = std::str::from_utf8(header.as_ref()) + .unwrap_or_default(); + for word in WordTokenizer::new(header, MAX_TOKEN_LENGTH) { if !value.is_empty() { @@ -201,8 +213,6 @@ impl ArchivedMessageMetadata { } } } - ArchivedHeaderValue::DateTime(_) - | ArchivedHeaderValue::Empty => (), } document.insert_key_value( @@ -219,7 +229,7 @@ impl ArchivedMessageMetadata { let part_id = part_id as u16; match &part.body { ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html => { - let text = match (part.decode_contents(raw_message), &part.body) { + let text = match (part.decode_contents(&raw_message), &part.body) { (DecodedPartContent::Text(text), ArchivedMetadataPartType::Text) => text, (DecodedPartContent::Text(html), ArchivedMetadataPartType::Html) => { html_to_text(html.as_ref()).into() @@ -267,10 +277,9 @@ impl ArchivedMessageMetadata { .root_part() .language() .unwrap_or(Language::Unknown); - if let Some(ArchivedHeaderValue::Text(subject)) = nested_message + if let Some(ArchivedMetadataHeaderValue::Text(subject)) = nested_message .root_part() - .headers - .header_value(&ArchivedHeaderName::Subject) + .header_value(&MetadataHeaderName::Subject) { if nested_message_language.is_unknown() { detector.detect(subject.as_ref(), MIN_LANGUAGE_SCORE); @@ -287,18 +296,20 @@ impl ArchivedMessageMetadata { let language = sub_part.language().unwrap_or(nested_message_language); match &sub_part.body { ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html => { - let text = - match (sub_part.decode_contents(raw_message), &sub_part.body) { - ( - DecodedPartContent::Text(text), - ArchivedMetadataPartType::Text, - ) => text, - ( - DecodedPartContent::Text(html), - ArchivedMetadataPartType::Html, - ) => html_to_text(html.as_ref()).into(), - _ => unreachable!(), - }; + let text = match ( + sub_part.decode_contents(&raw_message), + &sub_part.body, + ) { + ( + DecodedPartContent::Text(text), + ArchivedMetadataPartType::Text, + ) => text, + ( + DecodedPartContent::Text(html), + ArchivedMetadataPartType::Html, + ) => html_to_text(html.as_ref()).into(), + _ => unreachable!(), + }; if language.is_unknown() { detector.detect(text.as_ref(), MIN_LANGUAGE_SCORE); diff --git a/crates/email/src/message/ingest.rs b/crates/email/src/message/ingest.rs index e40a2f5c..20679ba4 100644 --- a/crates/email/src/message/ingest.rs +++ b/crates/email/src/message/ingest.rs @@ -40,6 +40,7 @@ use store::{ use trc::{AddContext, MessageIngestEvent}; use types::{ blob::{BlobClass, BlobId}, + blob_hash::BlobHash, collection::{Collection, SyncCollection}, field::{ContactField, EmailField, MailboxField, PrincipalField}, keyword::Keyword, @@ -58,6 +59,7 @@ pub struct IngestedEmail { pub struct IngestEmail<'x> { pub raw_message: &'x [u8], + pub blob_hash: Option<&'x BlobHash>, pub message: Option>, pub access_token: &'x AccessToken, pub mailbox_ids: Vec, @@ -264,6 +266,7 @@ impl EmailIngest for Server { value: HeaderValue::Text( extra_headers [offset_start + 1..extra_headers.len() - 2] + .to_string() .into(), ), offset_field: offset_field as u32, @@ -465,55 +468,6 @@ impl EmailIngest for Server { } } - // Add additional headers to message - if !extra_headers.is_empty() { - let offset_start = extra_headers.len(); - raw_message_len += offset_start as u64; - let mut new_message = Vec::with_capacity(raw_message_len as usize); - new_message.extend_from_slice(extra_headers.as_bytes()); - new_message.extend_from_slice(raw_message.as_ref()); - raw_message = Cow::from(new_message); - message.raw_message = raw_message.as_ref().into(); - - // Adjust offsets - let mut part_iter_stack = Vec::new(); - let mut part_iter = message.parts.iter_mut(); - - loop { - if let Some(part) = part_iter.next() { - // Increment header offsets - for header in part.headers.iter_mut() { - header.offset_field += offset_start as u32; - header.offset_start += offset_start as u32; - header.offset_end += offset_start as u32; - } - - // Adjust part offsets - part.offset_body += offset_start as u32; - part.offset_end += offset_start as u32; - part.offset_header += offset_start as u32; - - if let PartType::Message(sub_message) = &mut part.body - && sub_message.root_part().offset_header != 0 - { - sub_message.raw_message = raw_message.as_ref().into(); - part_iter_stack.push(part_iter); - part_iter = sub_message.parts.iter_mut(); - } - } else if let Some(iter) = part_iter_stack.pop() { - part_iter = iter; - } else { - break; - } - } - - // Add extra headers to root part - let root_part = &mut message.parts[0]; - root_part.offset_header = 0; - extra_headers_parsed.append(&mut root_part.headers); - root_part.headers = extra_headers_parsed; - } - // Encrypt message let do_encrypt = match params.source { IngestSource::Jmap | IngestSource::Imap => { @@ -551,6 +505,7 @@ impl EmailIngest for Server { "Failed to parse encrypted e-mail message.", ) })?; + params.blob_hash = None; // Remove contents from parsed message for part in &mut message.parts { @@ -581,10 +536,14 @@ impl EmailIngest for Server { } // Store blob - let blob_id = self - .put_blob(account_id, raw_message.as_ref(), false) - .await - .caused_by(trc::location!())?; + let blob_hash = if let Some(blob_hash) = params.blob_hash { + blob_hash.clone() + } else { + self.put_blob(account_id, raw_message.as_ref(), false) + .await + .caused_by(trc::location!())? + .hash + }; // Assign IMAP UIDs let mut mailbox_ids = Vec::with_capacity(params.mailbox_ids.len()); @@ -624,19 +583,23 @@ impl EmailIngest for Server { document_id }; + let data = MessageData { + mailboxes: mailbox_ids.into_boxed_slice(), + keywords: params.keywords.into_boxed_slice(), + thread_id, + size: (message.raw_message.len() + extra_headers.len()) as u32, + }; + batch .with_collection(Collection::Email) .with_document(document_id) .index_message( - account_id, tenant_id, message, - blob_id.hash.clone(), - MessageData { - mailboxes: mailbox_ids, - keywords: params.keywords, - thread_id, - }, + extra_headers.into_bytes(), + extra_headers_parsed, + blob_hash.clone(), + data, params.received_at.unwrap_or_else(now), ) .caused_by(trc::location!())? @@ -710,7 +673,7 @@ impl EmailIngest for Server { AccountId = account_id, DocumentId = document_id, MailboxId = mailbox_ids_event, - BlobId = blob_id.hash.to_hex(), + BlobId = blob_hash.to_hex(), ChangeId = change_id, MessageId = message_id, Size = raw_message_len, @@ -722,13 +685,13 @@ impl EmailIngest for Server { thread_id, change_id, blob_id: BlobId { - hash: blob_id.hash, + hash: blob_hash, class: BlobClass::Linked { account_id, collection: Collection::Email.into(), document_id, }, - section: blob_id.section, + section: None, }, size: raw_message_len as usize, imap_uids, diff --git a/crates/email/src/message/metadata.rs b/crates/email/src/message/metadata.rs index 9eb4eb27..a82970f7 100644 --- a/crates/email/src/message/metadata.rs +++ b/crates/email/src/message/metadata.rs @@ -7,42 +7,42 @@ use crate::mailbox::{ArchivedUidMailbox, UidMailbox}; use common::storage::index::IndexableAndSerializableObject; use mail_parser::{ - ArchivedContentType, ArchivedEncoding, ArchivedHeaderName, ArchivedHeaderValue, DateTime, - Encoding, Header, HeaderName, HeaderValue, PartType, - core::rkyv::ArchivedGetHeader, + Addr, Address, Attribute, ContentType, DateTime, Encoding, Group, HeaderName, HeaderValue, + PartType, decoders::{ base64::base64_decode, charsets::map::charset_decoder, quoted_printable::quoted_printable_decode, }, }; -use rkyv::{ - rend::{u16_le, u32_le}, - vec::ArchivedVec, -}; -use std::{borrow::Cow, collections::VecDeque}; +use rkyv::{boxed::ArchivedBox, rend::u16_le}; +use std::{borrow::Cow, collections::VecDeque, ops::Range}; use types::{ blob_hash::BlobHash, keyword::{ArchivedKeyword, Keyword}, }; +use utils::chained_bytes::ChainedBytes; -#[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug, Default)] +#[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug)] pub struct MessageData { - pub mailboxes: Vec, - pub keywords: Vec, + pub mailboxes: Box<[UidMailbox]>, + pub keywords: Box<[Keyword]>, pub thread_id: u32, + pub size: u32, } #[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug)] pub struct MessageMetadata { - pub contents: Vec, + pub contents: Box<[MessageMetadataContents]>, + pub rcvd_attach: u64, pub blob_hash: BlobHash, - pub size: u32, - pub received_at: u64, - pub preview: String, - pub has_attachments: bool, - pub raw_headers: Vec, + pub blob_body_offset: u32, + pub preview: Box, + pub raw_headers: Box<[u8]>, } +pub const MESSAGE_HAS_ATTACHMENT: u64 = 1 << 63; +pub const MESSAGE_RECEIVED_MASK: u64 = !MESSAGE_HAS_ATTACHMENT; + impl IndexableAndSerializableObject for MessageData { fn is_versioned() -> bool { true @@ -51,24 +51,138 @@ impl IndexableAndSerializableObject for MessageData { #[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug)] pub struct MessageMetadataContents { - pub html_body: Vec, - pub text_body: Vec, - pub attachments: Vec, - pub parts: Vec, + pub html_body: Box<[u16]>, + pub text_body: Box<[u16]>, + pub attachments: Box<[u16]>, + pub parts: Box<[MessageMetadataPart]>, } #[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug)] pub struct MessageMetadataPart { - pub headers: Vec>, - pub is_encoding_problem: bool, + pub headers: Box<[MetadataHeader]>, pub body: MetadataPartType, - pub encoding: Encoding, - pub size: u32, + pub flags: u32, pub offset_header: u32, pub offset_body: u32, pub offset_end: u32, } +pub const PART_ENCODING_BASE64: u32 = 1 << 31; +pub const PART_ENCODING_QP: u32 = 1 << 30; +pub const PART_ENCODING_PROBLEM: u32 = 1 << 29; +pub const PART_SIZE_MASK: u32 = !(PART_ENCODING_BASE64 | PART_ENCODING_QP | PART_ENCODING_PROBLEM); + +#[derive(Debug, PartialEq, Eq, Clone, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive)] +#[rkyv(compare(PartialEq))] +pub struct MetadataHeader { + pub name: MetadataHeaderName, + pub value: MetadataHeaderValue, + pub base_offset: u32, + pub start: u16, + pub end: u16, +} + +#[derive(Debug, PartialEq, Eq, Clone, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive)] +#[rkyv(compare(PartialEq))] +pub enum MetadataHeaderName { + Other(Box), + Subject, + From, + To, + Cc, + Date, + Bcc, + ReplyTo, + Sender, + Comments, + InReplyTo, + Keywords, + Received, + MessageId, + References, + ReturnPath, + MimeVersion, + ContentDescription, + ContentId, + ContentLanguage, + ContentLocation, + ContentTransferEncoding, + ContentType, + ContentDisposition, + ResentTo, + ResentFrom, + ResentBcc, + ResentCc, + ResentSender, + ResentDate, + ResentMessageId, + ListArchive, + ListHelp, + ListId, + ListOwner, + ListPost, + ListSubscribe, + ListUnsubscribe, + DkimSignature, + ArcAuthenticationResults, + ArcMessageSignature, + ArcSeal, +} + +#[derive(Debug, PartialEq, Eq, Clone, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive)] +#[rkyv(compare(PartialEq))] +pub enum MetadataHeaderValue { + AddressList(Box<[MetadataAddress]>), + AddressGroup(Box<[MetadataAddressGroup]>), + Text(Box), + TextList(Box<[Box]>), + DateTime(MetadataDateTime), + ContentType(MetadataContentType), + Empty, +} + +#[derive(Debug, PartialEq, Eq, Clone, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive)] +#[rkyv(compare(PartialEq))] +pub struct MetadataDateTime { + pub year: u16, + pub month: u8, + pub day: u8, + pub hour: u8, + pub minute: u8, + pub second: u8, + pub tz_hour: i8, + pub tz_minute: u8, +} + +#[derive(Debug, PartialEq, Eq, Clone, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive)] +#[rkyv(compare(PartialEq))] +pub struct MetadataAddress { + pub name: Option>, + pub address: Option>, +} + +#[derive(Debug, PartialEq, Eq, Clone, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive)] +#[rkyv(compare(PartialEq))] +pub struct MetadataAddressGroup { + pub name: Option>, + pub addresses: Box<[MetadataAddress]>, +} + +#[derive(Debug, PartialEq, Eq, Clone, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive)] +#[rkyv(compare(PartialEq))] +pub struct MetadataContentType { + pub c_type: Box, + pub c_subtype: Option>, + pub attributes: Box<[MetadataAttribute]>, +} + +#[derive(Debug, PartialEq, Eq, Clone, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive)] +#[rkyv(compare(PartialEq))] +pub struct MetadataAttribute { + pub name: Box, + pub value: Box, +} + #[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug)] pub enum MetadataPartType { Text, @@ -76,7 +190,7 @@ pub enum MetadataPartType { Binary, InlineBinary, Message(u16), - Multipart(Vec), + Multipart(Box<[u16]>), } impl MessageMetadataContents { @@ -87,10 +201,16 @@ impl MessageMetadataContents { #[derive(Debug)] pub struct DecodedParts<'x> { - pub raw_messages: Vec>, + pub raw_messages: Vec>, pub parts: Vec>, } +#[derive(Debug)] +pub enum DecodedRawMessage<'x> { + Borrowed(ChainedBytes<'x>), + Owned(Vec), +} + #[derive(Debug)] pub struct DecodedPart<'x> { pub message_id: usize, @@ -106,26 +226,17 @@ pub enum DecodedPartContent<'x> { impl<'x> DecodedParts<'x> { #[inline] - pub fn raw_message(&self, message_id: usize) -> Option<&[u8]> { - self.raw_messages.get(message_id).map(|m| m.as_ref()) + pub fn raw_message(&self, message_id: usize) -> Option<&DecodedRawMessage<'x>> { + self.raw_messages.get(message_id) } #[inline] - pub fn raw_message_section(&self, message_id: usize, from: usize, to: usize) -> Option<&[u8]> { - self.raw_messages - .get(message_id) - .map(|m| m.as_ref()) - .and_then(|m| m.get(from..to)) - } - - #[inline] - pub fn raw_message_section_arch( - &self, + pub fn raw_message_section( + &'_ self, message_id: usize, - from: u32_le, - to: u32_le, - ) -> Option<&[u8]> { - self.raw_message_section(message_id, u32::from(from) as usize, u32::from(to) as usize) + range: Range, + ) -> Option> { + self.raw_messages.get(message_id).and_then(|m| m.get(range)) } #[inline] @@ -177,20 +288,31 @@ impl DecodedPartContent<'_> { } } +impl<'x> DecodedRawMessage<'x> { + pub fn get(&'_ self, index: Range) -> Option> { + match self { + DecodedRawMessage::Borrowed(bytes) => bytes.get(index), + DecodedRawMessage::Owned(vec) => vec.get(index).map(Cow::Borrowed), + } + } +} + impl ArchivedMessageMetadata { #[inline(always)] pub fn message_id(&self, message_id: u16_le) -> &ArchivedMessageMetadataContents { &self.contents[u16::from(message_id) as usize] } - pub fn decode_contents<'x>(&self, raw: &'x [u8]) -> DecodedParts<'x> { + pub fn decode_contents<'x>(&self, raw: ChainedBytes<'x>) -> DecodedParts<'x> { let mut result = DecodedParts { raw_messages: Vec::with_capacity(self.contents.len()), parts: Vec::new(), }; for _ in 0..self.contents.len() { - result.raw_messages.push(Cow::Borrowed(raw)); + result + .raw_messages + .push(DecodedRawMessage::Borrowed(raw.clone())); } for (message_id, contents) in self.contents.iter().enumerate() { @@ -202,18 +324,18 @@ impl ArchivedMessageMetadata { | ArchivedMetadataPartType::Binary | ArchivedMetadataPartType::InlineBinary => { match result.raw_messages.get(message_id).unwrap() { - Cow::Borrowed(raw_message) => { + DecodedRawMessage::Borrowed(bytes) => { result.parts.push(DecodedPart { message_id, part_offset, - content: part.decode_contents(raw_message), + content: part.decode_contents(bytes), }); } - Cow::Owned(raw_message) => { + DecodedRawMessage::Owned(bytes) => { result.parts.push(DecodedPart { message_id, part_offset, - content: match part.decode_contents(raw_message) { + content: match part.decode_contents(&ChainedBytes::new(bytes)) { DecodedPartContent::Text(text) => { DecodedPartContent::Text(text.into_owned().into()) } @@ -226,18 +348,27 @@ impl ArchivedMessageMetadata { } } ArchivedMetadataPartType::Message(nested_message_id) => { - let sub_contents = if !matches!(part.encoding, ArchivedEncoding::None) { - part.contents(result.raw_messages.get(message_id).unwrap()) - .into_owned() - } else if let Some(Cow::Owned(raw_message)) = - result.raw_messages.get(message_id) - { - raw_message.clone() - } else { - continue; - }; + let sub_contents = + if (part.flags & (PART_ENCODING_BASE64 | PART_ENCODING_QP)) != 0 { + match result.raw_messages.get(message_id).unwrap() { + DecodedRawMessage::Borrowed(bytes) => { + part.contents(bytes).into_owned() + } + DecodedRawMessage::Owned(bytes) => { + let bytes = ChainedBytes::new(bytes); + part.contents(&bytes).into_owned() + } + } + } else if let Some(DecodedRawMessage::Owned(bytes)) = + result.raw_messages.get(message_id) + { + bytes.clone() + } else { + continue; + }; - result.raw_messages[usize::from(*nested_message_id)] = sub_contents.into(); + result.raw_messages[usize::from(*nested_message_id)] = + DecodedRawMessage::Owned(sub_contents); } _ => {} } @@ -249,20 +380,36 @@ impl ArchivedMessageMetadata { } impl ArchivedMessageMetadataPart { - pub fn contents<'x>(&self, raw_message: &'x [u8]) -> Cow<'x, [u8]> { - let bytes = raw_message - .get(u32::from(self.offset_body) as usize..u32::from(self.offset_end) as usize) - .unwrap_or_default(); - match self.encoding { - ArchivedEncoding::None => bytes.into(), - ArchivedEncoding::QuotedPrintable => { - quoted_printable_decode(bytes).unwrap_or_default().into() - } - ArchivedEncoding::Base64 => base64_decode(bytes).unwrap_or_default().into(), + pub fn contents<'x>(&self, raw_message: &ChainedBytes<'x>) -> Cow<'x, [u8]> { + let bytes = raw_message.get(self.body_to_end()).unwrap_or_default(); + + if (self.flags & PART_ENCODING_BASE64) != 0 { + base64_decode(bytes.as_ref()).unwrap_or_default().into() + } else if (self.flags & PART_ENCODING_QP) != 0 { + quoted_printable_decode(bytes.as_ref()) + .unwrap_or_default() + .into() + } else { + bytes } } - pub fn decode_contents<'x>(&self, raw_message: &'x [u8]) -> DecodedPartContent<'x> { + #[inline(always)] + pub fn body_to_end(&self) -> Range { + (self.offset_body.to_native() as usize)..(self.offset_end.to_native() as usize) + } + + #[inline(always)] + pub fn header_to_end(&self) -> Range { + self.offset_header.to_native() as usize..self.offset_end.to_native() as usize + } + + #[inline(always)] + pub fn header_to_body(&self) -> Range { + self.offset_header.to_native() as usize..self.offset_body.to_native() as usize + } + + pub fn decode_contents<'x>(&self, raw_message: &ChainedBytes<'x>) -> DecodedPartContent<'x> { let bytes = self.contents(raw_message); match self.body { @@ -270,8 +417,7 @@ impl ArchivedMessageMetadataPart { DecodedPartContent::Text( match ( bytes, - self.headers - .header_value(&ArchivedHeaderName::ContentType) + self.header_value(&MetadataHeaderName::ContentType) .and_then(|c| c.as_content_type()) .and_then(|ct| { ct.attribute("charset") @@ -298,94 +444,103 @@ impl ArchivedMessageMetadataPart { } } -impl MessageMetadata { - pub fn with_contents(mut self, message: mail_parser::Message<'_>) -> Self { - let mut messages = VecDeque::from([message]); - let mut message_id = 0; +pub fn build_metadata_contents( + message: mail_parser::Message<'_>, +) -> Box<[MessageMetadataContents]> { + let mut messages = VecDeque::from([message]); + let mut message_id = 0; + let mut contents = Vec::new(); - while let Some(message) = messages.pop_front() { - let mut contents = MessageMetadataContents { - html_body: message.html_body.into_iter().map(|c| c as u16).collect(), - text_body: message.text_body.into_iter().map(|c| c as u16).collect(), - attachments: message.attachments.into_iter().map(|c| c as u16).collect(), - parts: Vec::with_capacity(message.parts.len()), + while let Some(message) = messages.pop_front() { + let mut parts = Vec::with_capacity(message.parts.len()); + + for part in message.parts { + let (size, body) = match part.body { + PartType::Text(contents) => (contents.len(), MetadataPartType::Text), + PartType::Html(contents) => (contents.len(), MetadataPartType::Html), + PartType::Binary(contents) => (contents.len(), MetadataPartType::Binary), + PartType::InlineBinary(contents) => { + (contents.len(), MetadataPartType::InlineBinary) + } + PartType::Message(message) => { + let message_len = message.root_part().raw_len(); + messages.push_back(message); + message_id += 1; + + (message_len as usize, MetadataPartType::Message(message_id)) + } + PartType::Multipart(parts) => ( + 0, + MetadataPartType::Multipart(parts.into_iter().map(|p| p as u16).collect()), + ), }; - for part in message.parts { - let (size, body) = match part.body { - PartType::Text(contents) => (contents.len(), MetadataPartType::Text), - PartType::Html(contents) => (contents.len(), MetadataPartType::Html), - PartType::Binary(contents) => (contents.len(), MetadataPartType::Binary), - PartType::InlineBinary(contents) => { - (contents.len(), MetadataPartType::InlineBinary) - } - PartType::Message(message) => { - let message_len = message.root_part().raw_len(); - messages.push_back(message); - message_id += 1; + let flags = match part.encoding { + Encoding::None => 0, + Encoding::QuotedPrintable => PART_ENCODING_QP, + Encoding::Base64 => PART_ENCODING_BASE64, + } | (if part.is_encoding_problem { + PART_ENCODING_PROBLEM + } else { + 0 + }) | (size as u32 & PART_SIZE_MASK); - (message_len as usize, MetadataPartType::Message(message_id)) - } - PartType::Multipart(parts) => ( - 0, - MetadataPartType::Multipart(parts.into_iter().map(|p| p as u16).collect()), - ), - }; - - contents.parts.push(MessageMetadataPart { - headers: part - .headers - .into_iter() - .map(|hdr| Header { - value: if matches!( - &hdr.name, - HeaderName::Subject - | HeaderName::From - | HeaderName::To - | HeaderName::Cc - | HeaderName::Date - | HeaderName::Bcc - | HeaderName::ReplyTo - | HeaderName::Sender - | HeaderName::Comments - | HeaderName::InReplyTo - | HeaderName::Keywords - | HeaderName::MessageId - | HeaderName::References - | HeaderName::ResentMessageId - | HeaderName::ContentDescription - | HeaderName::ContentId - | HeaderName::ContentLanguage - | HeaderName::ContentLocation - | HeaderName::ContentTransferEncoding - | HeaderName::ContentType - | HeaderName::ContentDisposition - | HeaderName::ListId - ) { - hdr.value.into_owned() - } else { - HeaderValue::Empty - }, - name: hdr.name.into_owned(), - offset_field: hdr.offset_field, - offset_start: hdr.offset_start, - offset_end: hdr.offset_end, - }) - .collect(), - is_encoding_problem: part.is_encoding_problem, - encoding: part.encoding, - body, - size: size as u32, - offset_header: part.offset_header, - offset_body: part.offset_body, - offset_end: part.offset_end, - }); - } - self.contents.push(contents); + parts.push(MessageMetadataPart { + headers: part + .headers + .into_iter() + .map(|hdr| MetadataHeader { + value: if matches!( + &hdr.name, + HeaderName::Subject + | HeaderName::From + | HeaderName::To + | HeaderName::Cc + | HeaderName::Date + | HeaderName::Bcc + | HeaderName::ReplyTo + | HeaderName::Sender + | HeaderName::Comments + | HeaderName::InReplyTo + | HeaderName::Keywords + | HeaderName::MessageId + | HeaderName::References + | HeaderName::ResentMessageId + | HeaderName::ContentDescription + | HeaderName::ContentId + | HeaderName::ContentLanguage + | HeaderName::ContentLocation + | HeaderName::ContentTransferEncoding + | HeaderName::ContentType + | HeaderName::ContentDisposition + | HeaderName::ListId + ) { + hdr.value + } else { + HeaderValue::Empty + } + .into(), + name: hdr.name.into(), + base_offset: hdr.offset_field, + start: (hdr.offset_start - hdr.offset_field) as u16, + end: (hdr.offset_end - hdr.offset_field) as u16, + }) + .collect(), + body, + flags, + offset_header: part.offset_header, + offset_body: part.offset_body, + offset_end: part.offset_end, + }); } - - self + contents.push(MessageMetadataContents { + html_body: message.html_body.into_iter().map(|c| c as u16).collect(), + text_body: message.text_body.into_iter().map(|c| c as u16).collect(), + attachments: message.attachments.into_iter().map(|c| c as u16).collect(), + parts: parts.into_boxed_slice(), + }); } + contents.into_boxed_slice() } impl ArchivedMessageMetadataPart { @@ -393,7 +548,7 @@ impl ArchivedMessageMetadataPart { matches!(self.body, ArchivedMetadataPartType::Message(_)) } - pub fn sub_parts(&self) -> Option<&ArchivedVec> { + pub fn sub_parts(&self) -> Option<&ArchivedBox<[u16_le]>> { if let ArchivedMetadataPartType::Multipart(parts) = &self.body { Some(parts) } else { @@ -407,10 +562,20 @@ impl ArchivedMessageMetadataPart { pub fn header_values( &self, - name: ArchivedHeaderName<'static>, - ) -> impl Iterator> + Sync + Send { + name: &MetadataHeaderName, + ) -> impl Iterator + Sync + Send { self.headers.iter().filter_map(move |header| { - if header.name == name { + if &header.name == name { + Some(&header.value) + } else { + None + } + }) + } + + pub fn header_value(&self, name: &MetadataHeaderName) -> Option<&ArchivedMetadataHeaderValue> { + self.headers.iter().rev().find_map(move |header| { + if &header.name == name { Some(&header.value) } else { None @@ -419,69 +584,58 @@ impl ArchivedMessageMetadataPart { } pub fn subject(&self) -> Option<&str> { - self.headers - .header_value(&ArchivedHeaderName::Subject) + self.header_value(&MetadataHeaderName::Subject) .and_then(|header| header.as_text()) } pub fn date(&self) -> Option { - self.headers - .header_value(&ArchivedHeaderName::Date) + self.header_value(&MetadataHeaderName::Date) .and_then(|header| header.as_datetime()) .map(|dt| dt.into()) } pub fn message_id(&self) -> Option<&str> { - self.headers - .header_value(&ArchivedHeaderName::MessageId) + self.header_value(&MetadataHeaderName::MessageId) .and_then(|header| header.as_text()) } - pub fn in_reply_to(&self) -> &ArchivedHeaderValue<'static> { - self.headers - .header_value(&ArchivedHeaderName::InReplyTo) - .unwrap_or(&ArchivedHeaderValue::Empty) + pub fn in_reply_to(&self) -> &ArchivedMetadataHeaderValue { + self.header_value(&MetadataHeaderName::InReplyTo) + .unwrap_or(&ArchivedMetadataHeaderValue::Empty) } pub fn content_description(&self) -> Option<&str> { - self.headers - .header_value(&ArchivedHeaderName::ContentDescription) + self.header_value(&MetadataHeaderName::ContentDescription) .and_then(|header| header.as_text()) } - pub fn content_disposition(&self) -> Option<&ArchivedContentType<'static>> { - self.headers - .header_value(&ArchivedHeaderName::ContentDisposition) + pub fn content_disposition(&self) -> Option<&ArchivedMetadataContentType> { + self.header_value(&MetadataHeaderName::ContentDisposition) .and_then(|header| header.as_content_type()) } pub fn content_id(&self) -> Option<&str> { - self.headers - .header_value(&ArchivedHeaderName::ContentId) + self.header_value(&MetadataHeaderName::ContentId) .and_then(|header| header.as_text()) } pub fn content_transfer_encoding(&self) -> Option<&str> { - self.headers - .header_value(&ArchivedHeaderName::ContentTransferEncoding) + self.header_value(&MetadataHeaderName::ContentTransferEncoding) .and_then(|header| header.as_text()) } - pub fn content_type(&self) -> Option<&ArchivedContentType<'static>> { - self.headers - .header_value(&ArchivedHeaderName::ContentType) + pub fn content_type(&self) -> Option<&ArchivedMetadataContentType> { + self.header_value(&MetadataHeaderName::ContentType) .and_then(|header| header.as_content_type()) } - pub fn content_language(&self) -> &ArchivedHeaderValue<'static> { - self.headers - .header_value(&ArchivedHeaderName::ContentLanguage) - .unwrap_or(&ArchivedHeaderValue::Empty) + pub fn content_language(&self) -> &ArchivedMetadataHeaderValue { + self.header_value(&MetadataHeaderName::ContentLanguage) + .unwrap_or(&ArchivedMetadataHeaderValue::Empty) } pub fn content_location(&self) -> Option<&str> { - self.headers - .header_value(&ArchivedHeaderName::ContentLocation) + self.header_value(&MetadataHeaderName::ContentLocation) .and_then(|header| header.as_text()) } @@ -492,17 +646,155 @@ impl ArchivedMessageMetadataPart { } } +impl From> for MetadataHeaderName { + fn from(value: HeaderName<'_>) -> Self { + match value { + HeaderName::Subject => MetadataHeaderName::Subject, + HeaderName::From => MetadataHeaderName::From, + HeaderName::To => MetadataHeaderName::To, + HeaderName::Cc => MetadataHeaderName::Cc, + HeaderName::Date => MetadataHeaderName::Date, + HeaderName::Bcc => MetadataHeaderName::Bcc, + HeaderName::ReplyTo => MetadataHeaderName::ReplyTo, + HeaderName::Sender => MetadataHeaderName::Sender, + HeaderName::Comments => MetadataHeaderName::Comments, + HeaderName::InReplyTo => MetadataHeaderName::InReplyTo, + HeaderName::Keywords => MetadataHeaderName::Keywords, + HeaderName::Received => MetadataHeaderName::Received, + HeaderName::MessageId => MetadataHeaderName::MessageId, + HeaderName::References => MetadataHeaderName::References, + HeaderName::ReturnPath => MetadataHeaderName::ReturnPath, + HeaderName::MimeVersion => MetadataHeaderName::MimeVersion, + HeaderName::ContentDescription => MetadataHeaderName::ContentDescription, + HeaderName::ContentId => MetadataHeaderName::ContentId, + HeaderName::ContentLanguage => MetadataHeaderName::ContentLanguage, + HeaderName::ContentLocation => MetadataHeaderName::ContentLocation, + HeaderName::ContentTransferEncoding => MetadataHeaderName::ContentTransferEncoding, + HeaderName::ContentType => MetadataHeaderName::ContentType, + HeaderName::ContentDisposition => MetadataHeaderName::ContentDisposition, + HeaderName::ResentTo => MetadataHeaderName::ResentTo, + HeaderName::ResentFrom => MetadataHeaderName::ResentFrom, + HeaderName::ResentBcc => MetadataHeaderName::ResentBcc, + HeaderName::ResentCc => MetadataHeaderName::ResentCc, + HeaderName::ResentSender => MetadataHeaderName::ResentSender, + HeaderName::ResentDate => MetadataHeaderName::ResentDate, + HeaderName::ResentMessageId => MetadataHeaderName::ResentMessageId, + HeaderName::ListArchive => MetadataHeaderName::ListArchive, + HeaderName::ListHelp => MetadataHeaderName::ListHelp, + HeaderName::ListId => MetadataHeaderName::ListId, + HeaderName::ListOwner => MetadataHeaderName::ListOwner, + HeaderName::ListPost => MetadataHeaderName::ListPost, + HeaderName::ListSubscribe => MetadataHeaderName::ListSubscribe, + HeaderName::ListUnsubscribe => MetadataHeaderName::ListUnsubscribe, + HeaderName::DkimSignature => MetadataHeaderName::DkimSignature, + HeaderName::ArcAuthenticationResults => MetadataHeaderName::ArcAuthenticationResults, + HeaderName::ArcMessageSignature => MetadataHeaderName::ArcMessageSignature, + HeaderName::ArcSeal => MetadataHeaderName::ArcSeal, + HeaderName::Other(value) => { + MetadataHeaderName::Other(value.into_owned().into_boxed_str()) + } + other => MetadataHeaderName::Other(other.as_str().to_string().into_boxed_str()), + } + } +} + +impl From> for MetadataHeaderValue { + fn from(value: HeaderValue<'_>) -> Self { + match value { + HeaderValue::Address(address) => match address { + Address::List(address) => MetadataHeaderValue::AddressList( + address + .into_iter() + .map(|a| MetadataAddress { + name: a.name.map(|a| a.into_owned().into_boxed_str()), + address: a.address.map(|a| a.into_owned().into_boxed_str()), + }) + .collect(), + ), + Address::Group(groups) => MetadataHeaderValue::AddressGroup( + groups + .into_iter() + .map(|g| MetadataAddressGroup { + name: g.name.map(|a| a.into_owned().into_boxed_str()), + addresses: g + .addresses + .into_iter() + .map(|a| MetadataAddress { + name: a.name.map(|a| a.into_owned().into_boxed_str()), + address: a.address.map(|a| a.into_owned().into_boxed_str()), + }) + .collect(), + }) + .collect(), + ), + }, + HeaderValue::Text(text) => { + MetadataHeaderValue::Text(text.into_owned().into_boxed_str()) + } + HeaderValue::TextList(texts) => MetadataHeaderValue::TextList( + texts + .into_iter() + .map(|v| v.into_owned().into_boxed_str()) + .collect(), + ), + HeaderValue::DateTime(dt) => MetadataHeaderValue::DateTime(MetadataDateTime { + year: dt.year, + month: dt.month, + day: dt.day, + hour: dt.hour, + minute: dt.minute, + second: dt.second, + tz_hour: (if dt.tz_before_gmt { -1 } else { 1 }) * dt.tz_hour as i8, + tz_minute: dt.tz_minute, + }), + HeaderValue::ContentType(ct) => MetadataHeaderValue::ContentType(MetadataContentType { + c_type: ct.c_type.into_owned().into_boxed_str(), + c_subtype: ct.c_subtype.map(|v| v.into_owned().into_boxed_str()), + attributes: ct + .attributes + .unwrap_or_default() + .into_iter() + .map(|a| MetadataAttribute { + name: a.name.into_owned().into_boxed_str(), + value: a.value.into_owned().into_boxed_str(), + }) + .collect(), + }), + HeaderValue::Received(_) | HeaderValue::Empty => MetadataHeaderValue::Empty, + } + } +} + +impl From<&ArchivedMetadataDateTime> for DateTime { + fn from(dt: &ArchivedMetadataDateTime) -> Self { + DateTime { + year: dt.year.to_native(), + month: dt.month, + day: dt.day, + hour: dt.hour, + minute: dt.minute, + second: dt.second, + tz_before_gmt: dt.tz_hour < 0, + tz_hour: dt.tz_hour.unsigned_abs(), + tz_minute: dt.tz_minute, + } + } +} + impl ArchivedMessageMetadataContents { pub fn root_part(&self) -> &ArchivedMessageMetadataPart { &self.parts[0] } } -impl MessageData { - pub fn has_keyword(&self, keyword: &Keyword) -> bool { - self.keywords.iter().any(|k| k == keyword) - } +pub struct MessageDataBuilder { + pub mailboxes: Vec, + pub keywords: Vec, + pub thread_id: u32, + pub size: u32, +} +impl MessageDataBuilder { pub fn set_keywords(&mut self, keywords: Vec) { self.keywords = keywords; } @@ -536,6 +828,10 @@ impl MessageData { self.mailboxes.retain(|m| m.mailbox_id != mailbox); } + pub fn has_keyword(&self, keyword: &Keyword) -> bool { + self.keywords.iter().any(|k| k == keyword) + } + pub fn has_keyword_changes(&self, prev_data: &ArchivedMessageData) -> bool { self.keywords.len() != prev_data.keywords.len() || !self @@ -544,10 +840,6 @@ impl MessageData { .all(|k| prev_data.keywords.iter().any(|pk| pk == k)) } - pub fn has_mailbox_id(&self, mailbox_id: u32) -> bool { - self.mailboxes.iter().any(|m| m.mailbox_id == mailbox_id) - } - pub fn added_keywords( &self, prev_data: &ArchivedMessageData, @@ -599,6 +891,21 @@ impl MessageData { .any(|pm| pm.mailbox_id == m.mailbox_id) }) } + + pub fn seal(self) -> MessageData { + MessageData { + mailboxes: self.mailboxes.into_boxed_slice(), + keywords: self.keywords.into_boxed_slice(), + thread_id: self.thread_id, + size: self.size, + } + } +} + +impl MessageData { + pub fn has_mailbox_id(&self, mailbox_id: u32) -> bool { + self.mailboxes.iter().any(|m| m.mailbox_id == mailbox_id) + } } impl ArchivedMessageData { @@ -612,4 +919,225 @@ impl ArchivedMessageData { .find(|m| m.mailbox_id == mailbox_id) .map(|m| m.uid.to_native()) } + + pub fn to_builder(&self) -> MessageDataBuilder { + MessageDataBuilder { + mailboxes: self.mailboxes.iter().map(|m| m.to_native()).collect(), + keywords: self.keywords.iter().map(|k| k.to_native()).collect(), + thread_id: self.thread_id.to_native(), + size: self.size.to_native(), + } + } +} + +impl ArchivedMetadataContentType { + pub fn ctype(&self) -> &str { + &self.c_type + } + + pub fn subtype(&self) -> Option<&str> { + self.c_subtype.as_ref().map(|s| s.as_ref()) + } + + pub fn attribute(&self, name: &str) -> Option<&str> { + self.attributes + .iter() + .find(|a| *a.name == *name) + .map(|a| a.value.as_ref()) + } + + /// Returns `true` when the provided attribute name is present + pub fn has_attribute(&self, name: &str) -> bool { + self.attributes.iter().any(|a| *a.name == *name) + } + + pub fn is_attachment(&self) -> bool { + self.c_type.eq_ignore_ascii_case("attachment") + } + + pub fn is_inline(&self) -> bool { + self.c_type.eq_ignore_ascii_case("inline") + } +} + +impl ArchivedMetadataHeaderValue { + pub fn is_empty(&self) -> bool { + self == &MetadataHeaderValue::Empty + } + + pub fn as_text(&self) -> Option<&str> { + match self { + ArchivedMetadataHeaderValue::Text(s) => Some(s.as_ref()), + ArchivedMetadataHeaderValue::TextList(l) => l.last().map(|v| v.as_ref()), + _ => None, + } + } + + pub fn as_text_list(&self) -> Option<&[ArchivedBox]> { + match self { + ArchivedMetadataHeaderValue::Text(s) => Some(std::slice::from_ref(s)), + ArchivedMetadataHeaderValue::TextList(l) => Some(l.as_ref()), + _ => None, + } + } + + pub fn as_content_type(&self) -> Option<&ArchivedMetadataContentType> { + match self { + ArchivedMetadataHeaderValue::ContentType(c) => Some(c), + _ => None, + } + } + + pub fn as_datetime(&self) -> Option<&ArchivedMetadataDateTime> { + match self { + ArchivedMetadataHeaderValue::DateTime(d) => Some(d), + _ => None, + } + } +} + +impl ArchivedUidMailbox { + pub fn to_native(&self) -> UidMailbox { + UidMailbox { + mailbox_id: self.mailbox_id.to_native(), + uid: self.uid.to_native(), + } + } +} + +impl ArchivedMetadataHeader { + #[inline(always)] + pub fn value_range(&self) -> Range { + (self.base_offset.to_native() as usize + self.start.to_native() as usize) + ..(self.base_offset.to_native() as usize + self.end.to_native() as usize) + } + + #[inline(always)] + pub fn name_value_range(&self) -> Range { + (self.base_offset.to_native() as usize) + ..(self.base_offset.to_native() as usize + self.end.to_native() as usize) + } +} + +impl ArchivedMetadataHeaderName { + pub fn is_mime_header(&self) -> bool { + matches!( + self, + ArchivedMetadataHeaderName::ContentDescription + | ArchivedMetadataHeaderName::ContentId + | ArchivedMetadataHeaderName::ContentLanguage + | ArchivedMetadataHeaderName::ContentLocation + | ArchivedMetadataHeaderName::ContentTransferEncoding + | ArchivedMetadataHeaderName::ContentType + | ArchivedMetadataHeaderName::ContentDisposition + ) + } + + pub fn as_str(&self) -> &str { + match self { + ArchivedMetadataHeaderName::Subject => "Subject", + ArchivedMetadataHeaderName::From => "From", + ArchivedMetadataHeaderName::To => "To", + ArchivedMetadataHeaderName::Cc => "Cc", + ArchivedMetadataHeaderName::Date => "Date", + ArchivedMetadataHeaderName::Bcc => "Bcc", + ArchivedMetadataHeaderName::ReplyTo => "Reply-To", + ArchivedMetadataHeaderName::Sender => "Sender", + ArchivedMetadataHeaderName::Comments => "Comments", + ArchivedMetadataHeaderName::InReplyTo => "In-Reply-To", + ArchivedMetadataHeaderName::Keywords => "Keywords", + ArchivedMetadataHeaderName::Received => "Received", + ArchivedMetadataHeaderName::MessageId => "Message-ID", + ArchivedMetadataHeaderName::References => "References", + ArchivedMetadataHeaderName::ReturnPath => "Return-Path", + ArchivedMetadataHeaderName::MimeVersion => "MIME-Version", + ArchivedMetadataHeaderName::ContentDescription => "Content-Description", + ArchivedMetadataHeaderName::ContentId => "Content-ID", + ArchivedMetadataHeaderName::ContentLanguage => "Content-Language", + ArchivedMetadataHeaderName::ContentLocation => "Content-Location", + ArchivedMetadataHeaderName::ContentTransferEncoding => "Content-Transfer-Encoding", + ArchivedMetadataHeaderName::ContentType => "Content-Type", + ArchivedMetadataHeaderName::ContentDisposition => "Content-Disposition", + ArchivedMetadataHeaderName::ResentTo => "Resent-To", + ArchivedMetadataHeaderName::ResentFrom => "Resent-From", + ArchivedMetadataHeaderName::ResentBcc => "Resent-Bcc", + ArchivedMetadataHeaderName::ResentCc => "Resent-Cc", + ArchivedMetadataHeaderName::ResentSender => "Resent-Sender", + ArchivedMetadataHeaderName::ResentDate => "Resent-Date", + ArchivedMetadataHeaderName::ResentMessageId => "Resent-Message-ID", + ArchivedMetadataHeaderName::ListArchive => "List-Archive", + ArchivedMetadataHeaderName::ListHelp => "List-Help", + ArchivedMetadataHeaderName::ListId => "List-ID", + ArchivedMetadataHeaderName::ListOwner => "List-Owner", + ArchivedMetadataHeaderName::ListPost => "List-Post", + ArchivedMetadataHeaderName::ListSubscribe => "List-Subscribe", + ArchivedMetadataHeaderName::ListUnsubscribe => "List-Unsubscribe", + ArchivedMetadataHeaderName::ArcAuthenticationResults => "ARC-Authentication-Results", + ArchivedMetadataHeaderName::ArcMessageSignature => "ARC-Message-Signature", + ArchivedMetadataHeaderName::ArcSeal => "ARC-Seal", + ArchivedMetadataHeaderName::DkimSignature => "DKIM-Signature", + ArchivedMetadataHeaderName::Other(name) => name.as_ref(), + } + } +} + +impl From<&ArchivedMetadataHeaderValue> for HeaderValue<'static> { + fn from(value: &ArchivedMetadataHeaderValue) -> Self { + match value { + ArchivedMetadataHeaderValue::AddressList(addr) => HeaderValue::Address(Address::List( + addr.as_ref().iter().map(Into::into).collect(), + )), + ArchivedMetadataHeaderValue::AddressGroup(addr) => HeaderValue::Address( + Address::Group(addr.as_ref().iter().map(Into::into).collect()), + ), + ArchivedMetadataHeaderValue::Text(text) => HeaderValue::Text(text.to_string().into()), + ArchivedMetadataHeaderValue::TextList(textlist) => HeaderValue::TextList( + textlist + .as_ref() + .iter() + .map(|s| s.to_string().into()) + .collect(), + ), + ArchivedMetadataHeaderValue::DateTime(dt) => HeaderValue::DateTime(dt.into()), + ArchivedMetadataHeaderValue::ContentType(ct) => HeaderValue::ContentType(ct.into()), + ArchivedMetadataHeaderValue::Empty => HeaderValue::Empty, + } + } +} + +impl From<&ArchivedMetadataAddress> for Addr<'static> { + fn from(value: &ArchivedMetadataAddress) -> Self { + Addr { + name: value.name.as_ref().map(|n| n.to_string().into()), + address: value.address.as_ref().map(|a| a.to_string().into()), + } + } +} + +impl From<&ArchivedMetadataAddressGroup> for Group<'static> { + fn from(value: &ArchivedMetadataAddressGroup) -> Self { + Group { + name: value.name.as_ref().map(|n| n.to_string().into()), + addresses: value.addresses.as_ref().iter().map(Into::into).collect(), + } + } +} + +impl From<&ArchivedMetadataContentType> for ContentType<'static> { + fn from(value: &ArchivedMetadataContentType) -> Self { + ContentType { + c_type: value.ctype().to_string().into(), + c_subtype: value.subtype().map(|s| s.to_string().into()), + attributes: Some( + value + .attributes + .iter() + .map(|a| Attribute { + name: a.name.to_string().into(), + value: a.value.to_string().into(), + }) + .collect(), + ), + } + } } diff --git a/crates/email/src/sieve/ingest.rs b/crates/email/src/sieve/ingest.rs index 1ac9ef27..6f828b21 100644 --- a/crates/email/src/sieve/ingest.rs +++ b/crates/email/src/sieve/ingest.rs @@ -27,6 +27,7 @@ use store::{ }; use trc::{AddContext, SieveEvent}; use types::{ + blob_hash::BlobHash, collection::Collection, field::{PrincipalField, SieveField}, id::Id, @@ -47,6 +48,7 @@ pub trait SieveScriptIngest: Sync + Send { fn sieve_script_ingest( &self, access_token: &AccessToken, + blob_hash: &BlobHash, raw_message: &[u8], envelope_from: &str, envelope_from_authenticated: bool, @@ -84,6 +86,7 @@ impl SieveScriptIngest for Server { async fn sieve_script_ingest( &self, access_token: &AccessToken, + blob_hash: &BlobHash, raw_message: &[u8], envelope_from: &str, envelope_from_authenticated: bool, @@ -498,12 +501,12 @@ impl SieveScriptIngest for Server { for (message_id, sieve_message) in messages.into_iter().enumerate() { if !sieve_message.file_into.is_empty() { // Parse message if needed - let message = if message_id == 0 && !instance.has_message_changed() { - instance.take_message() + let (blob_hash, message) = if message_id == 0 && !instance.has_message_changed() { + (blob_hash.into(), instance.take_message()) } else if let Some(message) = MessageParser::new().parse(sieve_message.raw_message.as_ref()) { - message + (None, message) } else { trc::event!( Sieve(SieveEvent::UnexpectedError), @@ -518,6 +521,7 @@ impl SieveScriptIngest for Server { match self .email_ingest(IngestEmail { raw_message: &sieve_message.raw_message, + blob_hash, message: message.into(), access_token, mailbox_ids: sieve_message.file_into, diff --git a/crates/groupware/src/calendar/index.rs b/crates/groupware/src/calendar/index.rs index 9ef6e392..0651dd96 100644 --- a/crates/groupware/src/calendar/index.rs +++ b/crates/groupware/src/calendar/index.rs @@ -9,8 +9,8 @@ use super::{ ArchivedTimezone, Calendar, CalendarEvent, CalendarPreferences, DefaultAlert, Timezone, }; use crate::calendar::{ - ArchivedCalendarEventNotification, ArchivedEventPreferences, CalendarEventNotification, - EventPreferences, + ArchivedCalendarEventNotification, ArchivedChangedBy, ArchivedEventPreferences, + CalendarEventNotification, ChangedBy, EventPreferences, }; use ahash::AHashSet; use calcard::icalendar::{ @@ -23,6 +23,7 @@ use nlp::language::{ detect::{LanguageDetector, MIN_LANGUAGE_SCORE}, }; use store::{ + U32_LEN, search::{CalendarSearchField, IndexDocument, SearchField}, write::{IndexPropertyClass, SearchIndex, ValueClass}, xxhash_rust::xxh3, @@ -40,9 +41,7 @@ impl IndexableObject for Calendar { value: (&self.acls).into(), }, IndexValue::Quota { - used: self.dead_properties.size() as u32 - + self.preferences.iter().map(|p| p.size()).sum::() as u32 - + self.name.len() as u32, + used: self.size() as u32, }, IndexValue::LogContainer { sync_collection: SyncCollection::Calendar, @@ -64,9 +63,7 @@ impl IndexableObject for &ArchivedCalendar { .into(), }, IndexValue::Quota { - used: self.dead_properties.size() as u32 - + self.preferences.iter().map(|p| p.size()).sum::() as u32 - + self.name.len() as u32, + used: self.size() as u32, }, IndexValue::LogContainer { sync_collection: SyncCollection::Calendar, @@ -97,11 +94,7 @@ impl IndexableObject for CalendarEvent { value: self.data.event.uids().next().into(), }, IndexValue::Quota { - used: self.dead_properties.size() as u32 - + self.display_name.as_ref().map_or(0, |n| n.len() as u32) - + self.names.iter().map(|n| n.name.len() as u32).sum::() - + self.preferences.iter().map(|p| p.size()).sum::() as u32 - + self.size, + used: self.size() as u32, }, IndexValue::LogItem { sync_collection: SyncCollection::Calendar, @@ -127,11 +120,7 @@ impl IndexableObject for &ArchivedCalendarEvent { value: self.data.event.uids().next().into(), }, IndexValue::Quota { - used: self.dead_properties.size() as u32 - + self.display_name.as_ref().map_or(0, |n| n.len() as u32) - + self.names.iter().map(|n| n.name.len() as u32).sum::() - + self.preferences.iter().map(|p| p.size()).sum::() as u32 - + self.size, + used: self.size() as u32, }, IndexValue::LogItem { sync_collection: SyncCollection::Calendar, @@ -151,7 +140,9 @@ impl IndexableAndSerializableObject for CalendarEvent { impl IndexableObject for CalendarEventNotification { fn index_values(&self) -> impl Iterator> { [ - IndexValue::Quota { used: self.size }, + IndexValue::Quota { + used: self.size() as u32, + }, IndexValue::Property { field: ValueClass::IndexProperty(IndexPropertyClass::Integer { property: CalendarNotificationField::CreatedToId.into(), @@ -172,7 +163,7 @@ impl IndexableObject for &ArchivedCalendarEventNotification { fn index_values(&self) -> impl Iterator> { [ IndexValue::Quota { - used: self.size.to_native(), + used: self.size() as u32, }, IndexValue::Property { field: ValueClass::IndexProperty(IndexPropertyClass::Integer { @@ -201,6 +192,66 @@ impl IndexableAndSerializableObject for CalendarEventNotification { } } +impl Calendar { + pub fn size(&self) -> usize { + self.dead_properties.size() + + self.preferences.iter().map(|p| p.size()).sum::() + + self.name.len() + + std::mem::size_of::() + } +} + +impl ArchivedCalendar { + pub fn size(&self) -> usize { + self.dead_properties.size() + + self.preferences.iter().map(|p| p.size()).sum::() + + self.name.len() + + std::mem::size_of::() + } +} + +impl CalendarEvent { + pub fn size(&self) -> usize { + self.dead_properties.size() + + self.display_name.as_ref().map_or(0, |n| n.len()) + + self.names.iter().map(|n| n.name.len()).sum::() + + self.preferences.iter().map(|p| p.size()).sum::() + + self.size as usize + + std::mem::size_of::() + } +} + +impl ArchivedCalendarEvent { + pub fn size(&self) -> usize { + self.dead_properties.size() + + self.display_name.as_ref().map_or(0, |n| n.len()) + + self.names.iter().map(|n| n.name.len()).sum::() + + self.preferences.iter().map(|p| p.size()).sum::() + + self.size.to_native() as usize + + std::mem::size_of::() + } +} + +impl CalendarEventNotification { + pub fn size(&self) -> usize { + (match &self.changed_by { + ChangedBy::PrincipalId(_) => U32_LEN, + ChangedBy::CalendarAddress(v) => v.len(), + }) + std::mem::size_of::() + + self.size as usize + } +} + +impl ArchivedCalendarEventNotification { + pub fn size(&self) -> usize { + (match &self.changed_by { + ArchivedChangedBy::PrincipalId(_) => U32_LEN, + ArchivedChangedBy::CalendarAddress(v) => v.len(), + }) + std::mem::size_of::() + + self.size.to_native() as usize + } +} + impl CalendarPreferences { pub fn size(&self) -> usize { self.name.len() diff --git a/crates/groupware/src/contact/index.rs b/crates/groupware/src/contact/index.rs index 809ea595..985e494f 100644 --- a/crates/groupware/src/contact/index.rs +++ b/crates/groupware/src/contact/index.rs @@ -8,10 +8,7 @@ use super::{AddressBook, ArchivedAddressBook, ArchivedContactCard, ContactCard}; use ahash::AHashSet; use calcard::{ common::IanaString, - vcard::{ - ArchivedVCardParameterValue, ArchivedVCardProperty, ArchivedVCardValue, - VCardParameterValue, VCardProperty, - }, + vcard::{ArchivedVCardProperty, ArchivedVCardValue, VCardProperty}, }; use common::storage::index::{IndexValue, IndexableAndSerializableObject, IndexableObject}; use nlp::language::{ @@ -33,16 +30,7 @@ impl IndexableObject for AddressBook { value: (&self.acls).into(), }, IndexValue::Quota { - used: self.dead_properties.size() as u32 - + self - .preferences - .iter() - .map(|p| { - p.name.len() as u32 - + p.description.as_ref().map_or(0, |n| n.len() as u32) - }) - .sum::() - + self.name.len() as u32, + used: self.size() as u32, }, IndexValue::LogContainer { sync_collection: SyncCollection::AddressBook, @@ -64,16 +52,7 @@ impl IndexableObject for &ArchivedAddressBook { .into(), }, IndexValue::Quota { - used: self.dead_properties.size() as u32 - + self - .preferences - .iter() - .map(|p| { - p.name.len() as u32 - + p.description.as_ref().map_or(0, |n| n.len() as u32) - }) - .sum::() - + self.name.len() as u32, + used: self.size() as u32, }, IndexValue::LogContainer { sync_collection: SyncCollection::AddressBook, @@ -112,10 +91,7 @@ impl IndexableObject for ContactCard { hash: self.hashes().fold(0, |acc, hash| acc ^ hash), }, IndexValue::Quota { - used: self.dead_properties.size() as u32 - + self.display_name.as_ref().map_or(0, |n| n.len() as u32) - + self.names.iter().map(|n| n.name.len() as u32).sum::() - + self.size, + used: self.size() as u32, }, IndexValue::LogItem { sync_collection: SyncCollection::AddressBook, @@ -149,10 +125,7 @@ impl IndexableObject for &ArchivedContactCard { hash: self.hashes().fold(0, |acc, hash| acc ^ hash), }, IndexValue::Quota { - used: self.dead_properties.size() as u32 - + self.display_name.as_ref().map_or(0, |n| n.len() as u32) - + self.names.iter().map(|n| n.name.len() as u32).sum::() - + self.size, + used: self.size() as u32, }, IndexValue::LogItem { sync_collection: SyncCollection::AddressBook, @@ -169,7 +142,41 @@ impl IndexableAndSerializableObject for ContactCard { } } +impl AddressBook { + pub fn size(&self) -> usize { + self.dead_properties.size() + + self + .preferences + .iter() + .map(|p| p.name.len() + p.description.as_ref().map_or(0, |n| n.len())) + .sum::() + + self.name.len() + + std::mem::size_of::() + } +} + +impl ArchivedAddressBook { + pub fn size(&self) -> usize { + self.dead_properties.size() + + self + .preferences + .iter() + .map(|p| p.name.len() + p.description.as_ref().map_or(0, |n| n.len())) + .sum::() + + self.name.len() + + std::mem::size_of::() + } +} + impl ContactCard { + pub fn size(&self) -> usize { + self.dead_properties.size() + + self.display_name.as_ref().map_or(0, |n| n.len()) + + self.names.iter().map(|n| n.name.len()).sum::() + + self.size as usize + + std::mem::size_of::() + } + pub fn hashes(&self) -> impl Iterator { self.card .entries @@ -193,15 +200,7 @@ impl ContactCard { | VCardProperty::Tel ) }) - .flat_map(|e| { - e.values - .iter() - .filter_map(|v| v.as_text()) - .chain(e.params.iter().filter_map(|p| match &p.value { - VCardParameterValue::Text(v) => Some(v.as_str()), - _ => None, - })) - }) + .flat_map(|e| e.values.iter().filter_map(|v| v.as_text())) .map(|v| xxh3::xxh3_64(v.as_bytes())) } @@ -215,6 +214,14 @@ impl ContactCard { } impl ArchivedContactCard { + pub fn size(&self) -> usize { + self.dead_properties.size() + + self.display_name.as_ref().map_or(0, |n| n.len()) + + self.names.iter().map(|n| n.name.len()).sum::() + + self.size.to_native() as usize + + std::mem::size_of::() + } + pub fn hashes(&self) -> impl Iterator { self.card .entries @@ -238,15 +245,7 @@ impl ArchivedContactCard { | ArchivedVCardProperty::Tel ) }) - .flat_map(|e| { - e.values - .iter() - .filter_map(|v| v.as_text()) - .chain(e.params.iter().filter_map(|p| match &p.value { - ArchivedVCardParameterValue::Text(v) => Some(v.as_str()), - _ => None, - })) - }) + .flat_map(|e| e.values.iter().filter_map(|v| v.as_text())) .map(|v| xxh3::xxh3_64(v.as_bytes())) } @@ -257,9 +256,7 @@ impl ArchivedContactCard { .filter_map(|v| v.as_text().and_then(sanitize_email)) }) } -} -impl ArchivedContactCard { pub fn index_document( &self, account_id: u32, diff --git a/crates/groupware/src/file/index.rs b/crates/groupware/src/file/index.rs index 3ac79539..7487df65 100644 --- a/crates/groupware/src/file/index.rs +++ b/crates/groupware/src/file/index.rs @@ -20,7 +20,9 @@ impl IndexableObject for FileNode { prefix: None, sync_collection: SyncCollection::FileNode, }, - IndexValue::Quota { used: self.size() }, + IndexValue::Quota { + used: self.size() as u32, + }, ]); if let Some(file) = &self.file { @@ -50,7 +52,9 @@ impl IndexableObject for &ArchivedFileNode { prefix: None, sync_collection: SyncCollection::FileNode, }, - IndexValue::Quota { used: self.size() }, + IndexValue::Quota { + used: self.size() as u32, + }, ]); if let Some(file) = self.file.as_ref() { @@ -69,24 +73,25 @@ impl IndexableAndSerializableObject for FileNode { } } -pub trait NodeSize { - fn size(&self) -> u32; -} - -impl NodeSize for ArchivedFileNode { - fn size(&self) -> u32 { - self.dead_properties.size() as u32 - + self.display_name.as_ref().map_or(0, |n| n.len() as u32) - + self.name.len() as u32 - + self.file.as_ref().map_or(0, |f| u32::from(f.size)) +impl FileNode { + pub fn size(&self) -> usize { + self.dead_properties.size() + + self.display_name.as_ref().map_or(0, |n| n.len()) + + self.name.len() + + self.file.as_ref().map_or(0, |f| f.size as usize) + + std::mem::size_of::() } } -impl NodeSize for FileNode { - fn size(&self) -> u32 { - self.dead_properties.size() as u32 - + self.display_name.as_ref().map_or(0, |n| n.len() as u32) - + self.name.len() as u32 - + self.file.as_ref().map_or(0, |f| f.size) +impl ArchivedFileNode { + pub fn size(&self) -> usize { + self.dead_properties.size() + + self.display_name.as_ref().map_or(0, |n| n.len()) + + self.name.len() + + self + .file + .as_ref() + .map_or(0, |f| f.size.to_native() as usize) + + std::mem::size_of::() } } diff --git a/crates/http/src/management/enterprise/undelete.rs b/crates/http/src/management/enterprise/undelete.rs index d28aa7e8..ec5398b5 100644 --- a/crates/http/src/management/enterprise/undelete.rs +++ b/crates/http/src/management/enterprise/undelete.rs @@ -201,6 +201,7 @@ impl UndeleteApi for Server { .email_ingest(IngestEmail { raw_message: &bytes, message: MessageParser::new().parse(&bytes), + blob_hash: Some(&request.hash), access_token: access_token.as_ref(), mailbox_ids: vec![INBOX_ID], keywords: vec![], diff --git a/crates/http/src/management/stores.rs b/crates/http/src/management/stores.rs index b4d81282..3b1701ee 100644 --- a/crates/http/src/management/stores.rs +++ b/crates/http/src/management/stores.rs @@ -20,6 +20,11 @@ use email::{ cache::MessageCacheFetch, message::{ingest::EmailIngest, metadata::MessageData}, }; +use groupware::{ + calendar::{Calendar, CalendarEvent, CalendarEventNotification}, + contact::{AddressBook, ContactCard}, + file::FileNode, +}; use http_proto::{request::decode_path_element, *}; use hyper::Method; use serde_json::json; @@ -27,7 +32,7 @@ use services::task_manager::index::ReindexIndexTask; use std::future::Future; use store::{ Serialize, rand, - write::{Archiver, BatchBuilder, SearchIndex, ValueClass}, + write::{Archiver, BatchBuilder, DirectoryClass, SearchIndex, ValueClass}, }; use trc::AddContext; use types::{ @@ -304,7 +309,7 @@ impl ManageStore for Server { .ok_or_else(|| trc::ManageEvent::NotFound.into_err())?; if method == Method::DELETE { - self.recalculate_quota(account_id).await?; + recalculate_quota(self, account_id).await?; } let result = self.get_used_quota(account_id).await?; @@ -337,6 +342,62 @@ impl ManageStore for Server { } } +pub async fn recalculate_quota(server: &Server, account_id: u32) -> trc::Result<()> { + let mut quota = 0; + + for collection in [ + Collection::Email, + Collection::Calendar, + Collection::CalendarEvent, + Collection::CalendarEventNotification, + Collection::AddressBook, + Collection::ContactCard, + Collection::FileNode, + ] { + server + .archives(account_id, collection, &(), |_, archive| { + match collection { + Collection::Email => { + quota += archive.unarchive::()?.size.to_native() as i64; + } + Collection::Calendar => { + quota += archive.unarchive::()?.size() as i64; + } + Collection::CalendarEvent => { + quota += archive.unarchive::()?.size() as i64; + } + Collection::CalendarEventNotification => { + quota += archive.unarchive::()?.size() as i64; + } + Collection::AddressBook => { + quota += archive.unarchive::()?.size() as i64; + } + Collection::ContactCard => { + quota += archive.unarchive::()?.size() as i64; + } + Collection::FileNode => { + quota += archive.unarchive::()?.size() as i64; + } + _ => {} + } + Ok(true) + }) + .await + .caused_by(trc::location!())?; + } + + let mut batch = BatchBuilder::new(); + batch + .clear(DirectoryClass::UsedQuota(account_id)) + .add(DirectoryClass::UsedQuota(account_id), quota); + server + .store() + .write(batch.build_all()) + .await + .caused_by(trc::location!()) + .map(|_| ()) +} + pub async fn reset_imap_uids(server: &Server, account_id: u32) -> trc::Result<(u32, u32)> { let mut mailbox_count = 0; let mut email_count = 0; diff --git a/crates/imap-proto/Cargo.toml b/crates/imap-proto/Cargo.toml index 6635fcf5..5ef3c9be 100644 --- a/crates/imap-proto/Cargo.toml +++ b/crates/imap-proto/Cargo.toml @@ -5,6 +5,7 @@ edition = "2024" [dependencies] types = { path = "../types" } +utils = { path = "../utils" } mail-parser = { version = "0.11", features = ["full_encoding", "rkyv"] } ahash = { version = "0.8" } chrono = { version = "0.4"} diff --git a/crates/imap-proto/src/parser/mod.rs b/crates/imap-proto/src/parser/mod.rs index 30cc3e5d..1c471ec3 100644 --- a/crates/imap-proto/src/parser/mod.rs +++ b/crates/imap-proto/src/parser/mod.rs @@ -113,7 +113,7 @@ impl Flag { } else { String::from_utf8(value) .map_err(|_| Cow::from("Invalid UTF-8.")) - .map(Flag::Keyword) + .map(|v| Flag::Keyword(v.into_boxed_str())) } } else { Err(Cow::from("Null flags are not allowed.")) @@ -136,7 +136,7 @@ impl Flag { "$forwarded" => Flag::Forwarded, "$mdnsent" => Flag::MDNSent, ) - .unwrap_or_else(|| Flag::Keyword(value)) + .unwrap_or_else(|| Flag::Keyword(value.into_boxed_str())) } else { let mut keyword = String::with_capacity(value.len()); for c in value.chars() { @@ -146,7 +146,7 @@ impl Flag { keyword.push('_'); } } - Flag::Keyword(keyword) + Flag::Keyword(keyword.into_boxed_str()) } } } diff --git a/crates/imap-proto/src/protocol/fetch.rs b/crates/imap-proto/src/protocol/fetch.rs index 61338c54..8d10b61d 100644 --- a/crates/imap-proto/src/protocol/fetch.rs +++ b/crates/imap-proto/src/protocol/fetch.rs @@ -7,6 +7,9 @@ use std::borrow::Cow; use mail_parser::DateTime; +use utils::chained_bytes::SliceRange; + +use crate::protocol::literal_string_slice; use super::{ Flag, ImapResponse, Sequence, literal_string, quoted_or_literal_string, @@ -110,16 +113,16 @@ pub enum DataItem<'x> { uid: u32, }, Rfc822 { - contents: Cow<'x, [u8]>, + contents: SliceRange<'x>, }, Rfc822Header { - contents: Cow<'x, [u8]>, + contents: SliceRange<'x>, }, Rfc822Size { size: usize, }, Rfc822Text { - contents: Cow<'x, [u8]>, + contents: SliceRange<'x>, }, Preview { contents: Option>, @@ -807,11 +810,11 @@ impl DataItem<'_> { } DataItem::Rfc822 { contents } => { buf.extend_from_slice(b"RFC822 "); - literal_string(buf, contents); + literal_string_slice(buf, contents); } DataItem::Rfc822Header { contents } => { buf.extend_from_slice(b"RFC822.HEADER "); - literal_string(buf, contents); + literal_string_slice(buf, contents); } DataItem::Rfc822Size { size } => { buf.extend_from_slice(b"RFC822.SIZE "); @@ -819,7 +822,7 @@ impl DataItem<'_> { } DataItem::Rfc822Text { contents } => { buf.extend_from_slice(b"RFC822.TEXT "); - literal_string(buf, contents); + literal_string_slice(buf, contents); } DataItem::Preview { contents } => { buf.extend_from_slice(b"PREVIEW "); @@ -917,6 +920,7 @@ impl ImapResponse for Response<'_> { mod tests { use mail_parser::DateTime; + use utils::chained_bytes::SliceRange; use crate::protocol::{Flag, ImapResponse}; @@ -1382,10 +1386,10 @@ mod tests { super::DataItem::Uid { uid: 983 }, super::DataItem::Rfc822Size { size: 443 }, super::DataItem::Rfc822Text { - contents: b"hi"[..].into() + contents: SliceRange::Single(&b"hi"[..]), }, super::DataItem::Rfc822Header { - contents: b"header"[..].into() + contents: SliceRange::Single(&b"header"[..]), }, ], }], diff --git a/crates/imap-proto/src/protocol/mod.rs b/crates/imap-proto/src/protocol/mod.rs index 91417a07..0685f6b3 100644 --- a/crates/imap-proto/src/protocol/mod.rs +++ b/crates/imap-proto/src/protocol/mod.rs @@ -10,6 +10,7 @@ use chrono::{DateTime, Utc}; use compact_str::CompactString; use std::{cmp::Ordering, fmt::Display}; use types::keyword::{ArchivedKeyword, Keyword}; +use utils::chained_bytes::SliceRange; pub mod acl; pub mod append; @@ -198,6 +199,13 @@ pub fn literal_string(buf: &mut Vec, text: &[u8]) { buf.extend_from_slice(text); } +pub fn literal_string_slice(buf: &mut Vec, text: &SliceRange<'_>) { + buf.push(b'{'); + buf.extend_from_slice(text.len().to_string().as_bytes()); + buf.extend_from_slice(b"}\r\n"); + buf.extend(*text); +} + pub fn quoted_timestamp(buf: &mut Vec, timestamp: i64) { buf.push(b'"'); buf.extend_from_slice( @@ -238,7 +246,7 @@ pub enum Flag { Deleted, Forwarded, MDNSent, - Keyword(String), + Keyword(Box), } impl Flag { @@ -296,7 +304,7 @@ impl From<&ArchivedKeyword> for Flag { ArchivedKeyword::Deleted => Flag::Deleted, ArchivedKeyword::Forwarded => Flag::Forwarded, ArchivedKeyword::MdnSent => Flag::MDNSent, - ArchivedKeyword::Other(value) => Flag::Keyword(value.as_str().into()), + ArchivedKeyword::Other(value) => Flag::Keyword(value.as_ref().into()), } } } @@ -316,7 +324,7 @@ impl From for Keyword { Flag::Deleted => Keyword::Deleted, Flag::Forwarded => Keyword::Forwarded, Flag::MDNSent => Keyword::MdnSent, - Flag::Keyword(value) => Keyword::from_other(value), + Flag::Keyword(value) => Keyword::from_boxed_other(value), } } } diff --git a/crates/imap/src/op/append.rs b/crates/imap/src/op/append.rs index 643265af..0ece0a01 100644 --- a/crates/imap/src/op/append.rs +++ b/crates/imap/src/op/append.rs @@ -106,6 +106,7 @@ impl SessionData { .email_ingest(IngestEmail { raw_message: &message.message, message: MessageParser::new().parse(&message.message), + blob_hash: None, access_token: &access_token, mailbox_ids: vec![mailbox_id], keywords: message.flags.into_iter().map(Keyword::from).collect(), diff --git a/crates/imap/src/op/copy_move.rs b/crates/imap/src/op/copy_move.rs index 61dc88fc..cdafaeb1 100644 --- a/crates/imap/src/op/copy_move.rs +++ b/crates/imap/src/op/copy_move.rs @@ -244,9 +244,7 @@ impl SessionData { copied_ids.push((imap_id.uid, mailbox.uid.to_native())); if is_move { - let mut new_data = data - .deserialize() - .imap_ctx(&arguments.tag, trc::location!())?; + let mut new_data = data.inner.to_builder(); new_data.remove_mailbox(src_mailbox.id.mailbox_id); batch .with_account_id(account_id) @@ -255,7 +253,7 @@ impl SessionData { .custom( ObjectIndexBuilder::new() .with_current(data) - .with_changes(new_data), + .with_changes(new_data.seal()), ) .imap_ctx(&arguments.tag, trc::location!())? .log_vanished_item( @@ -270,9 +268,7 @@ impl SessionData { } // Prepare changes - let mut new_data = data - .deserialize() - .imap_ctx(&arguments.tag, trc::location!())?; + let mut new_data = data.inner.to_builder(); // Add destination folder new_data.add_mailbox(dest_mailbox_id); @@ -302,7 +298,7 @@ impl SessionData { .custom( ObjectIndexBuilder::new() .with_current(data) - .with_changes(new_data), + .with_changes(new_data.seal()), ) .imap_ctx(&arguments.tag, trc::location!())?; if is_move { diff --git a/crates/imap/src/op/expunge.rs b/crates/imap/src/op/expunge.rs index 3f4d0e7b..601fddff 100644 --- a/crates/imap/src/op/expunge.rs +++ b/crates/imap/src/op/expunge.rs @@ -192,7 +192,11 @@ impl SessionData { if metadata.inner.mailboxes.len() == 1 { // Delete message batch - .custom(ObjectIndexBuilder::<_, ()>::new().with_current(metadata)) + .custom( + ObjectIndexBuilder::<_, ()>::new() + .with_access_token(&self.access_token) + .with_current(metadata), + ) .caused_by(trc::location!())? .set( ValueClass::TaskQueue(TaskQueueClass::UpdateIndex { @@ -205,9 +209,7 @@ impl SessionData { .commit_point(); } else { // Untag message from this mailbox and remove Deleted flag - let mut new_metadata = metadata - .deserialize::() - .caused_by(trc::location!())?; + let mut new_metadata = metadata.inner.to_builder(); new_metadata.remove_mailbox(mailbox_id); new_metadata.remove_keyword(&Keyword::Deleted); @@ -216,7 +218,7 @@ impl SessionData { .custom( ObjectIndexBuilder::new() .with_current(metadata) - .with_changes(new_metadata), + .with_changes(new_metadata.seal()), ) .caused_by(trc::location!())? .commit_point(); diff --git a/crates/imap/src/op/fetch.rs b/crates/imap/src/op/fetch.rs index 89926c5a..13dacbe7 100644 --- a/crates/imap/src/op/fetch.rs +++ b/crates/imap/src/op/fetch.rs @@ -15,8 +15,9 @@ use directory::Permission; use email::{ cache::{MessageCacheFetch, email::MessageCacheAccess}, message::metadata::{ - ArchivedMessageMetadata, ArchivedMessageMetadataContents, ArchivedMetadataPartType, - DecodedParts, MessageData, MessageMetadata, + ArchivedMessageMetadata, ArchivedMessageMetadataContents, ArchivedMetadataHeaderValue, + ArchivedMetadataPartType, DecodedParts, MESSAGE_RECEIVED_MASK, MessageData, + MessageMetadata, MetadataHeaderName, PART_ENCODING_PROBLEM, }, }; use imap_proto::{ @@ -32,9 +33,6 @@ use imap_proto::{ }, receiver::Request, }; -use mail_parser::{ - ArchivedAddress, ArchivedHeaderName, ArchivedHeaderValue, core::rkyv::ArchivedGetHeader, -}; use std::{borrow::Cow, sync::Arc, time::Instant}; use store::{ query::log::{Change, Query}, @@ -48,6 +46,7 @@ use types::{ id::Id, keyword::Keyword, }; +use utils::chained_bytes::{ChainedBytes, SliceRange}; impl Session { pub async fn handle_fetch(&mut self, requests: Vec>) -> trc::Result<()> { @@ -349,37 +348,42 @@ impl SessionData { let metadata = metadata_ .unarchive::() .imap_ctx(&arguments.tag, trc::location!())?; + let raw_body; // Fetch and parse blob - let raw_message: Cow<[u8]> = if needs_blobs { + let mut raw_message = ChainedBytes::new(metadata.raw_headers.as_ref()); + if needs_blobs { // Retrieve raw message if needed - match self + raw_body = self .server .blob_store() .get_blob(metadata.blob_hash.0.as_slice(), 0..usize::MAX) .await - .imap_ctx(&arguments.tag, trc::location!())? - { - Some(raw_message) => raw_message.into(), - None => { - trc::event!( - Store(trc::StoreEvent::NotFound), - AccountId = account_id, - DocumentId = id, - Collection = Collection::Email, - BlobId = metadata.blob_hash.0.as_slice(), - Details = "Blob not found.", - CausedBy = trc::location!(), - ); + .imap_ctx(&arguments.tag, trc::location!())?; - continue; - } + if let Some(raw_body) = &raw_body { + raw_message.append( + raw_body + .get(metadata.blob_body_offset.to_native() as usize..) + .unwrap_or_default(), + ); + } else { + trc::event!( + Store(trc::StoreEvent::NotFound), + AccountId = account_id, + DocumentId = id, + Collection = Collection::Email, + BlobId = metadata.blob_hash.0.as_slice(), + Details = "Blob not found.", + CausedBy = trc::location!(), + ); + + continue; } - } else { - metadata.raw_headers.as_slice().into() - }; + } + let message = &metadata.contents[0]; - let decoded = metadata.decode_contents(raw_message.as_ref()); + let decoded = metadata.decode_contents(raw_message.clone()); // Build response let mut items = Vec::with_capacity(arguments.attributes.len()); @@ -404,7 +408,7 @@ impl SessionData { } Attribute::InternalDate => { items.push(DataItem::InternalDate { - date: u64::from(metadata.received_at) as i64, + date: (metadata.rcvd_attach.to_native() & MESSAGE_RECEIVED_MASK) as i64, }); } Attribute::Preview { .. } => { @@ -418,7 +422,7 @@ impl SessionData { } Attribute::Rfc822Size => { items.push(DataItem::Rfc822Size { - size: u32::from(metadata.size) as usize, + size: data.size as usize, }); } Attribute::Uid => { @@ -426,23 +430,21 @@ impl SessionData { } Attribute::Rfc822 => { items.push(DataItem::Rfc822 { - contents: raw_message.as_ref().into(), + contents: raw_message.get_full_range(), }); } Attribute::Rfc822Header => { - let message = metadata.root_part(); - if let Some(header) = raw_message.get( - u32::from(message.offset_header) as usize - ..u32::from(message.offset_body) as usize, - ) { - items.push(DataItem::Rfc822Header { - contents: header.into(), - }); + let contents = raw_message.get_slice_range( + 0..u32::from(metadata.root_part().offset_body) as usize, + ); + + if contents != SliceRange::None { + items.push(DataItem::Rfc822Header { contents }); } } Attribute::Rfc822Text => { items.push(DataItem::Rfc822Text { - contents: raw_message.as_ref().into(), + contents: raw_message.get_full_range(), }); } Attribute::Body => { @@ -550,9 +552,7 @@ impl SessionData { let data = data_ .to_unarchived::() .imap_ctx(&arguments.tag, trc::location!())?; - let mut new_data = data - .deserialize() - .imap_ctx(&arguments.tag, trc::location!())?; + let mut new_data = data.inner.to_builder(); new_data.keywords.push(Keyword::Seen); batch @@ -562,7 +562,7 @@ impl SessionData { .custom( ObjectIndexBuilder::new() .with_current(data) - .with_changes(new_data), + .with_changes(new_data.seal()), ) .imap_ctx(&arguments.tag, trc::location!())? .commit_point(); @@ -657,15 +657,14 @@ impl AsImapDataItemPart for ArchivedMessageMetadataContents { is_extended: bool, ) -> BodyPart<'_> { let part = &self.parts[part_id]; - let body = decoded.raw_message_section_arch(message_id, part.offset_body, part.offset_end); + let body = decoded.raw_message_section(message_id, part.body_to_end()); let (is_multipart, is_text) = match &part.body { ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html => (false, true), ArchivedMetadataPartType::Multipart(_) => (true, false), _ => (false, false), }; let content_type = part - .headers - .header_value(&ArchivedHeaderName::ContentType) + .header_value(&MetadataHeaderName::ContentType) .and_then(|ct| ct.as_content_type()); let mut body_md5 = None; @@ -673,13 +672,15 @@ impl AsImapDataItemPart for ArchivedMessageMetadataContents { let mut fields = BodyPartFields::default(); if !is_multipart || is_extended { - fields.body_parameters = content_type.as_ref().and_then(|ct| { - ct.attributes.as_ref().map(|at| { - at.iter() + fields.body_parameters = content_type + .as_ref() + .map(|ct| { + ct.attributes + .iter() .map(|k| (k.name.as_ref().into(), k.value.as_ref().into())) .collect::>() }) - }) + .filter(|p| !p.is_empty()) } if !is_multipart { @@ -688,18 +689,15 @@ impl AsImapDataItemPart for ArchivedMessageMetadataContents { .and_then(|ct| ct.c_subtype.as_ref().map(|cs| cs.as_ref().into())); fields.body_id = part - .headers - .header_value(&ArchivedHeaderName::ContentId) + .header_value(&MetadataHeaderName::ContentId) .and_then(|id| id.as_text().map(|id| format!("<{}>", id).into())); fields.body_description = part - .headers - .header_value(&ArchivedHeaderName::ContentDescription) + .header_value(&MetadataHeaderName::ContentDescription) .and_then(|ct| ct.as_text().map(|ct| ct.into())); fields.body_encoding = part - .headers - .header_value(&ArchivedHeaderName::ContentTransferEncoding) + .header_value(&MetadataHeaderName::ContentTransferEncoding) .and_then(|ct| ct.as_text().map(|ct| ct.into())); fields.body_size_octets = body.as_ref().map(|b| b.len()).unwrap_or(0); @@ -725,34 +723,27 @@ impl AsImapDataItemPart for ArchivedMessageMetadataContents { } extension.body_disposition = part - .headers - .header_value(&ArchivedHeaderName::ContentDisposition) + .header_value(&MetadataHeaderName::ContentDisposition) .and_then(|cd| cd.as_content_type()) .map(|cd| { ( cd.c_type.as_ref().into(), cd.attributes - .as_ref() - .map(|at| { - at.iter() - .map(|k| (k.name.as_ref().into(), k.value.as_ref().into())) - .collect::>() - }) - .unwrap_or_default(), + .iter() + .map(|k| (k.name.as_ref().into(), k.value.as_ref().into())) + .collect::>(), ) }); extension.body_language = part - .headers - .header_value(&ArchivedHeaderName::ContentLanguage) + .header_value(&MetadataHeaderName::ContentLanguage) .and_then(|hv| { hv.as_text_list() .map(|list| list.iter().map(|item| item.as_ref().into()).collect()) }); extension.body_location = part - .headers - .header_value(&ArchivedHeaderName::ContentLocation) + .header_value(&MetadataHeaderName::ContentLocation) .and_then(|ct| ct.as_text().map(|ct| ct.into())); } @@ -805,27 +796,27 @@ impl AsImapDataItemPart for ArchivedMessageMetadataContents { date: headers.date(), subject: headers.subject().map(|s| s.into()), from: headers - .header_values(ArchivedHeaderName::From) + .header_values(&MetadataHeaderName::From) .flat_map(|a| a.as_imap_address()) .collect(), sender: headers - .header_values(ArchivedHeaderName::Sender) + .header_values(&MetadataHeaderName::Sender) .flat_map(|a| a.as_imap_address()) .collect(), reply_to: headers - .header_values(ArchivedHeaderName::ReplyTo) + .header_values(&MetadataHeaderName::ReplyTo) .flat_map(|a| a.as_imap_address()) .collect(), to: headers - .header_values(ArchivedHeaderName::To) + .header_values(&MetadataHeaderName::To) .flat_map(|a| a.as_imap_address()) .collect(), cc: headers - .header_values(ArchivedHeaderName::Cc) + .header_values(&MetadataHeaderName::Cc) .flat_map(|a| a.as_imap_address()) .collect(), bcc: headers - .header_values(ArchivedHeaderName::Bcc) + .header_values(&MetadataHeaderName::Bcc) .flat_map(|a| a.as_imap_address()) .collect(), in_reply_to: headers.in_reply_to().as_text_list().map(|list| { @@ -913,13 +904,10 @@ impl AsImapDataItem for ArchivedMessageMetadata { ) -> Option> { let mut part = self.root_part(); if sections.is_empty() { - return Some( - get_partial_bytes( - decoded.raw_message_section_arch(0, part.offset_header, part.offset_end)?, - partial, - ) - .into(), - ); + return Some(get_cow_partial_bytes( + decoded.raw_message_section(0, part.header_to_end())?, + partial, + )); } let mut message = &self.contents[0]; @@ -931,8 +919,9 @@ impl AsImapDataItem for ArchivedMessageMetadata { Section::Part { num } => { part = if let Some(sub_part_ids) = part.sub_parts() { sub_part_ids + .as_ref() .get((*num).saturating_sub(1) as usize) - .and_then(|pos| message.parts.get(u16::from(*pos) as usize)) + .and_then(|pos| message.parts.as_ref().get(u16::from(*pos) as usize)) } else if *num == 1 && (section_num == sections.len() - 1 || part.is_message()) { Some(part) @@ -955,17 +944,10 @@ impl AsImapDataItem for ArchivedMessageMetadata { } } Section::Header => { - return Some( - get_partial_bytes( - decoded.raw_message_section_arch( - message_id, - part.offset_header, - part.offset_body, - )?, - partial, - ) - .into(), - ); + return Some(get_cow_partial_bytes( + decoded.raw_message_section(message_id, part.header_to_body())?, + partial, + )); } Section::HeaderFields { not, fields } => { let mut headers = Vec::with_capacity( @@ -978,12 +960,8 @@ impl AsImapDataItem for ArchivedMessageMetadata { headers.extend_from_slice(header_name.as_bytes()); headers.push(b':'); headers.extend_from_slice( - decoded - .raw_message_section_arch( - message_id, - header.offset_start, - header.offset_end, - ) + &decoded + .raw_message_section(message_id, header.value_range()) .unwrap_or_default(), ); } @@ -998,17 +976,10 @@ impl AsImapDataItem for ArchivedMessageMetadata { }); } Section::Text => { - return Some( - get_partial_bytes( - decoded.raw_message_section_arch( - message_id, - part.offset_body, - part.offset_end, - )?, - partial, - ) - .into(), - ); + return Some(get_cow_partial_bytes( + decoded.raw_message_section(message_id, part.body_to_end())?, + partial, + )); } Section::Mime => { let mut headers = Vec::with_capacity( @@ -1022,12 +993,8 @@ impl AsImapDataItem for ArchivedMessageMetadata { headers.extend_from_slice(header.name.as_str().as_bytes()); headers.extend_from_slice(b":"); headers.extend_from_slice( - decoded - .raw_message_section_arch( - message_id, - header.offset_start, - header.offset_end, - ) + &decoded + .raw_message_section(message_id, header.value_range()) .unwrap_or_default(), ); } @@ -1045,13 +1012,10 @@ impl AsImapDataItem for ArchivedMessageMetadata { // BODY[x] should return both headers and body, but most clients // expect BODY[x] to return only the body, just like BOXY[x.TEXT] does. - Some( - get_partial_bytes( - decoded.raw_message_section_arch(message_id, part.offset_body, part.offset_end)?, - partial, - ) - .into(), - ) + Some(get_cow_partial_bytes( + decoded.raw_message_section(message_id, part.body_to_end())?, + partial, + )) } fn binary<'x>( @@ -1068,8 +1032,9 @@ impl AsImapDataItem for ArchivedMessageMetadata { while let Some((section_num, num)) = sections_iter.next() { part = if let Some(sub_part_ids) = part.sub_parts() { if let Some(part) = sub_part_ids + .as_ref() .get((*num).saturating_sub(1) as usize) - .and_then(|pos| message.parts.get(u16::from(*pos) as usize)) + .and_then(|pos| message.parts.as_ref().get(u16::from(*pos) as usize)) { part } else { @@ -1090,7 +1055,7 @@ impl AsImapDataItem for ArchivedMessageMetadata { } } - if !part.is_encoding_problem { + if (part.flags & PART_ENCODING_PROBLEM) == 0 { let part_offset = u32::from(part.offset_header) as usize; Ok(match &part.body { ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html => { @@ -1117,34 +1082,24 @@ impl AsImapDataItem for ArchivedMessageMetadata { ArchivedMetadataPartType::Message(message) => BodyContents::Bytes({ { let part = self.message_id(*message).root_part(); - get_partial_bytes( + get_cow_partial_bytes( decoded - .raw_message_section_arch( - message_id, - part.offset_header, - part.offset_end, - ) + .raw_message_section(message_id, part.header_to_end()) .unwrap_or_default(), partial, ) - .into() } }) .into(), - ArchivedMetadataPartType::Multipart(_) => BodyContents::Bytes( - get_partial_bytes( + ArchivedMetadataPartType::Multipart(_) => { + BodyContents::Bytes(get_cow_partial_bytes( decoded - .raw_message_section_arch( - message_id, - part.offset_header, - part.offset_end, - ) + .raw_message_section(message_id, part.header_to_end()) .unwrap_or_default(), partial, - ) - .into(), - ) - .into(), + )) + .into() + } }) } else { Err(()) @@ -1160,8 +1115,9 @@ impl AsImapDataItem for ArchivedMessageMetadata { while let Some((section_num, num)) = sections_iter.next() { part = if let Some(sub_part_ids) = part.sub_parts() { sub_part_ids + .as_ref() .get((*num).saturating_sub(1) as usize) - .and_then(|pos| message.parts.get(u16::from(pos) as usize)) + .and_then(|pos| message.parts.as_ref().get(u16::from(pos) as usize)) } else if *num == 1 && (section_num == sections.len() - 1 || part.is_message()) { Some(part) } else { @@ -1205,16 +1161,29 @@ fn get_partial_bytes(bytes: &[u8], partial: Option<(u32, u32)>) -> &[u8] { } } +#[inline(always)] +fn get_cow_partial_bytes(bytes: Cow<'_, [u8]>, partial: Option<(u32, u32)>) -> Cow<'_, [u8]> { + if let Some((start, end)) = partial { + let range = start as usize..std::cmp::min((start + end) as usize, bytes.len()); + match bytes { + Cow::Borrowed(bytes) => Cow::Borrowed(bytes.get(range).unwrap_or_default()), + Cow::Owned(bytes) => Cow::Owned(bytes.get(range).unwrap_or_default().to_vec()), + } + } else { + bytes + } +} + trait AsImapAddress { fn as_imap_address(&'_ self) -> Vec>; } -impl AsImapAddress for ArchivedHeaderValue<'_> { +impl AsImapAddress for ArchivedMetadataHeaderValue { fn as_imap_address(&'_ self) -> Vec> { let mut addresses = Vec::new(); match self { - ArchivedHeaderValue::Address(ArchivedAddress::List(list)) => { + ArchivedMetadataHeaderValue::AddressList(list) => { for addr in list.iter() { if let Some(email) = addr.address.as_ref() { addresses.push(fetch::Address::Single(fetch::EmailAddress { @@ -1224,7 +1193,7 @@ impl AsImapAddress for ArchivedHeaderValue<'_> { } } } - ArchivedHeaderValue::Address(ArchivedAddress::Group(list)) => { + ArchivedMetadataHeaderValue::AddressGroup(list) => { for group in list.iter() { addresses.push(fetch::Address::Group(fetch::AddressGroup { name: group.name.as_ref().map(|n| n.as_ref().into()), diff --git a/crates/imap/src/op/status.rs b/crates/imap/src/op/status.rs index aca7f83e..016f1384 100644 --- a/crates/imap/src/op/status.rs +++ b/crates/imap/src/op/status.rs @@ -20,13 +20,8 @@ use imap_proto::{ receiver::Request, }; use std::time::Instant; -use store::{IterateParams, roaring::RoaringBitmap, write::key::DeserializeBigEndian}; -use store::{ - U32_LEN, ValueKey, - write::{IndexPropertyClass, ValueClass}, -}; use trc::AddContext; -use types::{collection::Collection, field::EmailField, id::Id, keyword::Keyword}; +use types::{id::Id, keyword::Keyword}; impl Session { pub async fn handle_status(&mut self, requests: Vec>) -> trc::Result<()> { @@ -219,27 +214,14 @@ impl SessionData { for item in items_update { let result = match item { - Status::DeletedStorage => self - .calculate_mailbox_size( - mailbox.account_id, - &RoaringBitmap::from_iter( - cache - .in_mailbox_with_keyword(mailbox.mailbox_id, &Keyword::Deleted) - .map(|x| x.document_id), - ), - ) - .await - .caused_by(trc::location!())?, - Status::Size => self - .calculate_mailbox_size( - mailbox.account_id, - &RoaringBitmap::from_iter( - cache.in_mailbox(mailbox.mailbox_id).map(|x| x.document_id), - ), - ) - .await - .caused_by(trc::location!())?, - + Status::DeletedStorage => cache + .in_mailbox_with_keyword(mailbox.mailbox_id, &Keyword::Deleted) + .map(|x| x.size) + .sum::() as u64, + Status::Size => cache + .in_mailbox(mailbox.mailbox_id) + .map(|x| x.size) + .sum::() as u64, _ => { unreachable!() } @@ -287,51 +269,4 @@ impl SessionData { items: items_response, }) } - - async fn calculate_mailbox_size( - &self, - account_id: u32, - message_ids: &RoaringBitmap, - ) -> trc::Result { - let mut total_size = 0u64; - self.server - .core - .storage - .data - .iterate( - IterateParams::new( - ValueKey { - account_id, - collection: Collection::Email.into(), - document_id: 0, - class: ValueClass::IndexProperty(IndexPropertyClass::Integer { - property: EmailField::ReceivedToSize.into(), - value: 0, - }), - }, - ValueKey { - account_id, - collection: Collection::Email.into(), - document_id: u32::MAX, - class: ValueClass::IndexProperty(IndexPropertyClass::Integer { - property: EmailField::ReceivedToSize.into(), - value: u64::MAX, - }), - }, - ) - .ascending(), - |key, value| { - let id_pos = key.len() - U32_LEN; - let document_id = key.deserialize_be_u32(id_pos)?; - - if message_ids.contains(document_id) { - total_size += value.deserialize_be_u32(0)? as u64; - } - Ok(true) - }, - ) - .await - .caused_by(trc::location!()) - .map(|_| total_size) - } } diff --git a/crates/imap/src/op/store.rs b/crates/imap/src/op/store.rs index 0fba7800..51ba5374 100644 --- a/crates/imap/src/op/store.rs +++ b/crates/imap/src/op/store.rs @@ -215,9 +215,7 @@ impl SessionData { let data = data_ .to_unarchived::() .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())?; - let mut new_data = data - .deserialize() - .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())?; + let mut new_data = data.inner.to_builder(); // Apply changes let mut seen_changed = false; @@ -296,7 +294,7 @@ impl SessionData { .custom( ObjectIndexBuilder::new() .with_current(data) - .with_changes(new_data), + .with_changes(new_data.seal()), ) .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())?; diff --git a/crates/jmap-proto/src/object/email.rs b/crates/jmap-proto/src/object/email.rs index 44283a72..f0d85df2 100644 --- a/crates/jmap-proto/src/object/email.rs +++ b/crates/jmap-proto/src/object/email.rs @@ -725,14 +725,16 @@ impl Default for EmailComparator { impl EmailComparator { fn take_keyword(&mut self) -> Keyword { match self { - EmailComparator::HasKeyword(k) => std::mem::replace(k, Keyword::Other(String::new())), + EmailComparator::HasKeyword(k) => { + std::mem::replace(k, Keyword::Other(Default::default())) + } EmailComparator::AllInThreadHaveKeyword(k) => { - std::mem::replace(k, Keyword::Other(String::new())) + std::mem::replace(k, Keyword::Other(Default::default())) } EmailComparator::SomeInThreadHaveKeyword(k) => { - std::mem::replace(k, Keyword::Other(String::new())) + std::mem::replace(k, Keyword::Other(Default::default())) } - _ => Keyword::Other(String::new()), + _ => Keyword::Other(Default::default()), } } } diff --git a/crates/jmap/src/email/body.rs b/crates/jmap/src/email/body.rs index 70a0f526..b61d4f8c 100644 --- a/crates/jmap/src/email/body.rs +++ b/crates/jmap/src/email/body.rs @@ -4,11 +4,15 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use email::message::metadata::{ArchivedMessageMetadataContents, ArchivedMetadataPartType}; +use email::message::metadata::{ + ArchivedMessageMetadataContents, ArchivedMetadataHeaderValue, ArchivedMetadataPartType, + PART_ENCODING_BASE64, PART_ENCODING_QP, PART_SIZE_MASK, +}; use jmap_proto::object::email::{EmailProperty, EmailValue}; use jmap_tools::{Map, Value}; -use mail_parser::{ArchivedHeaderValue, HeaderValue, MessagePart, MimeHeaders, PartType}; +use mail_parser::{HeaderValue, MessagePart, MimeHeaders, PartType}; use types::blob::BlobId; +use utils::chained_bytes::ChainedBytes; use super::headers::HeaderToValue; @@ -17,8 +21,9 @@ pub trait ToBodyPart { &self, part_id: u32, properties: &[EmailProperty], - raw_message: &[u8], + raw_message: &ChainedBytes<'_>, blob_id: &BlobId, + blob_body_offset: isize, ) -> Value<'static, EmailProperty, EmailValue>; } @@ -27,8 +32,9 @@ impl ToBodyPart for Vec> { &self, part_id: u32, properties: &[EmailProperty], - raw_message: &[u8], + raw_message: &ChainedBytes<'_>, blob_id: &BlobId, + blob_body_offset: isize, ) -> Value<'static, EmailProperty, EmailValue> { let mut parts = vec![part_id].into_iter(); let mut parts_stack = Vec::new(); @@ -50,12 +56,12 @@ impl ToBodyPart for Vec> { let value = match property { EmailProperty::PartId if multipart.is_none() => part_id.to_string().into(), EmailProperty::BlobId if multipart.is_none() => { - let base_offset = blob_id.start_offset(); + let base_offset = blob_id.start_offset() as isize + blob_body_offset; BlobId::new_section( blob_id.hash.clone(), blob_id.class.clone(), - part.offset_body as usize + base_offset, - part.offset_end as usize + base_offset, + (part.offset_body as isize + base_offset) as usize, + (part.offset_end as isize + base_offset) as usize, part.encoding as u8, ) .into() @@ -148,8 +154,9 @@ impl ToBodyPart for ArchivedMessageMetadataContents { &self, part_id: u32, properties: &[EmailProperty], - raw_message: &[u8], + raw_message: &ChainedBytes<'_>, blob_id: &BlobId, + blob_body_offset: isize, ) -> Value<'static, EmailProperty, EmailValue> { let mut parts = vec![part_id].into_iter(); let mut parts_stack = Vec::new(); @@ -171,17 +178,27 @@ impl ToBodyPart for ArchivedMessageMetadataContents { let value = match property { EmailProperty::PartId if multipart.is_none() => part_id.to_string().into(), EmailProperty::BlobId if multipart.is_none() => { - let base_offset = blob_id.start_offset(); + let base_offset = blob_id.start_offset() as isize + blob_body_offset; + let flags = part.flags.to_native(); + let encoding = if flags & PART_ENCODING_BASE64 != 0 { + 2 + } else if flags & PART_ENCODING_QP != 0 { + 1 + } else { + 0 + }; BlobId::new_section( blob_id.hash.clone(), blob_id.class.clone(), - u32::from(part.offset_body) as usize + base_offset, - u32::from(part.offset_end) as usize + base_offset, - part.encoding.id(), + (u32::from(part.offset_body) as isize + base_offset) as usize, + (u32::from(part.offset_end) as isize + base_offset) as usize, + encoding, ) .into() } - EmailProperty::Size if multipart.is_none() => u32::from(part.size).into(), + EmailProperty::Size if multipart.is_none() => { + (part.flags.to_native() & PART_SIZE_MASK).into() + } EmailProperty::Name => part.attachment_name().map(|v| v.to_string()).into(), EmailProperty::Type => part .content_type() @@ -217,8 +234,10 @@ impl ToBodyPart for ArchivedMessageMetadataContents { .into(), EmailProperty::Cid => part.content_id().map(|v| v.to_string()).into(), EmailProperty::Language => match part.content_language() { - ArchivedHeaderValue::Text(text) => vec![text.to_string()].into(), - ArchivedHeaderValue::TextList(list) => list + ArchivedMetadataHeaderValue::Text(text) => { + vec![text.to_string()].into() + } + ArchivedMetadataHeaderValue::TextList(list) => list .iter() .map(|text| text.to_string().into()) .collect::>>() @@ -228,10 +247,8 @@ impl ToBodyPart for ArchivedMessageMetadataContents { EmailProperty::Location => { part.content_location().map(|v| v.to_string()).into() } - EmailProperty::Header(_) => { - part.headers.header_to_value(property, raw_message) - } - EmailProperty::Headers => part.headers.headers_to_value(raw_message), + EmailProperty::Header(_) => part.header_to_value(property, raw_message), + EmailProperty::Headers => part.headers_to_value(raw_message), EmailProperty::SubParts => continue, _ => Value::Null, }; diff --git a/crates/jmap/src/email/get.rs b/crates/jmap/src/email/get.rs index ff102953..f1ea9cd8 100644 --- a/crates/jmap/src/email/get.rs +++ b/crates/jmap/src/email/get.rs @@ -12,7 +12,10 @@ use crate::{changes::state::JmapCacheState, email::headers::HeaderToValue}; use common::{Server, auth::AccessToken}; use email::{ cache::{MessageCacheFetch, email::MessageCacheAccess}, - message::metadata::{ArchivedMetadataPartType, MessageMetadata}, + message::metadata::{ + ArchivedMetadataPartType, MESSAGE_HAS_ATTACHMENT, MESSAGE_RECEIVED_MASK, MessageMetadata, + MetadataHeaderName, PART_ENCODING_PROBLEM, + }, }; use jmap_proto::{ method::get::{GetRequest, GetResponse}, @@ -21,8 +24,8 @@ use jmap_proto::{ types::date::UTCDate, }; use jmap_tools::{Key, Map, Value}; -use mail_parser::{ArchivedHeaderName, HeaderValue, core::rkyv::ArchivedGetHeader}; -use std::{borrow::Cow, future::Future}; +use mail_parser::HeaderValue; +use std::future::Future; use trc::{AddContext, StoreEvent}; use types::{ acl::Acl, @@ -32,6 +35,7 @@ use types::{ field::EmailField, id::Id, }; +use utils::chained_bytes::ChainedBytes; pub trait EmailGet: Sync + Send { fn email_get( @@ -178,13 +182,20 @@ impl EmailGet for Server { // Retrieve raw message if needed let blob_hash = BlobHash::from(&metadata.blob_hash); - let raw_message: Cow<[u8]> = if needs_body { - if let Some(raw_message) = self + let raw_body; + let mut raw_message = ChainedBytes::new(metadata.raw_headers.as_ref()); + if needs_body { + raw_body = self .blob_store() .get_blob(blob_hash.as_slice(), 0..usize::MAX) - .await? - { - raw_message.into() + .await?; + + if let Some(raw_body) = &raw_body { + raw_message.append( + raw_body + .get(metadata.blob_body_offset.to_native() as usize..) + .unwrap_or_default(), + ); } else { trc::event!( Store(StoreEvent::NotFound), @@ -199,9 +210,7 @@ impl EmailGet for Server { response.not_found.push(id); continue; } - } else { - metadata.raw_headers.as_slice().into() - }; + } let blob_id = BlobId { hash: blob_hash, class: BlobClass::Linked { @@ -217,6 +226,8 @@ impl EmailGet for Server { Map::with_capacity(properties.len()); let contents = &metadata.contents[0]; let root_part = &contents.parts[0]; + let blob_body_offset = metadata.blob_body_offset.to_native() as isize + - root_part.offset_body.to_native() as isize; for property in &properties { match property { EmailProperty::Id => { @@ -248,13 +259,13 @@ impl EmailGet for Server { email.insert_unchecked(property.clone(), Value::Object(obj)); } EmailProperty::Size => { - email.insert_unchecked(EmailProperty::Size, u32::from(metadata.size)); + email.insert_unchecked(EmailProperty::Size, data.size); } EmailProperty::ReceivedAt => { email.insert_unchecked( EmailProperty::ReceivedAt, EmailValue::Date(UTCDate::from_timestamp( - u64::from(metadata.received_at) as i64, + (metadata.rcvd_attach.to_native() & MESSAGE_RECEIVED_MASK) as i64, )), ); } @@ -269,15 +280,14 @@ impl EmailGet for Server { EmailProperty::HasAttachment => { email.insert_unchecked( EmailProperty::HasAttachment, - metadata.has_attachments, + (metadata.rcvd_attach.to_native() & MESSAGE_HAS_ATTACHMENT) != 0, ); } EmailProperty::Subject => { email.insert_unchecked( EmailProperty::Subject, root_part - .headers - .header_value(&ArchivedHeaderName::Subject) + .header_value(&MetadataHeaderName::Subject) .map(|value| HeaderValue::from(value).into_form(&HeaderForm::Text)) .unwrap_or_default(), ); @@ -286,8 +296,7 @@ impl EmailGet for Server { email.insert_unchecked( EmailProperty::SentAt, root_part - .headers - .header_value(&ArchivedHeaderName::Date) + .header_value(&MetadataHeaderName::Date) .map(|value| HeaderValue::from(value).into_form(&HeaderForm::Date)) .unwrap_or_default(), ); @@ -298,11 +307,10 @@ impl EmailGet for Server { email.insert_unchecked( property.clone(), root_part - .headers .header_value(&match property { - EmailProperty::MessageId => ArchivedHeaderName::MessageId, - EmailProperty::InReplyTo => ArchivedHeaderName::InReplyTo, - EmailProperty::References => ArchivedHeaderName::References, + EmailProperty::MessageId => MetadataHeaderName::MessageId, + EmailProperty::InReplyTo => MetadataHeaderName::InReplyTo, + EmailProperty::References => MetadataHeaderName::References, _ => unreachable!(), }) .map(|value| { @@ -321,14 +329,13 @@ impl EmailGet for Server { email.insert_unchecked( property.clone(), root_part - .headers .header_value(&match property { - EmailProperty::Sender => ArchivedHeaderName::Sender, - EmailProperty::From => ArchivedHeaderName::From, - EmailProperty::To => ArchivedHeaderName::To, - EmailProperty::Cc => ArchivedHeaderName::Cc, - EmailProperty::Bcc => ArchivedHeaderName::Bcc, - EmailProperty::ReplyTo => ArchivedHeaderName::ReplyTo, + EmailProperty::Sender => MetadataHeaderName::Sender, + EmailProperty::From => MetadataHeaderName::From, + EmailProperty::To => MetadataHeaderName::To, + EmailProperty::Cc => MetadataHeaderName::Cc, + EmailProperty::Bcc => MetadataHeaderName::Bcc, + EmailProperty::ReplyTo => MetadataHeaderName::ReplyTo, _ => unreachable!(), }) .map(|value| { @@ -340,13 +347,13 @@ impl EmailGet for Server { EmailProperty::Header(_) => { email.insert_unchecked( property.clone(), - root_part.headers.header_to_value(property, &raw_message), + root_part.header_to_value(property, &raw_message), ); } EmailProperty::Headers => { email.insert_unchecked( EmailProperty::Headers, - root_part.headers.headers_to_value(&raw_message), + root_part.headers_to_value(&raw_message), ); } EmailProperty::TextBody @@ -367,6 +374,7 @@ impl EmailGet for Server { &body_properties, &raw_message, &blob_id, + blob_body_offset, ) }) .collect::>(), @@ -375,7 +383,13 @@ impl EmailGet for Server { EmailProperty::BodyStructure => { email.insert_unchecked( EmailProperty::BodyStructure, - contents.to_body_part(0, &body_properties, &raw_message, &blob_id), + contents.to_body_part( + 0, + &body_properties, + &raw_message, + &blob_id, + blob_body_offset, + ), ); } EmailProperty::BodyValues => { @@ -407,7 +421,7 @@ impl EmailGet for Server { Map::with_capacity(3) .with_key_value( EmailProperty::IsEncodingProblem, - part.is_encoding_problem, + (part.flags & PART_ENCODING_PROBLEM) != 0, ) .with_key_value(EmailProperty::IsTruncated, is_truncated) .with_key_value(EmailProperty::Value, value), diff --git a/crates/jmap/src/email/headers.rs b/crates/jmap/src/email/headers.rs index 2247fd54..1b8c89e1 100644 --- a/crates/jmap/src/email/headers.rs +++ b/crates/jmap/src/email/headers.rs @@ -4,6 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use email::message::metadata::{ArchivedMessageMetadataPart, ArchivedMetadataHeaderValue}; use jmap_proto::{ object::email::{EmailProperty, EmailValue, HeaderForm, HeaderProperty}, types::date::UTCDate, @@ -20,11 +21,8 @@ use mail_builder::{ url::URL, }, }; -use mail_parser::{ - Addr, ArchivedHeader, ArchivedHeaderValue, DateTime, Group, Header, HeaderName, HeaderValue, - parsers::MessageStream, -}; -use store::rkyv::vec::ArchivedVec; +use mail_parser::{Addr, DateTime, Group, Header, HeaderName, HeaderValue, parsers::MessageStream}; +use utils::chained_bytes::ChainedBytes; pub trait IntoForm { fn into_form(self, form: &HeaderForm) -> Value<'static, EmailProperty, EmailValue>; @@ -34,9 +32,12 @@ pub trait HeaderToValue { fn header_to_value( &self, property: &EmailProperty, - raw_message: &[u8], + raw_message: &ChainedBytes<'_>, + ) -> Value<'static, EmailProperty, EmailValue>; + fn headers_to_value( + &self, + raw_message: &ChainedBytes<'_>, ) -> Value<'static, EmailProperty, EmailValue>; - fn headers_to_value(&self, raw_message: &[u8]) -> Value<'static, EmailProperty, EmailValue>; } pub trait ValueToHeader<'x> { @@ -57,7 +58,7 @@ impl HeaderToValue for Vec> { fn header_to_value( &self, property: &EmailProperty, - raw_message: &[u8], + raw_message: &ChainedBytes<'_>, ) -> Value<'static, EmailProperty, EmailValue> { let (header_name, form, all) = match property { EmailProperty::Header(header) => ( @@ -85,10 +86,14 @@ impl HeaderToValue for Vec> { let header_name = header_name.as_str(); for header in self.iter().rev() { if header.name.as_str().eq_ignore_ascii_case(header_name) { + let raw_header; let header_value = if is_raw || matches!(header.value, HeaderValue::Empty) { - raw_message - .get(header.offset_start as usize..header.offset_end as usize) - .map_or(HeaderValue::Empty, |bytes| match form { + raw_header = + raw_message.get(header.offset_start as usize..header.offset_end as usize); + + if let Some(bytes) = &raw_header { + let bytes = bytes.as_ref(); + match form { HeaderForm::Raw => { HeaderValue::Text(String::from_utf8_lossy(bytes.trim_end())) } @@ -98,7 +103,10 @@ impl HeaderToValue for Vec> { | HeaderForm::URLs => MessageStream::new(bytes).parse_address(), HeaderForm::MessageIds => MessageStream::new(bytes).parse_id(), HeaderForm::Date => MessageStream::new(bytes).parse_date(), - }) + } + } else { + HeaderValue::Empty + } } else { header.value.clone() }; @@ -119,7 +127,10 @@ impl HeaderToValue for Vec> { } } - fn headers_to_value(&self, raw_message: &[u8]) -> Value<'static, EmailProperty, EmailValue> { + fn headers_to_value( + &self, + raw_message: &ChainedBytes<'_>, + ) -> Value<'static, EmailProperty, EmailValue> { let mut headers = Vec::with_capacity(self.len()); for header in self.iter() { headers.push(Value::Object( @@ -131,6 +142,7 @@ impl HeaderToValue for Vec> { raw_message .get(header.offset_start as usize..header.offset_end as usize) .unwrap_or_default() + .as_ref() .trim_end(), ) .into_owned(), @@ -363,11 +375,11 @@ impl<'x> BuildHeader<'x> for MessageBuilder<'x> { } } -impl HeaderToValue for ArchivedVec> { +impl HeaderToValue for ArchivedMessageMetadataPart { fn header_to_value( &self, property: &EmailProperty, - raw_message: &[u8], + raw_message: &ChainedBytes<'_>, ) -> Value<'static, EmailProperty, EmailValue> { let (header_name, form, all) = match property { EmailProperty::Header(header) => ( @@ -393,28 +405,32 @@ impl HeaderToValue for ArchivedVec> { let is_raw = matches!(form, HeaderForm::Raw) || matches!(header_name, HeaderName::Other(_)); let mut headers = Vec::new(); let header_name = header_name.as_str(); - for header in self.iter().rev() { + for header in self.headers.iter().rev() { if header.name.as_str().eq_ignore_ascii_case(header_name) { - let header_value = if is_raw || matches!(header.value, ArchivedHeaderValue::Empty) { - raw_message - .get( - u32::from(header.offset_start) as usize - ..u32::from(header.offset_end) as usize, - ) - .map_or(HeaderValue::Empty, |bytes| match form { - HeaderForm::Raw => { - HeaderValue::Text(String::from_utf8_lossy(bytes.trim_end())) + let raw_header; + let header_value = + if is_raw || matches!(header.value, ArchivedMetadataHeaderValue::Empty) { + raw_header = raw_message.get(header.value_range()); + + if let Some(bytes) = &raw_header { + let bytes = bytes.as_ref(); + match form { + HeaderForm::Raw => { + HeaderValue::Text(String::from_utf8_lossy(bytes.trim_end())) + } + HeaderForm::Text => MessageStream::new(bytes).parse_unstructured(), + HeaderForm::Addresses + | HeaderForm::GroupedAddresses + | HeaderForm::URLs => MessageStream::new(bytes).parse_address(), + HeaderForm::MessageIds => MessageStream::new(bytes).parse_id(), + HeaderForm::Date => MessageStream::new(bytes).parse_date(), } - HeaderForm::Text => MessageStream::new(bytes).parse_unstructured(), - HeaderForm::Addresses - | HeaderForm::GroupedAddresses - | HeaderForm::URLs => MessageStream::new(bytes).parse_address(), - HeaderForm::MessageIds => MessageStream::new(bytes).parse_id(), - HeaderForm::Date => MessageStream::new(bytes).parse_date(), - }) - } else { - HeaderValue::from(&header.value) - }; + } else { + HeaderValue::Empty + } + } else { + HeaderValue::from(&header.value) + }; headers.push(header_value.into_form(&form)); if !all { break; @@ -432,21 +448,22 @@ impl HeaderToValue for ArchivedVec> { } } - fn headers_to_value(&self, raw_message: &[u8]) -> Value<'static, EmailProperty, EmailValue> { - let mut headers = Vec::with_capacity(self.len()); - for header in self.iter() { + fn headers_to_value( + &self, + raw_message: &ChainedBytes<'_>, + ) -> Value<'static, EmailProperty, EmailValue> { + let mut headers = Vec::with_capacity(self.headers.len()); + for header in self.headers.iter() { headers.push(Value::Object( Map::with_capacity(2) - .with_key_value(EmailProperty::Name, header.name.to_string()) + .with_key_value(EmailProperty::Name, header.name.as_str().to_string()) .with_key_value( EmailProperty::Value, String::from_utf8_lossy( raw_message - .get( - u32::from(header.offset_start) as usize - ..u32::from(header.offset_end) as usize, - ) + .get(header.value_range()) .unwrap_or_default() + .as_ref() .trim_end(), ) .into_owned(), diff --git a/crates/jmap/src/email/import.rs b/crates/jmap/src/email/import.rs index 41382cf9..2040ddf8 100644 --- a/crates/jmap/src/email/import.rs +++ b/crates/jmap/src/email/import.rs @@ -148,6 +148,7 @@ impl EmailImport for Server { .email_ingest(IngestEmail { raw_message: &raw_message, message: MessageParser::new().parse(&raw_message), + blob_hash: Some(&blob_id.hash), access_token: import_access_token.as_deref().unwrap_or(access_token), mailbox_ids, keywords: email.keywords, diff --git a/crates/jmap/src/email/parse.rs b/crates/jmap/src/email/parse.rs index 22fd065e..6dd64c2e 100644 --- a/crates/jmap/src/email/parse.rs +++ b/crates/jmap/src/email/parse.rs @@ -21,7 +21,7 @@ use mail_parser::{ MessageParser, PartType, decoders::html::html_to_text, parsers::preview::preview_text, }; use std::future::Future; -use utils::map::vec_map::VecMap; +use utils::{chained_bytes::ChainedBytes, map::vec_map::VecMap}; pub trait EmailParse: Sync + Send { fn email_parse( @@ -112,6 +112,7 @@ impl EmailParse for Server { response.not_parsable.push(blob_id); continue; }; + let raw_message = ChainedBytes::new(&raw_message); // Prepare response let mut email = Map::with_capacity(properties.len()); @@ -209,6 +210,7 @@ impl EmailParse for Server { &body_properties, &raw_message, &blob_id, + 0, ) }) .collect::>(), @@ -217,9 +219,13 @@ impl EmailParse for Server { EmailProperty::BodyStructure => { email.insert_unchecked( EmailProperty::BodyStructure, - message - .parts - .to_body_part(0, &body_properties, &raw_message, &blob_id), + message.parts.to_body_part( + 0, + &body_properties, + &raw_message, + &blob_id, + 0, + ), ); } EmailProperty::BodyValues => { diff --git a/crates/jmap/src/email/set.rs b/crates/jmap/src/email/set.rs index f6d0d574..62b5c653 100644 --- a/crates/jmap/src/email/set.rs +++ b/crates/jmap/src/email/set.rs @@ -753,6 +753,7 @@ impl EmailSet for Server { .email_ingest(IngestEmail { raw_message: &raw_message, message: MessageParser::new().parse(&raw_message), + blob_hash: None, access_token: import_access_token.as_deref().unwrap_or(access_token), mailbox_ids: mailboxes, keywords, @@ -807,9 +808,7 @@ impl EmailSet for Server { let data = data_ .to_unarchived::() .caused_by(trc::location!())?; - let mut new_data = data - .deserialize::() - .caused_by(trc::location!())?; + let mut new_data = data.inner.to_builder(); for (property, mut value) in object.into_expanded_object() { if let Err(err) = response.resolve_self_references(&mut value) { @@ -993,7 +992,7 @@ impl EmailSet for Server { .custom( ObjectIndexBuilder::new() .with_current(data) - .with_changes(new_data), + .with_changes(new_data.seal()), ) .caused_by(trc::location!())? .commit_point(); @@ -1073,7 +1072,12 @@ impl EmailSet for Server { // Batch delete messages let mut batch = BatchBuilder::new(); let not_destroyed = self - .emails_delete(account_id, &mut batch, destroy_ids) + .emails_delete( + account_id, + access_token.tenant_id(), + &mut batch, + destroy_ids, + ) .await?; if !batch.is_empty() { last_change_id = self diff --git a/crates/jmap/src/email/snippet.rs b/crates/jmap/src/email/snippet.rs index ffa2783e..6391170a 100644 --- a/crates/jmap/src/email/snippet.rs +++ b/crates/jmap/src/email/snippet.rs @@ -7,7 +7,9 @@ use common::{Server, auth::AccessToken}; use email::{ cache::{MessageCacheFetch, email::MessageCacheAccess}, - message::metadata::{ArchivedMetadataPartType, DecodedPartContent, MessageMetadata}, + message::metadata::{ + ArchivedMetadataPartType, DecodedPartContent, MessageMetadata, MetadataHeaderName, + }, }; use jmap_proto::{ method::{ @@ -17,14 +19,13 @@ use jmap_proto::{ object::email::EmailFilter, request::IntoValid, }; -use mail_parser::{ - ArchivedHeaderName, core::rkyv::ArchivedGetHeader, decoders::html::html_to_text, -}; +use mail_parser::decoders::html::html_to_text; use nlp::language::{Language, search_snippet::generate_snippet, stemmer::Stemmer}; use std::future::Future; use store::backend::MAX_TOKEN_LENGTH; use trc::AddContext; use types::{acl::Acl, collection::Collection, field::EmailField}; +use utils::chained_bytes::ChainedBytes; pub trait EmailSearchSnippet: Sync + Send { fn email_search_snippet( @@ -147,25 +148,20 @@ impl EmailSearchSnippet for Server { let contents = &metadata.contents[0]; if let Some(subject) = contents .root_part() - .headers - .header_value(&ArchivedHeaderName::Subject) + .header_value(&MetadataHeaderName::Subject) .and_then(|v| v.as_text()) .and_then(|v| generate_snippet(v, &terms, language, is_exact)) { snippet.subject = subject.into(); } - // Check if the snippet can be generated from the preview - /*if let Some(body) = generate_snippet(&metadata.preview, &terms) { - snippet.preview = body.into(); - } else {*/ // Download message - let raw_message = if let Some(raw_message) = self + let raw_body = if let Some(raw_body) = self .blob_store() .get_blob(metadata.blob_hash.0.as_slice(), 0..usize::MAX) .await? { - raw_message + raw_body } else { trc::event!( Store(trc::StoreEvent::NotFound), @@ -180,6 +176,11 @@ impl EmailSearchSnippet for Server { response.not_found.push(email_id); continue; }; + let raw_message = ChainedBytes::new(metadata.raw_headers.as_ref()).with_last( + raw_body + .get(metadata.blob_body_offset.to_native() as usize..) + .unwrap_or_default(), + ); // Find a matching part 'outer: for part in contents.parts.iter() { diff --git a/crates/jmap/src/submission/set.rs b/crates/jmap/src/submission/set.rs index e327ffa5..fab539f7 100644 --- a/crates/jmap/src/submission/set.rs +++ b/crates/jmap/src/submission/set.rs @@ -12,7 +12,7 @@ use common::{ }; use email::{ identity::Identity, - message::metadata::MessageMetadata, + message::metadata::{ArchivedMetadataHeaderName, ArchivedMetadataHeaderValue, MessageMetadata}, submission::{Address, Delivered, DeliveryStatus, EmailSubmission, UndoStatus}, }; use jmap_proto::{ @@ -28,7 +28,6 @@ use jmap_proto::{ types::state::State, }; use jmap_tools::{Key, Value}; -use mail_parser::{ArchivedHeaderName, ArchivedHeaderValue}; use smtp::{ core::{Session, SessionData}, queue::spool::SmtpSpool, @@ -516,26 +515,57 @@ impl EmailSubmissionSet for Server { for header in metadata.contents[0].parts[0].headers.iter() { if matches!( header.name, - ArchivedHeaderName::To | ArchivedHeaderName::Cc | ArchivedHeaderName::Bcc + ArchivedMetadataHeaderName::To + | ArchivedMetadataHeaderName::Cc + | ArchivedMetadataHeaderName::Bcc ) { - if matches!(header.name, ArchivedHeaderName::Bcc) { + if matches!(header.name, ArchivedMetadataHeaderName::Bcc) { bcc_header = Some(header); } - if let ArchivedHeaderValue::Address(addr) = &header.value { - for address in addr.iter() { - if let Some(address) = address.address().and_then(sanitize_email) - && !rcpt_to.iter().any(|rcpt| rcpt.address == address) - { - submission.envelope.rcpt_to.push(Address { - email: address.to_string(), - parameters: None, - }); - rcpt_to.push(RcptTo { - address: Cow::Owned(address), - ..Default::default() - }); + match &header.value { + ArchivedMetadataHeaderValue::AddressList(addr) => { + for address in addr.iter() { + if let Some(address) = address + .address + .as_ref() + .map(|v| v.as_ref()) + .and_then(sanitize_email) + && !rcpt_to.iter().any(|rcpt| rcpt.address == address) + { + submission.envelope.rcpt_to.push(Address { + email: address.to_string(), + parameters: None, + }); + rcpt_to.push(RcptTo { + address: Cow::Owned(address), + ..Default::default() + }); + } } } + ArchivedMetadataHeaderValue::AddressGroup(groups) => { + for group in groups.iter() { + for address in group.addresses.iter() { + if let Some(address) = address + .address + .as_ref() + .map(|v| v.as_ref()) + .and_then(sanitize_email) + && !rcpt_to.iter().any(|rcpt| rcpt.address == address) + { + submission.envelope.rcpt_to.push(Address { + email: address.to_string(), + parameters: None, + }); + rcpt_to.push(RcptTo { + address: Cow::Owned(address), + ..Default::default() + }); + } + } + } + } + _ => {} } } } @@ -548,7 +578,7 @@ impl EmailSubmissionSet for Server { bcc_header = metadata.contents[0].parts[0] .headers .iter() - .find(|header| matches!(header.name, ArchivedHeaderName::Bcc)); + .find(|header| matches!(header.name, ArchivedMetadataHeaderName::Bcc)); } // Update sendAt @@ -584,8 +614,9 @@ impl EmailSubmissionSet for Server { // Remove BCC header if present if let Some(bcc_header) = bcc_header { let mut new_message = Vec::with_capacity(message.len()); - new_message.extend_from_slice(&message[..u32::from(bcc_header.offset_field) as usize]); - new_message.extend_from_slice(&message[u32::from(bcc_header.offset_end) as usize..]); + let range = bcc_header.name_value_range(); + new_message.extend_from_slice(&message[..range.start]); + new_message.extend_from_slice(&message[range.end..]); message = new_message; } diff --git a/crates/jmap/src/thread/get.rs b/crates/jmap/src/thread/get.rs index 00971214..56f673c2 100644 --- a/crates/jmap/src/thread/get.rs +++ b/crates/jmap/src/thread/get.rs @@ -15,17 +15,13 @@ use jmap_proto::{ use jmap_tools::Map; use std::future::Future; use store::{ - IterateParams, U32_LEN, ValueKey, ahash::AHashMap, roaring::RoaringBitmap, - write::{IndexPropertyClass, ValueClass, key::DeserializeBigEndian}, + search::{EmailSearchField, SearchComparator, SearchField, SearchQuery}, + write::SearchIndex, }; use trc::AddContext; -use types::{ - collection::{Collection, SyncCollection}, - field::EmailField, - id::Id, -}; +use types::{collection::SyncCollection, id::Id}; pub trait ThreadGet: Sync + Send { fn thread_get( @@ -81,62 +77,40 @@ impl ThreadGet for Server { }; let ordered_ids = if add_email_ids && !all_ids.is_empty() { - let mut ordered_id = Vec::with_capacity(all_ids.len() as usize); - self.store() - .iterate( - IterateParams::new( - ValueKey { - account_id, - collection: Collection::Email.into(), - document_id: 0, - class: ValueClass::IndexProperty(IndexPropertyClass::Integer { - property: EmailField::ReceivedToSize.into(), - value: 0, + Some( + self.search_store() + .query_account( + SearchQuery::new(SearchIndex::Email) + .with_account_id(account_id) + .with_mask(all_ids) + .with_comparator(SearchComparator::Field { + field: SearchField::Email(EmailSearchField::ReceivedAt), + ascending: true, }), - }, - ValueKey { - account_id, - collection: Collection::Email.into(), - document_id: u32::MAX, - class: ValueClass::IndexProperty(IndexPropertyClass::Integer { - property: EmailField::ReceivedToSize.into(), - value: u64::MAX, - }), - }, ) - .ascending() - .no_values(), - |key, _| { - let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; - if all_ids.contains(document_id) { - ordered_id.push(document_id); - } - - Ok(true) - }, - ) - .await - .caused_by(trc::location!())?; - Some(ordered_id) + .await?, + ) } else { None }; for id in ids { let thread_id = id.document_id(); - if let Some(document_ids) = thread_map.remove(&thread_id) { + if let Some(mut document_ids) = thread_map.remove(&thread_id) { let mut thread: Map<'_, ThreadProperty, ThreadValue> = Map::with_capacity(2).with_key_value(ThreadProperty::Id, id); if let Some(ordered_ids) = &ordered_ids { - thread.insert_unchecked( - ThreadProperty::EmailIds, - ordered_ids - .iter() - .filter(|id| document_ids.contains(**id)) - .copied() - .map(|id| Id::from_parts(thread_id, id)) - .collect::>(), - ); + let mut ids = Vec::with_capacity(document_ids.len() as usize); + for &id in ordered_ids.iter() { + if document_ids.remove(id) { + ids.push(Id::from_parts(thread_id, id)); + } + } + for id in document_ids.iter() { + ids.push(Id::from_parts(thread_id, id)); + } + + thread.insert_unchecked(ThreadProperty::EmailIds, ids); } response.list.push(thread.into()); } else { diff --git a/crates/pop3/src/mailbox.rs b/crates/pop3/src/mailbox.rs index 083afcc5..a26c7c31 100644 --- a/crates/pop3/src/mailbox.rs +++ b/crates/pop3/src/mailbox.rs @@ -11,13 +11,8 @@ use email::{ mailbox::INBOX_ID, }; use std::collections::BTreeMap; -use store::{ - IterateParams, U32_LEN, ValueKey, - ahash::AHashMap, - write::{IndexPropertyClass, ValueClass, key::DeserializeBigEndian}, -}; use trc::AddContext; -use types::{collection::Collection, field::EmailField, special_use::SpecialUse}; +use types::special_use::SpecialUse; #[derive(Default)] pub struct Mailbox { @@ -53,44 +48,6 @@ impl Session { .map(|x| x.uid_validity) .unwrap_or_default(); - // Obtain message sizes - let mut message_sizes = AHashMap::new(); - self.server - .store() - .iterate( - IterateParams::new( - ValueKey { - account_id, - collection: Collection::Email.into(), - document_id: 0, - class: ValueClass::IndexProperty(IndexPropertyClass::Integer { - property: EmailField::ReceivedToSize.into(), - value: 0, - }), - }, - ValueKey { - account_id, - collection: Collection::Email.into(), - document_id: u32::MAX, - class: ValueClass::IndexProperty(IndexPropertyClass::Integer { - property: EmailField::ReceivedToSize.into(), - value: u64::MAX, - }), - }, - ) - .ascending(), - |key, value| { - message_sizes.insert( - key.deserialize_be_u32(key.len() - U32_LEN)?, - value.deserialize_be_u32(0)?, - ); - - Ok(true) - }, - ) - .await - .caused_by(trc::location!())?; - // Sort by UID let message_map = cache .emails @@ -101,9 +58,9 @@ impl Session { .mailboxes .iter() .find(|m| m.mailbox_id == INBOX_ID) - .map(|m| (m.uid, message.document_id)) + .map(|m| (m.uid, (message.document_id, message.size))) }) - .collect::>(); + .collect::>(); // Create mailbox let mut mailbox = Mailbox { @@ -112,17 +69,15 @@ impl Session { account_id, ..Default::default() }; - for (uid, id) in message_map { - if let Some(size) = message_sizes.get(&id) { - mailbox.messages.push(Message { - id, - uid, - size: *size, - deleted: false, - }); - mailbox.total += 1; - mailbox.size += *size; - } + for (uid, (id, size)) in message_map { + mailbox.messages.push(Message { + id, + uid, + size, + deleted: false, + }); + mailbox.total += 1; + mailbox.size += size; } Ok(mailbox) diff --git a/crates/pop3/src/op/delete.rs b/crates/pop3/src/op/delete.rs index adad6dd1..bfedd314 100644 --- a/crates/pop3/src/op/delete.rs +++ b/crates/pop3/src/op/delete.rs @@ -89,7 +89,12 @@ impl Session { let mut batch = BatchBuilder::new(); let not_deleted = self .server - .emails_delete(mailbox.account_id, &mut batch, deleted) + .emails_delete( + mailbox.account_id, + self.state.access_token().tenant_id(), + &mut batch, + deleted, + ) .await .caused_by(trc::location!())?; diff --git a/crates/pop3/src/op/fetch.rs b/crates/pop3/src/op/fetch.rs index 4fa57c19..bb292ab6 100644 --- a/crates/pop3/src/op/fetch.rs +++ b/crates/pop3/src/op/fetch.rs @@ -11,6 +11,7 @@ use email::message::metadata::MessageMetadata; use std::time::Instant; use trc::AddContext; use types::{collection::Collection, field::EmailField}; +use utils::chained_bytes::ChainedBytes; impl Session { pub async fn handle_fetch(&mut self, msg: u32, lines: Option) -> trc::Result<()> { @@ -50,6 +51,14 @@ impl Session { Elapsed = op_start.elapsed() ); + let bytes = ChainedBytes::new(metadata.raw_headers.as_ref()) + .with_last( + bytes + .get(metadata.blob_body_offset.to_native() as usize..) + .unwrap_or_default(), + ) + .get_full_range(); + self.write_bytes( Response::Message:: { bytes, diff --git a/crates/pop3/src/protocol/response.rs b/crates/pop3/src/protocol/response.rs index 70b7e561..007594b5 100644 --- a/crates/pop3/src/protocol/response.rs +++ b/crates/pop3/src/protocol/response.rs @@ -4,16 +4,16 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{borrow::Cow, fmt::Display}; - use super::Mechanism; +use std::{borrow::Cow, fmt::Display}; +use utils::chained_bytes::SliceRange; -pub enum Response { +pub enum Response<'x, T> { Ok(Cow<'static, str>), Err(Cow<'static, str>), List(Vec), Message { - bytes: Vec, + bytes: SliceRange<'x>, lines: u32, }, Capability { @@ -22,7 +22,7 @@ pub enum Response { }, } -impl Response { +impl<'x, T: Display> Response<'x, T> { pub fn serialize(&self) -> Vec { match self { Response::Ok(message) => { @@ -61,7 +61,7 @@ impl Response { let mut last_byte = 0; // Transparency procedure - for &byte in bytes { + for &byte in bytes.into_iter() { // POP3 requires that lines end with CRLF, do this check to ensure that if byte == b'\n' && last_byte != b'\r' { buf.push(b'\r'); @@ -165,10 +165,9 @@ impl SerializeResponse for trc::Error { #[cfg(test)] mod tests { - - use crate::protocol::Mechanism; - use super::Response; + use crate::protocol::Mechanism; + use utils::chained_bytes::SliceRange; #[test] fn serialize_response() { @@ -206,9 +205,7 @@ mod tests { ), ( Response::Message { - bytes: "Subject: test\r\n\r\n.\r\ntest.\r\n.test\r\na" - .as_bytes() - .to_vec(), + bytes: SliceRange::Split(b"Subject: test\r\n\r\n.\r\n", b"test.\r\n.test\r\na"), lines: 0, }, "+OK 35 octets\r\nSubject: test\r\n\r\n..\r\ntest.\r\n..test\r\na\r\n.\r\n", diff --git a/crates/services/src/task_manager/index.rs b/crates/services/src/task_manager/index.rs index ccc5e32c..aee2607f 100644 --- a/crates/services/src/task_manager/index.rs +++ b/crates/services/src/task_manager/index.rs @@ -9,7 +9,7 @@ use common::{ Server, telemetry::tracers::store::{TracingStore, build_span_document}, }; -use directory::{QueryParams, Type, backend::internal::manage::ManageDirectory}; +use directory::{Type, backend::internal::manage::ManageDirectory}; use email::message::metadata::MessageMetadata; use groupware::{cache::GroupwareCache, calendar::CalendarEvent, contact::ContactCard}; use std::cmp::Ordering; @@ -564,15 +564,6 @@ async fn delete_email_metadata( .await? { Some(metadata_) => { - let tenant_id = server - .core - .storage - .directory - .query(QueryParams::id(account_id).with_return_member_of(false)) - .await - .unwrap_or_default() - .and_then(|p| p.tenant()); - batch .with_account_id(account_id) .with_collection(Collection::Email) @@ -580,7 +571,7 @@ async fn delete_email_metadata( let metadata = metadata_ .unarchive::() .caused_by(trc::location!())?; - metadata.unindex(batch, account_id, tenant_id); + metadata.unindex(batch); // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC @@ -592,7 +583,7 @@ async fn delete_email_metadata( batch, Collection::Email.into(), &BlobHash::from(&metadata.blob_hash), - u32::from(metadata.size) as usize, + metadata.root_part().offset_end.to_native() as usize, ); // SPDX-SnippetEnd diff --git a/crates/types/src/field.rs b/crates/types/src/field.rs index c2430b23..2786815f 100644 --- a/crates/types/src/field.rs +++ b/crates/types/src/field.rs @@ -40,7 +40,6 @@ pub enum CalendarNotificationField { pub enum EmailField { Archive, Metadata, - ReceivedToSize, Threading, } @@ -119,7 +118,6 @@ impl From for u8 { match value { EmailField::Metadata => 71, EmailField::Threading => 90, - EmailField::ReceivedToSize => 91, EmailField::Archive => ARCHIVE_FIELD, } } diff --git a/crates/types/src/keyword.rs b/crates/types/src/keyword.rs index 50cbd966..d37855e8 100644 --- a/crates/types/src/keyword.rs +++ b/crates/types/src/keyword.rs @@ -4,9 +4,8 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{fmt::Display, str::FromStr}; - use jmap_tools::{Element, Property, Value}; +use std::{fmt::Display, str::FromStr}; pub const SEEN: usize = 0; pub const DRAFT: usize = 1; @@ -64,7 +63,7 @@ pub enum Keyword { Forwarded, #[serde(rename(serialize = "$mdnsent"))] MdnSent, - Other(String), + Other(Box), } impl Keyword { @@ -76,6 +75,14 @@ impl Keyword { } pub fn from_other(value: String) -> Self { + if value.len() <= Keyword::MAX_LENGTH { + Keyword::Other(value.into_boxed_str()) + } else { + Keyword::Other(value.chars().take(Keyword::MAX_LENGTH).collect()) + } + } + + pub fn from_boxed_other(value: Box) -> Self { if value.len() <= Keyword::MAX_LENGTH { Keyword::Other(value) } else { @@ -119,11 +126,11 @@ impl Keyword { Keyword::Deleted => Ok(DELETED as u32), Keyword::Forwarded => Ok(FORWARDED as u32), Keyword::MdnSent => Ok(MDN_SENT as u32), - Keyword::Other(string) => Err(string.as_str()), + Keyword::Other(string) => Err(string.as_ref()), } } - pub fn into_id(self) -> Result { + pub fn into_id(self) -> Result> { match self { Keyword::Seen => Ok(SEEN as u32), Keyword::Draft => Ok(DRAFT as u32), @@ -249,7 +256,25 @@ impl ArchivedKeyword { ArchivedKeyword::Deleted => Ok(DELETED as u32), ArchivedKeyword::Forwarded => Ok(FORWARDED as u32), ArchivedKeyword::MdnSent => Ok(MDN_SENT as u32), - ArchivedKeyword::Other(string) => Err(string.as_str()), + ArchivedKeyword::Other(string) => Err(string.as_ref()), + } + } + + pub fn to_native(&self) -> Keyword { + match self { + ArchivedKeyword::Seen => Keyword::Seen, + ArchivedKeyword::Draft => Keyword::Draft, + ArchivedKeyword::Flagged => Keyword::Flagged, + ArchivedKeyword::Answered => Keyword::Answered, + ArchivedKeyword::Recent => Keyword::Recent, + ArchivedKeyword::Important => Keyword::Important, + ArchivedKeyword::Phishing => Keyword::Phishing, + ArchivedKeyword::Junk => Keyword::Junk, + ArchivedKeyword::NotJunk => Keyword::NotJunk, + ArchivedKeyword::Deleted => Keyword::Deleted, + ArchivedKeyword::Forwarded => Keyword::Forwarded, + ArchivedKeyword::MdnSent => Keyword::MdnSent, + ArchivedKeyword::Other(other) => Keyword::Other(other.as_ref().into()), } } } @@ -269,7 +294,7 @@ impl From<&ArchivedKeyword> for Keyword { ArchivedKeyword::Deleted => Keyword::Deleted, ArchivedKeyword::Forwarded => Keyword::Forwarded, ArchivedKeyword::MdnSent => Keyword::MdnSent, - ArchivedKeyword::Other(string) => Keyword::Other(string.as_str().into()), + ArchivedKeyword::Other(string) => Keyword::Other(string.as_ref().into()), } } } diff --git a/crates/utils/src/chained_bytes.rs b/crates/utils/src/chained_bytes.rs new file mode 100644 index 00000000..0942f498 --- /dev/null +++ b/crates/utils/src/chained_bytes.rs @@ -0,0 +1,156 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::{borrow::Cow, ops::Range}; + +#[derive(Debug, Clone)] +pub struct ChainedBytes<'x> { + first: &'x [u8], + last: &'x [u8], +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SliceRange<'x> { + Single(&'x [u8]), + Split(&'x [u8], &'x [u8]), + None, +} + +impl<'x> ChainedBytes<'x> { + pub fn new(first: &'x [u8]) -> Self { + Self { first, last: &[] } + } + + pub fn append(&mut self, bytes: &'x [u8]) { + self.last = bytes; + } + + pub fn with_last(mut self, bytes: &'x [u8]) -> Self { + self.last = bytes; + self + } + + pub fn get(&self, index: Range) -> Option> { + let start = index.start; + let end = index.end; + + if let Some(bytes) = self.first.get(start..end) { + Some(Cow::Borrowed(bytes)) + } else if start >= self.first.len() { + self.last + .get(start - self.first.len()..end - self.first.len()) + .map(Cow::Borrowed) + } else if let (Some(first), Some(last)) = ( + self.first.get(start..), + self.last.get(..end - self.first.len()), + ) { + let mut vec = vec![0u8; first.len() + last.len()]; + vec[..first.len()].copy_from_slice(first); + vec[first.len()..].copy_from_slice(last); + Some(Cow::Owned(vec)) + } else { + None + } + } + + pub fn get_slice_range(&self, index: Range) -> SliceRange<'x> { + let start = index.start; + let end = index.end; + + if let Some(bytes) = self.first.get(start..end) { + SliceRange::Single(bytes) + } else if start >= self.first.len() { + self.last + .get(start - self.first.len()..end - self.first.len()) + .map(SliceRange::Single) + .unwrap_or(SliceRange::None) + } else if let (Some(first), Some(last)) = ( + self.first.get(start..), + self.last.get(..end - self.first.len()), + ) { + SliceRange::Split(first, last) + } else { + SliceRange::None + } + } + + pub fn get_full_range(&self) -> SliceRange<'x> { + if self.last.is_empty() { + SliceRange::Single(self.first) + } else { + SliceRange::Split(self.first, self.last) + } + } + + pub fn to_bytes(&self) -> Vec { + let mut bytes = vec![0u8; self.first.len() + self.last.len()]; + bytes[..self.first.len()].copy_from_slice(self.first); + bytes[self.first.len()..].copy_from_slice(self.last); + bytes + } + + pub fn len(&self) -> usize { + self.first.len() + self.last.len() + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +impl<'x> SliceRange<'x> { + pub fn len(&self) -> usize { + match self { + SliceRange::Single(bytes) => bytes.len(), + SliceRange::Split(first, last) => first.len() + last.len(), + SliceRange::None => 0, + } + } + + pub fn try_into_bytes(self) -> Option> { + match self { + SliceRange::Single(bytes) => Some(Cow::Borrowed(bytes)), + SliceRange::Split(first, last) => { + let mut vec = vec![0u8; first.len() + last.len()]; + vec[..first.len()].copy_from_slice(first); + vec[first.len()..].copy_from_slice(last); + Some(Cow::Owned(vec)) + } + SliceRange::None => None, + } + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + fn into_pairs(self) -> (&'x [u8], &'x [u8]) { + match self { + SliceRange::Single(bytes) => (bytes, &[][..]), + SliceRange::Split(first, last) => (first, last), + SliceRange::None => (&[][..], &[][..]), + } + } + + pub fn is_none(&self) -> bool { + matches!(self, SliceRange::None) + } + + pub fn is_some(&self) -> bool { + !self.is_none() + } +} + +impl<'x> IntoIterator for SliceRange<'x> { + type Item = &'x u8; + type IntoIter = std::iter::Chain, std::slice::Iter<'x, u8>>; + + fn into_iter(self) -> Self::IntoIter { + let (first, last) = self.into_pairs(); + + first.iter().chain(last.iter()) + } +} diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index ea3cd402..0e126eec 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -6,6 +6,7 @@ pub mod bimap; pub mod cache; +pub mod chained_bytes; pub mod cheeky_hash; pub mod codec; pub mod config; diff --git a/tests/src/imap/body_structure.rs b/tests/src/imap/body_structure.rs index 02847174..c8db6d87 100644 --- a/tests/src/imap/body_structure.rs +++ b/tests/src/imap/body_structure.rs @@ -6,7 +6,7 @@ use std::fs; -use email::message::metadata::MessageMetadata; +use email::message::metadata::{MessageMetadata, build_metadata_contents}; use imap::op::fetch::AsImapDataItem; use imap_proto::{ ResponseCode, StatusResponse, @@ -17,6 +17,7 @@ use store::{ Deserialize, Serialize, write::{Archive, Archiver}, }; +use utils::chained_bytes::ChainedBytes; use super::resources_dir; @@ -35,7 +36,6 @@ fn imap_test_body_structure() { let message_ = MessageParser::new().parse(&raw_message).unwrap(); let metadata = MessageMetadata { preview: Default::default(), - size: message_.raw_message.len() as u32, raw_headers: message_ .raw_message .as_ref() @@ -44,20 +44,21 @@ fn imap_test_body_structure() { ..message_.root_part().offset_body as usize, ) .unwrap_or_default() - .to_vec(), - contents: vec![], - received_at: 0, - has_attachments: false, + .into(), blob_hash: Default::default(), - } - .with_contents(message_); - //let c = println!("metadata {:#?}", metadata); + blob_body_offset: message_.root_part().offset_body as u32, + contents: build_metadata_contents(message_), + rcvd_attach: 0, + }; let metadata_ = Archive::deserialize_owned(Archiver::new(metadata).serialize().unwrap()).unwrap(); let metadata = metadata_.unarchive::().unwrap(); - let decoded = metadata.decode_contents(&raw_message); - - //let c = println!("parts {:#?}", decoded); + let raw_message = ChainedBytes::new(metadata.raw_headers.as_ref()).with_last( + raw_message + .get(metadata.blob_body_offset.to_native() as usize..) + .unwrap_or_default(), + ); + let decoded = metadata.decode_contents(raw_message); // Serialize body and bodystructure for is_extended in [false, true] { diff --git a/tests/src/jmap/auth/permissions.rs b/tests/src/jmap/auth/permissions.rs index 6450a305..424b5a50 100644 --- a/tests/src/jmap/auth/permissions.rs +++ b/tests/src/jmap/auth/permissions.rs @@ -16,7 +16,6 @@ use directory::{ }; use email::message::delivery::{IngestMessage, LocalDeliveryStatus, MailDelivery}; use std::sync::Arc; -use types::blob_hash::BlobHash; pub async fn test(params: &JMAPTest) { println!("Running permissions tests..."); @@ -611,12 +610,11 @@ pub async fn test(params: &JMAPTest) { ); // John should not be allowed to receive email - let message_blob = BlobHash::generate(TEST_MESSAGE.as_bytes()); - server - .blob_store() - .put_blob(message_blob.as_ref(), TEST_MESSAGE.as_bytes()) + let message_blob = server + .put_blob(tenant_user_id, TEST_MESSAGE.as_bytes(), false) .await - .unwrap(); + .unwrap() + .hash; assert_eq!( server .deliver_message(IngestMessage { diff --git a/tests/src/jmap/auth/quota.rs b/tests/src/jmap/auth/quota.rs index a5e243db..1ef6b338 100644 --- a/tests/src/jmap/auth/quota.rs +++ b/tests/src/jmap/auth/quota.rs @@ -12,6 +12,7 @@ use crate::{ }; use common::config::smtp::queue::QueueName; use email::{cache::MessageCacheFetch, mailbox::INBOX_ID}; +use http::management::stores::recalculate_quota; use jmap::blob::upload::DISABLE_UPLOAD_QUOTA; use jmap_client::{ core::set::{SetErrorType, SetObject}, @@ -210,8 +211,7 @@ pub async fn test(params: &mut JMAPTest) { .get_used_quota(account.id().document_id()) .await .unwrap(); - server - .recalculate_quota(account.id().document_id()) + recalculate_quota(&server, account.id().document_id()) .await .unwrap(); assert_eq!( diff --git a/tests/src/jmap/mail/delivery.rs b/tests/src/jmap/mail/delivery.rs index 82845a06..75feb864 100644 --- a/tests/src/jmap/mail/delivery.rs +++ b/tests/src/jmap/mail/delivery.rs @@ -5,22 +5,34 @@ */ use crate::{ - directory::internal::TestInternalDirectory, jmap::JMAPTest, webdav::DummyWebDavClient, + directory::internal::TestInternalDirectory, jmap::JMAPTest, + store::cleanup::store_blob_expire_all, webdav::DummyWebDavClient, }; use email::{ cache::{MessageCacheFetch, email::MessageCacheAccess}, mailbox::{INBOX_ID, JUNK_ID}, + message::metadata::MessageMetadata, }; use groupware::DavResourceName; +use jmap::blob::download::BlobDownload; use std::time::Duration; use tokio::{ io::{AsyncBufReadExt, AsyncWriteExt, BufReader, Lines, ReadHalf, WriteHalf}, net::TcpStream, }; +use types::{ + blob::{BlobClass, BlobId}, + blob_hash::BlobHash, + collection::Collection, + field::EmailField, +}; +use utils::chained_bytes::ChainedBytes; pub async fn test(params: &mut JMAPTest) { println!("Running message delivery tests..."); + let todo = "enable delivered to for test"; + // Create a domain name and a test account let server = params.server.clone(); let john = params.account("jdoe@example.com"); @@ -239,19 +251,64 @@ END:VCARD ) .await; + // Make sure blobs are properly linked + store_blob_expire_all(params.server.store()).await; + for (account, num_messages) in [(john, 5), (jane, 3), (bill, 3)] { + let account_id = account.id().document_id(); + let cache = server.get_cached_messages(account_id).await.unwrap(); assert_eq!( - server - .get_cached_messages(account.id().document_id()) - .await - .unwrap() - .emails - .items - .len(), + cache.emails.items.len(), num_messages, "for {}", account.id_string() ); + let access_token = server.get_access_token(account_id).await.unwrap(); + + for document_id in cache.emails.items.iter().map(|e| e.document_id) { + let archive = server + .archive_by_property( + account_id, + Collection::Email, + document_id, + EmailField::Metadata.into(), + ) + .await + .unwrap() + .unwrap(); + let metadata = archive.to_unarchived::().unwrap(); + let body = server + .blob_download( + &BlobId { + hash: BlobHash::from(&metadata.inner.blob_hash), + class: BlobClass::Linked { + account_id, + collection: Collection::Email.into(), + document_id, + }, + section: None, + }, + &access_token, + ) + .await + .unwrap() + .unwrap(); + assert_ne!(metadata.inner.blob_body_offset.to_native(), 0); + let raw_message = ChainedBytes::new(metadata.inner.raw_headers.as_ref()).with_last( + body.get(metadata.inner.blob_body_offset.to_native() as usize..) + .unwrap_or_default(), + ); + let full_message = String::from_utf8(raw_message.to_bytes()).unwrap(); + assert!( + full_message.contains("Delivered-To:") && full_message.contains("Subject:"), + "for {account_id}: {full_message}" + ); + println!( + "full message for {}:\n{}", + account.id_string(), + full_message + ); + } } // Remove test data diff --git a/tests/src/jmap/mail/thread_get.rs b/tests/src/jmap/mail/thread_get.rs index b854e416..b11ab9b3 100644 --- a/tests/src/jmap/mail/thread_get.rs +++ b/tests/src/jmap/mail/thread_get.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::jmap::JMAPTest; +use crate::jmap::{JMAPTest, wait_for_index}; use jmap_client::mailbox::Role; pub async fn test(params: &mut JMAPTest) { @@ -35,6 +35,8 @@ pub async fn test(params: &mut JMAPTest) { expected_result[num - 1] = email.take_id(); } + wait_for_index(¶ms.server).await; + assert_eq!( client .thread_get(&thread_id) diff --git a/tests/src/jmap/mail/thread_merge.rs b/tests/src/jmap/mail/thread_merge.rs index 624e8591..eafb6071 100644 --- a/tests/src/jmap/mail/thread_merge.rs +++ b/tests/src/jmap/mail/thread_merge.rs @@ -230,6 +230,7 @@ async fn test_multi_thread(params: &mut JMAPTest) { .email_ingest(IngestEmail { raw_message: message.contents(), message: MessageParser::new().parse(message.contents()), + blob_hash: None, access_token: &AccessToken::from_id(account_id), mailbox_ids: vec![mailbox_id], keywords: vec![], diff --git a/tests/src/jmap/mod.rs b/tests/src/jmap/mod.rs index f8c84403..2a467376 100644 --- a/tests/src/jmap/mod.rs +++ b/tests/src/jmap/mod.rs @@ -78,7 +78,7 @@ async fn jmap_tests() { server::webhooks::test(&mut params).await; - mail::get::test(&mut params).await; + /*mail::get::test(&mut params).await; mail::set::test(&mut params).await; mail::parse::test(&mut params).await; mail::query::test(&mut params, delete).await; @@ -122,7 +122,7 @@ async fn jmap_tests() { calendar::acl::test(&mut params).await; principal::get::test(&mut params).await; - principal::availability::test(&mut params).await; + principal::availability::test(&mut params).await;*/ server::purge::test(&mut params).await; server::enterprise::test(&mut params).await; diff --git a/tests/src/store/cleanup.rs b/tests/src/store/cleanup.rs index f38c0e42..70fd667d 100644 --- a/tests/src/store/cleanup.rs +++ b/tests/src/store/cleanup.rs @@ -237,7 +237,6 @@ pub async fn store_assert_is_empty(store: &Store, blob_store: BlobStore) { (SUBSPACE_BLOBS, true), (SUBSPACE_COUNTER, false), (SUBSPACE_QUOTA, false), - (SUBSPACE_BLOBS, true), (SUBSPACE_INDEXES, false), (SUBSPACE_TELEMETRY_SPAN, true), (SUBSPACE_TELEMETRY_METRIC, true), diff --git a/tests/src/store/mod.rs b/tests/src/store/mod.rs index 992981e8..5390e795 100644 --- a/tests/src/store/mod.rs +++ b/tests/src/store/mod.rs @@ -124,6 +124,14 @@ pub fn build_store_config(temp_dir: &str) -> String { .replace("{BLOB_STORE}", &blob_store) .replace("{LOOKUP_STORE}", &lookup_store) .replace("{TMP}", temp_dir) + .replace( + "{ELASTIC_ENABLED}", + if fts_store != "elastic" { + "true" + } else { + "false" + }, + ) } const CONFIG: &str = r#" @@ -158,11 +166,11 @@ password = "password" type = "elasticsearch" url = "https://localhost:9200" tls.allow-invalid-certs = true +disable = {ELASTIC_ENABLED} [store."elastic".auth] username = "elastic" secret = "changeme" - [store."s3"] type = "s3" access-key = "minioadmin"