From 9ec047d2844d3465f5b938170160c37f5a8c63c6 Mon Sep 17 00:00:00 2001 From: mdecimus Date: Sat, 1 Mar 2025 10:00:04 +0100 Subject: [PATCH] Zero copy IMAP passing tests --- Cargo.lock | 1 + crates/common/src/listener/acme/resolver.rs | 4 +- crates/common/src/storage/index.rs | 24 +- crates/email/src/mailbox/destroy.rs | 5 +- crates/email/src/mailbox/index.rs | 7 +- crates/email/src/mailbox/manage.rs | 2 +- crates/email/src/message/bayes.rs | 4 +- crates/email/src/message/copy.rs | 6 +- crates/email/src/message/crypto.rs | 4 +- crates/email/src/message/delete.rs | 13 +- crates/email/src/message/index.rs | 28 +- crates/email/src/message/ingest.rs | 4 +- crates/email/src/message/metadata.rs | 332 ++++++++++++-------- crates/email/src/sieve/activate.rs | 6 +- crates/email/src/sieve/delete.rs | 4 +- crates/email/src/sieve/index.rs | 2 +- crates/email/src/sieve/ingest.rs | 6 +- crates/email/src/sieve/mod.rs | 2 +- crates/email/src/submission/index.rs | 2 +- crates/imap/src/core/mailbox.rs | 6 +- crates/imap/src/core/message.rs | 8 +- crates/imap/src/op/acl.rs | 9 +- crates/imap/src/op/copy_move.rs | 5 +- crates/imap/src/op/expunge.rs | 19 +- crates/imap/src/op/fetch.rs | 318 ++++++++++--------- crates/imap/src/op/rename.rs | 3 +- crates/imap/src/op/status.rs | 3 +- crates/imap/src/op/store.rs | 16 +- crates/imap/src/op/subscribe.rs | 3 +- crates/jmap/src/api/management/stores.rs | 10 +- crates/jmap/src/blob/get.rs | 6 +- crates/jmap/src/email/crypto.rs | 9 +- crates/jmap/src/email/get.rs | 55 ++-- crates/jmap/src/email/set.rs | 9 +- crates/jmap/src/email/snippet.rs | 14 +- crates/jmap/src/identity/get.rs | 4 +- crates/jmap/src/identity/set.rs | 4 +- crates/jmap/src/mailbox/get.rs | 6 +- crates/jmap/src/mailbox/query.rs | 13 +- crates/jmap/src/mailbox/set.rs | 6 +- crates/jmap/src/push/get.rs | 5 +- crates/jmap/src/push/set.rs | 4 +- crates/jmap/src/services/index.rs | 8 +- crates/jmap/src/sieve/get.rs | 9 +- crates/jmap/src/sieve/query.rs | 5 +- crates/jmap/src/sieve/set.rs | 4 +- crates/jmap/src/submission/get.rs | 10 +- crates/jmap/src/submission/set.rs | 20 +- crates/jmap/src/vacation/get.rs | 4 +- crates/jmap/src/vacation/set.rs | 4 +- crates/managesieve/src/op/getscript.rs | 4 +- crates/managesieve/src/op/listscripts.rs | 4 +- crates/managesieve/src/op/putscript.rs | 12 +- crates/managesieve/src/op/renamescript.rs | 4 +- crates/pop3/src/mailbox.rs | 7 +- crates/pop3/src/op/fetch.rs | 4 +- crates/store/src/query/mod.rs | 4 +- crates/store/src/write/mod.rs | 9 +- crates/store/src/write/serialize.rs | 202 ++++++------ tests/Cargo.toml | 1 + tests/src/imap/body_structure.rs | 61 ++-- tests/src/imap/mod.rs | 8 +- tests/src/jmap/crypto.rs | 5 +- tests/src/jmap/mod.rs | 6 +- tests/src/jmap/stress_test.rs | 7 +- tests/src/store/query.rs | 6 +- 66 files changed, 717 insertions(+), 682 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e9bb4465..c9e9cea3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7100,6 +7100,7 @@ dependencies = [ "rayon", "reqwest 0.12.12", "ring 0.17.8", + "rkyv 0.8.10", "rustls 0.23.21", "rustls-pemfile 2.2.0", "rustls-pki-types", diff --git a/crates/common/src/listener/acme/resolver.rs b/crates/common/src/listener/acme/resolver.rs index 8759c74e..ce43d6c0 100644 --- a/crates/common/src/listener/acme/resolver.rs +++ b/crates/common/src/listener/acme/resolver.rs @@ -20,7 +20,7 @@ use crate::{KV_ACME, Server}; use super::{ AcmeProvider, StaticResolver, - directory::{ACME_TLS_ALPN_NAME, ArchivedSerializedCert}, + directory::{ACME_TLS_ALPN_NAME, SerializedCert}, }; impl Server { @@ -51,7 +51,7 @@ impl Server { .key_get::(KeyValue::<()>::build_key(KV_ACME, domain)) .await { - Ok(Some(cert_)) => match cert_.unarchive::() { + Ok(Some(cert_)) => match cert_.unarchive::() { Ok(cert) => { match any_ecdsa_type(&PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from( cert.private_key.as_ref(), diff --git a/crates/common/src/storage/index.rs b/crates/common/src/storage/index.rs index 66d1163c..3818df53 100644 --- a/crates/common/src/storage/index.rs +++ b/crates/common/src/storage/index.rs @@ -16,7 +16,7 @@ use store::{ #[derive(Debug, Clone, PartialEq, Eq)] pub enum IndexValue<'x> { - Text { field: u8, value: Cow<'x, str> }, + Text { field: u8, value: Cow<'x, [u8]> }, U32 { field: u8, value: Option }, U64 { field: u8, value: Option }, U32List { field: u8, value: &'x [u32] }, @@ -49,6 +49,20 @@ pub struct ObjectIndexBuilder { changes: Option, } +impl<'x> IndexValue<'x> { + pub fn queryable_text(field: impl Into, text: &'x str) -> Self { + let mut value = Vec::with_capacity((text.len() * 2) + 1); + value.extend_from_slice(text.as_bytes()); + value.push(0); + value.extend_from_slice(text.to_lowercase().as_bytes()); + + IndexValue::Text { + field: field.into(), + value: value.into(), + } + } +} + impl Default for ObjectIndexBuilder { fn default() -> Self { Self::new() @@ -139,7 +153,7 @@ fn build_batch( if !value.is_empty() { batch.ops.push(Operation::Index { field, - key: value.as_ref().serialize(), + key: value.into_owned(), set, }); } @@ -236,7 +250,7 @@ fn merge_batch( if !old_value.is_empty() { batch.ops.push(Operation::Index { field, - key: old_value.as_ref().serialize(), + key: old_value.into_owned(), set: false, }); } @@ -244,7 +258,7 @@ fn merge_batch( if !new_value.is_empty() { batch.ops.push(Operation::Index { field, - key: new_value.as_ref().serialize(), + key: new_value.into_owned(), set: true, }); } @@ -423,7 +437,7 @@ fn merge_batch( if has_changes { batch.ops.push(Operation::Value { class: Property::Value.into(), - op: ValueOp::Set(Archiver::new(current).serialize()?.into()), + op: ValueOp::Set(Archiver::new(changes).serialize()?.into()), }); } diff --git a/crates/email/src/mailbox/destroy.rs b/crates/email/src/mailbox/destroy.rs index ffced6c3..e2b89aa4 100644 --- a/crates/email/src/mailbox/destroy.rs +++ b/crates/email/src/mailbox/destroy.rs @@ -12,7 +12,6 @@ use jmap_proto::{ error::set::{SetError, SetErrorType}, types::{acl::Acl, collection::Collection, id::Id, property::Property}, }; -use rkyv::vec::ArchivedVec; use store::{ Serialize, SerializeInfallible, query::Filter, @@ -112,7 +111,7 @@ impl MailboxDestroy for Server { { // Remove mailbox from list let mut mailbox_ids = mailbox_ids - .into_deserialized::, Vec>() + .into_deserialized::>() .caused_by(trc::location!())?; let orig_len = mailbox_ids.inner.len(); mailbox_ids.inner.retain(|id| id.mailbox_id != document_id); @@ -202,7 +201,7 @@ impl MailboxDestroy for Server { .caused_by(trc::location!())? { let mailbox = mailbox - .into_deserialized::() + .into_deserialized::() .caused_by(trc::location!())?; // Validate ACLs if access_token.is_shared(account_id) { diff --git a/crates/email/src/mailbox/index.rs b/crates/email/src/mailbox/index.rs index 164676c3..391925fc 100644 --- a/crates/email/src/mailbox/index.rs +++ b/crates/email/src/mailbox/index.rs @@ -16,13 +16,10 @@ use super::{ArchivedUidMailbox, Mailbox, UidMailbox}; impl IndexableObject for Mailbox { fn index_values(&self) -> impl Iterator> { [ - IndexValue::Text { - field: Property::Name.into(), - value: self.name.to_lowercase().into(), - }, + IndexValue::queryable_text(Property::Name, &self.name), IndexValue::Text { field: Property::Role.into(), - value: self.role.as_str().unwrap_or_default().into(), + value: self.role.as_str().unwrap_or_default().as_bytes().into(), }, IndexValue::Tag { field: Property::Role.into(), diff --git a/crates/email/src/mailbox/manage.rs b/crates/email/src/mailbox/manage.rs index ca16d8f1..498134e5 100644 --- a/crates/email/src/mailbox/manage.rs +++ b/crates/email/src/mailbox/manage.rs @@ -313,7 +313,7 @@ impl MailboxFnc for Server { ) .await? { - let obj = obj.unarchive::()?; + let obj = obj.unarchive::()?; found_names.push(( obj.name.to_string(), u32::from(obj.parent_id), diff --git a/crates/email/src/message/bayes.rs b/crates/email/src/message/bayes.rs index cbe526a4..4682b0ed 100644 --- a/crates/email/src/message/bayes.rs +++ b/crates/email/src/message/bayes.rs @@ -16,7 +16,7 @@ use store::write::{Archive, TaskQueueClass}; use trc::StoreEvent; use utils::BlobHash; -use super::metadata::ArchivedMessageMetadata; +use super::metadata::MessageMetadata; pub trait EmailBayesTrain: Sync + Send { fn email_bayes_train( @@ -75,7 +75,7 @@ impl EmailBayesTrain for Server { Ok(TaskQueueClass::BayesTrain { seq: self.generate_snowflake_id()?, - hash: BlobHash::from(&metadata.unarchive::()?.blob_hash), + hash: BlobHash::from(&metadata.unarchive::()?.blob_hash), learn_spam, }) } diff --git a/crates/email/src/message/copy.rs b/crates/email/src/message/copy.rs index 07a249e2..a6cfbfef 100644 --- a/crates/email/src/message/copy.rs +++ b/crates/email/src/message/copy.rs @@ -27,7 +27,7 @@ use crate::mailbox::UidMailbox; use super::{ index::{MAX_ID_LENGTH, MAX_SORT_FIELD_LENGTH, TrimTextValue}, ingest::{EmailIngest, IngestedEmail, LogEmailInsert}, - metadata::{ArchivedMessageMetadata, HeaderName, HeaderValue, MessageMetadata}, + metadata::{HeaderName, HeaderValue, MessageMetadata}, }; pub trait EmailCopy: Sync + Send { @@ -68,7 +68,7 @@ impl EmailCopy for Server { .await? { metadata - .deserialize::() + .deserialize::() .caused_by(trc::location!())? } else { return Ok(Err(SetError::not_found().with_description(format!( @@ -103,7 +103,7 @@ impl EmailCopy for Server { // Obtain threadId let mut references = Vec::with_capacity(5); let mut subject = ""; - for header in &metadata.contents.parts[0].headers { + for header in &metadata.contents[0].parts[0].headers { match &header.name { HeaderName::MessageId | HeaderName::InReplyTo diff --git a/crates/email/src/message/crypto.rs b/crates/email/src/message/crypto.rs index ac88b686..285279fc 100644 --- a/crates/email/src/message/crypto.rs +++ b/crates/email/src/message/crypto.rs @@ -529,9 +529,7 @@ fn try_parse_pem( .ok_or(Cow::from("Failed to decode base64")) .and_then(|bytes| { Archive::deserialize_owned(bytes) - .and_then(|arch| { - arch.deserialize::() - }) + .and_then(|arch| arch.deserialize::()) .map_err(|_| Cow::from("Failed to deserialize internal certificate")) }) .map(|params| Some((params.method, params.certs))); diff --git a/crates/email/src/message/delete.rs b/crates/email/src/message/delete.rs index dc4e3216..6ea0e244 100644 --- a/crates/email/src/message/delete.rs +++ b/crates/email/src/message/delete.rs @@ -8,10 +8,9 @@ use std::time::Duration; use common::{KV_LOCK_PURGE_ACCOUNT, Server}; use jmap_proto::types::{ - collection::Collection, id::Id, keyword::ArchivedKeyword, property::Property, - state::StateChange, type_state::DataType, + collection::Collection, id::Id, keyword::Keyword, property::Property, state::StateChange, + type_state::DataType, }; -use rkyv::vec::ArchivedVec; use store::{ BitmapKey, IterateParams, U32_LEN, ValueKey, ahash::AHashMap, @@ -27,7 +26,7 @@ use utils::{BlobHash, codec::leb128::Leb128Reader}; use std::future::Future; use store::rand::prelude::SliceRandom; -use crate::{mailbox::*, message::metadata::ArchivedMessageMetadata}; +use crate::{mailbox::*, message::metadata::MessageMetadata}; pub trait EmailDeletion: Sync + Send { fn emails_tombstone( @@ -77,7 +76,7 @@ impl EmailDeletion for Server { document_id, DeleteProperties { mailboxes: mailboxes - .unarchive::>() + .unarchive::>() .caused_by(trc::location!())? .iter() .map(|m| u32::from(m.mailbox_id)) @@ -455,7 +454,7 @@ impl EmailDeletion for Server { .await? { let keywords = keywords_ - .unarchive::>() + .unarchive::>() .caused_by(trc::location!())?; batch .untag_many(Property::Keywords, keywords.iter()) @@ -484,7 +483,7 @@ impl EmailDeletion for Server { .await? { let metadata = metadata_ - .unarchive::() + .unarchive::() .caused_by(trc::location!())?; // SPDX-SnippetBegin diff --git a/crates/email/src/message/index.rs b/crates/email/src/message/index.rs index 4810f5fb..86a85d52 100644 --- a/crates/email/src/message/index.rs +++ b/crates/email/src/message/index.rs @@ -38,7 +38,7 @@ pub const PREVIEW_LENGTH: usize = 256; impl MessageMetadata { #[inline(always)] pub fn root_part(&self) -> &MessageMetadataPart { - &self.contents.parts[0] + &self.contents[0].parts[0] } pub fn index( @@ -173,7 +173,9 @@ impl MessageMetadata { HeaderName::Date => { if !seen_headers[header.name.id() as usize] { if let HeaderValue::DateTime(datetime) = &header.value { - let value = (*datetime as u64).serialize(); + let value = (mail_parser::DateTime::from(datetime).to_timestamp() + as u64) + .serialize(); if set { batch.index(Property::SentAt, value); } else { @@ -231,7 +233,7 @@ impl MessageMetadata { impl ArchivedMessageMetadata { #[inline(always)] pub fn root_part(&self) -> &ArchivedMessageMetadataPart { - &self.contents.parts[0] + &self.contents[0].parts[0] } pub fn index( @@ -367,7 +369,9 @@ impl ArchivedMessageMetadata { ArchivedHeaderName::Date => { if !seen_headers[header.name.id() as usize] { if let ArchivedHeaderValue::DateTime(datetime) = &header.value { - let value = (i64::from(*datetime) as u64).serialize(); + let value = (mail_parser::DateTime::from(datetime).to_timestamp() + as u64) + .serialize(); if set { batch.index(Property::SentAt, value); } else { @@ -553,11 +557,12 @@ impl IndexMessage for BatchBuilder { .get(root_part.offset_header..root_part.offset_body) .unwrap_or_default() .to_vec(), - contents: message.into(), + contents: vec![], received_at, has_attachments, blob_hash, - }; + } + .with_contents(message); metadata.index_headers(self, true); // Store and index hasAttachment property @@ -596,9 +601,9 @@ impl<'x> IndexMessageText<'x> for FtsDocument<'x, mail_parser::HeaderName<'x>> { raw_message: &'x [u8], ) -> Self { let mut language = Language::Unknown; + let message_contents = &message.contents[0]; - for (part_id, part) in message - .contents + for (part_id, part) in message_contents .parts .iter() .take(MAX_MESSAGE_PARTS) @@ -679,15 +684,16 @@ impl<'x> IndexMessageText<'x> for FtsDocument<'x, mail_parser::HeaderName<'x>> { _ => unreachable!(), }; - if message.contents.is_html_part(part_id) - || message.contents.is_text_part(part_id) + if message_contents.is_html_part(part_id) + || message_contents.is_text_part(part_id) { self.index(Field::Body, text, part_language); } else { self.index(Field::Attachment, text, part_language); } } - ArchivedMetadataPartType::Message(nested_message) => { + ArchivedMetadataPartType::Message(nested_message_id) => { + let nested_message = message.message_id(*nested_message_id); let nested_message_language = nested_message .root_part() .language() diff --git a/crates/email/src/message/ingest.rs b/crates/email/src/message/ingest.rs index bb282c42..78d6c026 100644 --- a/crates/email/src/message/ingest.rs +++ b/crates/email/src/message/ingest.rs @@ -50,7 +50,7 @@ use utils::map::vec_map::VecMap; use crate::{ mailbox::{INBOX_ID, JUNK_ID, UidMailbox}, message::{ - crypto::ArchivedEncryptionParams, + crypto::EncryptionParams, index::{IndexMessage, MAX_ID_LENGTH, VisitValues}, }, thread::cache::ThreadCache, @@ -394,7 +394,7 @@ impl EmailIngest for Server { .caused_by(trc::location!())? { let encrypt_params = encrypt_params_ - .unarchive::() + .unarchive::() .caused_by(trc::location!())?; match message.encrypt(encrypt_params).await { Ok(new_raw_message) => { diff --git a/crates/email/src/message/metadata.rs b/crates/email/src/message/metadata.rs index 2817fd77..fe7995d8 100644 --- a/crates/email/src/message/metadata.rs +++ b/crates/email/src/message/metadata.rs @@ -4,10 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{borrow::Cow, fmt::Display}; +use std::{borrow::Cow, collections::VecDeque, fmt::Display}; use mail_parser::{ - DateTime, PartType, + PartType, decoders::{ base64::base64_decode, charsets::map::charset_decoder, quoted_printable::quoted_printable_decode, @@ -22,7 +22,7 @@ use utils::BlobHash; #[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug)] pub struct MessageMetadata { - pub contents: MessageMetadataContents, + pub contents: Vec, pub blob_hash: BlobHash, pub size: u32, pub received_at: u64, @@ -60,22 +60,12 @@ pub enum Encoding { } #[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug)] -#[rkyv(serialize_bounds( - __S: rkyv::ser::Writer + rkyv::ser::Allocator, - __S::Error: rkyv::rancor::Source, -))] -#[rkyv(deserialize_bounds(__D::Error: rkyv::rancor::Source))] -#[rkyv(bytecheck( - bounds( - __C: rkyv::validation::ArchiveContext, - ) -))] pub enum MetadataPartType { Text, Html, Binary, InlineBinary, - Message(#[rkyv(omit_bounds)] MessageMetadataContents), + Message(u16), Multipart(Vec), } @@ -156,12 +146,17 @@ pub enum HeaderValue { Address(Address), Text(String), TextList(Vec), - DateTime(i64), + DateTime(CompactDateTime), ContentType(ContentType), #[default] Empty, } +#[derive( + rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug, PartialEq, Eq, Clone, Default, +)] +pub struct CompactDateTime(pub u64); + #[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug, PartialEq, Eq, Clone)] pub enum Address { List(Vec), @@ -183,17 +178,20 @@ impl MessageMetadataContents { } } +#[derive(Debug)] pub struct DecodedParts<'x> { pub raw_messages: Vec>, pub parts: Vec>, } +#[derive(Debug)] pub struct DecodedPart<'x> { pub message_id: usize, - pub part_id: usize, + pub part_offset: usize, pub content: DecodedPartContent<'x>, } +#[derive(Debug)] pub enum DecodedPartContent<'x> { Text(Cow<'x, str>), Binary(Cow<'x, [u8]>), @@ -224,26 +222,26 @@ impl<'x> DecodedParts<'x> { } #[inline] - pub fn part(&self, message_id: usize, part_id: usize) -> Option<&DecodedPartContent<'x>> { + pub fn part(&self, message_id: usize, part_offset: usize) -> Option<&DecodedPartContent<'x>> { self.parts .iter() - .find(|p| p.message_id == message_id && p.part_id == part_id) + .find(|p| p.message_id == message_id && p.part_offset == part_offset) .map(|p| &p.content) } #[inline] - pub fn text_part(&self, message_id: usize, part_id: usize) -> Option<&str> { - self.part(message_id, part_id).and_then(|p| match p { + pub fn text_part(&self, message_id: usize, part_offset: usize) -> Option<&str> { + self.part(message_id, part_offset).and_then(|p| match p { DecodedPartContent::Text(text) => Some(text.as_ref()), DecodedPartContent::Binary(_) => None, }) } #[inline] - pub fn binary_part(&self, message_id: usize, part_id: usize) -> Option<&[u8]> { - self.part(message_id, part_id).and_then(|p| match p { - DecodedPartContent::Text(_) => None, - DecodedPartContent::Binary(binary) => Some(binary.as_ref()), + pub fn binary_part(&self, message_id: usize, part_offset: usize) -> Option<&[u8]> { + self.part(message_id, part_offset).map(|p| match p { + DecodedPartContent::Text(part) => part.as_bytes(), + DecodedPartContent::Binary(binary) => binary.as_ref(), }) } } @@ -272,35 +270,42 @@ impl DecodedPartContent<'_> { } } -impl ArchivedMessageMetadataContents { - pub fn decode_contents<'x>(&self, raw_message: &'x [u8]) -> DecodedParts<'x> { +impl ArchivedMessageMetadata { + #[inline(always)] + pub fn message_id(&self, message_id: u16_le) -> &ArchivedMessageMetadataContents { + &self.contents[u16::from(message_id) as usize] + } + + pub fn decode_contents<'x>(&self, raw: &'x [u8]) -> DecodedParts<'x> { let mut result = DecodedParts { - raw_messages: vec![raw_message.into()], + raw_messages: Vec::with_capacity(self.contents.len()), parts: Vec::new(), }; - let mut parts_iter = self.parts.iter().enumerate(); - let mut iter_stack = Vec::new(); - let mut message_id = 0; - loop { - while let Some((part_id, part)) = parts_iter.next() { + for _ in 0..self.contents.len() { + result.raw_messages.push(Cow::Borrowed(raw)); + } + + for (message_id, contents) in self.contents.iter().enumerate() { + for part in contents.parts.iter() { + let part_offset = u32::from(part.offset_header) as usize; match &part.body { ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html | ArchivedMetadataPartType::Binary | ArchivedMetadataPartType::InlineBinary => { - match result.raw_messages.last().unwrap() { + match result.raw_messages.get(message_id).unwrap() { Cow::Borrowed(raw_message) => { result.parts.push(DecodedPart { message_id, - part_id, + part_offset, content: part.decode_contents(raw_message), }); } Cow::Owned(raw_message) => { result.parts.push(DecodedPart { message_id, - part_id, + part_offset, content: match part.decode_contents(raw_message) { DecodedPartContent::Text(text) => { DecodedPartContent::Text(text.into_owned().into()) @@ -313,28 +318,23 @@ impl ArchivedMessageMetadataContents { } } } - ArchivedMetadataPartType::Message(message) => { - match part.contents(raw_message) { - Cow::Borrowed(raw_message) => { - result.raw_messages.push(raw_message.into()); - } - Cow::Owned(raw_message) => { - result.raw_messages.push(raw_message.into()); - } - } - iter_stack.push((parts_iter, message_id)); - message_id += 1; - parts_iter = message.parts.iter().enumerate(); + ArchivedMetadataPartType::Message(nested_message_id) => { + let sub_contents = if !matches!(part.encoding, ArchivedEncoding::None) { + part.contents(result.raw_messages.get(message_id).unwrap()) + .into_owned() + } else if let Some(Cow::Owned(raw_message)) = + result.raw_messages.get(message_id) + { + raw_message.clone() + } else { + continue; + }; + + result.raw_messages[usize::from(*nested_message_id)] = sub_contents.into(); } _ => {} } } - if let Some((iter, prev_message_id)) = iter_stack.pop() { - parts_iter = iter; - message_id = prev_message_id; - } else { - break; - } } result @@ -391,87 +391,94 @@ impl ArchivedMessageMetadataPart { } } -impl From> for MessageMetadataContents { - fn from(value: mail_parser::Message<'_>) -> Self { - MessageMetadataContents { - html_body: value.html_body.into_iter().map(|c| c as u16).collect(), - text_body: value.text_body.into_iter().map(|c| c as u16).collect(), - attachments: value.attachments.into_iter().map(|c| c as u16).collect(), - parts: value - .parts - .into_iter() - .map(|part| { - let (size, body) = match part.body { - PartType::Text(contents) => (contents.len(), MetadataPartType::Text), - PartType::Html(contents) => (contents.len(), MetadataPartType::Html), - PartType::Binary(contents) => (contents.len(), MetadataPartType::Binary), - PartType::InlineBinary(contents) => { - (contents.len(), MetadataPartType::InlineBinary) - } - PartType::Message(message) => ( - message.root_part().raw_len(), - MetadataPartType::Message(message.into()), - ), - PartType::Multipart(parts) => ( - 0, - MetadataPartType::Multipart( - parts.into_iter().map(|p| p as u16).collect(), - ), - ), - }; +impl MessageMetadata { + pub fn with_contents(mut self, message: mail_parser::Message<'_>) -> Self { + let mut messages = VecDeque::from([message]); + let mut message_id = 0; - MessageMetadataPart { - headers: part - .headers - .into_iter() - .map(|hdr| Header { - value: if matches!( - &hdr.name, - mail_parser::HeaderName::Subject - | mail_parser::HeaderName::From - | mail_parser::HeaderName::To - | mail_parser::HeaderName::Cc - | mail_parser::HeaderName::Date - | mail_parser::HeaderName::Bcc - | mail_parser::HeaderName::ReplyTo - | mail_parser::HeaderName::Sender - | mail_parser::HeaderName::Comments - | mail_parser::HeaderName::InReplyTo - | mail_parser::HeaderName::Keywords - | mail_parser::HeaderName::MessageId - | mail_parser::HeaderName::References - | mail_parser::HeaderName::ResentMessageId - | mail_parser::HeaderName::ContentDescription - | mail_parser::HeaderName::ContentId - | mail_parser::HeaderName::ContentLanguage - | mail_parser::HeaderName::ContentLocation - | mail_parser::HeaderName::ContentTransferEncoding - | mail_parser::HeaderName::ContentType - | mail_parser::HeaderName::ContentDisposition - | mail_parser::HeaderName::ListId - ) { - hdr.value - } else { - mail_parser::HeaderValue::Empty - } - .into(), - name: hdr.name.into(), - offset_field: hdr.offset_field as u32, - offset_start: hdr.offset_start as u32, - offset_end: hdr.offset_end as u32, - }) - .collect(), - is_encoding_problem: part.is_encoding_problem, - encoding: part.encoding.into(), - body, - size: size as u32, - offset_header: part.offset_header as u32, - offset_body: part.offset_body as u32, - offset_end: part.offset_end as u32, + while let Some(message) = messages.pop_front() { + let mut contents = MessageMetadataContents { + html_body: message.html_body.into_iter().map(|c| c as u16).collect(), + text_body: message.text_body.into_iter().map(|c| c as u16).collect(), + attachments: message.attachments.into_iter().map(|c| c as u16).collect(), + parts: Vec::with_capacity(message.parts.len()), + }; + + for part in message.parts { + let (size, body) = match part.body { + PartType::Text(contents) => (contents.len(), MetadataPartType::Text), + PartType::Html(contents) => (contents.len(), MetadataPartType::Html), + PartType::Binary(contents) => (contents.len(), MetadataPartType::Binary), + PartType::InlineBinary(contents) => { + (contents.len(), MetadataPartType::InlineBinary) } - }) - .collect(), + PartType::Message(message) => { + let message_len = message.root_part().raw_len(); + messages.push_back(message); + message_id += 1; + + (message_len, MetadataPartType::Message(message_id)) + } + PartType::Multipart(parts) => ( + 0, + MetadataPartType::Multipart(parts.into_iter().map(|p| p as u16).collect()), + ), + }; + + contents.parts.push(MessageMetadataPart { + headers: part + .headers + .into_iter() + .map(|hdr| Header { + value: if matches!( + &hdr.name, + mail_parser::HeaderName::Subject + | mail_parser::HeaderName::From + | mail_parser::HeaderName::To + | mail_parser::HeaderName::Cc + | mail_parser::HeaderName::Date + | mail_parser::HeaderName::Bcc + | mail_parser::HeaderName::ReplyTo + | mail_parser::HeaderName::Sender + | mail_parser::HeaderName::Comments + | mail_parser::HeaderName::InReplyTo + | mail_parser::HeaderName::Keywords + | mail_parser::HeaderName::MessageId + | mail_parser::HeaderName::References + | mail_parser::HeaderName::ResentMessageId + | mail_parser::HeaderName::ContentDescription + | mail_parser::HeaderName::ContentId + | mail_parser::HeaderName::ContentLanguage + | mail_parser::HeaderName::ContentLocation + | mail_parser::HeaderName::ContentTransferEncoding + | mail_parser::HeaderName::ContentType + | mail_parser::HeaderName::ContentDisposition + | mail_parser::HeaderName::ListId + ) { + hdr.value + } else { + mail_parser::HeaderValue::Empty + } + .into(), + name: hdr.name.into(), + offset_field: hdr.offset_field as u32, + offset_start: hdr.offset_start as u32, + offset_end: hdr.offset_end as u32, + }) + .collect(), + is_encoding_problem: part.is_encoding_problem, + encoding: part.encoding.into(), + body, + size: size as u32, + offset_header: part.offset_header as u32, + offset_body: part.offset_body as u32, + offset_end: part.offset_end as u32, + }); + } + self.contents.push(contents); } + + self } } @@ -546,7 +553,7 @@ impl From> for HeaderValue { HeaderValue::TextList(cows.into_iter().map(|cow| cow.into_owned()).collect()) } mail_parser::HeaderValue::DateTime(date_time) => { - HeaderValue::DateTime(date_time.to_timestamp()) + HeaderValue::DateTime(date_time.into()) } mail_parser::HeaderValue::ContentType(content_type) => { HeaderValue::ContentType(content_type.into()) @@ -644,7 +651,7 @@ impl ArchivedMessageMetadataPart { .and_then(|header| header.as_text()) } - pub fn date(&self) -> Option { + pub fn date(&self) -> Option { self.headers .header_value(&ArchivedHeaderName::Date) .and_then(|header| header.as_datetime()) @@ -806,9 +813,9 @@ impl ArchivedHeaderValue { } } - pub fn as_datetime(&self) -> Option { + pub fn as_datetime(&self) -> Option { match self { - ArchivedHeaderValue::DateTime(d) => Some(i64::from(d)), + ArchivedHeaderValue::DateTime(d) => Some(d.into()), _ => None, } } @@ -1087,9 +1094,7 @@ impl From<&ArchivedHeaderValue> for mail_parser::HeaderValue<'static> { ArchivedHeaderValue::TextList(list) => mail_parser::HeaderValue::TextList( list.iter().map(|s| s.to_string().into()).collect(), ), - ArchivedHeaderValue::DateTime(d) => { - mail_parser::HeaderValue::DateTime(DateTime::from_timestamp(i64::from(d))) - } + ArchivedHeaderValue::DateTime(d) => mail_parser::HeaderValue::DateTime(d.into()), ArchivedHeaderValue::ContentType(ct) => { mail_parser::HeaderValue::ContentType(ct.into()) } @@ -1164,3 +1169,54 @@ impl ArchivedEncoding { } } } + +impl From for CompactDateTime { + fn from(dt: mail_parser::DateTime) -> Self { + let mut value: u64 = 0; + value |= (dt.year as u64) << 48; + value |= (dt.month as u64) << 44; + value |= (dt.day as u64) << 39; + value |= (dt.hour as u64) << 34; + value |= (dt.minute as u64) << 28; + value |= (dt.second as u64) << 22; + value |= (if dt.tz_before_gmt { 1 } else { 0 }) << 21; + value |= (dt.tz_hour as u64) << 16; + value |= (dt.tz_minute as u64) << 10; + + CompactDateTime(value) + } +} + +impl From<&ArchivedCompactDateTime> for mail_parser::DateTime { + fn from(value: &ArchivedCompactDateTime) -> Self { + let value = u64::from(value.0); + mail_parser::DateTime { + year: (value >> 48) as u16, + month: ((value >> 44) & 0xF) as u8, + day: ((value >> 39) & 0x1F) as u8, + hour: ((value >> 34) & 0x1F) as u8, + minute: ((value >> 28) & 0x3F) as u8, + second: ((value >> 22) & 0x3F) as u8, + tz_before_gmt: ((value >> 21) & 0x1) == 1, + tz_hour: ((value >> 16) & 0x1F) as u8, + tz_minute: ((value >> 10) & 0x3F) as u8, + } + } +} + +impl From<&CompactDateTime> for mail_parser::DateTime { + fn from(value: &CompactDateTime) -> Self { + let value = value.0; + mail_parser::DateTime { + year: (value >> 48) as u16, + month: ((value >> 44) & 0xF) as u8, + day: ((value >> 39) & 0x1F) as u8, + hour: ((value >> 34) & 0x1F) as u8, + minute: ((value >> 28) & 0x3F) as u8, + second: ((value >> 22) & 0x3F) as u8, + tz_before_gmt: ((value >> 21) & 0x1) == 1, + tz_hour: ((value >> 16) & 0x1F) as u8, + tz_minute: ((value >> 10) & 0x3F) as u8, + } + } +} diff --git a/crates/email/src/sieve/activate.rs b/crates/email/src/sieve/activate.rs index 479f5a68..f13b27c9 100644 --- a/crates/email/src/sieve/activate.rs +++ b/crates/email/src/sieve/activate.rs @@ -13,7 +13,7 @@ use store::{ }; use trc::AddContext; -use super::{ArchivedSieveScript, SieveScript}; +use super::SieveScript; pub trait SieveScriptActivate: Sync + Send { fn sieve_activate_script( @@ -68,7 +68,7 @@ impl SieveScriptActivate for Server { .await? { let sieve = sieve - .into_deserialized::() + .into_deserialized::() .caused_by(trc::location!())?; let mut new_sieve = sieve.inner.clone(); new_sieve.is_active = false; @@ -97,7 +97,7 @@ impl SieveScriptActivate for Server { .await? { let sieve = sieve - .into_deserialized::() + .into_deserialized::() .caused_by(trc::location!())?; let mut new_sieve = sieve.inner.clone(); new_sieve.is_active = true; diff --git a/crates/email/src/sieve/delete.rs b/crates/email/src/sieve/delete.rs index 131ed8b0..9a07821e 100644 --- a/crates/email/src/sieve/delete.rs +++ b/crates/email/src/sieve/delete.rs @@ -9,7 +9,7 @@ use jmap_proto::types::{collection::Collection, property::Property}; use store::write::{Archive, BatchBuilder, BlobOp, assert::HashedValue}; use trc::AddContext; -use super::{ArchivedSieveScript, SieveScript}; +use super::SieveScript; pub trait SieveScriptDelete: Sync + Send { fn sieve_script_delete( @@ -43,7 +43,7 @@ impl SieveScriptDelete for Server { .caused_by(trc::location!()) .document_id(document_id) })? - .into_deserialized::() + .into_deserialized::() .caused_by(trc::location!())?; // Make sure the script is not active diff --git a/crates/email/src/sieve/index.rs b/crates/email/src/sieve/index.rs index 7156ae0d..40e42e73 100644 --- a/crates/email/src/sieve/index.rs +++ b/crates/email/src/sieve/index.rs @@ -14,7 +14,7 @@ impl IndexableObject for SieveScript { [ IndexValue::Text { field: Property::Name.into(), - value: self.name.to_lowercase().into(), + value: self.name.as_bytes().into(), }, IndexValue::U32 { field: Property::IsActive.into(), diff --git a/crates/email/src/sieve/ingest.rs b/crates/email/src/sieve/ingest.rs index 7af3d64e..0fd08b73 100644 --- a/crates/email/src/sieve/ingest.rs +++ b/crates/email/src/sieve/ingest.rs @@ -31,7 +31,7 @@ use utils::config::utils::ParseValue; use std::future::Future; -use super::{ActiveScript, ArchivedSeenIds, ArchivedSieveScript, SeenIdHash, SeenIds}; +use super::{ActiveScript, SeenIdHash, SeenIds, SieveScript}; struct SieveMessage<'x> { pub raw_message: Cow<'x, [u8]>, @@ -149,7 +149,7 @@ impl SieveScriptIngest for Server { }; let mut seen_ids = if let Some(seen_ids) = active_script.seen_ids { seen_ids - .deserialize::() + .deserialize::() .caused_by(trc::location!())? } else { SeenIds::default() @@ -648,7 +648,7 @@ impl SieveScriptIngest for Server { // Obtain the sieve script length let unarchived_script = script_object .inner - .unarchive::() + .unarchive::() .caused_by(trc::location!())?; let script_offset = u32::from(unarchived_script.size) as usize; diff --git a/crates/email/src/sieve/mod.rs b/crates/email/src/sieve/mod.rs index 8a26328d..6261d432 100644 --- a/crates/email/src/sieve/mod.rs +++ b/crates/email/src/sieve/mod.rs @@ -16,7 +16,7 @@ pub mod index; pub mod ingest; pub mod serialize; -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone)] pub struct ActiveScript { pub document_id: u32, pub script_name: String, diff --git a/crates/email/src/submission/index.rs b/crates/email/src/submission/index.rs index df356427..b60ea1f3 100644 --- a/crates/email/src/submission/index.rs +++ b/crates/email/src/submission/index.rs @@ -14,7 +14,7 @@ impl IndexableObject for EmailSubmission { [ IndexValue::Text { field: Property::UndoStatus.into(), - value: self.undo_status.as_index().into(), + value: self.undo_status.as_index().as_bytes().into(), }, IndexValue::U32 { field: Property::EmailId.into(), diff --git a/crates/imap/src/core/mailbox.rs b/crates/imap/src/core/mailbox.rs index 372edae8..33e72ded 100644 --- a/crates/imap/src/core/mailbox.rs +++ b/crates/imap/src/core/mailbox.rs @@ -12,7 +12,7 @@ use common::{ sharing::EffectiveAcl, }; use directory::{QueryBy, backend::internal::PrincipalField}; -use email::mailbox::{ArchivedMailbox, INBOX_ID, manage::MailboxFnc}; +use email::mailbox::{INBOX_ID, manage::MailboxFnc}; use imap_proto::protocol::list::Attribute; use jmap_proto::types::{acl::Acl, collection::Collection, id::Id, property::Property}; use parking_lot::Mutex; @@ -165,7 +165,7 @@ impl SessionData { .caused_by(trc::location!())? { let mailbox = mailbox_ - .unarchive::() + .unarchive::() .caused_by(trc::location!())?; // Map special uses let role = SpecialUse::from(&mailbox.role); @@ -631,7 +631,7 @@ impl SessionData { if let Some(mailbox) = mailbox { Ok(Some( mailbox - .unarchive::()? + .unarchive::()? .acls .effective_acl(&access_token) .contains(item), diff --git a/crates/imap/src/core/message.rs b/crates/imap/src/core/message.rs index 0b3f7d1f..68532935 100644 --- a/crates/imap/src/core/message.rs +++ b/crates/imap/src/core/message.rs @@ -8,10 +8,10 @@ use std::{collections::BTreeMap, sync::Arc}; use ahash::AHashMap; use common::{NextMailboxState, listener::SessionStream}; -use email::mailbox::{ArchivedMailbox, ArchivedUidMailbox}; +use email::mailbox::UidMailbox; use imap_proto::protocol::{Sequence, expunge, select::Exists}; use jmap_proto::types::{collection::Collection, property::Property}; -use store::{rkyv::vec::ArchivedVec, write::Archive}; +use store::write::Archive; use trc::AddContext; use crate::core::ImapId; @@ -62,7 +62,7 @@ impl SessionData { .into_iter() { let uid_mailbox = uid_mailbox_ - .unarchive::>() + .unarchive::>() .caused_by(trc::location!())?; // Make sure the message is still in this mailbox if let Some(item) = uid_mailbox @@ -243,7 +243,7 @@ impl SessionData { .document_id(mailbox.mailbox_id) }) .and_then(|m| { - m.unarchive::() + m.unarchive::() .map(|m| u32::from(m.uid_validity)) }) } diff --git a/crates/imap/src/op/acl.rs b/crates/imap/src/op/acl.rs index 4ca8cb59..18cc36fc 100644 --- a/crates/imap/src/op/acl.rs +++ b/crates/imap/src/op/acl.rs @@ -14,7 +14,6 @@ use directory::{ Permission, QueryBy, Type, backend::internal::{PrincipalField, manage::ChangedPrincipals}, }; -use email::mailbox::ArchivedMailbox; use imap_proto::{ Command, ResponseCode, StatusResponse, protocol::acl::{ @@ -54,7 +53,7 @@ impl Session { .imap_ctx(&arguments.tag, trc::location!())?; let mut permissions = Vec::new(); let mailbox = mailbox_ - .to_unarchived::() + .to_unarchived::() .imap_ctx(&arguments.tag, trc::location!())?; for item in mailbox.inner.acls.iter() { @@ -152,7 +151,7 @@ impl Session { .await .imap_ctx(&arguments.tag, trc::location!())?; let mailbox = mailbox_ - .to_unarchived::() + .to_unarchived::() .imap_ctx(&arguments.tag, trc::location!())?; let rights = if access_token.is_shared(mailbox_id.account_id) { let acl = mailbox.inner.acls.effective_acl(&access_token); @@ -241,7 +240,7 @@ impl Session { .await .imap_ctx(&arguments.tag, trc::location!())?; let current_mailbox = current_mailbox - .into_deserialized::() + .into_deserialized::() .imap_ctx(&arguments.tag, trc::location!())?; // Obtain principal id @@ -454,7 +453,7 @@ impl SessionData { || access_token.is_member(mailbox.account_id) || values .inner - .unarchive::() + .unarchive::() .caused_by(trc::location!())? .acls .effective_acl(&access_token) diff --git a/crates/imap/src/op/copy_move.rs b/crates/imap/src/op/copy_move.rs index d68d53fd..ad271629 100644 --- a/crates/imap/src/op/copy_move.rs +++ b/crates/imap/src/op/copy_move.rs @@ -8,7 +8,7 @@ use std::{sync::Arc, time::Instant}; use directory::Permission; use email::{ - mailbox::{ArchivedUidMailbox, JUNK_ID, UidMailbox}, + mailbox::{JUNK_ID, UidMailbox}, message::{bayes::EmailBayesTrain, copy::EmailCopy, ingest::EmailIngest}, }; use imap_proto::{ @@ -31,7 +31,6 @@ use jmap_proto::{ }; use store::{ SerializeInfallible, - rkyv::vec::ArchivedVec, roaring::RoaringBitmap, write::{Archive, BatchBuilder, ValueClass, assert::HashedValue, log::ChangeLogBuilder}, }; @@ -489,7 +488,7 @@ impl SessionData { Ok(Some(( TagManager::new( mailboxes - .into_deserialized::, Vec>() + .into_deserialized::>() .caused_by(trc::location!())?, ), thread_id, diff --git a/crates/imap/src/op/expunge.rs b/crates/imap/src/op/expunge.rs index 7ca4575b..f4062f23 100644 --- a/crates/imap/src/op/expunge.rs +++ b/crates/imap/src/op/expunge.rs @@ -8,10 +8,7 @@ use std::{sync::Arc, time::Instant}; use ahash::AHashMap; use directory::Permission; -use email::{ - mailbox::{ArchivedUidMailbox, UidMailbox}, - message::delete::EmailDeletion, -}; +use email::{mailbox::UidMailbox, message::delete::EmailDeletion}; use imap_proto::{ Command, ResponseCode, ResponseType, StatusResponse, parser::parse_sequence_set, @@ -22,17 +19,11 @@ use trc::AddContext; use crate::core::{SavedSearch, SelectedMailbox, Session, SessionData}; use common::{ImapId, listener::SessionStream, storage::tag::TagManager}; use jmap_proto::types::{ - acl::Acl, - collection::Collection, - id::Id, - keyword::{ArchivedKeyword, Keyword}, - property::Property, - state::StateChange, - type_state::DataType, + acl::Acl, collection::Collection, id::Id, keyword::Keyword, property::Property, + state::StateChange, type_state::DataType, }; use store::{ SerializeInfallible, - rkyv::vec::ArchivedVec, roaring::RoaringBitmap, write::{Archive, BatchBuilder, assert::HashedValue, log::ChangeLogBuilder}, }; @@ -214,7 +205,7 @@ impl SessionData { { let mut mailboxes = TagManager::new( mailbox_ids - .into_deserialized::, Vec>() + .into_deserialized::>() .caused_by(trc::location!())?, ); @@ -244,7 +235,7 @@ impl SessionData { ( TagManager::new( keywords - .into_deserialized::, Vec>() + .into_deserialized::>() .caused_by(trc::location!())?, ), thread_id, diff --git a/crates/imap/src/op/fetch.rs b/crates/imap/src/op/fetch.rs index 1f043431..fd4eb3b6 100644 --- a/crates/imap/src/op/fetch.rs +++ b/crates/imap/src/op/fetch.rs @@ -16,7 +16,7 @@ use directory::Permission; use email::message::metadata::{ ArchivedAddress, ArchivedGetHeader, ArchivedHeaderName, ArchivedHeaderValue, ArchivedMessageMetadata, ArchivedMessageMetadataContents, ArchivedMetadataPartType, - DecodedParts, + DecodedParts, MessageMetadata, }; use imap_proto::{ Command, ResponseCode, ResponseType, StatusResponse, @@ -40,11 +40,10 @@ use jmap_proto::types::{ state::StateChange, type_state::DataType, }; -use mail_parser::DateTime; use store::{ Serialize, SerializeInfallible, query::log::{Change, Query}, - rkyv::{rend::u16_le, vec::ArchivedVec}, + rkyv::rend::u16_le, write::{Archive, Archiver, BatchBuilder, assert::HashedValue, serialize::rkyv_deserialize}, }; @@ -309,7 +308,7 @@ impl SessionData { for (seqnum, uid, id) in ids { // Obtain attributes and keywords - let (email_, keywords_) = if let (Some(email), Some(keywords)) = ( + let (metadata_, keywords_) = if let (Some(email), Some(keywords)) = ( self.server .get_property::( account_id, @@ -341,12 +340,12 @@ impl SessionData { ); continue; }; - let email = email_ - .unarchive::() + let metadata = metadata_ + .unarchive::() .imap_ctx(&arguments.tag, trc::location!())?; let keywords = keywords_ .inner - .unarchive::>() + .unarchive::>() .imap_ctx(&arguments.tag, trc::location!())?; // Fetch and parse blob @@ -355,7 +354,7 @@ impl SessionData { match self .server .blob_store() - .get_blob(email.blob_hash.0.as_slice(), 0..usize::MAX) + .get_blob(metadata.blob_hash.0.as_slice(), 0..usize::MAX) .await .imap_ctx(&arguments.tag, trc::location!())? { @@ -366,7 +365,7 @@ impl SessionData { AccountId = account_id, DocumentId = id, Collection = Collection::Email, - BlobId = email.blob_hash.0.as_slice(), + BlobId = metadata.blob_hash.0.as_slice(), Details = "Blob not found.", CausedBy = trc::location!(), ); @@ -375,10 +374,10 @@ impl SessionData { } } } else { - email.raw_headers.as_slice().into() + metadata.raw_headers.as_slice().into() }; - let message = &email.contents; - let decoded = message.decode_contents(raw_message.as_ref()); + let message = &metadata.contents[0]; + let decoded = metadata.decode_contents(raw_message.as_ref()); // Build response let mut items = Vec::with_capacity(arguments.attributes.len()); @@ -414,13 +413,13 @@ impl SessionData { } Attribute::InternalDate => { items.push(DataItem::InternalDate { - date: u64::from(email.received_at) as i64, + date: u64::from(metadata.received_at) as i64, }); } Attribute::Preview { .. } => { items.push(DataItem::Preview { - contents: if !email.preview.is_empty() { - Some(email.preview.as_bytes().into()) + contents: if !metadata.preview.is_empty() { + Some(metadata.preview.as_bytes().into()) } else { None }, @@ -428,7 +427,7 @@ impl SessionData { } Attribute::Rfc822Size => { items.push(DataItem::Rfc822Size { - size: u32::from(email.size) as usize, + size: u32::from(metadata.size) as usize, }); } Attribute::Uid => { @@ -440,7 +439,7 @@ impl SessionData { }); } Attribute::Rfc822Header => { - let message = email.root_part(); + let message = metadata.root_part(); if let Some(header) = raw_message.get( u32::from(message.offset_header) as usize ..u32::from(message.offset_body) as usize, @@ -457,18 +456,19 @@ impl SessionData { } Attribute::Body => { items.push(DataItem::Body { - part: message.body_structure(&decoded, false), + part: metadata.body_structure(&decoded, false), }); } Attribute::BodyStructure => { items.push(DataItem::BodyStructure { - part: message.body_structure(&decoded, true), + part: metadata.body_structure(&decoded, true), }); } Attribute::BodySection { sections, partial, .. } => { - if let Some(contents) = message.body_section(&decoded, sections, *partial) { + if let Some(contents) = metadata.body_section(&decoded, sections, *partial) + { items.push(DataItem::BodySection { sections: sections.to_vec(), origin_octet: partial.map(|(start, _)| start), @@ -479,7 +479,7 @@ impl SessionData { Attribute::Binary { sections, partial, .. - } => match message.binary(&decoded, sections, *partial) { + } => match metadata.binary(&decoded, sections, *partial) { Ok(Some(contents)) => { items.push(DataItem::Binary { sections: sections.to_vec(), @@ -508,7 +508,7 @@ impl SessionData { _ => (), }, Attribute::BinarySize { sections } => { - if let Some(size) = message.binary_size(&decoded, sections) { + if let Some(size) = metadata.binary_size(&decoded, sections) { items.push(DataItem::BinarySize { sections: sections.to_vec(), size, @@ -661,6 +661,10 @@ pub trait AsImapDataItem { partial: Option<(u32, u32)>, ) -> Result>, ()>; fn binary_size(&self, decoded: &DecodedParts<'_>, sections: &[u32]) -> Option; +} + +#[allow(clippy::result_unit_err)] +pub trait AsImapDataItemPart { fn as_body_part( &self, decoded: &DecodedParts<'_>, @@ -668,68 +672,11 @@ pub trait AsImapDataItem { part_id: usize, is_extended: bool, ) -> BodyPart; + fn envelope(&self) -> Envelope; } -impl AsImapDataItem for ArchivedMessageMetadataContents { - fn body_structure(&self, decoded: &DecodedParts<'_>, is_extended: bool) -> BodyPart { - let mut stack = Vec::new(); - let base_part = [u16_le::from_native(0)]; - let mut parts = base_part.as_slice().iter(); - let mut message = self; - let mut root_part = None; - let mut message_id = 0; - - loop { - while let Some(part_id) = parts.next() { - let part_id = u16::from(part_id) as usize; - let mut part = message.as_body_part(decoded, message_id, part_id, is_extended); - - match &message.parts[part_id].body { - ArchivedMetadataPartType::Message(nested_message) => { - part.set_envelope(nested_message.envelope()); - if let Some(root_part) = root_part { - stack.push((root_part, parts, (message, message_id).into())); - message_id += 1; - } - root_part = part.into(); - parts = base_part.as_slice().iter(); - message = nested_message; - continue; - } - ArchivedMetadataPartType::Multipart(subparts) => { - if let Some(root_part) = root_part { - stack.push((root_part, parts, None)); - } - root_part = part.into(); - parts = subparts.iter(); - continue; - } - _ => (), - } - if let Some(root_part) = &mut root_part { - root_part.add_part(part); - } else { - return part; - } - } - if let Some((mut prev_root_part, prev_parts, prev_message)) = stack.pop() { - if let Some((prev_message, prev_message_id)) = prev_message { - message = prev_message; - message_id = prev_message_id; - } - - prev_root_part.add_part(root_part.unwrap()); - parts = prev_parts; - root_part = prev_root_part.into(); - } else { - break; - } - } - - root_part.unwrap() - } - +impl AsImapDataItemPart for ArchivedMessageMetadataContents { fn as_body_part( &self, decoded: &DecodedParts<'_>, @@ -880,6 +827,112 @@ impl AsImapDataItem for ArchivedMessageMetadataContents { } } + fn envelope(&self) -> Envelope { + let headers = self.root_part(); + Envelope { + date: headers.date(), + subject: headers.subject().map(|s| s.into()), + from: headers + .header_values(ArchivedHeaderName::From) + .flat_map(|a| a.as_imap_address()) + .collect(), + sender: headers + .header_values(ArchivedHeaderName::Sender) + .flat_map(|a| a.as_imap_address()) + .collect(), + reply_to: headers + .header_values(ArchivedHeaderName::ReplyTo) + .flat_map(|a| a.as_imap_address()) + .collect(), + to: headers + .header_values(ArchivedHeaderName::To) + .flat_map(|a| a.as_imap_address()) + .collect(), + cc: headers + .header_values(ArchivedHeaderName::Cc) + .flat_map(|a| a.as_imap_address()) + .collect(), + bcc: headers + .header_values(ArchivedHeaderName::Bcc) + .flat_map(|a| a.as_imap_address()) + .collect(), + in_reply_to: headers.in_reply_to().as_text_list().map(|list| { + let mut irt = String::with_capacity(list.len() * 10); + for (pos, l) in list.iter().enumerate() { + if pos > 0 { + irt.push(' '); + } + irt.push('<'); + irt.push_str(l.as_ref()); + irt.push('>'); + } + irt.into() + }), + message_id: headers.message_id().map(|id| format!("<{}>", id).into()), + } + } +} + +impl AsImapDataItem for ArchivedMessageMetadata { + fn body_structure(&self, decoded: &DecodedParts<'_>, is_extended: bool) -> BodyPart { + let mut stack = Vec::new(); + let base_part = [u16_le::from_native(0)]; + let mut parts = base_part.as_slice().iter(); + let mut message = &self.contents[0]; + let mut root_part = None; + let mut message_id = 0; + + loop { + while let Some(part_id) = parts.next() { + let part_id = u16::from(part_id) as usize; + let mut part = message.as_body_part(decoded, message_id, part_id, is_extended); + + match &message.parts[part_id].body { + ArchivedMetadataPartType::Message(nested_message_id) => { + let nested_message = self.message_id(*nested_message_id); + part.set_envelope(nested_message.envelope()); + if let Some(root_part) = root_part { + stack.push((root_part, parts, (message, message_id).into())); + } + root_part = part.into(); + parts = base_part.as_slice().iter(); + message = nested_message; + message_id = u16::from(*nested_message_id) as usize; + continue; + } + ArchivedMetadataPartType::Multipart(subparts) => { + if let Some(root_part) = root_part { + stack.push((root_part, parts, None)); + } + root_part = part.into(); + parts = subparts.iter(); + continue; + } + _ => (), + } + if let Some(root_part) = &mut root_part { + root_part.add_part(part); + } else { + return part; + } + } + if let Some((mut prev_root_part, prev_parts, prev_message)) = stack.pop() { + if let Some((prev_message, prev_message_id)) = prev_message { + message = prev_message; + message_id = prev_message_id; + } + + prev_root_part.add_part(root_part.unwrap()); + parts = prev_parts; + root_part = prev_root_part.into(); + } else { + break; + } + } + + root_part.unwrap() + } + fn body_section<'x>( &self, decoded: &'x DecodedParts<'x>, @@ -897,7 +950,7 @@ impl AsImapDataItem for ArchivedMessageMetadataContents { ); } - let mut message = self; + let mut message = &self.contents[0]; let mut message_id = 0; let mut sections_iter = sections.iter().enumerate().peekable(); @@ -915,20 +968,19 @@ impl AsImapDataItem for ArchivedMessageMetadataContents { None }?; - if let ( - ArchivedMetadataPartType::Message(nested_message), - Some(( + if let ArchivedMetadataPartType::Message(nested_message_id) = &part.body { + if let Some(( _, Section::Part { .. } | Section::Header | Section::HeaderFields { .. } | Section::Text, - )), - ) = (&part.body, sections_iter.peek()) - { - message = nested_message; - part = message.root_part(); - message_id += 1; + )) = sections_iter.peek() + { + message = self.message_id(*nested_message_id); + part = message.root_part(); + message_id = u16::from(nested_message_id) as usize; + } } } Section::Header => { @@ -1037,17 +1089,15 @@ impl AsImapDataItem for ArchivedMessageMetadataContents { sections: &[u32], partial: Option<(u32, u32)>, ) -> Result>, ()> { - let mut message = self; + let mut message = &self.contents[0]; let mut message_id = 0; let mut part = self.root_part(); - let mut part_id = 0; let mut sections_iter = sections.iter().enumerate().peekable(); while let Some((section_num, num)) = sections_iter.next() { part = if let Some(sub_part_ids) = part.sub_parts() { - part_id = (*num).saturating_sub(1) as usize; if let Some(part) = sub_part_ids - .get(part_id) + .get((*num).saturating_sub(1) as usize) .and_then(|pos| message.parts.get(u16::from(*pos) as usize)) { part @@ -1055,7 +1105,6 @@ impl AsImapDataItem for ArchivedMessageMetadataContents { return Ok(None); } } else if *num == 1 && (section_num == sections.len() - 1 || part.is_message()) { - part_id = 0; part } else { return Ok(None); @@ -1064,17 +1113,20 @@ impl AsImapDataItem for ArchivedMessageMetadataContents { if let (ArchivedMetadataPartType::Message(nested_message), Some(_)) = (&part.body, sections_iter.peek()) { - message = nested_message; + message = self.message_id(*nested_message); part = message.root_part(); - message_id += 1; + message_id = u16::from(nested_message) as usize; } } if !part.is_encoding_problem { + let part_offset = u32::from(part.offset_header) as usize; Ok(match &part.body { ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html => { BodyContents::Text(String::from_utf8_lossy(get_partial_bytes( - decoded.binary_part(message_id, part_id).unwrap_or_default(), + decoded + .binary_part(message_id, part_offset) + .unwrap_or_default(), partial, ))) .into() @@ -1082,7 +1134,9 @@ impl AsImapDataItem for ArchivedMessageMetadataContents { ArchivedMetadataPartType::Binary | ArchivedMetadataPartType::InlineBinary => { BodyContents::Bytes( get_partial_bytes( - decoded.binary_part(message_id, part_id).unwrap_or_default(), + decoded + .binary_part(message_id, part_offset) + .unwrap_or_default(), partial, ) .into(), @@ -1091,7 +1145,7 @@ impl AsImapDataItem for ArchivedMessageMetadataContents { } ArchivedMetadataPartType::Message(message) => BodyContents::Bytes({ { - let part = message.root_part(); + let part = self.message_id(*message).root_part(); get_partial_bytes( decoded .raw_message_section_arch( @@ -1127,17 +1181,15 @@ impl AsImapDataItem for ArchivedMessageMetadataContents { } fn binary_size(&self, decoded: &DecodedParts<'_>, sections: &[u32]) -> Option { - let mut message = self; + let mut message = &self.contents[0]; let mut message_id = 0; let mut part = self.root_part(); - let mut part_id = 0; let mut sections_iter = sections.iter().enumerate().peekable(); while let Some((section_num, num)) = sections_iter.next() { - part_id = (*num).saturating_sub(1) as usize; part = if let Some(sub_part_ids) = part.sub_parts() { sub_part_ids - .get(part_id) + .get((*num).saturating_sub(1) as usize) .and_then(|pos| message.parts.get(u16::from(pos) as usize)) } else if *num == 1 && (section_num == sections.len() - 1 || part.is_message()) { Some(part) @@ -1148,10 +1200,9 @@ impl AsImapDataItem for ArchivedMessageMetadataContents { if let (ArchivedMetadataPartType::Message(nested_message), Some(_)) = (&part.body, sections_iter.peek()) { - message = nested_message; - message_id += 1; + message = self.message_id(*nested_message); + message_id = u16::from(nested_message) as usize; part = message.root_part(); - part_id = 0; } } @@ -1160,59 +1211,16 @@ impl AsImapDataItem for ArchivedMessageMetadataContents { | ArchivedMetadataPartType::Html | ArchivedMetadataPartType::Binary | ArchivedMetadataPartType::InlineBinary => decoded - .part(message_id, part_id) + .part(message_id, u32::from(part.offset_header) as usize) .map(|p| p.len()) .unwrap_or_default(), - ArchivedMetadataPartType::Message(message) => message.root_part().raw_len(), + ArchivedMetadataPartType::Message(message) => { + self.message_id(*message).root_part().raw_len() + } ArchivedMetadataPartType::Multipart(_) => part.raw_len(), } .into() } - - fn envelope(&self) -> Envelope { - let headers = self.root_part(); - Envelope { - date: headers.date().map(DateTime::from_timestamp), - subject: headers.subject().map(|s| s.into()), - from: headers - .header_values(ArchivedHeaderName::From) - .flat_map(|a| a.as_imap_address()) - .collect(), - sender: headers - .header_values(ArchivedHeaderName::Sender) - .flat_map(|a| a.as_imap_address()) - .collect(), - reply_to: headers - .header_values(ArchivedHeaderName::ReplyTo) - .flat_map(|a| a.as_imap_address()) - .collect(), - to: headers - .header_values(ArchivedHeaderName::To) - .flat_map(|a| a.as_imap_address()) - .collect(), - cc: headers - .header_values(ArchivedHeaderName::Cc) - .flat_map(|a| a.as_imap_address()) - .collect(), - bcc: headers - .header_values(ArchivedHeaderName::Bcc) - .flat_map(|a| a.as_imap_address()) - .collect(), - in_reply_to: headers.in_reply_to().as_text_list().map(|list| { - let mut irt = String::with_capacity(list.len() * 10); - for (pos, l) in list.iter().enumerate() { - if pos > 0 { - irt.push(' '); - } - irt.push('<'); - irt.push_str(l.as_ref()); - irt.push('>'); - } - irt.into() - }), - message_id: headers.message_id().map(|id| format!("<{}>", id).into()), - } - } } #[inline(always)] diff --git a/crates/imap/src/op/rename.rs b/crates/imap/src/op/rename.rs index 2b7bb922..a9213b75 100644 --- a/crates/imap/src/op/rename.rs +++ b/crates/imap/src/op/rename.rs @@ -12,7 +12,6 @@ use crate::{ }; use common::{listener::SessionStream, sharing::EffectiveAcl, storage::index::ObjectIndexBuilder}; use directory::Permission; -use email::mailbox::ArchivedMailbox; use imap_proto::{ Command, ResponseCode, StatusResponse, protocol::rename::Arguments, receiver::Request, }; @@ -105,7 +104,7 @@ impl SessionData { .code(ResponseCode::NonExistent) .id(arguments.tag.clone()) })? - .into_deserialized::() + .into_deserialized::() .imap_ctx(&arguments.tag, trc::location!())?; // Validate ACL diff --git a/crates/imap/src/op/status.rs b/crates/imap/src/op/status.rs index eb44ead2..d122fedb 100644 --- a/crates/imap/src/op/status.rs +++ b/crates/imap/src/op/status.rs @@ -13,7 +13,6 @@ use crate::{ }; use common::{Mailbox, listener::SessionStream}; use directory::Permission; -use email::mailbox::ArchivedMailbox; use imap_proto::{ Command, ResponseCode, StatusResponse, parser::PushUnique, @@ -271,7 +270,7 @@ impl SessionData { .account_id(mailbox.account_id) .document_id(mailbox.mailbox_id) })? - .unarchive::() + .unarchive::() .caused_by(trc::location!())? .uid_validity, ) as u64, diff --git a/crates/imap/src/op/store.rs b/crates/imap/src/op/store.rs index 2d402c37..ed990e01 100644 --- a/crates/imap/src/op/store.rs +++ b/crates/imap/src/op/store.rs @@ -14,7 +14,7 @@ use ahash::AHashSet; use common::{listener::SessionStream, storage::tag::TagManager}; use directory::Permission; use email::{ - mailbox::ArchivedUidMailbox, + mailbox::UidMailbox, message::{bayes::EmailBayesTrain, ingest::EmailIngest}, }; use imap_proto::{ @@ -27,18 +27,12 @@ use imap_proto::{ receiver::Request, }; use jmap_proto::types::{ - acl::Acl, - collection::Collection, - id::Id, - keyword::{ArchivedKeyword, Keyword}, - property::Property, - state::StateChange, - type_state::DataType, + acl::Acl, collection::Collection, id::Id, keyword::Keyword, property::Property, + state::StateChange, type_state::DataType, }; use store::{ SerializeInfallible, query::log::{Change, Query}, - rkyv::vec::ArchivedVec, write::{Archive, BatchBuilder, ValueClass, assert::HashedValue, log::ChangeLogBuilder}, }; use trc::AddContext; @@ -229,7 +223,7 @@ impl SessionData { ( TagManager::new( keywords - .into_deserialized::, Vec>() + .into_deserialized::>() .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())?, ), thread_id, @@ -346,7 +340,7 @@ impl SessionData { .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())? { for mailbox_id in mailboxes - .unarchive::>() + .unarchive::>() .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())? .iter() { diff --git a/crates/imap/src/op/subscribe.rs b/crates/imap/src/op/subscribe.rs index e14aa0e8..caa5c62f 100644 --- a/crates/imap/src/op/subscribe.rs +++ b/crates/imap/src/op/subscribe.rs @@ -12,7 +12,6 @@ use crate::{ }; use common::{listener::SessionStream, storage::index::ObjectIndexBuilder}; use directory::Permission; -use email::mailbox::ArchivedMailbox; use imap_proto::{Command, ResponseCode, StatusResponse, receiver::Request}; use jmap_proto::types::{ collection::Collection, property::Property, state::StateChange, type_state::DataType, @@ -113,7 +112,7 @@ impl SessionData { .id(tag.clone()) .caused_by(trc::location!()) })? - .into_deserialized::() + .into_deserialized::() .imap_ctx(&tag, trc::location!())?; if (subscribe && !mailbox.inner.is_subscribed(self.account_id)) diff --git a/crates/jmap/src/api/management/stores.rs b/crates/jmap/src/api/management/stores.rs index 47bc9a1f..9d97e620 100644 --- a/crates/jmap/src/api/management/stores.rs +++ b/crates/jmap/src/api/management/stores.rs @@ -16,16 +16,12 @@ use directory::{ Permission, backend::internal::manage::{self, ManageDirectory}, }; -use email::{ - mailbox::{ArchivedMailbox, ArchivedUidMailbox, UidMailbox}, - message::ingest::EmailIngest, -}; +use email::{mailbox::UidMailbox, message::ingest::EmailIngest}; use hyper::Method; use jmap_proto::types::{collection::Collection, property::Property}; use serde_json::json; use store::{ Serialize, - rkyv::vec::ArchivedVec, write::{Archive, Archiver, BatchBuilder, ValueClass, assert::HashedValue}, }; use trc::AddContext; @@ -351,7 +347,7 @@ pub async fn reset_imap_uids(server: &Server, account_id: u32) -> trc::Result<(u .await .caused_by(trc::location!())? .ok_or_else(|| trc::ImapEvent::Error.into_err().caused_by(trc::location!()))? - .into_deserialized::() + .into_deserialized::() .caused_by(trc::location!())?; let mut new_mailbox = mailbox.inner.clone(); new_mailbox.uid_validity = rand::random::(); @@ -392,7 +388,7 @@ pub async fn reset_imap_uids(server: &Server, account_id: u32) -> trc::Result<(u .await .caused_by(trc::location!())?; let mut uids = if let Some(uids) = uids { - uids.into_deserialized::, Vec>() + uids.into_deserialized::>() .caused_by(trc::location!())? } else { continue; diff --git a/crates/jmap/src/blob/get.rs b/crates/jmap/src/blob/get.rs index 8289e55b..42400569 100644 --- a/crates/jmap/src/blob/get.rs +++ b/crates/jmap/src/blob/get.rs @@ -5,7 +5,7 @@ */ use common::{Server, auth::AccessToken}; -use email::mailbox::ArchivedUidMailbox; +use email::mailbox::UidMailbox; use jmap_proto::{ method::{ get::{GetRequest, GetResponse}, @@ -24,7 +24,7 @@ use jmap_proto::{ use mail_builder::encoders::base64::base64_encode; use sha1::{Digest, Sha1}; use sha2::{Sha256, Sha512}; -use store::{BlobClass, rkyv::vec::ArchivedVec, write::Archive}; +use store::{BlobClass, write::Archive}; use trc::AddContext; use utils::map::vec_map::VecMap; @@ -250,7 +250,7 @@ impl BlobOperations for Server { matched_ids.append( DataType::Mailbox, mailboxes - .unarchive::>() + .unarchive::>() .caused_by(trc::location!())? .iter() .map(|m| { diff --git a/crates/jmap/src/email/crypto.rs b/crates/jmap/src/email/crypto.rs index 9656ff8d..1c592f5a 100644 --- a/crates/jmap/src/email/crypto.rs +++ b/crates/jmap/src/email/crypto.rs @@ -10,9 +10,8 @@ use crate::api::{HttpResponse, JsonResponse, http::ToHttpResponse}; use common::{Server, auth::AccessToken}; use directory::backend::internal::manage; use email::message::crypto::{ - Algorithm, ArchivedAlgorithm, ArchivedEncryptionMethod, ArchivedEncryptionParams, - EncryptMessage, EncryptMessageError, EncryptionMethod, EncryptionParams, EncryptionType, - try_parse_certs, + Algorithm, ArchivedAlgorithm, ArchivedEncryptionMethod, EncryptMessage, EncryptMessageError, + EncryptionMethod, EncryptionParams, EncryptionType, try_parse_certs, }; use jmap_proto::types::{collection::Collection, property::Property}; use mail_builder::encoders::base64::base64_encode_mime; @@ -49,7 +48,7 @@ impl CryptoHandler for Server { .await? { let params = params_ - .unarchive::() + .unarchive::() .caused_by(trc::location!())?; let algo = match ¶ms.algo { ArchivedAlgorithm::Aes128 => Algorithm::Aes128, @@ -132,7 +131,7 @@ impl CryptoHandler for Server { if let Err(EncryptMessageError::Error(message)) = MessageParser::new() .parse("Subject: test\r\ntest\r\n".as_bytes()) .unwrap() - .encrypt(rkyv_unarchive(¶ms)?) + .encrypt(rkyv_unarchive::(¶ms)?) .await { return Err(manage::error(message, None::)); diff --git a/crates/jmap/src/email/get.rs b/crates/jmap/src/email/get.rs index f5b22425..8b973d57 100644 --- a/crates/jmap/src/email/get.rs +++ b/crates/jmap/src/email/get.rs @@ -7,9 +7,9 @@ use common::{Server, auth::AccessToken}; use email::{ - mailbox::ArchivedUidMailbox, + mailbox::UidMailbox, message::metadata::{ - ArchivedGetHeader, ArchivedHeaderName, ArchivedMessageMetadata, ArchivedMetadataPartType, + ArchivedGetHeader, ArchivedHeaderName, ArchivedMetadataPartType, MessageMetadata, }, thread::cache::ThreadCache, }; @@ -22,13 +22,13 @@ use jmap_proto::{ collection::Collection, date::UTCDate, id::Id, - keyword::ArchivedKeyword, + keyword::Keyword, property::{HeaderForm, Property}, value::{Object, Value}, }, }; -use store::{BlobClass, rkyv::vec::ArchivedVec, write::Archive}; +use store::{BlobClass, write::Archive}; use trc::{AddContext, StoreEvent}; use utils::BlobHash; @@ -172,7 +172,7 @@ impl EmailGet for Server { } }; let metadata = metadata_ - .unarchive::() + .unarchive::() .caused_by(trc::location!())?; // Retrieve raw message if needed @@ -209,6 +209,8 @@ impl EmailGet for Server { // Prepare response let mut email = Object::with_capacity(properties.len()); + let contents = &metadata.contents[0]; + let root_part = &contents.parts[0]; for property in &properties { match property { Property::Id => { @@ -231,7 +233,7 @@ impl EmailGet for Server { .await? { let mailboxes = mailboxes_ - .unarchive::>() + .unarchive::>() .caused_by(trc::location!())?; let mut obj = Object::with_capacity(mailboxes.len()); for id in mailboxes.iter() { @@ -268,7 +270,7 @@ impl EmailGet for Server { .await? { let keywords = keywords_ - .unarchive::>() + .unarchive::>() .caused_by(trc::location!())?; let mut obj = Object::with_capacity(keywords.len()); for keyword in keywords.iter() { @@ -311,7 +313,7 @@ impl EmailGet for Server { Property::Subject => { email.append( Property::Subject, - metadata.contents.parts[0] + root_part .headers .convert_header(&ArchivedHeaderName::Subject) .map(|value| value.into_form(&HeaderForm::Text)) @@ -321,7 +323,7 @@ impl EmailGet for Server { Property::SentAt => { email.append( Property::SentAt, - metadata.contents.parts[0] + root_part .headers .convert_header(&ArchivedHeaderName::Date) .map(|value| value.into_form(&HeaderForm::Date)) @@ -331,7 +333,7 @@ impl EmailGet for Server { Property::MessageId | Property::InReplyTo | Property::References => { email.append( property.clone(), - metadata.contents.parts[0] + root_part .headers .convert_header(&match property { Property::MessageId => ArchivedHeaderName::MessageId, @@ -352,7 +354,7 @@ impl EmailGet for Server { | Property::ReplyTo => { email.append( property.clone(), - metadata.contents.parts[0] + root_part .headers .convert_header(&match property { Property::Sender => ArchivedHeaderName::Sender, @@ -370,31 +372,27 @@ impl EmailGet for Server { Property::Header(_) => { email.append( property.clone(), - metadata.contents.parts[0] - .headers - .header_to_value(property, &raw_message), + root_part.headers.header_to_value(property, &raw_message), ); } Property::Headers => { email.append( Property::Headers, - metadata.contents.parts[0] - .headers - .headers_to_value(&raw_message), + root_part.headers.headers_to_value(&raw_message), ); } Property::TextBody | Property::HtmlBody | Property::Attachments => { let list = match property { - Property::TextBody => &metadata.contents.text_body, - Property::HtmlBody => &metadata.contents.html_body, - Property::Attachments => &metadata.contents.attachments, + Property::TextBody => &contents.text_body, + Property::HtmlBody => &contents.html_body, + Property::Attachments => &contents.attachments, _ => unreachable!(), } .iter(); email.append( property.clone(), list.map(|part_id| { - metadata.contents.to_body_part( + contents.to_body_part( u16::from(part_id) as usize, &body_properties, &raw_message, @@ -407,20 +405,15 @@ impl EmailGet for Server { Property::BodyStructure => { email.append( Property::BodyStructure, - metadata.contents.to_body_part( - 0, - &body_properties, - &raw_message, - &blob_id, - ), + contents.to_body_part(0, &body_properties, &raw_message, &blob_id), ); } Property::BodyValues => { - let mut body_values = Object::with_capacity(metadata.contents.parts.len()); - for (part_id, part) in metadata.contents.parts.iter().enumerate() { - if ((metadata.contents.is_html_part(part_id as u16) + let mut body_values = Object::with_capacity(contents.parts.len()); + for (part_id, part) in contents.parts.iter().enumerate() { + if ((contents.is_html_part(part_id as u16) && (fetch_all_body_values || fetch_html_body_values)) - || (metadata.contents.is_text_part(part_id as u16) + || (contents.is_text_part(part_id as u16) && (fetch_all_body_values || fetch_text_body_values))) && matches!( part.body, diff --git a/crates/jmap/src/email/set.rs b/crates/jmap/src/email/set.rs index 21beff44..cadb00ef 100644 --- a/crates/jmap/src/email/set.rs +++ b/crates/jmap/src/email/set.rs @@ -8,7 +8,7 @@ use std::{borrow::Cow, collections::HashMap}; use common::{Server, auth::AccessToken, storage::tag::TagManager}; use email::{ - mailbox::{ArchivedUidMailbox, UidMailbox, manage::MailboxFnc}, + mailbox::{UidMailbox, manage::MailboxFnc}, message::{ delete::EmailDeletion, ingest::{EmailIngest, IngestEmail, IngestSource}, @@ -21,7 +21,7 @@ use jmap_proto::{ types::{ acl::Acl, collection::Collection, - keyword::{ArchivedKeyword, Keyword}, + keyword::Keyword, property::Property, state::{State, StateChange}, type_state::DataType, @@ -40,7 +40,6 @@ use mail_parser::MessageParser; use store::{ SerializeInfallible, ahash::AHashSet, - rkyv::vec::ArchivedVec, roaring::RoaringBitmap, write::{Archive, BatchBuilder, assert::HashedValue, log::ChangeLogBuilder}, }; @@ -783,12 +782,12 @@ impl EmailSet for Server { ( TagManager::new( mailboxes - .into_deserialized::, Vec>() + .into_deserialized::>() .caused_by(trc::location!())?, ), TagManager::new( keywords - .into_deserialized::, Vec>() + .into_deserialized::>() .caused_by(trc::location!())?, ), ) diff --git a/crates/jmap/src/email/snippet.rs b/crates/jmap/src/email/snippet.rs index 97919ce8..7c0f2bfc 100644 --- a/crates/jmap/src/email/snippet.rs +++ b/crates/jmap/src/email/snippet.rs @@ -6,8 +6,8 @@ use common::{Server, auth::AccessToken}; use email::message::metadata::{ - ArchivedGetHeader, ArchivedHeaderName, ArchivedMessageMetadata, ArchivedMetadataPartType, - DecodedPartContent, + ArchivedGetHeader, ArchivedHeaderName, ArchivedMetadataPartType, DecodedPartContent, + MessageMetadata, }; use jmap_proto::{ method::{ @@ -135,12 +135,12 @@ impl EmailSearchSnippet for Server { } }; let metadata = metadata_ - .unarchive::() + .unarchive::() .caused_by(trc::location!())?; // Add subject snippet - if let Some(subject) = metadata - .contents + let contents = &metadata.contents[0]; + if let Some(subject) = contents .root_part() .headers .header_value(&ArchivedHeaderName::Subject) @@ -176,7 +176,7 @@ impl EmailSearchSnippet for Server { }; // Find a matching part - 'outer: for part in metadata.contents.parts.iter() { + 'outer: for part in contents.parts.iter() { match &part.body { ArchivedMetadataPartType::Text => { let text = match part.decode_contents(&raw_message) { @@ -201,7 +201,7 @@ impl EmailSearchSnippet for Server { } } ArchivedMetadataPartType::Message(message) => { - for part in message.parts.iter() { + for part in metadata.contents[u16::from(message) as usize].parts.iter() { if let ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html = part.body { diff --git a/crates/jmap/src/identity/get.rs b/crates/jmap/src/identity/get.rs index 5c118073..7b9cc4b0 100644 --- a/crates/jmap/src/identity/get.rs +++ b/crates/jmap/src/identity/get.rs @@ -6,7 +6,7 @@ use common::Server; use directory::{QueryBy, backend::internal::PrincipalField}; -use email::identity::{ArchivedEmailAddress, ArchivedIdentity, Identity}; +use email::identity::{ArchivedEmailAddress, Identity}; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, types::{ @@ -99,7 +99,7 @@ impl IdentityGet for Server { continue; }; let identity = _identity - .unarchive::() + .unarchive::() .caused_by(trc::location!())?; let mut result = Object::with_capacity(properties.len()); for property in &properties { diff --git a/crates/jmap/src/identity/set.rs b/crates/jmap/src/identity/set.rs index eee73596..03e38cdd 100644 --- a/crates/jmap/src/identity/set.rs +++ b/crates/jmap/src/identity/set.rs @@ -6,7 +6,7 @@ use common::Server; use directory::{QueryBy, backend::internal::PrincipalField}; -use email::identity::{ArchivedIdentity, EmailAddress, Identity}; +use email::identity::{EmailAddress, Identity}; use jmap_proto::{ error::set::SetError, method::set::{RequestArguments, SetRequest, SetResponse}, @@ -128,7 +128,7 @@ impl IdentitySet for Server { .await? { identity - .deserialize::() + .deserialize::() .caused_by(trc::location!())? } else { response.not_updated.append(id, SetError::not_found()); diff --git a/crates/jmap/src/mailbox/get.rs b/crates/jmap/src/mailbox/get.rs index bf9add60..d810b078 100644 --- a/crates/jmap/src/mailbox/get.rs +++ b/crates/jmap/src/mailbox/get.rs @@ -5,7 +5,7 @@ */ use common::{Server, auth::AccessToken, sharing::EffectiveAcl}; -use email::mailbox::{ArchivedMailbox, manage::MailboxFnc}; +use email::mailbox::manage::MailboxFnc; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, types::{ @@ -15,7 +15,7 @@ use jmap_proto::{ value::{Object, Value}, }, }; -use store::write::{Archive}; +use store::write::Archive; use trc::AddContext; use crate::changes::state::StateManager; @@ -118,7 +118,7 @@ impl MailboxGet for Server { }; let archived_mailbox = if let Some(archived_mailbox) = &archived_mailbox_ { archived_mailbox - .unarchive::() + .unarchive::() .caused_by(trc::location!())? .into() } else { diff --git a/crates/jmap/src/mailbox/query.rs b/crates/jmap/src/mailbox/query.rs index 50e347a0..f87b39d7 100644 --- a/crates/jmap/src/mailbox/query.rs +++ b/crates/jmap/src/mailbox/query.rs @@ -5,18 +5,18 @@ */ use common::{Server, auth::AccessToken}; -use email::mailbox::{ArchivedMailbox, manage::MailboxFnc}; +use email::mailbox::manage::MailboxFnc; use jmap_proto::{ method::query::{Comparator, Filter, QueryRequest, QueryResponse, SortProperty}, object::mailbox::QueryArguments, types::{acl::Acl, collection::Collection, property::Property}, }; use store::{ - Serialize, SerializeInfallible, + SerializeInfallible, ahash::{AHashMap, AHashSet}, query::{self, sort::Pagination}, roaring::RoaringBitmap, - write::{Archive}, + write::Archive, }; use crate::{JmapMethods, UpdateResults}; @@ -59,10 +59,7 @@ impl MailboxQuery for Server { tokio::time::sleep(std::time::Duration::from_secs(1)).await; } } - filters.push(query::Filter::contains( - Property::Name, - name.to_lowercase().into_bytes(), - )); + filters.push(query::Filter::contains(Property::Name, &name)); } Filter::Role(role) => { if let Some(role) = role { @@ -134,7 +131,7 @@ impl MailboxQuery for Server { .await? { let todo = "use index"; - let mailbox = value.unarchive::()?; + let mailbox = value.unarchive::()?; let parent_id = u32::from(mailbox.parent_id); hierarchy.insert(document_id + 1, parent_id); tree.entry(parent_id) diff --git a/crates/jmap/src/mailbox/set.rs b/crates/jmap/src/mailbox/set.rs index 972a6b3c..d7dc3476 100644 --- a/crates/jmap/src/mailbox/set.rs +++ b/crates/jmap/src/mailbox/set.rs @@ -9,7 +9,7 @@ use common::{ storage::index::ObjectIndexBuilder, }; -use email::mailbox::{ArchivedMailbox, Mailbox, destroy::MailboxDestroy, manage::MailboxFnc}; +use email::mailbox::{Mailbox, destroy::MailboxDestroy, manage::MailboxFnc}; use jmap_proto::{ error::set::SetError, method::set::{SetRequest, SetResponse}, @@ -168,7 +168,7 @@ impl MailboxSet for Server { { // Validate ACL let mailbox = mailbox - .into_deserialized::() + .into_deserialized::() .caused_by(trc::location!())?; if ctx.is_shared { let acl = mailbox.inner.acls.effective_acl(access_token); @@ -417,7 +417,7 @@ impl MailboxSet for Server { .await? { let mailbox = mailbox_ - .unarchive::() + .unarchive::() .caused_by(trc::location!())?; if depth == 0 && ctx.is_shared diff --git a/crates/jmap/src/push/get.rs b/crates/jmap/src/push/get.rs index 1a9948ae..5b4aeac9 100644 --- a/crates/jmap/src/push/get.rs +++ b/crates/jmap/src/push/get.rs @@ -9,7 +9,6 @@ use common::{ auth::AccessToken, ipc::{StateEvent, UpdateSubscription}, }; -use email::push::ArchivedPushSubscription; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, types::{ @@ -99,7 +98,7 @@ impl PushSubscriptionFetch for Server { continue; }; let push = push_ - .unarchive::() + .unarchive::() .caused_by(trc::location!())?; let mut result = Object::with_capacity(properties.len()); for property in &properties { @@ -181,7 +180,7 @@ impl PushSubscriptionFetch for Server { .caused_by(trc::location!()) .document_id(document_id) })? - .deserialize::() + .deserialize::() .caused_by(trc::location!())?; if subscription.expires > current_time { diff --git a/crates/jmap/src/push/set.rs b/crates/jmap/src/push/set.rs index 8951e28a..20e5e9de 100644 --- a/crates/jmap/src/push/set.rs +++ b/crates/jmap/src/push/set.rs @@ -6,7 +6,7 @@ use base64::{Engine, engine::general_purpose}; use common::{Server, auth::AccessToken}; -use email::push::{ArchivedPushSubscription, Keys, PushSubscription}; +use email::push::{Keys, PushSubscription}; use jmap_proto::{ error::set::SetError, method::set::{RequestArguments, SetRequest, SetResponse}, @@ -144,7 +144,7 @@ impl PushSubscriptionSet for Server { ) .await? { - push.deserialize::() + push.deserialize::() .caused_by(trc::location!())? } else { response.not_updated.append(id, SetError::not_found()); diff --git a/crates/jmap/src/services/index.rs b/crates/jmap/src/services/index.rs index ad71cbb8..dfd5a315 100644 --- a/crates/jmap/src/services/index.rs +++ b/crates/jmap/src/services/index.rs @@ -11,9 +11,7 @@ use directory::{ Type, backend::internal::{PrincipalField, manage::ManageDirectory}, }; -use email::message::{ - bayes::EmailBayesTrain, index::IndexMessageText, metadata::ArchivedMessageMetadata, -}; +use email::message::{bayes::EmailBayesTrain, index::IndexMessageText, metadata::MessageMetadata}; use jmap_proto::types::{collection::Collection, property::Property}; use mail_parser::MessageParser; use store::{ @@ -176,7 +174,7 @@ impl Indexer for Server { .await { Ok(Some(metadata_)) => { - match metadata_.unarchive::() { + match metadata_.unarchive::() { Ok(metadata) if metadata.blob_hash.0.as_slice() == event.hash.as_slice() => { @@ -187,7 +185,7 @@ impl Indexer for Server { .with_account_id(event.account_id) .with_collection(Collection::Email) .with_document_id(event.document_id) - .index_message(&metadata, &raw_message); + .index_message(metadata, &raw_message); if let Err(err) = self.core.storage.fts.index(document).await { trc::error!( err.account_id(event.account_id) diff --git a/crates/jmap/src/sieve/get.rs b/crates/jmap/src/sieve/get.rs index 4d229afa..e5d874b1 100644 --- a/crates/jmap/src/sieve/get.rs +++ b/crates/jmap/src/sieve/get.rs @@ -5,7 +5,7 @@ */ use common::Server; -use email::sieve::ArchivedSieveScript; +use email::sieve::SieveScript; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, types::{ @@ -15,10 +15,7 @@ use jmap_proto::{ value::{Object, Value}, }, }; -use store::{ - BlobClass, - write::{Archive}, -}; +use store::{BlobClass, write::Archive}; use trc::AddContext; use crate::changes::state::StateManager; @@ -90,7 +87,7 @@ impl SieveScriptGet for Server { continue; }; let sieve = sieve_ - .unarchive::() + .unarchive::() .caused_by(trc::location!())?; let mut result = Object::with_capacity(properties.len()); for property in &properties { diff --git a/crates/jmap/src/sieve/query.rs b/crates/jmap/src/sieve/query.rs index a83dec72..87eb126e 100644 --- a/crates/jmap/src/sieve/query.rs +++ b/crates/jmap/src/sieve/query.rs @@ -36,10 +36,7 @@ impl SieveScriptQuery for Server { for cond in std::mem::take(&mut request.filter) { match cond { - Filter::Name(name) => filters.push(query::Filter::contains( - Property::Name, - name.to_lowercase().into_bytes(), - )), + Filter::Name(name) => filters.push(query::Filter::contains(Property::Name, &name)), Filter::IsActive(is_active) => filters.push(query::Filter::eq( Property::IsActive, (is_active as u32).serialize(), diff --git a/crates/jmap/src/sieve/set.rs b/crates/jmap/src/sieve/set.rs index 201d345b..289c1851 100644 --- a/crates/jmap/src/sieve/set.rs +++ b/crates/jmap/src/sieve/set.rs @@ -10,7 +10,7 @@ use common::{ storage::index::ObjectIndexBuilder, }; use email::sieve::{ - ArchivedSieveScript, SieveScript, activate::SieveScriptActivate, delete::SieveScriptDelete, + SieveScript, activate::SieveScriptActivate, delete::SieveScriptDelete, }; use jmap_proto::{ error::set::{SetError, SetErrorType}, @@ -193,7 +193,7 @@ impl SieveScriptSet for Server { .await? { let sieve = sieve - .into_deserialized::() + .into_deserialized::() .caused_by(trc::location!())?; let prev_blob_hash = sieve.inner.blob_hash.clone(); diff --git a/crates/jmap/src/submission/get.rs b/crates/jmap/src/submission/get.rs index e6cdff9a..0daa1bb1 100644 --- a/crates/jmap/src/submission/get.rs +++ b/crates/jmap/src/submission/get.rs @@ -6,8 +6,7 @@ use common::Server; use email::submission::{ - ArchivedAddress, ArchivedEmailSubmission, ArchivedEnvelope, Delivered, DeliveryStatus, - UndoStatus, + ArchivedAddress, ArchivedEnvelope, Delivered, DeliveryStatus, EmailSubmission, UndoStatus, }; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, @@ -21,10 +20,7 @@ use jmap_proto::{ }; use smtp::queue::{self, spool::SmtpSpool}; use std::future::Future; -use store::{ - rkyv::option::ArchivedOption, - write::{Archive}, -}; +use store::{rkyv::option::ArchivedOption, write::Archive}; use trc::AddContext; use utils::map::vec_map::VecMap; @@ -101,7 +97,7 @@ impl EmailSubmissionGet for Server { continue; }; let submission = submission_ - .unarchive::() + .unarchive::() .caused_by(trc::location!())?; // Obtain queueId diff --git a/crates/jmap/src/submission/set.rs b/crates/jmap/src/submission/set.rs index 28d6b28d..444f8ada 100644 --- a/crates/jmap/src/submission/set.rs +++ b/crates/jmap/src/submission/set.rs @@ -12,11 +12,9 @@ use common::{ storage::index::ObjectIndexBuilder, }; use email::{ - identity::ArchivedIdentity, - message::metadata::{ArchivedHeaderName, ArchivedHeaderValue, ArchivedMessageMetadata}, - submission::{ - Address, ArchivedEmailSubmission, Delivered, DeliveryStatus, EmailSubmission, UndoStatus, - }, + identity::Identity, + message::metadata::{ArchivedHeaderName, ArchivedHeaderValue, MessageMetadata}, + submission::{Address, Delivered, DeliveryStatus, EmailSubmission, UndoStatus}, }; use jmap_proto::{ error::set::{SetError, SetErrorType}, @@ -132,7 +130,7 @@ impl EmailSubmissionSet for Server { .await? { submission - .into_deserialized::() + .into_deserialized::() .caused_by(trc::location!())? } else { response.not_updated.append(id, SetError::not_found()); @@ -244,7 +242,7 @@ impl EmailSubmissionSet for Server { .custom( ObjectIndexBuilder::new().with_current( submission - .into_deserialized::() + .into_deserialized::() .caused_by(trc::location!())?, ), ) @@ -469,7 +467,7 @@ impl EmailSubmissionSet for Server { .await? { identity - .unarchive::() + .unarchive::() .caused_by(trc::location!())? .email .to_string() @@ -516,13 +514,13 @@ impl EmailSubmissionSet for Server { .with_description("Email not found."))); }; let metadata = metadata_ - .unarchive::() + .unarchive::() .caused_by(trc::location!())?; // Add recipients to envelope if missing let mut bcc_header = None; if rcpt_to.is_empty() { - for header in metadata.contents.parts[0].headers.iter() { + for header in metadata.contents[0].parts[0].headers.iter() { if matches!( header.name, ArchivedHeaderName::To | ArchivedHeaderName::Cc | ArchivedHeaderName::Bcc @@ -554,7 +552,7 @@ impl EmailSubmissionSet for Server { .with_description("No recipients found in email."))); } } else { - bcc_header = metadata.contents.parts[0] + bcc_header = metadata.contents[0].parts[0] .headers .iter() .find(|header| matches!(header.name, ArchivedHeaderName::Bcc)); diff --git a/crates/jmap/src/vacation/get.rs b/crates/jmap/src/vacation/get.rs index cb468cfa..59f8aba8 100644 --- a/crates/jmap/src/vacation/get.rs +++ b/crates/jmap/src/vacation/get.rs @@ -5,7 +5,7 @@ */ use common::Server; -use email::sieve::ArchivedSieveScript; +use email::sieve::SieveScript; use jmap_proto::{ method::get::{GetRequest, GetResponse, RequestArguments}, request::reference::MaybeReference, @@ -90,7 +90,7 @@ impl VacationResponseGet for Server { .await? { let sieve = sieve_ - .unarchive::() + .unarchive::() .caused_by(trc::location!())?; let vacation = sieve.vacation_response.as_ref(); let mut result = Object::with_capacity(properties.len()); diff --git a/crates/jmap/src/vacation/set.rs b/crates/jmap/src/vacation/set.rs index a8fb957c..068c6bd6 100644 --- a/crates/jmap/src/vacation/set.rs +++ b/crates/jmap/src/vacation/set.rs @@ -8,7 +8,7 @@ use std::borrow::Cow; use common::{Server, auth::AccessToken, storage::index::ObjectIndexBuilder}; use email::sieve::{ - ArchivedSieveScript, SieveScript, VacationResponse, activate::SieveScriptActivate, + SieveScript, VacationResponse, activate::SieveScriptActivate, delete::SieveScriptDelete, }; use jmap_proto::{ @@ -232,7 +232,7 @@ impl VacationResponseSet for Server { .caused_by(trc::location!()) })?; let prev_sieve = prev_sieve - .into_deserialized::() + .into_deserialized::() .caused_by(trc::location!())?; was_active = prev_sieve.inner.is_active; let mut sieve = prev_sieve.inner.clone(); diff --git a/crates/managesieve/src/op/getscript.rs b/crates/managesieve/src/op/getscript.rs index 559eaa09..d1cf6255 100644 --- a/crates/managesieve/src/op/getscript.rs +++ b/crates/managesieve/src/op/getscript.rs @@ -8,7 +8,7 @@ use std::time::Instant; use common::listener::SessionStream; use directory::Permission; -use email::sieve::ArchivedSieveScript; +use email::sieve::SieveScript; use imap_proto::receiver::Request; use jmap_proto::types::{blob::BlobSection, collection::Collection, property::Property}; use store::write::Archive; @@ -52,7 +52,7 @@ impl Session { .code(ResponseCode::NonExistent) })?; let sieve = sieve_ - .unarchive::() + .unarchive::() .caused_by(trc::location!())?; let blob_size = u32::from(sieve.size) as usize; let script = self diff --git a/crates/managesieve/src/op/listscripts.rs b/crates/managesieve/src/op/listscripts.rs index 06638bb7..bac826b3 100644 --- a/crates/managesieve/src/op/listscripts.rs +++ b/crates/managesieve/src/op/listscripts.rs @@ -8,7 +8,7 @@ use std::time::Instant; use common::listener::SessionStream; use directory::Permission; -use email::sieve::ArchivedSieveScript; +use email::sieve::SieveScript; use jmap_proto::types::{collection::Collection, property::Property}; use store::write::Archive; use trc::AddContext; @@ -49,7 +49,7 @@ impl Session { .caused_by(trc::location!())? { let script = script_ - .unarchive::() + .unarchive::() .caused_by(trc::location!())?; response.push(b'\"'); for ch in script.name.as_bytes() { diff --git a/crates/managesieve/src/op/putscript.rs b/crates/managesieve/src/op/putscript.rs index f5af603b..6762767a 100644 --- a/crates/managesieve/src/op/putscript.rs +++ b/crates/managesieve/src/op/putscript.rs @@ -8,7 +8,7 @@ use std::time::Instant; use common::{listener::SessionStream, storage::index::ObjectIndexBuilder}; use directory::Permission; -use email::sieve::{ArchivedSieveScript, SieveScript}; +use email::sieve::SieveScript; use imap_proto::receiver::Request; use jmap_proto::types::{blob::BlobId, collection::Collection, property::Property}; use sieve::compiler::ErrorType; @@ -115,7 +115,7 @@ impl Session { .details("Script not found") .code(ResponseCode::NonExistent) })? - .into_deserialized::() + .into_deserialized::() .caused_by(trc::location!())?; // Write script blob @@ -137,7 +137,13 @@ impl Session { // Write record let mut obj = ObjectIndexBuilder::new() - .with_changes(script.inner.clone().with_blob_hash(blob_hash.clone())) + .with_changes( + script + .inner + .clone() + .with_size(script_size as u32) + .with_blob_hash(blob_hash.clone()), + ) .with_current(script); // Update tenant quota diff --git a/crates/managesieve/src/op/renamescript.rs b/crates/managesieve/src/op/renamescript.rs index 9c6e34b4..49154943 100644 --- a/crates/managesieve/src/op/renamescript.rs +++ b/crates/managesieve/src/op/renamescript.rs @@ -8,7 +8,7 @@ use std::time::Instant; use common::{listener::SessionStream, storage::index::ObjectIndexBuilder}; use directory::Permission; -use email::sieve::{ArchivedSieveScript, SieveScript}; +use email::sieve::{SieveScript}; use imap_proto::receiver::Request; use jmap_proto::types::{collection::Collection, property::Property}; use store::write::{Archive, BatchBuilder, assert::HashedValue, log::ChangeLogBuilder}; @@ -74,7 +74,7 @@ impl Session { .details("Script not found") .code(ResponseCode::NonExistent) })? - .into_deserialized::() + .into_deserialized::() .caused_by(trc::location!())?; // Write record diff --git a/crates/pop3/src/mailbox.rs b/crates/pop3/src/mailbox.rs index 188bb8b0..88431dcb 100644 --- a/crates/pop3/src/mailbox.rs +++ b/crates/pop3/src/mailbox.rs @@ -7,12 +7,11 @@ use std::collections::BTreeMap; use common::listener::SessionStream; -use email::mailbox::{ArchivedMailbox, ArchivedUidMailbox, INBOX_ID, manage::MailboxFnc}; +use email::mailbox::{INBOX_ID, UidMailbox, manage::MailboxFnc}; use jmap_proto::types::{collection::Collection, property::Property}; use store::{ IndexKey, IterateParams, SerializeInfallible, U32_LEN, ahash::AHashMap, - rkyv::vec::ArchivedVec, write::{Archive, key::DeserializeBigEndian}, }; use trc::AddContext; @@ -79,7 +78,7 @@ impl Session { .account_id(account_id) .document_id(INBOX_ID) })? - .unarchive::() + .unarchive::() .caused_by(trc::location!())? .uid_validity, ); @@ -137,7 +136,7 @@ impl Session { { // Make sure the message is still in Inbox if let Some(item) = uid_mailbox - .unarchive::>() + .unarchive::>() .caused_by(trc::location!())? .iter() .find(|item| item.mailbox_id == INBOX_ID) diff --git a/crates/pop3/src/op/fetch.rs b/crates/pop3/src/op/fetch.rs index 2bb69014..4eb085a8 100644 --- a/crates/pop3/src/op/fetch.rs +++ b/crates/pop3/src/op/fetch.rs @@ -8,7 +8,7 @@ use std::time::Instant; use common::listener::SessionStream; use directory::Permission; -use email::message::metadata::ArchivedMessageMetadata; +use email::message::metadata::MessageMetadata; use jmap_proto::types::{collection::Collection, property::Property}; use store::write::Archive; use trc::AddContext; @@ -37,7 +37,7 @@ impl Session { .caused_by(trc::location!())? { let metadata = metadata_ - .unarchive::() + .unarchive::() .caused_by(trc::location!())?; if let Some(bytes) = self .server diff --git a/crates/store/src/query/mod.rs b/crates/store/src/query/mod.rs index 885e8c89..cbebefbe 100644 --- a/crates/store/src/query/mod.rs +++ b/crates/store/src/query/mod.rs @@ -128,11 +128,11 @@ impl Filter { } } - pub fn contains(field: impl Into, value: Vec) -> Self { + pub fn contains(field: impl Into, value: &str) -> Self { Filter::MatchValue { field: field.into(), op: Operator::Contains, - value, + value: value.to_lowercase().into_bytes(), } } diff --git a/crates/store/src/write/mod.rs b/crates/store/src/write/mod.rs index 35d99257..e21ffff9 100644 --- a/crates/store/src/write/mod.rs +++ b/crates/store/src/write/mod.rs @@ -14,6 +14,7 @@ use std::{ use nlp::tokenizers::word::WordTokenizer; use rand::Rng; +use rkyv::util::AlignedVec; use roaring::RoaringBitmap; use utils::BlobHash; @@ -29,10 +30,12 @@ pub mod key; pub mod log; pub mod serialize; -#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) const ARCHIVE_ALIGNMENT: usize = 16; + +#[derive(Debug, Clone)] pub enum Archive { - Raw(Vec), - Uncompressed(Vec), + Aligned(AlignedVec), + Vec(Vec), } #[repr(transparent)] diff --git a/crates/store/src/write/serialize.rs b/crates/store/src/write/serialize.rs index 079199dc..c56e9e52 100644 --- a/crates/store/src/write/serialize.rs +++ b/crates/store/src/write/serialize.rs @@ -4,31 +4,27 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{Deserialize, Serialize, SerializeInfallible, Value}; +use rkyv::util::AlignedVec; -use super::{Archive, Archiver, LegacyBincode, assert::HashedValue}; +use crate::{Deserialize, Serialize, SerializeInfallible, U32_LEN, Value}; + +use super::{ARCHIVE_ALIGNMENT, Archive, Archiver, LegacyBincode, assert::HashedValue}; const MAGIC_MARKER: u8 = 1 << 7; const LZ4_COMPRESSES: u8 = 1 << 6; const ARCHIVE_UNCOMPRESSED: u8 = MAGIC_MARKER; const ARCHIVE_LZ4_COMPRESSED: u8 = MAGIC_MARKER | LZ4_COMPRESSES; const COMPRESS_WATERMARK: usize = 8192; -const COMPRESS_DATA_OFFSET: usize = std::mem::size_of::() + 1; impl Deserialize for Archive { fn deserialize(bytes: &[u8]) -> trc::Result { - match bytes.first().copied() { - Some(ARCHIVE_UNCOMPRESSED) => Ok(Archive::Raw(bytes.to_vec())), - Some(ARCHIVE_LZ4_COMPRESSED) => { - lz4_flex::decompress_size_prepended(bytes.get(1..).unwrap_or_default()) - .map_err(|err| { - trc::StoreEvent::DecompressError - .ctx(trc::Key::Value, bytes) - .caused_by(trc::location!()) - .reason(err) - }) - .map(Archive::Uncompressed) + match bytes.split_last() { + Some((&ARCHIVE_UNCOMPRESSED, archive)) => { + let mut bytes = AlignedVec::with_capacity(archive.len()); + bytes.extend_from_slice(archive); + Ok(Archive::Aligned(bytes)) } + Some((&ARCHIVE_LZ4_COMPRESSED, archive)) => aligned_lz4_deflate(archive), _ => Err(trc::StoreEvent::DataCorruption .into_err() .details("Invalid archive marker.") @@ -37,18 +33,20 @@ impl Deserialize for Archive { } } - fn deserialize_owned(bytes: Vec) -> trc::Result { - match bytes.first().copied() { - Some(ARCHIVE_UNCOMPRESSED) => Ok(Archive::Raw(bytes)), - Some(ARCHIVE_LZ4_COMPRESSED) => { - lz4_flex::decompress_size_prepended(bytes.get(1..).unwrap_or_default()) - .map_err(|err| { - trc::StoreEvent::DecompressError - .ctx(trc::Key::Value, bytes) - .caused_by(trc::location!()) - .reason(err) - }) - .map(Archive::Uncompressed) + fn deserialize_owned(mut bytes: Vec) -> trc::Result { + match bytes.last() { + Some(&ARCHIVE_UNCOMPRESSED) => { + bytes.pop(); + if bytes.as_ptr().addr() & (ARCHIVE_ALIGNMENT - 1) == 0 { + Ok(Archive::Vec(bytes)) + } else { + let mut aligned = AlignedVec::with_capacity(bytes.len()); + aligned.extend_from_slice(&bytes); + Ok(Archive::Aligned(aligned)) + } + } + Some(&ARCHIVE_LZ4_COMPRESSED) => { + aligned_lz4_deflate(bytes.get(..bytes.len() - 1).unwrap_or_default()) } _ => Err(trc::StoreEvent::DataCorruption .into_err() @@ -59,6 +57,22 @@ impl Deserialize for Archive { } } +#[inline] +fn aligned_lz4_deflate(archive: &[u8]) -> trc::Result { + lz4_flex::block::uncompressed_size(archive) + .and_then(|(uncompressed_size, archive)| { + let mut bytes = AlignedVec::with_capacity(uncompressed_size); + lz4_flex::decompress_into(archive, &mut bytes)?; + Ok(Archive::Aligned(bytes)) + }) + .map_err(|err| { + trc::StoreEvent::DecompressError + .ctx(trc::Key::Value, archive) + .caused_by(trc::location!()) + .reason(err) + }) +} + impl Serialize for Archiver where T: rkyv::Archive @@ -81,28 +95,27 @@ where let input = input.as_ref(); let input_len = input.len(); if input_len > COMPRESS_WATERMARK { - let mut bytes = vec![ - ARCHIVE_LZ4_COMPRESSED; - lz4_flex::block::get_maximum_output_size(input_len) - + COMPRESS_DATA_OFFSET - ]; - bytes[1..COMPRESS_DATA_OFFSET] - .copy_from_slice(&(input_len as u32).to_le_bytes()); - let bytes_len = - lz4_flex::compress_into(input, &mut bytes[COMPRESS_DATA_OFFSET..]).unwrap() - + COMPRESS_DATA_OFFSET; + let mut bytes = + vec![ + ARCHIVE_LZ4_COMPRESSED; + lz4_flex::block::get_maximum_output_size(input_len) + U32_LEN + 1 + ]; + bytes[0..U32_LEN].copy_from_slice(&(input_len as u32).to_le_bytes()); + let bytes_len = lz4_flex::compress_into(input, &mut bytes[U32_LEN..]).unwrap() + + U32_LEN + + 1; if bytes_len < input_len { bytes.truncate(bytes_len); } else { bytes.clear(); - bytes.push(ARCHIVE_UNCOMPRESSED); bytes.extend_from_slice(input); + bytes.push(ARCHIVE_UNCOMPRESSED); } bytes } else { let mut bytes = Vec::with_capacity(input_len + 1); - bytes.push(ARCHIVE_UNCOMPRESSED); bytes.extend_from_slice(input); + bytes.push(ARCHIVE_UNCOMPRESSED); bytes } }) @@ -110,62 +123,51 @@ where } impl Archive { - pub fn unarchive(&self) -> trc::Result<&T> - where - T: rkyv::Portable - + for<'a> rkyv::bytecheck::CheckBytes< - rkyv::api::high::HighValidator<'a, rkyv::rancor::Error>, - > + Sync - + Send, - { + #[inline] + pub fn as_bytes(&self) -> &[u8] { match self { - Archive::Raw(bytes) => rkyv::access::(bytes.get(1..).unwrap()) - .map_err(|err| { - trc::StoreEvent::DataCorruption - .caused_by(trc::location!()) - .ctx(trc::Key::Value, bytes.as_slice()) - .reason(err) - }), - Archive::Uncompressed(bytes) => Ok(unsafe { rkyv::access_unchecked::(bytes) }), + Archive::Vec(bytes) => bytes.as_slice(), + Archive::Aligned(bytes) => bytes.as_slice(), } } - pub fn deserialize(&self) -> trc::Result + pub fn unarchive(&self) -> trc::Result<&::Archived> where - T: rkyv::Portable - + for<'a> rkyv::bytecheck::CheckBytes< + T: rkyv::Archive, + T::Archived: for<'a> rkyv::bytecheck::CheckBytes< rkyv::api::high::HighValidator<'a, rkyv::rancor::Error>, - > + Sync - + Send - + rkyv::Deserialize>, + > + rkyv::Deserialize>, { - self.unarchive::().and_then(|value| { - rkyv::deserialize::(value).map_err(|err| { - trc::StoreEvent::DeserializeError - .ctx( - trc::Key::Value, - match self { - Archive::Raw(bytes) => bytes, - Archive::Uncompressed(bytes) => bytes, - } - .as_slice(), - ) - .caused_by(trc::location!()) - .reason(err) - }) + rkyv::access::(self.as_bytes()).map_err(|err| { + trc::StoreEvent::DataCorruption + .caused_by(trc::location!()) + .ctx(trc::Key::Value, self.as_bytes()) + .reason(err) + }) + } + + pub fn deserialize(&self) -> trc::Result + where + T: rkyv::Archive, + T::Archived: for<'a> rkyv::bytecheck::CheckBytes< + rkyv::api::high::HighValidator<'a, rkyv::rancor::Error>, + > + rkyv::Deserialize>, + { + rkyv::from_bytes(self.as_bytes()).map_err(|err| { + trc::StoreEvent::DeserializeError + .ctx(trc::Key::Value, self.as_bytes()) + .caused_by(trc::location!()) + .reason(err) }) } pub fn into_inner(self) -> Vec { - match self { - Archive::Raw(bytes) => bytes, - Archive::Uncompressed(bytes) => { - let mut result = Vec::with_capacity(bytes.len() + 1); - result.push(ARCHIVE_UNCOMPRESSED); - result.extend_from_slice(&bytes); - result - } - } + let mut bytes = match self { + Archive::Vec(bytes) => bytes, + Archive::Aligned(bytes) => bytes.to_vec(), + }; + bytes.push(ARCHIVE_UNCOMPRESSED); + bytes } } @@ -190,30 +192,27 @@ where } impl HashedValue { - pub fn to_unarchived(&self) -> trc::Result> + pub fn to_unarchived(&self) -> trc::Result::Archived>> where - T: rkyv::Portable - + for<'a> rkyv::bytecheck::CheckBytes< + T: rkyv::Archive, + T::Archived: for<'a> rkyv::bytecheck::CheckBytes< rkyv::api::high::HighValidator<'a, rkyv::rancor::Error>, - > + Sync - + Send, + > + rkyv::Deserialize>, { - self.inner.unarchive().map(|inner| HashedValue { + self.inner.unarchive::().map(|inner| HashedValue { hash: self.hash, inner, }) } - pub fn into_deserialized(self) -> trc::Result> + pub fn into_deserialized(&self) -> trc::Result> where - T: rkyv::Portable - + for<'a> rkyv::bytecheck::CheckBytes< + T: rkyv::Archive, + T::Archived: for<'a> rkyv::bytecheck::CheckBytes< rkyv::api::high::HighValidator<'a, rkyv::rancor::Error>, - > + Sync - + Send - + rkyv::Deserialize>, + > + rkyv::Deserialize>, { - self.inner.deserialize::().map(|inner| HashedValue { + self.inner.deserialize::().map(|inner| HashedValue { hash: self.hash, inner, }) @@ -236,14 +235,13 @@ where }) } -pub fn rkyv_unarchive(input: &[u8]) -> trc::Result<&T> +pub fn rkyv_unarchive(input: &[u8]) -> trc::Result<&::Archived> where - T: rkyv::Portable - + for<'a> rkyv::bytecheck::CheckBytes> - + Sync - + Send, + T: rkyv::Archive, + T::Archived: for<'a> rkyv::bytecheck::CheckBytes> + + rkyv::Deserialize>, { - rkyv::access::(input).map_err(|err| { + rkyv::access::(input).map_err(|err| { trc::StoreEvent::DataCorruption .caused_by(trc::location!()) .ctx(trc::Key::Value, input) diff --git a/tests/Cargo.toml b/tests/Cargo.toml index ee34c523..8145a924 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -66,6 +66,7 @@ chrono = "0.4" ring = { version = "0.17" } biscuit = "0.7.0" form_urlencoded = "1.1.0" +rkyv = { version = "0.8.10", features = ["little_endian"] } [target.'cfg(not(target_env = "msvc"))'.dependencies] jemallocator = "0.5.0" diff --git a/tests/src/imap/body_structure.rs b/tests/src/imap/body_structure.rs index 83f08d6a..b76280d4 100644 --- a/tests/src/imap/body_structure.rs +++ b/tests/src/imap/body_structure.rs @@ -6,7 +6,7 @@ use std::fs; -use email::message::metadata::{ArchivedMessageMetadata, MessageMetadata}; +use email::message::metadata::MessageMetadata; use imap::op::fetch::AsImapDataItem; use imap_proto::{ ResponseCode, StatusResponse, @@ -21,7 +21,7 @@ use store::{ use super::resources_dir; #[test] -fn body_structure() { +fn imap_test_body_structure() { println!("Running BODYSTRUCTURE..."); for file_name in fs::read_dir(resources_dir()).unwrap() { @@ -33,33 +33,33 @@ fn body_structure() { let mut buf = Vec::new(); let raw_message = fs::read(&file_name).unwrap(); let message_ = MessageParser::new().parse(&raw_message).unwrap(); - let metadata_ = Archive::deserialize_owned( - Archiver::new(MessageMetadata { - preview: Default::default(), - size: message_.raw_message.len() as u32, - raw_headers: message_ - .raw_message - .as_ref() - .get(message_.root_part().offset_header..message_.root_part().offset_body) - .unwrap_or_default() - .to_vec(), - contents: message_.into(), - received_at: 0, - has_attachments: false, - blob_hash: Default::default(), - }) - .serialize() - .unwrap(), - ) - .unwrap(); - let metadata = metadata_.unarchive::().unwrap(); - let message = &metadata.contents; - let decoded = message.decode_contents(&raw_message); + let metadata = MessageMetadata { + preview: Default::default(), + size: message_.raw_message.len() as u32, + raw_headers: message_ + .raw_message + .as_ref() + .get(message_.root_part().offset_header..message_.root_part().offset_body) + .unwrap_or_default() + .to_vec(), + contents: vec![], + received_at: 0, + has_attachments: false, + blob_hash: Default::default(), + } + .with_contents(message_); + //let c = println!("metadata {:#?}", metadata); + let metadata_ = + Archive::deserialize_owned(Archiver::new(metadata).serialize().unwrap()).unwrap(); + let metadata = metadata_.unarchive::().unwrap(); + let decoded = metadata.decode_contents(&raw_message); + + //let c = println!("parts {:#?}", decoded); // Serialize body and bodystructure for is_extended in [false, true] { let mut buf_ = Vec::new(); - message + metadata .body_structure(&decoded, is_extended) .serialize(&mut buf_, is_extended); if is_extended { @@ -119,7 +119,8 @@ fn body_structure() { true }; - if let Some(contents) = message.body_section(&decoded, &body_sections, None) + if let Some(contents) = + metadata.body_section(&decoded, &body_sections, None) { DataItem::BodySection { sections: body_sections, @@ -129,7 +130,7 @@ fn body_structure() { .serialize(&mut buf); if is_first { - match message.binary(&decoded, §ions, None) { + match metadata.binary(&decoded, §ions, None) { Ok(Some(contents)) => { buf.push(b'\n'); DataItem::Binary { @@ -163,7 +164,7 @@ fn body_structure() { } } - if let Some(size) = message.binary_size(&decoded, §ions) { + if let Some(size) = metadata.binary_size(&decoded, §ions) { buf.push(b'\n'); DataItem::BinarySize { sections: sections.clone(), @@ -203,14 +204,14 @@ fn body_structure() { }], ] { DataItem::BodySection { - contents: message.body_section(&decoded, §ions, None).unwrap(), + contents: metadata.body_section(&decoded, §ions, None).unwrap(), sections: sections.clone(), origin_octet: None, } .serialize(&mut buf); buf.extend_from_slice(b"\n----------------------------------\n"); DataItem::BodySection { - contents: message + contents: metadata .body_section(&decoded, §ions, (10, 25).into()) .unwrap(), sections, diff --git a/tests/src/imap/mod.rs b/tests/src/imap/mod.rs index 4f5ed953..8dc0236a 100644 --- a/tests/src/imap/mod.rs +++ b/tests/src/imap/mod.rs @@ -28,22 +28,22 @@ use std::{ use ::managesieve::core::ManageSieveSessionManager; use common::{ + Caches, Core, Data, Inner, Server, config::{ server::{Listeners, ServerProtocol}, telemetry::Telemetry, }, core::BuildServer, manager::boot::build_ipc, - Caches, Core, Data, Inner, Server, }; use ::store::Stores; use ahash::AHashSet; use imap::core::ImapSessionManager; use imap_proto::ResponseType; -use jmap::{api::JmapSessionManager, SpawnServices}; +use jmap::{SpawnServices, api::JmapSessionManager}; use pop3::Pop3SessionManager; -use smtp::{core::SmtpSessionManager, SpawnQueueManager}; +use smtp::{SpawnQueueManager, core::SmtpSessionManager}; use tokio::{ io::{AsyncBufReadExt, AsyncWriteExt, BufReader, Lines, ReadHalf, WriteHalf}, net::TcpStream, @@ -52,7 +52,7 @@ use tokio::{ use utils::config::Config; use crate::{ - add_test_certs, directory::internal::TestInternalDirectory, store::TempDir, AssertConfig, + AssertConfig, add_test_certs, directory::internal::TestInternalDirectory, store::TempDir, }; const SERVER: &str = r#" diff --git a/tests/src/jmap/crypto.rs b/tests/src/jmap/crypto.rs index d8f19af2..b66e0615 100644 --- a/tests/src/jmap/crypto.rs +++ b/tests/src/jmap/crypto.rs @@ -220,7 +220,10 @@ pub async fn import_certs_and_encrypt() { let arch = Archive::deserialize_owned(Archiver::new(params.clone()).serialize().unwrap()) .unwrap(); - message.encrypt(arch.unarchive().unwrap()).await.unwrap(); + message + .encrypt(arch.unarchive::().unwrap()) + .await + .unwrap(); } } diff --git a/tests/src/jmap/mod.rs b/tests/src/jmap/mod.rs index 484498c2..8adf886f 100644 --- a/tests/src/jmap/mod.rs +++ b/tests/src/jmap/mod.rs @@ -372,8 +372,8 @@ pub async fn jmap_tests() { ) .await; - webhooks::test(&mut params).await; - /*email_query::test(&mut params, delete).await; + //webhooks::test(&mut params).await; + //email_query::test(&mut params, delete).await; email_get::test(&mut params).await; email_set::test(&mut params).await; email_parse::test(&mut params).await; @@ -385,7 +385,7 @@ pub async fn jmap_tests() { thread_merge::test(&mut params).await; mailbox::test(&mut params).await; delivery::test(&mut params).await; - auth_acl::test(&mut params).await;*/ + auth_acl::test(&mut params).await; auth_limits::test(&mut params).await; auth_oauth::test(&mut params).await; event_source::test(&mut params).await; diff --git a/tests/src/jmap/stress_test.rs b/tests/src/jmap/stress_test.rs index 4c8e53b7..4aa0cd91 100644 --- a/tests/src/jmap/stress_test.rs +++ b/tests/src/jmap/stress_test.rs @@ -9,7 +9,7 @@ use std::{sync::Arc, time::Duration}; use crate::jmap::{mailbox::destroy_all_mailboxes_no_wait, wait_for_index}; use common::Server; use directory::backend::internal::manage::ManageDirectory; -use email::mailbox::{ArchivedUidMailbox, UidMailbox}; +use email::mailbox::UidMailbox; use futures::future::join_all; use jmap_client::{ client::Client, @@ -19,7 +19,6 @@ use jmap_client::{ use jmap_proto::types::{collection::Collection, id::Id, property::Property}; use store::{ rand::{self, Rng}, - rkyv::vec::ArchivedVec, write::Archive, }; @@ -242,9 +241,7 @@ async fn email_tests(server: Server, client: Arc) { .await .unwrap() { - let mailbox_tags = mailbox_tags - .deserialize::, Vec>() - .unwrap(); + let mailbox_tags = mailbox_tags.deserialize::>().unwrap(); if mailbox_tags.len() != 1 { panic!( "Email ORM has more than one mailbox {:?}! Id {} in mailbox {} with messages {:?}", diff --git a/tests/src/store/query.rs b/tests/src/store/query.rs index 8853079a..58cea389 100644 --- a/tests/src/store/query.rs +++ b/tests/src/store/query.rs @@ -145,7 +145,7 @@ pub async fn test(db: Store, fts_store: FtsStore, do_insert: bool) { match FIELDS_OPTIONS[pos] { FieldType::Text => { if !field.is_empty() { - builder.tag(field_id, field.to_lowercase()).set( + builder.index(field_id, field.to_lowercase()).set( ValueClass::Property(field_id), field.to_lowercase().into_bytes(), ); @@ -326,7 +326,7 @@ pub async fn test_filter(db: Store, fts: FtsStore) { ), ( vec![ - Filter::has_text(fields_u8["artist"], "mauro kunst"), + Filter::contains(fields_u8["artist"], "kunst, mauro"), Filter::is_in_bitmap( fields_u8["artistRole"], Keyword::Other("artist".to_string()), @@ -374,7 +374,7 @@ pub async fn test_filter(db: Store, fts: FtsStore) { ( vec![ Filter::And, - Filter::has_text(fields_u8["artist"], "warhol"), + Filter::contains(fields_u8["artist"], "warhol"), Filter::Not, Filter::is_in_set( fts.query(