From 84be2524136ba8339f9d5a663b9ba4b2becb725f Mon Sep 17 00:00:00 2001 From: mdecimus Date: Thu, 27 Feb 2025 16:23:34 +0100 Subject: [PATCH] Zero copy everything --- crates/common/src/core.rs | 2 +- crates/common/src/listener/acme/directory.rs | 8 +- crates/common/src/listener/acme/resolver.rs | 59 +- crates/common/src/storage/index.rs | 21 +- crates/common/src/storage/tag.rs | 33 +- crates/email/src/mailbox/destroy.rs | 20 +- crates/email/src/mailbox/index.rs | 8 +- crates/email/src/mailbox/manage.rs | 6 +- crates/email/src/mailbox/mod.rs | 4 +- crates/email/src/mailbox/serialize.rs | 47 - crates/email/src/message/bayes.rs | 9 +- crates/email/src/message/copy.rs | 37 +- crates/email/src/message/crypto.rs | 122 +- crates/email/src/message/delete.rs | 58 +- crates/email/src/message/index.rs | 1000 +++++++++----- crates/email/src/message/ingest.rs | 25 +- crates/email/src/message/metadata.rs | 1283 +++++++++++++++--- crates/email/src/sieve/activate.rs | 16 +- crates/email/src/sieve/delete.rs | 8 +- crates/email/src/sieve/ingest.rs | 45 +- crates/email/src/sieve/mod.rs | 4 +- crates/imap-proto/src/protocol/mod.rs | 22 +- crates/imap/src/core/mailbox.rs | 21 +- crates/imap/src/core/message.rs | 21 +- crates/imap/src/op/acl.rs | 18 +- crates/imap/src/op/copy_move.rs | 17 +- crates/imap/src/op/expunge.rs | 36 +- crates/imap/src/op/fetch.rs | 408 +++--- crates/imap/src/op/rename.rs | 6 +- crates/imap/src/op/status.rs | 6 +- crates/imap/src/op/store.rs | 35 +- crates/imap/src/op/subscribe.rs | 6 +- crates/jmap-proto/src/types/blob.rs | 21 +- crates/jmap-proto/src/types/keyword.rs | 117 +- crates/jmap-proto/src/types/property.rs | 236 +--- crates/jmap-proto/src/types/type_state.rs | 28 - crates/jmap/src/api/form.rs | 2 +- crates/jmap/src/api/management/queue.rs | 48 +- crates/jmap/src/api/management/report.rs | 36 +- crates/jmap/src/api/management/stores.rs | 20 +- crates/jmap/src/auth/oauth/auth.rs | 8 +- crates/jmap/src/auth/oauth/token.rs | 25 +- crates/jmap/src/blob/get.rs | 13 +- crates/jmap/src/email/body.rs | 227 ++-- crates/jmap/src/email/crypto.rs | 72 +- crates/jmap/src/email/get.rs | 150 +- crates/jmap/src/email/headers.rs | 92 ++ crates/jmap/src/email/set.rs | 24 +- crates/jmap/src/email/snippet.rs | 67 +- crates/jmap/src/identity/get.rs | 8 +- crates/jmap/src/identity/set.rs | 10 +- crates/jmap/src/mailbox/get.rs | 6 +- crates/jmap/src/mailbox/query.rs | 6 +- crates/jmap/src/mailbox/set.rs | 14 +- crates/jmap/src/push/get.rs | 12 +- crates/jmap/src/push/set.rs | 7 +- crates/jmap/src/services/index.rs | 166 ++- crates/jmap/src/sieve/get.rs | 11 +- crates/jmap/src/sieve/set.rs | 8 +- crates/jmap/src/submission/get.rs | 11 +- crates/jmap/src/submission/set.rs | 80 +- crates/jmap/src/vacation/get.rs | 8 +- crates/jmap/src/vacation/set.rs | 8 +- crates/managesieve/src/op/getscript.rs | 8 +- crates/managesieve/src/op/listscripts.rs | 8 +- crates/managesieve/src/op/putscript.rs | 6 +- crates/managesieve/src/op/renamescript.rs | 8 +- crates/pop3/src/mailbox.rs | 20 +- crates/pop3/src/op/fetch.rs | 13 +- crates/smtp/src/queue/spool.rs | 14 +- crates/smtp/src/reporting/analysis.rs | 8 +- crates/smtp/src/reporting/dmarc.rs | 10 +- crates/smtp/src/reporting/tls.rs | 10 +- crates/store/src/fts/index.rs | 10 +- crates/store/src/fts/mod.rs | 6 +- crates/store/src/write/mod.rs | 336 +---- crates/store/src/write/serialize.rs | 382 ++++++ crates/utils/src/lib.rs | 24 +- tests/src/imap/body_structure.rs | 50 +- tests/src/jmap/crypto.rs | 9 +- tests/src/jmap/permissions.rs | 2 +- tests/src/jmap/stress_test.rs | 22 +- tests/src/smtp/inbound/mod.rs | 8 +- tests/src/smtp/queue/dsn.rs | 8 +- tests/src/store/blob.rs | 18 +- tests/src/store/import_export.rs | 4 +- 86 files changed, 3749 insertions(+), 2187 deletions(-) delete mode 100644 crates/email/src/mailbox/serialize.rs create mode 100644 crates/store/src/write/serialize.rs diff --git a/crates/common/src/core.rs b/crates/common/src/core.rs index f6cefa87..1a13f486 100644 --- a/crates/common/src/core.rs +++ b/crates/common/src/core.rs @@ -612,7 +612,7 @@ impl Server { set_quota: bool, ) -> trc::Result { // First reserve the hash - let hash = BlobHash::from(data); + let hash = BlobHash::generate(data); let mut batch = BatchBuilder::new(); let until = now() + self.core.jmap.upload_tmp_ttl; diff --git a/crates/common/src/listener/acme/directory.rs b/crates/common/src/listener/acme/directory.rs index dd66ecbd..fb9cc519 100644 --- a/crates/common/src/listener/acme/directory.rs +++ b/crates/common/src/listener/acme/directory.rs @@ -12,7 +12,7 @@ use ring::rand::SystemRandom; use ring::signature::{ECDSA_P256_SHA256_FIXED_SIGNING, EcdsaKeyPair, EcdsaSigningAlgorithm}; use serde::Deserialize; use store::Serialize; -use store::write::Bincode; +use store::write::Archiver; use trc::AddContext; use trc::event::conv::AssertSuccess; @@ -190,7 +190,7 @@ impl Account { .reason(err) })?; - Bincode::new(SerializedCert { + Archiver::new(SerializedCert { certificate: cert.serialize_der().map_err(|err| { trc::EventType::Acme(trc::AcmeEvent::Error) .caused_by(trc::location!()) @@ -202,7 +202,9 @@ impl Account { } } -#[derive(Debug, Clone, serde::Serialize, Deserialize)] +#[derive( + rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug, Clone, serde::Serialize, Deserialize, +)] pub struct SerializedCert { pub certificate: Vec, pub private_key: Vec, diff --git a/crates/common/src/listener/acme/resolver.rs b/crates/common/src/listener/acme/resolver.rs index cc4e74b9..8759c74e 100644 --- a/crates/common/src/listener/acme/resolver.rs +++ b/crates/common/src/listener/acme/resolver.rs @@ -7,18 +7,21 @@ use std::sync::Arc; use rustls::{ + ServerConfig, crypto::ring::sign::any_ecdsa_type, server::{ClientHello, ResolvesServerCert}, sign::CertifiedKey, - ServerConfig, }; use rustls_pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer}; -use store::{dispatch::lookup::KeyValue, write::Bincode}; +use store::{dispatch::lookup::KeyValue, write::Archive}; use trc::AcmeEvent; -use crate::{listener::acme::directory::SerializedCert, Server, KV_ACME}; +use crate::{KV_ACME, Server}; -use super::{directory::ACME_TLS_ALPN_NAME, AcmeProvider, StaticResolver}; +use super::{ + AcmeProvider, StaticResolver, + directory::{ACME_TLS_ALPN_NAME, ArchivedSerializedCert}, +}; impl Server { pub(crate) fn set_cert(&self, provider: &AcmeProvider, cert: Arc) { @@ -45,28 +48,40 @@ impl Server { pub(crate) async fn build_acme_certificate(&self, domain: &str) -> Option> { match self .in_memory_store() - .key_get::>(KeyValue::<()>::build_key(KV_ACME, domain)) + .key_get::(KeyValue::<()>::build_key(KV_ACME, domain)) .await { - Ok(Some(cert)) => { - match any_ecdsa_type(&PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from( - cert.inner.private_key, - ))) { - Ok(key) => Some(Arc::new(CertifiedKey::new( - vec![CertificateDer::from(cert.inner.certificate)], - key, - ))), - Err(err) => { - trc::event!( - Acme(AcmeEvent::Error), - Domain = domain.to_string(), - Reason = err.to_string(), - Details = "Failed to parse private key" - ); - None + Ok(Some(cert_)) => match cert_.unarchive::() { + Ok(cert) => { + match any_ecdsa_type(&PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from( + cert.private_key.as_ref(), + ))) { + Ok(key) => Some(Arc::new(CertifiedKey::new( + vec![CertificateDer::from(cert.certificate.to_vec())], + key, + ))), + Err(err) => { + trc::event!( + Acme(AcmeEvent::Error), + Domain = domain.to_string(), + Reason = err.to_string(), + Details = "Failed to parse private key" + ); + None + } } } - } + + Err(err) => { + trc::event!( + Acme(AcmeEvent::Error), + Domain = domain.to_string(), + CausedBy = err, + Details = "Failed to unarchive certificate" + ); + None + } + }, Err(err) => { trc::event!( Acme(AcmeEvent::Error), diff --git a/crates/common/src/storage/index.rs b/crates/common/src/storage/index.rs index 1b771453..66d1163c 100644 --- a/crates/common/src/storage/index.rs +++ b/crates/common/src/storage/index.rs @@ -9,7 +9,7 @@ use std::{borrow::Cow, collections::HashSet, fmt::Debug}; use store::{ Serialize, SerializeInfallible, write::{ - BatchBuilder, BitmapClass, DirectoryClass, IntoOperations, Operation, ValueOp, + Archiver, BatchBuilder, BitmapClass, DirectoryClass, IntoOperations, Operation, ValueOp, assert::HashedValue, }, }; @@ -25,7 +25,20 @@ pub enum IndexValue<'x> { Acl { value: &'x [AclGrant] }, } -pub trait IndexableObject: Debug + Eq + Serialize + Sync + Send { +pub trait IndexableObject: + Debug + + Eq + + Sync + + Send + + rkyv::Archive + + for<'a> rkyv::Serialize< + rkyv::api::high::HighSerializer< + rkyv::util::AlignedVec, + rkyv::ser::allocator::ArenaHandle<'a>, + rkyv::rancor::Error, + >, + > +{ fn index_values(&self) -> impl Iterator>; } @@ -94,7 +107,7 @@ impl IntoOperations for ObjectIndexBuilder { (None, Some(changes)) => { // Insertion build_batch(batch, &changes, self.tenant_id, true); - batch.set(Property::Value, changes.serialize()?); + batch.set(Property::Value, Archiver::new(changes).serialize()?); } (Some(current), Some(changes)) => { // Update @@ -410,7 +423,7 @@ fn merge_batch( if has_changes { batch.ops.push(Operation::Value { class: Property::Value.into(), - op: ValueOp::Set(current.serialize()?.into()), + op: ValueOp::Set(Archiver::new(current).serialize()?.into()), }); } diff --git a/crates/common/src/storage/tag.rs b/crates/common/src/storage/tag.rs index b5911eaf..ea35b64d 100644 --- a/crates/common/src/storage/tag.rs +++ b/crates/common/src/storage/tag.rs @@ -9,21 +9,23 @@ use std::slice::IterMut; use jmap_proto::types::property::Property; use store::{ Serialize, - write::{ - BatchBuilder, DeserializeFrom, MaybeDynamicId, SerializeInto, TagValue, ValueClass, - assert::HashedValue, - }, + write::{Archiver, BatchBuilder, MaybeDynamicId, TagValue, ValueClass, assert::HashedValue}, }; pub struct TagManager< T: Into> + PartialEq + Clone - + SerializeInto - + Serialize - + DeserializeFrom + Sync - + Send, + + Send + + rkyv::Archive + + for<'a> rkyv::Serialize< + rkyv::api::high::HighSerializer< + rkyv::util::AlignedVec, + rkyv::ser::allocator::ArenaHandle<'a>, + rkyv::rancor::Error, + >, + >, > { current: HashedValue>, added: Vec, @@ -41,11 +43,16 @@ impl< T: Into> + PartialEq + Clone - + SerializeInto - + Serialize - + DeserializeFrom + Sync - + Send, + + Send + + rkyv::Archive + + for<'a> rkyv::Serialize< + rkyv::api::high::HighSerializer< + rkyv::util::AlignedVec, + rkyv::ser::allocator::ArenaHandle<'a>, + rkyv::rancor::Error, + >, + >, > TagManager { pub fn new(current: HashedValue>) -> Self { @@ -129,7 +136,7 @@ impl< .assert_value(ValueClass::Property(property), &self.current) .set( ValueClass::Property(property), - self.current.inner.serialize()?, + Archiver::new(self.current.inner).serialize()?, ); for added in self.added { batch.tag(property, added); diff --git a/crates/email/src/mailbox/destroy.rs b/crates/email/src/mailbox/destroy.rs index d34d4dce..ffced6c3 100644 --- a/crates/email/src/mailbox/destroy.rs +++ b/crates/email/src/mailbox/destroy.rs @@ -12,11 +12,12 @@ use jmap_proto::{ error::set::{SetError, SetErrorType}, types::{acl::Acl, collection::Collection, id::Id, property::Property}, }; +use rkyv::vec::ArchivedVec; use store::{ Serialize, SerializeInfallible, query::Filter, roaring::RoaringBitmap, - write::{ArchivedValue, BatchBuilder, assert::HashedValue, log::ChangeLogBuilder}, + write::{Archive, Archiver, BatchBuilder, assert::HashedValue, log::ChangeLogBuilder}, }; use trc::AddContext; @@ -100,8 +101,8 @@ impl MailboxDestroy for Server { // If the message is in multiple mailboxes, untag it from the current mailbox, // otherwise delete it. let mut destroy_ids = RoaringBitmap::new(); - for (message_id, mut mailbox_ids) in self - .get_properties::>, _, _>( + for (message_id, mailbox_ids) in self + .get_properties::, _, _>( account_id, Collection::Email, &message_ids, @@ -110,6 +111,9 @@ impl MailboxDestroy for Server { .await? { // Remove mailbox from list + let mut mailbox_ids = mailbox_ids + .into_deserialized::, Vec>() + .caused_by(trc::location!())?; let orig_len = mailbox_ids.inner.len(); mailbox_ids.inner.retain(|id| id.mailbox_id != document_id); if mailbox_ids.inner.len() == orig_len { @@ -136,7 +140,9 @@ impl MailboxDestroy for Server { .assert_value(Property::MailboxIds, &mailbox_ids) .set( Property::MailboxIds, - mailbox_ids.inner.serialize().caused_by(trc::location!())?, + Archiver::new(mailbox_ids.inner) + .serialize() + .caused_by(trc::location!())?, ) .untag(Property::MailboxIds, document_id); match self.core.storage.data.write(batch.build()).await { @@ -186,7 +192,7 @@ impl MailboxDestroy for Server { // Obtain mailbox if let Some(mailbox) = self - .get_property::>>( + .get_property::>( account_id, Collection::Mailbox, document_id, @@ -195,7 +201,9 @@ impl MailboxDestroy for Server { .await .caused_by(trc::location!())? { - let mailbox = mailbox.into_deserialized().caused_by(trc::location!())?; + let mailbox = mailbox + .into_deserialized::() + .caused_by(trc::location!())?; // Validate ACLs if access_token.is_shared(account_id) { let acl = mailbox.inner.acls.effective_acl(access_token); diff --git a/crates/email/src/mailbox/index.rs b/crates/email/src/mailbox/index.rs index 6f9d7b7e..164676c3 100644 --- a/crates/email/src/mailbox/index.rs +++ b/crates/email/src/mailbox/index.rs @@ -11,7 +11,7 @@ use common::{ use jmap_proto::types::property::Property; use store::write::{MaybeDynamicId, TagValue}; -use super::{Mailbox, UidMailbox}; +use super::{ArchivedUidMailbox, Mailbox, UidMailbox}; impl IndexableObject for Mailbox { fn index_values(&self) -> impl Iterator> { @@ -57,3 +57,9 @@ impl From for TagValue { TagValue::Id(MaybeDynamicId::Static(value.mailbox_id)) } } + +impl From<&ArchivedUidMailbox> for TagValue { + fn from(value: &ArchivedUidMailbox) -> Self { + TagValue::Id(MaybeDynamicId::Static(value.mailbox_id.into())) + } +} diff --git a/crates/email/src/mailbox/manage.rs b/crates/email/src/mailbox/manage.rs index 4a46a6ad..ca16d8f1 100644 --- a/crates/email/src/mailbox/manage.rs +++ b/crates/email/src/mailbox/manage.rs @@ -13,7 +13,7 @@ use store::{ ahash::AHashSet, query::Filter, roaring::RoaringBitmap, - write::{ArchivedValue, BatchBuilder}, + write::{Archive, BatchBuilder}, }; use trc::AddContext; @@ -305,7 +305,7 @@ impl MailboxFnc for Server { let mut found_names = Vec::new(); for document_id in document_ids { if let Some(obj) = self - .get_property::>( + .get_property::( account_id, Collection::Mailbox, document_id, @@ -313,7 +313,7 @@ impl MailboxFnc for Server { ) .await? { - let obj = obj.unarchive()?; + let obj = obj.unarchive::()?; found_names.push(( obj.name.to_string(), u32::from(obj.parent_id), diff --git a/crates/email/src/mailbox/mod.rs b/crates/email/src/mailbox/mod.rs index 42d87671..1d894987 100644 --- a/crates/email/src/mailbox/mod.rs +++ b/crates/email/src/mailbox/mod.rs @@ -10,7 +10,6 @@ use jmap_proto::types::value::AclGrant; pub mod destroy; pub mod index; pub mod manage; -pub mod serialize; pub const INBOX_ID: u32 = 0; pub const TRASH_ID: u32 = 1; @@ -31,7 +30,8 @@ pub struct Mailbox { pub acls: Vec, } -#[derive(Debug, Clone, Copy)] +#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, Copy)] +#[rkyv(derive(Debug, Clone, Copy))] pub struct UidMailbox { pub mailbox_id: u32, pub uid: u32, diff --git a/crates/email/src/mailbox/serialize.rs b/crates/email/src/mailbox/serialize.rs deleted file mode 100644 index 2496f02f..00000000 --- a/crates/email/src/mailbox/serialize.rs +++ /dev/null @@ -1,47 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use std::slice::Iter; - -use store::{ - Serialize, U32_LEN, - write::{DeserializeFrom, SerializeInto}, -}; -use utils::codec::leb128::{Leb128Iterator, Leb128Vec}; - -use super::{Mailbox, UidMailbox}; - -impl SerializeInto for UidMailbox { - fn serialize_into(&self, buf: &mut Vec) { - buf.push_leb128(self.mailbox_id); - buf.push_leb128(self.uid); - } -} - -impl DeserializeFrom for UidMailbox { - fn deserialize_from(bytes: &mut Iter<'_, u8>) -> Option { - Some(UidMailbox { - mailbox_id: bytes.next_leb128()?, - uid: bytes.next_leb128()?, - }) - } -} - -impl Serialize for UidMailbox { - fn serialize(&self) -> trc::Result> { - let mut buf = Vec::with_capacity(U32_LEN * 2); - self.serialize_into(&mut buf); - Ok(buf) - } -} - -impl Serialize for Mailbox { - fn serialize(&self) -> trc::Result> { - rkyv::to_bytes::(self) - .map(|r| r.into_vec()) - .map_err(Into::into) - } -} diff --git a/crates/email/src/message/bayes.rs b/crates/email/src/message/bayes.rs index 0c9ad8cf..cbe526a4 100644 --- a/crates/email/src/message/bayes.rs +++ b/crates/email/src/message/bayes.rs @@ -12,10 +12,11 @@ use mail_parser::Message; use spam_filter::{ SpamFilterInput, analysis::init::SpamFilterInit, modules::bayes::BayesClassifier, }; -use store::write::{Bincode, TaskQueueClass}; +use store::write::{Archive, TaskQueueClass}; use trc::StoreEvent; +use utils::BlobHash; -use super::metadata::MessageMetadata; +use super::metadata::ArchivedMessageMetadata; pub trait EmailBayesTrain: Sync + Send { fn email_bayes_train( @@ -58,7 +59,7 @@ impl EmailBayesTrain for Server { learn_spam: bool, ) -> trc::Result { let metadata = self - .get_property::>( + .get_property::( account_id, Collection::Email, document_id, @@ -74,7 +75,7 @@ impl EmailBayesTrain for Server { Ok(TaskQueueClass::BayesTrain { seq: self.generate_snowflake_id()?, - hash: metadata.inner.blob_hash, + hash: BlobHash::from(&metadata.unarchive::()?.blob_hash), learn_spam, }) } diff --git a/crates/email/src/message/copy.rs b/crates/email/src/message/copy.rs index 77d59be9..07a249e2 100644 --- a/crates/email/src/message/copy.rs +++ b/crates/email/src/message/copy.rs @@ -12,11 +12,11 @@ use jmap_proto::{ property::Property, }, }; -use mail_parser::{HeaderName, HeaderValue, parsers::fields::thread::thread_name}; +use mail_parser::parsers::fields::thread::thread_name; use store::{ BlobClass, Serialize, SerializeInfallible, write::{ - BatchBuilder, Bincode, MaybeDynamicId, TagValue, TaskQueueClass, ValueClass, + Archive, Archiver, BatchBuilder, MaybeDynamicId, TagValue, TaskQueueClass, ValueClass, log::{Changes, LogInsert}, }, }; @@ -25,9 +25,9 @@ use trc::AddContext; use crate::mailbox::UidMailbox; use super::{ - index::{EmailIndexBuilder, MAX_ID_LENGTH, MAX_SORT_FIELD_LENGTH, TrimTextValue, VisitValues}, + index::{MAX_ID_LENGTH, MAX_SORT_FIELD_LENGTH, TrimTextValue}, ingest::{EmailIngest, IngestedEmail, LogEmailInsert}, - metadata::MessageMetadata, + metadata::{ArchivedMessageMetadata, HeaderName, HeaderValue, MessageMetadata}, }; pub trait EmailCopy: Sync + Send { @@ -59,7 +59,7 @@ impl EmailCopy for Server { // Obtain metadata let account_id = resource_token.account_id; let mut metadata = if let Some(metadata) = self - .get_property::>( + .get_property::( from_account_id, Collection::Email, from_message_id, @@ -67,7 +67,9 @@ impl EmailCopy for Server { ) .await? { - metadata.inner + metadata + .deserialize::() + .caused_by(trc::location!())? } else { return Ok(Err(SetError::not_found().with_description(format!( "Message not found not found in account {}.", @@ -137,7 +139,7 @@ impl EmailCopy for Server { // Assign id let mut email = IngestedEmail { - size: metadata.size, + size: metadata.size as usize, ..Default::default() }; let blob_hash = metadata.blob_hash.clone(); @@ -179,16 +181,20 @@ impl EmailCopy for Server { .log(LogEmailInsert::new(thread_id)) .set(Property::ThreadId, maybe_thread_id) .tag(Property::ThreadId, TagValue::Id(maybe_thread_id)) + .tag_many(Property::MailboxIds, mailbox_ids.iter()) .set( Property::MailboxIds, - mailbox_ids.serialize().caused_by(trc::location!())?, + Archiver::new(mailbox_ids) + .serialize() + .caused_by(trc::location!())?, ) + .tag_many(Property::Keywords, keywords.iter()) .set( Property::Keywords, - keywords.serialize().caused_by(trc::location!())?, + Archiver::new(keywords) + .serialize() + .caused_by(trc::location!())?, ) - .tag_many(Property::MailboxIds, mailbox_ids.iter()) - .tag_many(Property::Keywords, keywords.into_iter()) .set(Property::Cid, change_id.serialize()) .set( ValueClass::TaskQueue(TaskQueueClass::IndexEmail { @@ -197,8 +203,13 @@ impl EmailCopy for Server { }), vec![], ); - EmailIndexBuilder::set(metadata) - .build(&mut batch, account_id, resource_token.tenant.map(|t| t.id)) + metadata + .index( + &mut batch, + account_id, + resource_token.tenant.map(|t| t.id), + true, + ) .caused_by(trc::location!())?; // Insert and obtain ids diff --git a/crates/email/src/message/crypto.rs b/crates/email/src/message/crypto.rs index 761280df..ac88b686 100644 --- a/crates/email/src/message/crypto.rs +++ b/crates/email/src/message/crypto.rs @@ -26,7 +26,7 @@ use rasn_cms::{ }; use rsa::{Pkcs1v15Encrypt, RsaPublicKey, pkcs1::DecodeRsaPublicKey}; use sequoia_openpgp as openpgp; -use store::{Deserialize, Serialize, write::Bincode}; +use store::{Deserialize, write::Archive}; const P: openpgp::policy::StandardPolicy<'static> = openpgp::policy::StandardPolicy::new(); @@ -36,26 +36,63 @@ pub enum EncryptMessageError { Error(String), } -#[derive(Debug, Clone, Copy, serde::Serialize, serde::Deserialize)] +#[derive( + rkyv::Serialize, + rkyv::Deserialize, + rkyv::Archive, + Debug, + Clone, + Copy, + serde::Serialize, + serde::Deserialize, +)] +#[rkyv(derive(Clone, Copy))] pub enum Algorithm { Aes128, Aes256, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[derive( + rkyv::Serialize, + rkyv::Deserialize, + rkyv::Archive, + Debug, + Clone, + Copy, + PartialEq, + Eq, + serde::Serialize, + serde::Deserialize, +)] pub enum EncryptionMethod { PGP, SMIME, } -#[derive(Debug, serde::Serialize, serde::Deserialize)] +#[derive( + Clone, + rkyv::Serialize, + rkyv::Deserialize, + rkyv::Archive, + Debug, + serde::Serialize, + serde::Deserialize, +)] pub struct EncryptionParams { pub method: EncryptionMethod, pub algo: Algorithm, pub certs: Vec>, } -#[derive(Debug, serde::Serialize, serde::Deserialize, Default)] +#[derive( + rkyv::Serialize, + rkyv::Deserialize, + rkyv::Archive, + Debug, + serde::Serialize, + serde::Deserialize, + Default, +)] #[serde(tag = "type")] #[serde(rename_all = "camelCase")] pub enum EncryptionType { @@ -73,12 +110,18 @@ pub enum EncryptionType { #[allow(async_fn_in_trait)] pub trait EncryptMessage { - async fn encrypt(&self, params: &EncryptionParams) -> Result, EncryptMessageError>; + async fn encrypt( + &self, + params: &ArchivedEncryptionParams, + ) -> Result, EncryptMessageError>; fn is_encrypted(&self) -> bool; } impl EncryptMessage for Message<'_> { - async fn encrypt(&self, params: &EncryptionParams) -> Result, EncryptMessageError> { + async fn encrypt( + &self, + params: &ArchivedEncryptionParams, + ) -> Result, EncryptMessageError> { let root = self.root_part(); let raw_message = self.raw_message(); let mut outer_message = Vec::with_capacity((raw_message.len() as f64 * 1.5) as usize); @@ -98,7 +141,7 @@ impl EncryptMessage for Message<'_> { // Encrypt inner message match params.method { - EncryptionMethod::PGP => { + ArchivedEncryptionMethod::PGP => { // Prepare encrypted message let boundary = make_boundary("_"); outer_message.extend_from_slice( @@ -179,8 +222,8 @@ impl EncryptMessage for Message<'_> { })?; let message = stream::Encryptor2::for_recipients(message, keys) .symmetric_algo(match algo { - Algorithm::Aes128 => SymmetricAlgorithm::AES128, - Algorithm::Aes256 => SymmetricAlgorithm::AES256, + ArchivedAlgorithm::Aes128 => SymmetricAlgorithm::AES128, + ArchivedAlgorithm::Aes256 => SymmetricAlgorithm::AES256, }) .build() .map_err(|err| { @@ -224,7 +267,7 @@ impl EncryptMessage for Message<'_> { outer_message.extend_from_slice(boundary.as_bytes()); outer_message.extend_from_slice(b"--\r\n"); } - EncryptionMethod::SMIME => { + ArchivedEncryptionMethod::SMIME => { // Generate random IV let mut rng = StdRng::from_entropy(); let mut iv = vec![0u8; 16]; @@ -247,7 +290,7 @@ impl EncryptMessage for Message<'_> { // Encrypt key using public keys #[allow(clippy::mutable_key_type)] let mut recipient_infos = BTreeSet::new(); - for cert in ¶ms.certs { + for cert in params.certs.iter() { let cert = rasn::der::decode::(cert).map_err(|err| { EncryptMessageError::Error(format!( @@ -412,26 +455,26 @@ impl EncryptMessage for Message<'_> { } } -impl Algorithm { +impl ArchivedAlgorithm { fn key_size(&self) -> usize { match self { - Algorithm::Aes128 => 16, - Algorithm::Aes256 => 32, + ArchivedAlgorithm::Aes128 => 16, + ArchivedAlgorithm::Aes256 => 32, } } fn to_algorithm_identifier(self) -> ObjectIdentifier { match self { - Algorithm::Aes128 => AES128_CBC.into(), - Algorithm::Aes256 => AES256_CBC.into(), + ArchivedAlgorithm::Aes128 => AES128_CBC.into(), + ArchivedAlgorithm::Aes256 => AES256_CBC.into(), } } fn encrypt(&self, key: &[u8], iv: &[u8], contents: &[u8]) -> Vec { match self { - Algorithm::Aes128 => cbc::Encryptor::::new(key.into(), iv.into()) + ArchivedAlgorithm::Aes128 => cbc::Encryptor::::new(key.into(), iv.into()) .encrypt_padded_vec_mut::(contents), - Algorithm::Aes256 => cbc::Encryptor::::new(key.into(), iv.into()) + ArchivedAlgorithm::Aes256 => cbc::Encryptor::::new(key.into(), iv.into()) .encrypt_padded_vec_mut::(contents), } } @@ -485,10 +528,13 @@ fn try_parse_pem( return base64_decode(internal.as_bytes()) .ok_or(Cow::from("Failed to decode base64")) .and_then(|bytes| { - Bincode::::deserialize(&bytes) + Archive::deserialize_owned(bytes) + .and_then(|arch| { + arch.deserialize::() + }) .map_err(|_| Cow::from("Failed to deserialize internal certificate")) }) - .map(|params| Some((params.inner.method, params.inner.certs))); + .map(|params| Some((params.method, params.certs))); } let mut bytes = bytes_.iter().enumerate(); @@ -612,40 +658,6 @@ fn try_parse_pem( Ok(method.map(|method| (method, certs))) } -impl Serialize for EncryptionParams { - fn serialize(&self) -> trc::Result> { - let len = bincode::serialized_size(&self).unwrap_or_default(); - let mut buf = Vec::with_capacity(len as usize + 1); - buf.push(1); - bincode::serialize_into(&mut buf, &self).map_err(|err| { - trc::EventType::Store(trc::StoreEvent::DeserializeError) - .reason(err) - .caused_by(trc::location!()) - })?; - Ok(buf) - } -} - -impl Deserialize for EncryptionParams { - fn deserialize(bytes: &[u8]) -> trc::Result { - let version = *bytes - .first() - .ok_or_else(|| trc::StoreEvent::DataCorruption.caused_by(trc::location!()))?; - match version { - 1 if bytes.len() > 1 => bincode::deserialize(&bytes[1..]).map_err(|err| { - trc::EventType::Store(trc::StoreEvent::DeserializeError) - .from_bincode_error(err) - .caused_by(trc::location!()) - }), - - _ => Err(trc::StoreEvent::DeserializeError - .into_err() - .caused_by(trc::location!()) - .ctx(trc::Key::Value, version as u64)), - } - } -} - impl Display for EncryptionMethod { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { diff --git a/crates/email/src/message/delete.rs b/crates/email/src/message/delete.rs index ebdc9788..dc4e3216 100644 --- a/crates/email/src/message/delete.rs +++ b/crates/email/src/message/delete.rs @@ -8,28 +8,26 @@ use std::time::Duration; use common::{KV_LOCK_PURGE_ACCOUNT, Server}; use jmap_proto::types::{ - collection::Collection, id::Id, keyword::Keyword, property::Property, state::StateChange, - type_state::DataType, + collection::Collection, id::Id, keyword::ArchivedKeyword, property::Property, + state::StateChange, type_state::DataType, }; +use rkyv::vec::ArchivedVec; use store::{ BitmapKey, IterateParams, U32_LEN, ValueKey, ahash::AHashMap, roaring::RoaringBitmap, write::{ - BatchBuilder, Bincode, BitmapClass, MaybeDynamicId, TagValue, ValueClass, + Archive, BatchBuilder, BitmapClass, MaybeDynamicId, TagValue, ValueClass, log::ChangeLogBuilder, }, }; use trc::{AddContext, StoreEvent}; -use utils::codec::leb128::Leb128Reader; +use utils::{BlobHash, codec::leb128::Leb128Reader}; use std::future::Future; use store::rand::prelude::SliceRandom; -use crate::{ - mailbox::*, - message::{index::EmailIndexBuilder, metadata::MessageMetadata}, -}; +use crate::{mailbox::*, message::metadata::ArchivedMessageMetadata}; pub trait EmailDeletion: Sync + Send { fn emails_tombstone( @@ -67,7 +65,7 @@ impl EmailDeletion for Server { // Fetch mailboxes and threadIds let mut thread_ids: AHashMap = AHashMap::new(); for (document_id, mailboxes) in self - .get_properties::, _, _>( + .get_properties::( account_id, Collection::Email, &document_ids, @@ -78,7 +76,12 @@ impl EmailDeletion for Server { delete_properties.insert( document_id, DeleteProperties { - mailboxes, + mailboxes: mailboxes + .unarchive::>() + .caused_by(trc::location!())? + .iter() + .map(|m| u32::from(m.mailbox_id)) + .collect(), thread_id: None, }, ); @@ -151,12 +154,14 @@ impl EmailDeletion for Server { if !delete_properties.mailboxes.is_empty() { for mailbox_id in &delete_properties.mailboxes { - debug_assert!(mailbox_id.uid != 0); - changes.log_child_update(Collection::Mailbox, mailbox_id.mailbox_id); + changes.log_child_update(Collection::Mailbox, *mailbox_id); } batch - .untag_many(Property::MailboxIds, delete_properties.mailboxes.iter()) + .untag_many( + Property::MailboxIds, + delete_properties.mailboxes.iter().copied(), + ) .clear(Property::MailboxIds); } else { trc::event!( @@ -437,11 +442,11 @@ impl EmailDeletion for Server { ); // Remove keywords - if let Some(keywords) = self + if let Some(keywords_) = self .core .storage .data - .get_value::>(ValueKey { + .get_value::(ValueKey { account_id, collection: Collection::Email.into(), document_id, @@ -449,8 +454,11 @@ impl EmailDeletion for Server { }) .await? { + let keywords = keywords_ + .unarchive::>() + .caused_by(trc::location!())?; batch - .untag_many(Property::Keywords, keywords.into_iter()) + .untag_many(Property::Keywords, keywords.iter()) .clear(Property::Keywords); } else { trc::event!( @@ -463,11 +471,11 @@ impl EmailDeletion for Server { } // Remove message metadata - if let Some(metadata) = self + if let Some(metadata_) = self .core .storage .data - .get_value::>(ValueKey { + .get_value::(ValueKey { account_id, collection: Collection::Email.into(), document_id, @@ -475,6 +483,10 @@ impl EmailDeletion for Server { }) .await? { + let metadata = metadata_ + .unarchive::() + .caused_by(trc::location!())?; + // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd // SPDX-License-Identifier: LicenseRef-SEL @@ -484,15 +496,15 @@ impl EmailDeletion for Server { self.core.hold_undelete( &mut batch, Collection::Email.into(), - &metadata.inner.blob_hash, - metadata.inner.size, + &BlobHash::from(&metadata.blob_hash), + u32::from(metadata.size) as usize, ); // SPDX-SnippetEnd // Delete message - EmailIndexBuilder::clear(metadata.inner) - .build(&mut batch, account_id, tenant_id) + metadata + .index(&mut batch, account_id, tenant_id, false) .caused_by(trc::location!())?; // Commit batch @@ -514,6 +526,6 @@ impl EmailDeletion for Server { #[derive(Default, Debug)] struct DeleteProperties { - mailboxes: Vec, + mailboxes: Vec, thread_id: Option, } diff --git a/crates/email/src/message/index.rs b/crates/email/src/message/index.rs index 20b7a109..4810f5fb 100644 --- a/crates/email/src/message/index.rs +++ b/crates/email/src/message/index.rs @@ -4,28 +4,30 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::borrow::Cow; - use jmap_proto::types::{keyword::Keyword, property::Property}; use mail_parser::{ - Addr, Address, GetHeader, Group, Header, HeaderName, HeaderValue, Message, MessagePart, - PartType, decoders::html::html_to_text, parsers::{fields::thread::thread_name, preview::preview_text}, }; use nlp::language::Language; +use rkyv::option::ArchivedOption; use store::{ Serialize, SerializeInfallible, backend::MAX_TOKEN_LENGTH, fts::{Field, index::FtsDocument}, - write::{BatchBuilder, Bincode, BlobOp, DirectoryClass}, + write::{Archiver, BatchBuilder, BlobOp, DirectoryClass}, }; use trc::AddContext; use utils::BlobHash; use crate::mailbox::UidMailbox; -use super::metadata::MessageMetadata; +use super::metadata::{ + Addr, Address, ArchivedAddress, ArchivedGetHeader, ArchivedHeaderName, ArchivedHeaderValue, + ArchivedMessageMetadata, ArchivedMessageMetadataContents, ArchivedMessageMetadataPart, + ArchivedMetadataPartType, DecodedPartContent, Group, HeaderName, HeaderValue, MessageMetadata, + MessageMetadataPart, +}; pub const MAX_MESSAGE_PARTS: usize = 1000; pub const MAX_ID_LENGTH: usize = 100; @@ -33,6 +35,402 @@ pub const MAX_SORT_FIELD_LENGTH: usize = 255; pub const MAX_STORED_FIELD_LENGTH: usize = 512; pub const PREVIEW_LENGTH: usize = 256; +impl MessageMetadata { + #[inline(always)] + pub fn root_part(&self) -> &MessageMetadataPart { + &self.contents.parts[0] + } + + pub fn index( + self, + batch: &mut BatchBuilder, + account_id: u32, + tenant_id: Option, + set: bool, + ) -> trc::Result<()> { + if set { + // Serialize metadata + batch + .index(Property::Size, self.size.serialize()) + .index(Property::ReceivedAt, (self.received_at).serialize()); + } else { + // Delete metadata + batch + .clear(Property::BodyStructure) + .unindex(Property::Size, self.size.serialize()) + .unindex(Property::ReceivedAt, (self.received_at).serialize()); + } + + // Index properties + let quota = if set { + self.size as i64 + } else { + -(self.size as i64) + }; + batch.add(DirectoryClass::UsedQuota(account_id), quota); + if let Some(tenant_id) = tenant_id { + batch.add(DirectoryClass::UsedQuota(tenant_id), quota); + } + + if self.has_attachments { + if set { + batch.tag(Property::HasAttachment, ()); + } else { + batch.untag(Property::HasAttachment, ()); + } + } + + // Index headers + self.index_headers(batch, set); + + // Link blob + if set { + batch.set( + BlobOp::Link { + hash: self.blob_hash.clone(), + }, + Vec::new(), + ); + } else { + batch.clear(BlobOp::Link { + hash: self.blob_hash.clone(), + }); + } + + if set { + batch.set(Property::BodyStructure, Archiver::new(self).serialize()?); + } + + Ok(()) + } + + fn index_headers(&self, batch: &mut BatchBuilder, set: bool) { + let mut seen_headers = [false; 40]; + for header in self.root_part().headers.iter().rev() { + if matches!(header.name, HeaderName::Other(_)) { + continue; + } + + match header.name { + HeaderName::MessageId => { + header.value.visit_text(|id| { + // Add ids to inverted index + if id.len() < MAX_ID_LENGTH { + if set { + batch + .index(Property::MessageId, id.serialize()) + .index(Property::References, id.serialize()); + } else { + batch + .unindex(Property::MessageId, id.serialize()) + .unindex(Property::References, id.serialize()); + } + } + }); + } + HeaderName::InReplyTo | HeaderName::References | HeaderName::ResentMessageId => { + header.value.visit_text(|id| { + // Add ids to inverted index + if id.len() < MAX_ID_LENGTH { + if set { + batch.index(Property::References, id.serialize()); + } else { + batch.unindex(Property::References, id.serialize()); + } + } + }); + } + HeaderName::From | HeaderName::To | HeaderName::Cc | HeaderName::Bcc => { + if !seen_headers[header.name.id() as usize] { + let property = property_from_header(&header.name); + let mut sort_text = SortedAddressBuilder::new(); + let mut found_addr = false; + + header.value.visit_addresses(|element, value| { + if !found_addr { + match element { + AddressElement::Name => { + found_addr = !sort_text.push(value); + } + AddressElement::Address => { + sort_text.push(value); + found_addr = true; + } + AddressElement::GroupName => (), + } + } + }); + + // Add address to inverted index + if set { + batch.index(u8::from(&property), sort_text.build()); + } else { + batch.unindex(u8::from(&property), sort_text.build()); + } + seen_headers[header.name.id() as usize] = true; + } + } + HeaderName::Date => { + if !seen_headers[header.name.id() as usize] { + if let HeaderValue::DateTime(datetime) = &header.value { + let value = (*datetime as u64).serialize(); + if set { + batch.index(Property::SentAt, value); + } else { + batch.unindex(Property::SentAt, value); + } + } + seen_headers[header.name.id() as usize] = true; + } + } + HeaderName::Subject => { + if !seen_headers[header.name.id() as usize] { + // Index subject + let subject = match &header.value { + HeaderValue::Text(text) => text.clone(), + HeaderValue::TextList(list) if !list.is_empty() => { + list.first().unwrap().clone() + } + _ => "".into(), + }; + + // Index thread name + let thread_name = thread_name(&subject); + let thread_name = if !thread_name.is_empty() { + thread_name.trim_text(MAX_SORT_FIELD_LENGTH) + } else { + "!" + } + .serialize(); + + if set { + batch.index(Property::Subject, thread_name); + } else { + batch.unindex(Property::Subject, thread_name); + } + + seen_headers[header.name.id() as usize] = true; + } + } + + _ => (), + } + } + + // Add subject to index if missing + if !seen_headers[HeaderName::Subject.id() as usize] { + if set { + batch.index(Property::Subject, "!".serialize()); + } else { + batch.unindex(Property::Subject, "!".serialize()); + } + } + } +} + +impl ArchivedMessageMetadata { + #[inline(always)] + pub fn root_part(&self) -> &ArchivedMessageMetadataPart { + &self.contents.parts[0] + } + + pub fn index( + &self, + batch: &mut BatchBuilder, + account_id: u32, + tenant_id: Option, + set: bool, + ) -> trc::Result<()> { + if set { + // Serialize metadata + batch + .index(Property::Size, u32::from(self.size).serialize()) + .index( + Property::ReceivedAt, + u64::from(self.received_at).serialize(), + ); + } else { + // Delete metadata + batch + .clear(Property::BodyStructure) + .unindex(Property::Size, u32::from(self.size).serialize()) + .unindex( + Property::ReceivedAt, + u64::from(self.received_at).serialize(), + ); + } + + // Index properties + let quota = if set { + u32::from(self.size) as i64 + } else { + -(u32::from(self.size) as i64) + }; + batch.add(DirectoryClass::UsedQuota(account_id), quota); + if let Some(tenant_id) = tenant_id { + batch.add(DirectoryClass::UsedQuota(tenant_id), quota); + } + + if self.has_attachments { + if set { + batch.tag(Property::HasAttachment, ()); + } else { + batch.untag(Property::HasAttachment, ()); + } + } + + // Index headers + self.index_headers(batch, set); + + // Link blob + let hash = BlobHash::from(&self.blob_hash); + if set { + batch.set(BlobOp::Link { hash }, Vec::new()); + } else { + batch.clear(BlobOp::Link { hash }); + } + + Ok(()) + } + + fn index_headers(&self, batch: &mut BatchBuilder, set: bool) { + let mut seen_headers = [false; 40]; + for header in self.root_part().headers.iter().rev() { + if matches!(header.name, ArchivedHeaderName::Other(_)) { + continue; + } + + match header.name { + ArchivedHeaderName::MessageId => { + header.value.visit_text(|id| { + // Add ids to inverted index + if id.len() < MAX_ID_LENGTH { + if set { + batch + .index(Property::MessageId, id.serialize()) + .index(Property::References, id.serialize()); + } else { + batch + .unindex(Property::MessageId, id.serialize()) + .unindex(Property::References, id.serialize()); + } + } + }); + } + ArchivedHeaderName::InReplyTo + | ArchivedHeaderName::References + | ArchivedHeaderName::ResentMessageId => { + header.value.visit_text(|id| { + // Add ids to inverted index + if id.len() < MAX_ID_LENGTH { + if set { + batch.index(Property::References, id.serialize()); + } else { + batch.unindex(Property::References, id.serialize()); + } + } + }); + } + ArchivedHeaderName::From + | ArchivedHeaderName::To + | ArchivedHeaderName::Cc + | ArchivedHeaderName::Bcc => { + if !seen_headers[header.name.id() as usize] { + let property = property_from_archived_header(&header.name); + let mut sort_text = SortedAddressBuilder::new(); + let mut found_addr = false; + + header.value.visit_addresses(|element, value| { + if !found_addr { + match element { + AddressElement::Name => { + found_addr = !sort_text.push(value); + } + AddressElement::Address => { + sort_text.push(value); + found_addr = true; + } + AddressElement::GroupName => (), + } + } + }); + + // Add address to inverted index + if set { + batch.index(u8::from(&property), sort_text.build()); + } else { + batch.unindex(u8::from(&property), sort_text.build()); + } + seen_headers[header.name.id() as usize] = true; + } + } + ArchivedHeaderName::Date => { + if !seen_headers[header.name.id() as usize] { + if let ArchivedHeaderValue::DateTime(datetime) = &header.value { + let value = (i64::from(*datetime) as u64).serialize(); + if set { + batch.index(Property::SentAt, value); + } else { + batch.unindex(Property::SentAt, value); + } + } + seen_headers[header.name.id() as usize] = true; + } + } + ArchivedHeaderName::Subject => { + if !seen_headers[header.name.id() as usize] { + // Index subject + let subject = match &header.value { + ArchivedHeaderValue::Text(text) => text.as_str(), + ArchivedHeaderValue::TextList(list) if !list.is_empty() => { + list.first().unwrap().as_str() + } + _ => "", + }; + + // Index thread name + let thread_name = thread_name(subject); + let thread_name = if !thread_name.is_empty() { + thread_name.trim_text(MAX_SORT_FIELD_LENGTH) + } else { + "!" + } + .serialize(); + + if set { + batch.index(Property::Subject, thread_name); + } else { + batch.unindex(Property::Subject, thread_name); + } + + seen_headers[header.name.id() as usize] = true; + } + } + + _ => (), + } + } + + // Add subject to index if missing + if !seen_headers[HeaderName::Subject.id() as usize] { + if set { + batch.index(Property::Subject, "!".serialize()); + } else { + batch.unindex(Property::Subject, "!".serialize()); + } + } + } +} + +impl ArchivedMessageMetadataContents { + pub fn is_html_part(&self, part_id: u16) -> bool { + self.html_body.iter().any(|&id| id == part_id) + } + + pub fn is_text_part(&self, part_id: u16) -> bool { + self.text_body.iter().any(|&id| id == part_id) + } +} #[derive(Debug)] pub struct SortedAddressBuilder { last_is_space: bool, @@ -45,18 +443,12 @@ pub(super) trait IndexMessage { &mut self, account_id: u32, tenant_id: Option, - message: Message, + message: mail_parser::Message<'_>, blob_hash: BlobHash, keywords: Vec, mailbox_ids: Vec, received_at: u64, ) -> trc::Result<&mut Self>; - - fn index_headers(&mut self, headers: &[Header<'_>], set: bool); -} - -pub trait IndexMessageText<'x>: Sized { - fn index_message(self, message: &'x Message<'x>) -> Self; } impl IndexMessage for BatchBuilder { @@ -64,25 +456,27 @@ impl IndexMessage for BatchBuilder { &mut self, account_id: u32, tenant_id: Option, - message: Message, + message: mail_parser::Message<'_>, blob_hash: BlobHash, keywords: Vec, mailbox_ids: Vec, received_at: u64, ) -> trc::Result<&mut Self> { // Index keywords + let keywords = Archiver::new(keywords); self.set( Property::Keywords, keywords.serialize().caused_by(trc::location!())?, ) - .tag_many(Property::Keywords, keywords.into_iter()); + .tag_many(Property::Keywords, keywords.into_inner().into_iter()); // Index mailboxIds - self.set( + self.tag_many(Property::MailboxIds, mailbox_ids.iter()).set( Property::MailboxIds, - mailbox_ids.serialize().caused_by(trc::location!())?, - ) - .tag_many(Property::MailboxIds, mailbox_ids.iter()); + Archiver::new(mailbox_ids) + .serialize() + .caused_by(trc::location!())?, + ); // Index size self.index( @@ -113,12 +507,8 @@ impl IndexMessage for BatchBuilder { .unwrap_or(usize::MAX); for (part_id, part) in message.parts.iter().take(MAX_MESSAGE_PARTS).enumerate() { - if part_id == 0 { - self.index_headers(&part.headers, true); - } - match &part.body { - PartType::Text(text) => { + mail_parser::PartType::Text(text) => { if part_id == preview_part_id { preview = preview_text(text.replace('\r', "").into(), PREVIEW_LENGTH).into(); @@ -130,7 +520,7 @@ impl IndexMessage for BatchBuilder { has_attachments = true; } } - PartType::Html(html) => { + mail_parser::PartType::Html(html) => { let text = html_to_text(html); if part_id == preview_part_id { preview = @@ -143,13 +533,33 @@ impl IndexMessage for BatchBuilder { has_attachments = true; } } - PartType::Binary(_) | PartType::Message(_) if !has_attachments => { + mail_parser::PartType::Binary(_) | mail_parser::PartType::Message(_) + if !has_attachments => + { has_attachments = true; } _ => {} } } + // Build metadata + let root_part = message.root_part(); + let metadata = MessageMetadata { + preview: preview.unwrap_or_default().into_owned(), + size: message.raw_message.len() as u32, + raw_headers: message + .raw_message + .as_ref() + .get(root_part.offset_header..root_part.offset_body) + .unwrap_or_default() + .to_vec(), + contents: message.into(), + received_at, + has_attachments, + blob_hash, + }; + metadata.index_headers(self, true); + // Store and index hasAttachment property if has_attachments { self.tag(Property::HasAttachment, ()); @@ -158,209 +568,97 @@ impl IndexMessage for BatchBuilder { // Link blob self.set( BlobOp::Link { - hash: blob_hash.clone(), + hash: metadata.blob_hash.clone(), }, Vec::new(), ); // Store message metadata - let root_part = message.root_part(); self.set( Property::BodyStructure, - Bincode::new(MessageMetadata { - preview: preview.unwrap_or_default().into_owned(), - size: message.raw_message.len(), - raw_headers: message - .raw_message - .as_ref() - .get(root_part.offset_header..root_part.offset_body) - .unwrap_or_default() - .to_vec(), - contents: message.into(), - received_at, - has_attachments, - blob_hash, - }) - .serialize() - .caused_by(trc::location!())?, + Archiver::new(metadata) + .serialize() + .caused_by(trc::location!())?, ); Ok(self) } - - fn index_headers(&mut self, headers: &[Header<'_>], set: bool) { - let mut seen_headers = [false; 40]; - for header in headers.iter().rev() { - if matches!(header.name, HeaderName::Other(_)) { - continue; - } - - match header.name { - HeaderName::MessageId => { - header.value.visit_text(|id| { - // Add ids to inverted index - if id.len() < MAX_ID_LENGTH { - if set { - self.index(Property::MessageId, id.serialize()) - .index(Property::References, id.serialize()); - } else { - self.unindex(Property::MessageId, id.serialize()) - .unindex(Property::References, id.serialize()); - } - } - }); - } - HeaderName::InReplyTo | HeaderName::References | HeaderName::ResentMessageId => { - header.value.visit_text(|id| { - // Add ids to inverted index - if id.len() < MAX_ID_LENGTH { - if set { - self.index(Property::References, id.serialize()); - } else { - self.unindex(Property::References, id.serialize()); - } - } - }); - } - HeaderName::From | HeaderName::To | HeaderName::Cc | HeaderName::Bcc => { - if !seen_headers[header.name.id() as usize] { - let property = Property::from_header(&header.name); - let mut sort_text = SortedAddressBuilder::new(); - let mut found_addr = false; - - header.value.visit_addresses(|element, value| { - if !found_addr { - match element { - AddressElement::Name => { - found_addr = !sort_text.push(value); - } - AddressElement::Address => { - sort_text.push(value); - found_addr = true; - } - AddressElement::GroupName => (), - } - } - }); - - // Add address to inverted index - if set { - self.index(u8::from(&property), sort_text.build()); - } else { - self.unindex(u8::from(&property), sort_text.build()); - } - seen_headers[header.name.id() as usize] = true; - } - } - HeaderName::Date => { - if !seen_headers[header.name.id() as usize] { - if let HeaderValue::DateTime(datetime) = &header.value { - let value = (datetime.to_timestamp() as u64).serialize(); - if set { - self.index(Property::SentAt, value); - } else { - self.unindex(Property::SentAt, value); - } - } - seen_headers[header.name.id() as usize] = true; - } - } - HeaderName::Subject => { - if !seen_headers[header.name.id() as usize] { - // Index subject - let subject = match &header.value { - HeaderValue::Text(text) => text.clone(), - HeaderValue::TextList(list) if !list.is_empty() => { - list.first().unwrap().clone() - } - _ => "".into(), - }; - - // Index thread name - let thread_name = thread_name(&subject); - let thread_name = if !thread_name.is_empty() { - thread_name.trim_text(MAX_SORT_FIELD_LENGTH) - } else { - "!" - } - .serialize(); - - if set { - self.index(Property::Subject, thread_name); - } else { - self.unindex(Property::Subject, thread_name); - } - - seen_headers[header.name.id() as usize] = true; - } - } - - _ => (), - } - } - - // Add subject to index if missing - if !seen_headers[HeaderName::Subject.id() as usize] { - if set { - self.index(Property::Subject, "!".serialize()); - } else { - self.unindex(Property::Subject, "!".serialize()); - } - } - } } -impl<'x> IndexMessageText<'x> for FtsDocument<'x, HeaderName<'x>> { - fn index_message(mut self, message: &'x Message<'x>) -> Self { +pub trait IndexMessageText<'x>: Sized { + fn index_message(self, message: &'x ArchivedMessageMetadata, raw_message: &'x [u8]) -> Self; +} + +impl<'x> IndexMessageText<'x> for FtsDocument<'x, mail_parser::HeaderName<'x>> { + fn index_message( + mut self, + message: &'x ArchivedMessageMetadata, + raw_message: &'x [u8], + ) -> Self { let mut language = Language::Unknown; - for (part_id, part) in message.parts.iter().take(MAX_MESSAGE_PARTS).enumerate() { + for (part_id, part) in message + .contents + .parts + .iter() + .take(MAX_MESSAGE_PARTS) + .enumerate() + { let part_language = part.language().unwrap_or(language); if part_id == 0 { language = part_language; for header in part.headers.iter().rev() { - if matches!(header.name, HeaderName::Other(_)) { + if matches!(header.name, ArchivedHeaderName::Other(_)) { continue; } // Index hasHeader property self.index_keyword(Field::Keyword, header.name.as_str().to_ascii_lowercase()); match &header.name { - HeaderName::MessageId - | HeaderName::InReplyTo - | HeaderName::References - | HeaderName::ResentMessageId => { + ArchivedHeaderName::MessageId + | ArchivedHeaderName::InReplyTo + | ArchivedHeaderName::References + | ArchivedHeaderName::ResentMessageId => { header.value.visit_text(|id| { // Index ids without stemming if id.len() < MAX_TOKEN_LENGTH { self.index_keyword( - Field::Header(header.name.clone()), + Field::Header(mail_parser::HeaderName::from(&header.name)), id.to_string(), ); } }); } - HeaderName::From | HeaderName::To | HeaderName::Cc | HeaderName::Bcc => { + ArchivedHeaderName::From + | ArchivedHeaderName::To + | ArchivedHeaderName::Cc + | ArchivedHeaderName::Bcc => { header.value.visit_addresses(|_, value| { // Index an address name or email without stemming self.index_tokenized( - Field::Header(header.name.clone()), + Field::Header(mail_parser::HeaderName::from(&header.name)), value.to_string(), ); }); } - HeaderName::Subject => { + ArchivedHeaderName::Subject => { // Index subject for FTS if let Some(subject) = header.value.as_text() { - self.index(Field::Header(HeaderName::Subject), subject, language); + self.index( + Field::Header(mail_parser::HeaderName::Subject), + subject, + language, + ); } } - HeaderName::Comments | HeaderName::Keywords | HeaderName::ListId => { + ArchivedHeaderName::Comments + | ArchivedHeaderName::Keywords + | ArchivedHeaderName::ListId => { // Index headers header.value.visit_text(|text| { self.index_tokenized( - Field::Header(header.name.clone()), + Field::Header(mail_parser::HeaderName::from(&header.name)), text.to_string(), ); }); @@ -370,32 +668,34 @@ impl<'x> IndexMessageText<'x> for FtsDocument<'x, HeaderName<'x>> { } } + let part_id = part_id as u16; match &part.body { - PartType::Text(text) => { - if message.text_body.contains(&part_id) || message.html_body.contains(&part_id) - { - self.index(Field::Body, text.as_ref(), part_language); - } else { - self.index(Field::Attachment, text.as_ref(), part_language); - } - } - PartType::Html(html) => { - let text = html_to_text(html); + ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html => { + let text = match (part.decode_contents(raw_message), &part.body) { + (DecodedPartContent::Text(text), ArchivedMetadataPartType::Text) => text, + (DecodedPartContent::Text(html), ArchivedMetadataPartType::Html) => { + html_to_text(html.as_ref()).into() + } + _ => unreachable!(), + }; - if message.text_body.contains(&part_id) || message.html_body.contains(&part_id) + if message.contents.is_html_part(part_id) + || message.contents.is_text_part(part_id) { self.index(Field::Body, text, part_language); } else { self.index(Field::Attachment, text, part_language); } } - PartType::Message(nested_message) => { + ArchivedMetadataPartType::Message(nested_message) => { let nested_message_language = nested_message .root_part() .language() .unwrap_or(Language::Unknown); - if let Some(HeaderValue::Text(subject)) = - nested_message.header(HeaderName::Subject) + if let Some(ArchivedHeaderValue::Text(subject)) = nested_message + .root_part() + .headers + .header_value(&ArchivedHeaderName::Subject) { self.index(Field::Attachment, subject.as_ref(), nested_message_language); } @@ -403,11 +703,20 @@ impl<'x> IndexMessageText<'x> for FtsDocument<'x, HeaderName<'x>> { for sub_part in nested_message.parts.iter().take(MAX_MESSAGE_PARTS) { let language = sub_part.language().unwrap_or(nested_message_language); match &sub_part.body { - PartType::Text(text) => { - self.index(Field::Attachment, text.as_ref(), language); - } - PartType::Html(html) => { - self.index(Field::Attachment, html_to_text(html), language); + ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html => { + let text = + match (sub_part.decode_contents(raw_message), &sub_part.body) { + ( + DecodedPartContent::Text(text), + ArchivedMetadataPartType::Text, + ) => text, + ( + DecodedPartContent::Text(html), + ArchivedMetadataPartType::Html, + ) => html_to_text(html.as_ref()).into(), + _ => unreachable!(), + }; + self.index(Field::Attachment, text, language); } _ => (), } @@ -420,88 +729,6 @@ impl<'x> IndexMessageText<'x> for FtsDocument<'x, HeaderName<'x>> { } } -pub struct EmailIndexBuilder<'x> { - inner: Bincode>, - set: bool, -} - -impl<'x> EmailIndexBuilder<'x> { - pub fn set(inner: MessageMetadata<'x>) -> Self { - Self { - inner: Bincode { inner }, - set: true, - } - } - - pub fn clear(inner: MessageMetadata<'x>) -> Self { - Self { - inner: Bincode { inner }, - set: false, - } - } -} - -impl EmailIndexBuilder<'_> { - pub fn build( - self, - batch: &mut BatchBuilder, - account_id: u32, - tenant_id: Option, - ) -> trc::Result<()> { - let metadata = &self.inner.inner; - if self.set { - // Serialize metadata - batch - .set(Property::BodyStructure, (self.inner).serialize()?) - .index(Property::Size, (metadata.size as u32).serialize()) - .index(Property::ReceivedAt, (metadata.received_at).serialize()); - } else { - // Delete metadata - batch - .clear(Property::BodyStructure) - .unindex(Property::Size, (metadata.size as u32).serialize()) - .unindex(Property::ReceivedAt, (metadata.received_at).serialize()); - } - - // Index properties - let quota = if self.set { - metadata.size as i64 - } else { - -(metadata.size as i64) - }; - batch.add(DirectoryClass::UsedQuota(account_id), quota); - if let Some(tenant_id) = tenant_id { - batch.add(DirectoryClass::UsedQuota(tenant_id), quota); - } - - if metadata.has_attachments { - if self.set { - batch.tag(Property::HasAttachment, ()); - } else { - batch.untag(Property::HasAttachment, ()); - } - } - - // Index headers - batch.index_headers(&metadata.contents.parts[0].headers, self.set); - - // Link blob - if self.set { - batch.set( - BlobOp::Link { - hash: metadata.blob_hash.clone(), - }, - Vec::new(), - ); - } else { - batch.clear(BlobOp::Link { - hash: metadata.blob_hash.clone(), - }); - } - Ok(()) - } -} - impl SortedAddressBuilder { pub fn new() -> Self { Self { @@ -548,11 +775,7 @@ impl Default for SortedAddressBuilder { } } -trait GetContentLanguage { - fn language(&self) -> Option; -} - -impl GetContentLanguage for MessagePart<'_> { +/*impl MessageMetadataPart { fn language(&self) -> Option { self.headers .header_value(&HeaderName::ContentLanguage) @@ -568,12 +791,24 @@ impl GetContentLanguage for MessagePart<'_> { .into() }) } -} +}*/ -pub trait VisitValues<'x> { - fn visit_addresses<'y: 'x>(&'y self, visitor: impl FnMut(AddressElement, &'x str)); - fn visit_text<'y: 'x>(&'y self, visitor: impl FnMut(&'x str)); - fn into_visit_text(self, visitor: impl FnMut(String)); +impl ArchivedMessageMetadataPart { + fn language(&self) -> Option { + self.headers + .header_value(&ArchivedHeaderName::ContentLanguage) + .and_then(|v| { + Language::from_iso_639(match v { + ArchivedHeaderValue::Text(v) => v.as_ref(), + ArchivedHeaderValue::TextList(v) => v.first()?, + _ => { + return None; + } + }) + .unwrap_or(Language::Unknown) + .into() + }) + } } #[derive(Debug, PartialEq, Eq)] @@ -583,8 +818,8 @@ pub enum AddressElement { GroupName, } -impl<'x> VisitValues<'x> for HeaderValue<'x> { - fn visit_addresses<'y: 'x>(&'y self, mut visitor: impl FnMut(AddressElement, &'x str)) { +impl HeaderValue { + pub fn visit_addresses(&self, mut visitor: impl FnMut(AddressElement, &str)) { match self { HeaderValue::Address(Address::List(addr_list)) => { for addr in addr_list { @@ -616,7 +851,7 @@ impl<'x> VisitValues<'x> for HeaderValue<'x> { } } - fn visit_text<'y: 'x>(&'y self, mut visitor: impl FnMut(&'x str)) { + pub fn visit_text<'x>(&'x self, mut visitor: impl FnMut(&'x str)) { match &self { HeaderValue::Text(text) => { visitor(text.as_ref()); @@ -630,14 +865,62 @@ impl<'x> VisitValues<'x> for HeaderValue<'x> { } } - fn into_visit_text(self, mut visitor: impl FnMut(String)) { + pub fn into_visit_text(self, mut visitor: impl FnMut(String)) { match self { HeaderValue::Text(text) => { - visitor(text.into_owned()); + visitor(text); } HeaderValue::TextList(texts) => { for text in texts { - visitor(text.into_owned()); + visitor(text); + } + } + _ => (), + } + } +} + +impl ArchivedHeaderValue { + fn visit_addresses(&self, mut visitor: impl FnMut(AddressElement, &str)) { + match self { + ArchivedHeaderValue::Address(ArchivedAddress::List(addr_list)) => { + for addr in addr_list.iter() { + if let ArchivedOption::Some(name) = &addr.name { + visitor(AddressElement::Name, name); + } + if let ArchivedOption::Some(addr) = &addr.address { + visitor(AddressElement::Address, addr); + } + } + } + ArchivedHeaderValue::Address(ArchivedAddress::Group(groups)) => { + for group in groups.iter() { + if let ArchivedOption::Some(name) = &group.name { + visitor(AddressElement::GroupName, name); + } + + for addr in group.addresses.iter() { + if let ArchivedOption::Some(name) = &addr.name { + visitor(AddressElement::Name, name); + } + if let ArchivedOption::Some(addr) = &addr.address { + visitor(AddressElement::Address, addr); + } + } + } + } + _ => (), + } + } + + fn visit_text(&self, mut visitor: impl FnMut(&str)) { + match &self { + ArchivedHeaderValue::Text(text) => { + visitor(text.as_ref()); + } + ArchivedHeaderValue::TextList(texts) => { + for text in texts.iter() { + visitor(text.as_ref()); } } _ => (), @@ -649,7 +932,7 @@ pub trait TrimTextValue { fn trim_text(self, length: usize) -> Self; } -impl TrimTextValue for HeaderValue<'_> { +impl TrimTextValue for HeaderValue { fn trim_text(self, length: usize) -> Self { match self { HeaderValue::Address(Address::List(v)) => { @@ -665,7 +948,7 @@ impl TrimTextValue for HeaderValue<'_> { } } -impl TrimTextValue for Addr<'_> { +impl TrimTextValue for Addr { fn trim_text(self, length: usize) -> Self { Self { name: self.name.map(|v| v.trim_text(length)), @@ -674,7 +957,7 @@ impl TrimTextValue for Addr<'_> { } } -impl TrimTextValue for Group<'_> { +impl TrimTextValue for Group { fn trim_text(self, length: usize) -> Self { Self { name: self.name.map(|v| v.trim_text(length)), @@ -683,19 +966,6 @@ impl TrimTextValue for Group<'_> { } } -impl TrimTextValue for Cow<'_, str> { - fn trim_text(self, length: usize) -> Self { - if self.len() < length { - self - } else { - match self { - Cow::Borrowed(v) => v.trim_text(length).into(), - Cow::Owned(v) => v.trim_text(length).into(), - } - } - } -} - impl TrimTextValue for &str { fn trim_text(self, length: usize) -> Self { if self.len() < length { @@ -737,3 +1007,107 @@ impl TrimTextValue for Vec { self.into_iter().map(|v| v.trim_text(length)).collect() } } + +pub fn property_from_header(header: &HeaderName) -> Property { + match header { + HeaderName::Subject => Property::Subject, + HeaderName::From => Property::From, + HeaderName::To => Property::To, + HeaderName::Cc => Property::Cc, + HeaderName::Date => Property::SentAt, + HeaderName::Bcc => Property::Bcc, + HeaderName::ReplyTo => Property::ReplyTo, + HeaderName::Sender => Property::Sender, + HeaderName::InReplyTo => Property::InReplyTo, + HeaderName::MessageId => Property::MessageId, + HeaderName::References => Property::References, + HeaderName::ResentMessageId => Property::EmailIds, + _ => unreachable!(), + } +} + +pub fn property_from_archived_header(header: &ArchivedHeaderName) -> Property { + match header { + ArchivedHeaderName::Subject => Property::Subject, + ArchivedHeaderName::From => Property::From, + ArchivedHeaderName::To => Property::To, + ArchivedHeaderName::Cc => Property::Cc, + ArchivedHeaderName::Date => Property::SentAt, + ArchivedHeaderName::Bcc => Property::Bcc, + ArchivedHeaderName::ReplyTo => Property::ReplyTo, + ArchivedHeaderName::Sender => Property::Sender, + ArchivedHeaderName::InReplyTo => Property::InReplyTo, + ArchivedHeaderName::MessageId => Property::MessageId, + ArchivedHeaderName::References => Property::References, + ArchivedHeaderName::ResentMessageId => Property::EmailIds, + _ => unreachable!(), + } +} + +pub trait VisitValues<'x> { + fn visit_addresses<'y: 'x>(&'y self, visitor: impl FnMut(AddressElement, &'x str)); + fn visit_text<'y: 'x>(&'y self, visitor: impl FnMut(&'x str)); + fn into_visit_text(self, visitor: impl FnMut(String)); +} + +impl<'x> VisitValues<'x> for mail_parser::HeaderValue<'x> { + fn visit_addresses<'y: 'x>(&'y self, mut visitor: impl FnMut(AddressElement, &'x str)) { + match self { + mail_parser::HeaderValue::Address(mail_parser::Address::List(addr_list)) => { + for addr in addr_list { + if let Some(name) = &addr.name { + visitor(AddressElement::Name, name); + } + if let Some(addr) = &addr.address { + visitor(AddressElement::Address, addr); + } + } + } + mail_parser::HeaderValue::Address(mail_parser::Address::Group(groups)) => { + for group in groups { + if let Some(name) = &group.name { + visitor(AddressElement::GroupName, name); + } + + for addr in &group.addresses { + if let Some(name) = &addr.name { + visitor(AddressElement::Name, name); + } + if let Some(addr) = &addr.address { + visitor(AddressElement::Address, addr); + } + } + } + } + _ => (), + } + } + + fn visit_text<'y: 'x>(&'y self, mut visitor: impl FnMut(&'x str)) { + match &self { + mail_parser::HeaderValue::Text(text) => { + visitor(text.as_ref()); + } + mail_parser::HeaderValue::TextList(texts) => { + for text in texts { + visitor(text.as_ref()); + } + } + _ => (), + } + } + + fn into_visit_text(self, mut visitor: impl FnMut(String)) { + match self { + mail_parser::HeaderValue::Text(text) => { + visitor(text.into_owned()); + } + mail_parser::HeaderValue::TextList(texts) => { + for text in texts { + visitor(text.into_owned()); + } + } + _ => (), + } + } +} diff --git a/crates/email/src/message/ingest.rs b/crates/email/src/message/ingest.rs index a9d68a98..bb282c42 100644 --- a/crates/email/src/message/ingest.rs +++ b/crates/email/src/message/ingest.rs @@ -37,8 +37,8 @@ use store::{ ahash::AHashSet, query::Filter, write::{ - AssignedIds, BatchBuilder, BitmapClass, MaybeDynamicId, MaybeDynamicValue, SerializeWithId, - TagValue, TaskQueueClass, ValueClass, + Archive, AssignedIds, BatchBuilder, BitmapClass, MaybeDynamicId, MaybeDynamicValue, + SerializeWithId, TagValue, TaskQueueClass, ValueClass, log::{ChangeLogBuilder, Changes, LogInsert}, now, }, @@ -49,12 +49,15 @@ use utils::map::vec_map::VecMap; use crate::{ mailbox::{INBOX_ID, JUNK_ID, UidMailbox}, - message::index::{IndexMessage, MAX_ID_LENGTH, VisitValues}, + message::{ + crypto::ArchivedEncryptionParams, + index::{IndexMessage, MAX_ID_LENGTH, VisitValues}, + }, thread::cache::ThreadCache, }; use super::{ - crypto::{EncryptMessage, EncryptMessageError, EncryptionParams}, + crypto::{EncryptMessage, EncryptMessageError}, index::{MAX_SORT_FIELD_LENGTH, TrimTextValue}, }; @@ -385,17 +388,15 @@ impl EmailIngest for Server { IngestSource::Restore => false, }; if do_encrypt && !message.is_encrypted() { - if let Some(encrypt_params) = self - .get_property::( - account_id, - Collection::Principal, - 0, - Property::Parameters, - ) + if let Some(encrypt_params_) = self + .get_property::(account_id, Collection::Principal, 0, Property::Parameters) .await .caused_by(trc::location!())? { - match message.encrypt(&encrypt_params).await { + let encrypt_params = encrypt_params_ + .unarchive::() + .caused_by(trc::location!())?; + match message.encrypt(encrypt_params).await { Ok(new_raw_message) => { raw_message = Cow::from(new_raw_message); raw_message_len = raw_message.len() as u64; diff --git a/crates/email/src/message/metadata.rs b/crates/email/src/message/metadata.rs index 8ed52003..2817fd77 100644 --- a/crates/email/src/message/metadata.rs +++ b/crates/email/src/message/metadata.rs @@ -4,175 +4,399 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::borrow::Cow; +use std::{borrow::Cow, fmt::Display}; use mail_parser::{ + DateTime, PartType, decoders::{ base64::base64_decode, charsets::map::charset_decoder, quoted_printable::quoted_printable_decode, }, - ContentType, Encoding, GetHeader, Header, HeaderName, HeaderValue, Message, MessagePart, - MessagePartId, MimeHeaders, PartType, }; -use serde::{Deserialize, Serialize}; +use rkyv::{ + rend::{u16_le, u32_le}, + string::ArchivedString, + vec::ArchivedVec, +}; use utils::BlobHash; -#[derive(Debug, Serialize, Deserialize)] -pub struct MessageMetadata<'x> { - pub contents: MessageMetadataContents<'x>, +#[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug)] +pub struct MessageMetadata { + pub contents: MessageMetadataContents, pub blob_hash: BlobHash, - pub size: usize, + pub size: u32, pub received_at: u64, pub preview: String, pub has_attachments: bool, pub raw_headers: Vec, } -#[derive(Debug, Serialize, Deserialize)] -pub struct MessageMetadataContents<'x> { - pub html_body: Vec, - pub text_body: Vec, - pub attachments: Vec, - pub parts: Vec>, +#[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug)] +pub struct MessageMetadataContents { + pub html_body: Vec, + pub text_body: Vec, + pub attachments: Vec, + pub parts: Vec, } -#[derive(Debug, Serialize, Deserialize)] -pub struct MessageMetadataPart<'x> { - pub headers: Vec>, +#[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug)] +pub struct MessageMetadataPart { + pub headers: Vec
, pub is_encoding_problem: bool, - pub body: MetadataPartType<'x>, + pub body: MetadataPartType, pub encoding: Encoding, - pub size: usize, - pub offset_header: usize, - pub offset_body: usize, - pub offset_end: usize, + pub size: u32, + pub offset_header: u32, + pub offset_body: u32, + pub offset_end: u32, } -#[derive(Debug, Serialize, Deserialize)] -pub enum MetadataPartType<'x> { +#[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug, Default)] +pub enum Encoding { + #[default] + None = 0, + QuotedPrintable = 1, + Base64 = 2, +} + +#[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug)] +#[rkyv(serialize_bounds( + __S: rkyv::ser::Writer + rkyv::ser::Allocator, + __S::Error: rkyv::rancor::Source, +))] +#[rkyv(deserialize_bounds(__D::Error: rkyv::rancor::Source))] +#[rkyv(bytecheck( + bounds( + __C: rkyv::validation::ArchiveContext, + ) +))] +pub enum MetadataPartType { Text, Html, Binary, InlineBinary, - Message(MessageMetadataContents<'x>), - Multipart(Vec), + Message(#[rkyv(omit_bounds)] MessageMetadataContents), + Multipart(Vec), } -impl<'x> MessageMetadataContents<'x> { - pub fn into_message(self, raw_message: &'x [u8]) -> Message<'x> { - Message { - html_body: self.html_body, - text_body: self.text_body, - attachments: self.attachments, - parts: self - .parts - .into_iter() - .map(|part| MessagePart { - body: match part.body { - MetadataPartType::Text - | MetadataPartType::Html - | MetadataPartType::Binary - | MetadataPartType::InlineBinary - if !raw_message.is_empty() => - { - part.decode_contents(raw_message) - } - MetadataPartType::Message(_) if !raw_message.is_empty() => { - match part.contents(raw_message) { - Cow::Borrowed(_) => PartType::Message( - part.body.unwrap_message().into_message(raw_message), - ), - Cow::Owned(raw_message) => PartType::Message( - part.body - .unwrap_message() - .into_message(&raw_message) - .into_owned(), - ), - } - } - MetadataPartType::Multipart(parts) => PartType::Multipart(parts), - _ => PartType::Binary(Cow::Borrowed(&[])), - }, - headers: part.headers, - is_encoding_problem: part.is_encoding_problem, - encoding: part.encoding, - offset_header: part.offset_header, - offset_body: part.offset_body, - offset_end: part.offset_end, - }) - .collect(), - raw_message: raw_message.into(), - } - } +#[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug, PartialEq, Eq, Clone)] +pub struct Header { + pub name: HeaderName, + pub value: HeaderValue, + pub offset_field: u32, + pub offset_start: u32, + pub offset_end: u32, +} - pub fn root_part(&self) -> &MessageMetadataPart<'x> { +#[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug, PartialEq, Eq, Clone)] +pub struct Addr { + pub name: Option, + pub address: Option, +} + +#[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug, PartialEq, Eq, Clone)] +pub struct Group { + pub name: Option, + pub addresses: Vec, +} + +#[derive( + rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug, PartialEq, Eq, Clone, PartialOrd, Ord, +)] +#[rkyv(derive(PartialEq, Debug))] +pub enum HeaderName { + Subject, + From, + To, + Cc, + Date, + Bcc, + ReplyTo, + Sender, + Comments, + InReplyTo, + Keywords, + Received, + MessageId, + References, + ReturnPath, + MimeVersion, + ContentDescription, + ContentId, + ContentLanguage, + ContentLocation, + ContentTransferEncoding, + ContentType, + ContentDisposition, + ResentTo, + ResentFrom, + ResentBcc, + ResentCc, + ResentSender, + ResentDate, + ResentMessageId, + ListArchive, + ListHelp, + ListId, + ListOwner, + ListPost, + ListSubscribe, + ListUnsubscribe, + Other(String), + DkimSignature, + ArcAuthenticationResults, + ArcMessageSignature, + ArcSeal, +} + +#[derive( + rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug, PartialEq, Eq, Clone, Default, +)] +pub enum HeaderValue { + Address(Address), + Text(String), + TextList(Vec), + DateTime(i64), + ContentType(ContentType), + #[default] + Empty, +} + +#[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug, PartialEq, Eq, Clone)] +pub enum Address { + List(Vec), + Group(Vec), +} + +#[derive( + rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug, PartialEq, Eq, Clone, Default, +)] +pub struct ContentType { + pub c_type: String, + pub c_subtype: Option, + pub attributes: Option>, +} + +impl MessageMetadataContents { + pub fn root_part(&self) -> &MessageMetadataPart { &self.parts[0] } } -impl<'x> MessageMetadataPart<'x> { - pub fn contents<'y>(&self, raw_message: &'y [u8]) -> Cow<'y, [u8]> { - let bytes = raw_message - .get(self.offset_body..self.offset_end) - .unwrap_or_default(); - match self.encoding { - Encoding::None => bytes.into(), - Encoding::QuotedPrintable => quoted_printable_decode(bytes).unwrap_or_default().into(), - Encoding::Base64 => base64_decode(bytes).unwrap_or_default().into(), - } +pub struct DecodedParts<'x> { + pub raw_messages: Vec>, + pub parts: Vec>, +} + +pub struct DecodedPart<'x> { + pub message_id: usize, + pub part_id: usize, + pub content: DecodedPartContent<'x>, +} + +pub enum DecodedPartContent<'x> { + Text(Cow<'x, str>), + Binary(Cow<'x, [u8]>), +} + +impl<'x> DecodedParts<'x> { + #[inline] + pub fn raw_message(&self, message_id: usize) -> Option<&[u8]> { + self.raw_messages.get(message_id).map(|m| m.as_ref()) } - pub fn decode_contents<'y>(&self, raw_message: &'y [u8]) -> PartType<'y> { - let bytes = self.contents(raw_message); - - match self.body { - MetadataPartType::Text | MetadataPartType::Html => { - let text = match ( - bytes, - self.headers - .header_value(&HeaderName::ContentType) - .and_then(|c| c.as_content_type()) - .and_then(|ct| { - ct.attribute("charset") - .and_then(|c| charset_decoder(c.as_bytes())) - }), - ) { - (Cow::Owned(vec), Some(charset_decoder)) => charset_decoder(&vec).into(), - (Cow::Owned(vec), None) => String::from_utf8(vec) - .unwrap_or_else(|e| String::from_utf8_lossy(e.as_bytes()).into_owned()) - .into(), - (Cow::Borrowed(bytes), Some(charset_decoder)) => charset_decoder(bytes).into(), - (Cow::Borrowed(bytes), None) => String::from_utf8_lossy(bytes), - }; - - if matches!(self.body, MetadataPartType::Text) { - PartType::Text(text) - } else { - PartType::Html(text) - } - } - MetadataPartType::Binary => PartType::Binary(bytes), - MetadataPartType::InlineBinary => PartType::InlineBinary(bytes), - MetadataPartType::Message(_) | MetadataPartType::Multipart(_) => unreachable!(), - } + #[inline] + pub fn raw_message_section(&self, message_id: usize, from: usize, to: usize) -> Option<&[u8]> { + self.raw_messages + .get(message_id) + .map(|m| m.as_ref()) + .and_then(|m| m.get(from..to)) } - pub fn remove_header(&mut self, header_name: &HeaderName) -> Option> { - for header in self.headers.iter_mut().rev() { - if header.name == *header_name { - return Some(std::mem::take(&mut header.value)); - } - } - None + #[inline] + pub fn raw_message_section_arch( + &self, + message_id: usize, + from: u32_le, + to: u32_le, + ) -> Option<&[u8]> { + self.raw_message_section(message_id, u32::from(from) as usize, u32::from(to) as usize) + } + + #[inline] + pub fn part(&self, message_id: usize, part_id: usize) -> Option<&DecodedPartContent<'x>> { + self.parts + .iter() + .find(|p| p.message_id == message_id && p.part_id == part_id) + .map(|p| &p.content) + } + + #[inline] + pub fn text_part(&self, message_id: usize, part_id: usize) -> Option<&str> { + self.part(message_id, part_id).and_then(|p| match p { + DecodedPartContent::Text(text) => Some(text.as_ref()), + DecodedPartContent::Binary(_) => None, + }) + } + + #[inline] + pub fn binary_part(&self, message_id: usize, part_id: usize) -> Option<&[u8]> { + self.part(message_id, part_id).and_then(|p| match p { + DecodedPartContent::Text(_) => None, + DecodedPartContent::Binary(binary) => Some(binary.as_ref()), + }) } } -impl<'x> From> for MessageMetadataContents<'x> { - fn from(value: Message<'x>) -> Self { +impl DecodedPartContent<'_> { + pub fn as_bytes(&self) -> &[u8] { + match self { + DecodedPartContent::Text(text) => text.as_bytes(), + DecodedPartContent::Binary(binary) => binary, + } + } + + #[allow(clippy::len_without_is_empty)] + pub fn len(&self) -> usize { + match self { + DecodedPartContent::Text(text) => text.len(), + DecodedPartContent::Binary(binary) => binary.len(), + } + } + + pub fn as_str(&self) -> &str { + match self { + DecodedPartContent::Text(text) => text, + DecodedPartContent::Binary(binary) => std::str::from_utf8(binary).unwrap_or_default(), + } + } +} + +impl ArchivedMessageMetadataContents { + pub fn decode_contents<'x>(&self, raw_message: &'x [u8]) -> DecodedParts<'x> { + let mut result = DecodedParts { + raw_messages: vec![raw_message.into()], + parts: Vec::new(), + }; + let mut parts_iter = self.parts.iter().enumerate(); + let mut iter_stack = Vec::new(); + let mut message_id = 0; + + loop { + while let Some((part_id, part)) = parts_iter.next() { + match &part.body { + ArchivedMetadataPartType::Text + | ArchivedMetadataPartType::Html + | ArchivedMetadataPartType::Binary + | ArchivedMetadataPartType::InlineBinary => { + match result.raw_messages.last().unwrap() { + Cow::Borrowed(raw_message) => { + result.parts.push(DecodedPart { + message_id, + part_id, + content: part.decode_contents(raw_message), + }); + } + Cow::Owned(raw_message) => { + result.parts.push(DecodedPart { + message_id, + part_id, + content: match part.decode_contents(raw_message) { + DecodedPartContent::Text(text) => { + DecodedPartContent::Text(text.into_owned().into()) + } + DecodedPartContent::Binary(binary) => { + DecodedPartContent::Binary(binary.into_owned().into()) + } + }, + }); + } + } + } + ArchivedMetadataPartType::Message(message) => { + match part.contents(raw_message) { + Cow::Borrowed(raw_message) => { + result.raw_messages.push(raw_message.into()); + } + Cow::Owned(raw_message) => { + result.raw_messages.push(raw_message.into()); + } + } + iter_stack.push((parts_iter, message_id)); + message_id += 1; + parts_iter = message.parts.iter().enumerate(); + } + _ => {} + } + } + if let Some((iter, prev_message_id)) = iter_stack.pop() { + parts_iter = iter; + message_id = prev_message_id; + } else { + break; + } + } + + result + } +} + +impl ArchivedMessageMetadataPart { + pub fn contents<'x>(&self, raw_message: &'x [u8]) -> Cow<'x, [u8]> { + let bytes = raw_message + .get(u32::from(self.offset_body) as usize..u32::from(self.offset_end) as usize) + .unwrap_or_default(); + match self.encoding { + ArchivedEncoding::None => bytes.into(), + ArchivedEncoding::QuotedPrintable => { + quoted_printable_decode(bytes).unwrap_or_default().into() + } + ArchivedEncoding::Base64 => base64_decode(bytes).unwrap_or_default().into(), + } + } + + pub fn decode_contents<'x>(&self, raw_message: &'x [u8]) -> DecodedPartContent<'x> { + let bytes = self.contents(raw_message); + + match self.body { + ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html => { + DecodedPartContent::Text( + match ( + bytes, + self.headers + .header_value(&ArchivedHeaderName::ContentType) + .and_then(|c| c.as_content_type()) + .and_then(|ct| { + ct.attribute("charset") + .and_then(|c| charset_decoder(c.as_bytes())) + }), + ) { + (Cow::Owned(vec), Some(charset_decoder)) => charset_decoder(&vec).into(), + (Cow::Owned(vec), None) => String::from_utf8(vec) + .unwrap_or_else(|e| String::from_utf8_lossy(e.as_bytes()).into_owned()) + .into(), + (Cow::Borrowed(bytes), Some(charset_decoder)) => { + charset_decoder(bytes).into() + } + (Cow::Borrowed(bytes), None) => String::from_utf8_lossy(bytes), + }, + ) + } + ArchivedMetadataPartType::Binary => DecodedPartContent::Binary(bytes), + ArchivedMetadataPartType::InlineBinary => DecodedPartContent::Binary(bytes), + ArchivedMetadataPartType::Message(_) | ArchivedMetadataPartType::Multipart(_) => { + unreachable!() + } + } + } +} + +impl From> for MessageMetadataContents { + fn from(value: mail_parser::Message<'_>) -> Self { MessageMetadataContents { - html_body: value.html_body, - text_body: value.text_body, - attachments: value.attachments, + html_body: value.html_body.into_iter().map(|c| c as u16).collect(), + text_body: value.text_body.into_iter().map(|c| c as u16).collect(), + attachments: value.attachments.into_iter().map(|c| c as u16).collect(), parts: value .parts .into_iter() @@ -188,7 +412,12 @@ impl<'x> From> for MessageMetadataContents<'x> { message.root_part().raw_len(), MetadataPartType::Message(message.into()), ), - PartType::Multipart(parts) => (0, MetadataPartType::Multipart(parts)), + PartType::Multipart(parts) => ( + 0, + MetadataPartType::Multipart( + parts.into_iter().map(|p| p as u16).collect(), + ), + ), }; MessageMetadataPart { @@ -198,46 +427,47 @@ impl<'x> From> for MessageMetadataContents<'x> { .map(|hdr| Header { value: if matches!( &hdr.name, - HeaderName::Subject - | HeaderName::From - | HeaderName::To - | HeaderName::Cc - | HeaderName::Date - | HeaderName::Bcc - | HeaderName::ReplyTo - | HeaderName::Sender - | HeaderName::Comments - | HeaderName::InReplyTo - | HeaderName::Keywords - | HeaderName::MessageId - | HeaderName::References - | HeaderName::ResentMessageId - | HeaderName::ContentDescription - | HeaderName::ContentId - | HeaderName::ContentLanguage - | HeaderName::ContentLocation - | HeaderName::ContentTransferEncoding - | HeaderName::ContentType - | HeaderName::ContentDisposition - | HeaderName::ListId + mail_parser::HeaderName::Subject + | mail_parser::HeaderName::From + | mail_parser::HeaderName::To + | mail_parser::HeaderName::Cc + | mail_parser::HeaderName::Date + | mail_parser::HeaderName::Bcc + | mail_parser::HeaderName::ReplyTo + | mail_parser::HeaderName::Sender + | mail_parser::HeaderName::Comments + | mail_parser::HeaderName::InReplyTo + | mail_parser::HeaderName::Keywords + | mail_parser::HeaderName::MessageId + | mail_parser::HeaderName::References + | mail_parser::HeaderName::ResentMessageId + | mail_parser::HeaderName::ContentDescription + | mail_parser::HeaderName::ContentId + | mail_parser::HeaderName::ContentLanguage + | mail_parser::HeaderName::ContentLocation + | mail_parser::HeaderName::ContentTransferEncoding + | mail_parser::HeaderName::ContentType + | mail_parser::HeaderName::ContentDisposition + | mail_parser::HeaderName::ListId ) { hdr.value } else { - HeaderValue::Empty - }, - name: hdr.name, - offset_field: hdr.offset_field, - offset_start: hdr.offset_start, - offset_end: hdr.offset_end, + mail_parser::HeaderValue::Empty + } + .into(), + name: hdr.name.into(), + offset_field: hdr.offset_field as u32, + offset_start: hdr.offset_start as u32, + offset_end: hdr.offset_end as u32, }) .collect(), is_encoding_problem: part.is_encoding_problem, - encoding: part.encoding, + encoding: part.encoding.into(), body, - size, - offset_header: part.offset_header, - offset_body: part.offset_body, - offset_end: part.offset_end, + size: size as u32, + offset_header: part.offset_header as u32, + offset_body: part.offset_body as u32, + offset_end: part.offset_end as u32, } }) .collect(), @@ -245,55 +475,692 @@ impl<'x> From> for MessageMetadataContents<'x> { } } -impl<'x> MetadataPartType<'x> { - fn unwrap_message(self) -> MessageMetadataContents<'x> { - match self { - MetadataPartType::Message(message) => message, - _ => panic!("unwrap_message called on non-message part"), +impl From for Encoding { + fn from(value: mail_parser::Encoding) -> Self { + match value { + mail_parser::Encoding::None => Encoding::None, + mail_parser::Encoding::QuotedPrintable => Encoding::QuotedPrintable, + mail_parser::Encoding::Base64 => Encoding::Base64, } } } -impl<'x> MimeHeaders<'x> for MessageMetadataPart<'x> { - fn content_description(&self) -> Option<&str> { - self.headers - .header_value(&HeaderName::ContentDescription) - .and_then(|header| header.as_text()) - } - - fn content_disposition(&self) -> Option<&ContentType> { - self.headers - .header_value(&HeaderName::ContentDisposition) - .and_then(|header| header.as_content_type()) - } - - fn content_id(&self) -> Option<&str> { - self.headers - .header_value(&HeaderName::ContentId) - .and_then(|header| header.as_text()) - } - - fn content_transfer_encoding(&self) -> Option<&str> { - self.headers - .header_value(&HeaderName::ContentTransferEncoding) - .and_then(|header| header.as_text()) - } - - fn content_type(&self) -> Option<&ContentType> { - self.headers - .header_value(&HeaderName::ContentType) - .and_then(|header| header.as_content_type()) - } - - fn content_language(&self) -> &HeaderValue { - self.headers - .header_value(&HeaderName::ContentLanguage) - .unwrap_or(&HeaderValue::Empty) - } - - fn content_location(&self) -> Option<&str> { - self.headers - .header_value(&HeaderName::ContentLocation) - .and_then(|header| header.as_text()) +impl From> for HeaderName { + fn from(value: mail_parser::HeaderName<'_>) -> Self { + match value { + mail_parser::HeaderName::Subject => HeaderName::Subject, + mail_parser::HeaderName::From => HeaderName::From, + mail_parser::HeaderName::To => HeaderName::To, + mail_parser::HeaderName::Cc => HeaderName::Cc, + mail_parser::HeaderName::Date => HeaderName::Date, + mail_parser::HeaderName::Bcc => HeaderName::Bcc, + mail_parser::HeaderName::ReplyTo => HeaderName::ReplyTo, + mail_parser::HeaderName::Sender => HeaderName::Sender, + mail_parser::HeaderName::Comments => HeaderName::Comments, + mail_parser::HeaderName::InReplyTo => HeaderName::InReplyTo, + mail_parser::HeaderName::Keywords => HeaderName::Keywords, + mail_parser::HeaderName::Received => HeaderName::Received, + mail_parser::HeaderName::MessageId => HeaderName::MessageId, + mail_parser::HeaderName::References => HeaderName::References, + mail_parser::HeaderName::ReturnPath => HeaderName::ReturnPath, + mail_parser::HeaderName::MimeVersion => HeaderName::MimeVersion, + mail_parser::HeaderName::ContentDescription => HeaderName::ContentDescription, + mail_parser::HeaderName::ContentId => HeaderName::ContentId, + mail_parser::HeaderName::ContentLanguage => HeaderName::ContentLanguage, + mail_parser::HeaderName::ContentLocation => HeaderName::ContentLocation, + mail_parser::HeaderName::ContentTransferEncoding => HeaderName::ContentTransferEncoding, + mail_parser::HeaderName::ContentType => HeaderName::ContentType, + mail_parser::HeaderName::ContentDisposition => HeaderName::ContentDisposition, + mail_parser::HeaderName::ResentTo => HeaderName::ResentTo, + mail_parser::HeaderName::ResentFrom => HeaderName::ResentFrom, + mail_parser::HeaderName::ResentBcc => HeaderName::ResentBcc, + mail_parser::HeaderName::ResentCc => HeaderName::ResentCc, + mail_parser::HeaderName::ResentSender => HeaderName::ResentSender, + mail_parser::HeaderName::ResentDate => HeaderName::ResentDate, + mail_parser::HeaderName::ResentMessageId => HeaderName::ResentMessageId, + mail_parser::HeaderName::ListArchive => HeaderName::ListArchive, + mail_parser::HeaderName::ListHelp => HeaderName::ListHelp, + mail_parser::HeaderName::ListId => HeaderName::ListId, + mail_parser::HeaderName::ListOwner => HeaderName::ListOwner, + mail_parser::HeaderName::ListPost => HeaderName::ListPost, + mail_parser::HeaderName::ListSubscribe => HeaderName::ListSubscribe, + mail_parser::HeaderName::ListUnsubscribe => HeaderName::ListUnsubscribe, + mail_parser::HeaderName::Other(other) => HeaderName::Other(other.into_owned()), + mail_parser::HeaderName::DkimSignature => HeaderName::DkimSignature, + mail_parser::HeaderName::ArcAuthenticationResults => { + HeaderName::ArcAuthenticationResults + } + mail_parser::HeaderName::ArcMessageSignature => HeaderName::ArcMessageSignature, + mail_parser::HeaderName::ArcSeal => HeaderName::ArcSeal, + _ => unreachable!(), + } + } +} + +impl From> for HeaderValue { + fn from(value: mail_parser::HeaderValue) -> Self { + match value { + mail_parser::HeaderValue::Address(address) => HeaderValue::Address(address.into()), + mail_parser::HeaderValue::Text(cow) => HeaderValue::Text(cow.into_owned()), + mail_parser::HeaderValue::TextList(cows) => { + HeaderValue::TextList(cows.into_iter().map(|cow| cow.into_owned()).collect()) + } + mail_parser::HeaderValue::DateTime(date_time) => { + HeaderValue::DateTime(date_time.to_timestamp()) + } + mail_parser::HeaderValue::ContentType(content_type) => { + HeaderValue::ContentType(content_type.into()) + } + mail_parser::HeaderValue::Received(_) | mail_parser::HeaderValue::Empty => { + HeaderValue::Empty + } + } + } +} + +impl From> for ContentType { + fn from(value: mail_parser::ContentType<'_>) -> Self { + ContentType { + c_type: value.c_type.into_owned(), + c_subtype: value.c_subtype.map(|cow| cow.into_owned()), + attributes: value.attributes.map(|attrs| { + attrs + .into_iter() + .map(|(k, v)| (k.into_owned(), v.into_owned())) + .collect() + }), + } + } +} + +impl From> for Address { + fn from(value: mail_parser::Address<'_>) -> Self { + match value { + mail_parser::Address::List(addrs) => { + Address::List(addrs.into_iter().map(|addr| addr.into()).collect()) + } + mail_parser::Address::Group(groups) => { + Address::Group(groups.into_iter().map(|group| group.into()).collect()) + } + } + } +} + +impl From> for Addr { + fn from(value: mail_parser::Addr<'_>) -> Self { + Addr { + name: value.name.map(|cow| cow.into_owned()), + address: value.address.map(|cow| cow.into_owned()), + } + } +} + +impl From> for Group { + fn from(value: mail_parser::Group<'_>) -> Self { + Group { + name: value.name.map(|cow| cow.into_owned()), + addresses: value + .addresses + .into_iter() + .map(|addr| addr.into()) + .collect(), + } + } +} + +impl ArchivedMessageMetadataPart { + pub fn is_message(&self) -> bool { + matches!(self.body, ArchivedMetadataPartType::Message(_)) + } + + pub fn sub_parts(&self) -> Option<&ArchivedVec> { + if let ArchivedMetadataPartType::Multipart(parts) = &self.body { + Some(parts) + } else { + None + } + } + + pub fn raw_len(&self) -> usize { + (u32::from(self.offset_end)).saturating_sub(u32::from(self.offset_header)) as usize + } + + pub fn header_values( + &self, + name: ArchivedHeaderName, + ) -> impl Iterator + Sync + Send { + self.headers.iter().filter_map(move |header| { + if header.name == name { + Some(&header.value) + } else { + None + } + }) + } + + pub fn subject(&self) -> Option<&str> { + self.headers + .header_value(&ArchivedHeaderName::Subject) + .and_then(|header| header.as_text()) + } + + pub fn date(&self) -> Option { + self.headers + .header_value(&ArchivedHeaderName::Date) + .and_then(|header| header.as_datetime()) + } + + pub fn message_id(&self) -> Option<&str> { + self.headers + .header_value(&ArchivedHeaderName::MessageId) + .and_then(|header| header.as_text()) + } + + pub fn in_reply_to(&self) -> &ArchivedHeaderValue { + self.headers + .header_value(&ArchivedHeaderName::InReplyTo) + .unwrap_or(&ArchivedHeaderValue::Empty) + } + + pub fn content_description(&self) -> Option<&str> { + self.headers + .header_value(&ArchivedHeaderName::ContentDescription) + .and_then(|header| header.as_text()) + } + + pub fn content_disposition(&self) -> Option<&ArchivedContentType> { + self.headers + .header_value(&ArchivedHeaderName::ContentDisposition) + .and_then(|header| header.as_content_type()) + } + + pub fn content_id(&self) -> Option<&str> { + self.headers + .header_value(&ArchivedHeaderName::ContentId) + .and_then(|header| header.as_text()) + } + + pub fn content_transfer_encoding(&self) -> Option<&str> { + self.headers + .header_value(&ArchivedHeaderName::ContentTransferEncoding) + .and_then(|header| header.as_text()) + } + + pub fn content_type(&self) -> Option<&ArchivedContentType> { + self.headers + .header_value(&ArchivedHeaderName::ContentType) + .and_then(|header| header.as_content_type()) + } + + pub fn content_language(&self) -> &ArchivedHeaderValue { + self.headers + .header_value(&ArchivedHeaderName::ContentLanguage) + .unwrap_or(&ArchivedHeaderValue::Empty) + } + + pub fn content_location(&self) -> Option<&str> { + self.headers + .header_value(&ArchivedHeaderName::ContentLocation) + .and_then(|header| header.as_text()) + } + + pub fn attachment_name(&self) -> Option<&str> { + self.content_disposition() + .and_then(|cd| cd.attribute("filename")) + .or_else(|| self.content_type().and_then(|ct| ct.attribute("name"))) + } +} + +pub trait GetHeader { + fn header_value(&self, name: &HeaderName) -> Option<&HeaderValue>; + fn header(&self, name: impl Into) -> Option<&Header>; +} + +impl GetHeader for Vec
{ + fn header_value(&self, name: &HeaderName) -> Option<&HeaderValue> { + self.iter() + .rev() + .find(|header| &header.name == name) + .map(|header| &header.value) + } + + fn header(&self, name: impl Into) -> Option<&Header> { + let name = name.into(); + self.iter().rev().find(|header| header.name == name) + } +} + +pub trait ArchivedGetHeader { + fn header_value(&self, name: &ArchivedHeaderName) -> Option<&ArchivedHeaderValue>; + fn header(&self, name: impl Into) -> Option<&ArchivedHeader>; + fn convert_header( + &self, + header_name: &ArchivedHeaderName, + ) -> Option>; +} + +impl ArchivedGetHeader for ArchivedVec { + fn header_value(&self, name: &ArchivedHeaderName) -> Option<&ArchivedHeaderValue> { + self.iter() + .rev() + .find(|header| &header.name == name) + .map(|header| &header.value) + } + + fn header(&self, name: impl Into) -> Option<&ArchivedHeader> { + let name = name.into(); + self.iter().rev().find(|header| header.name == name) + } + + fn convert_header( + &self, + header_name: &ArchivedHeaderName, + ) -> Option> { + for header in self.iter().rev() { + if header.name == *header_name { + return Some(mail_parser::HeaderValue::from(&header.value)); + } + } + None + } +} + +impl HeaderValue { + pub fn as_text(&self) -> Option<&str> { + match *self { + HeaderValue::Text(ref s) => Some(s), + HeaderValue::TextList(ref l) => l.last().map(|s| s.as_str()), + _ => None, + } + } + + pub fn as_content_type(&self) -> Option<&ContentType> { + match *self { + HeaderValue::ContentType(ref c) => Some(c), + _ => None, + } + } +} + +impl ArchivedHeaderValue { + pub fn as_text(&self) -> Option<&str> { + match *self { + ArchivedHeaderValue::Text(ref s) => Some(s), + ArchivedHeaderValue::TextList(ref l) => l.last().map(|s| s.as_str()), + _ => None, + } + } + + pub fn as_content_type(&self) -> Option<&ArchivedContentType> { + match *self { + ArchivedHeaderValue::ContentType(ref c) => Some(c), + _ => None, + } + } + + pub fn as_text_list(&self) -> Option<&[ArchivedString]> { + match *self { + ArchivedHeaderValue::Text(ref s) => Some(std::slice::from_ref(s)), + ArchivedHeaderValue::TextList(ref l) => Some(l.as_slice()), + _ => None, + } + } + + pub fn as_datetime(&self) -> Option { + match self { + ArchivedHeaderValue::DateTime(d) => Some(i64::from(d)), + _ => None, + } + } +} + +impl HeaderName { + pub fn id(&self) -> u8 { + match self { + HeaderName::Subject => 0, + HeaderName::From => 1, + HeaderName::To => 2, + HeaderName::Cc => 3, + HeaderName::Date => 4, + HeaderName::Bcc => 5, + HeaderName::ReplyTo => 6, + HeaderName::Sender => 7, + HeaderName::Comments => 8, + HeaderName::InReplyTo => 9, + HeaderName::Keywords => 10, + HeaderName::Received => 11, + HeaderName::MessageId => 12, + HeaderName::References => 13, + HeaderName::ReturnPath => 14, + HeaderName::MimeVersion => 15, + HeaderName::ContentDescription => 16, + HeaderName::ContentId => 17, + HeaderName::ContentLanguage => 18, + HeaderName::ContentLocation => 19, + HeaderName::ContentTransferEncoding => 20, + HeaderName::ContentType => 21, + HeaderName::ContentDisposition => 22, + HeaderName::ResentTo => 23, + HeaderName::ResentFrom => 24, + HeaderName::ResentBcc => 25, + HeaderName::ResentCc => 26, + HeaderName::ResentSender => 27, + HeaderName::ResentDate => 28, + HeaderName::ResentMessageId => 29, + HeaderName::ListArchive => 30, + HeaderName::ListHelp => 31, + HeaderName::ListId => 32, + HeaderName::ListOwner => 33, + HeaderName::ListPost => 34, + HeaderName::ListSubscribe => 35, + HeaderName::ListUnsubscribe => 36, + HeaderName::Other(_) => 37, + HeaderName::ArcAuthenticationResults => 38, + HeaderName::ArcMessageSignature => 39, + HeaderName::ArcSeal => 40, + HeaderName::DkimSignature => 41, + } + } +} + +impl ArchivedHeaderName { + pub fn id(&self) -> u8 { + match self { + ArchivedHeaderName::Subject => 0, + ArchivedHeaderName::From => 1, + ArchivedHeaderName::To => 2, + ArchivedHeaderName::Cc => 3, + ArchivedHeaderName::Date => 4, + ArchivedHeaderName::Bcc => 5, + ArchivedHeaderName::ReplyTo => 6, + ArchivedHeaderName::Sender => 7, + ArchivedHeaderName::Comments => 8, + ArchivedHeaderName::InReplyTo => 9, + ArchivedHeaderName::Keywords => 10, + ArchivedHeaderName::Received => 11, + ArchivedHeaderName::MessageId => 12, + ArchivedHeaderName::References => 13, + ArchivedHeaderName::ReturnPath => 14, + ArchivedHeaderName::MimeVersion => 15, + ArchivedHeaderName::ContentDescription => 16, + ArchivedHeaderName::ContentId => 17, + ArchivedHeaderName::ContentLanguage => 18, + ArchivedHeaderName::ContentLocation => 19, + ArchivedHeaderName::ContentTransferEncoding => 20, + ArchivedHeaderName::ContentType => 21, + ArchivedHeaderName::ContentDisposition => 22, + ArchivedHeaderName::ResentTo => 23, + ArchivedHeaderName::ResentFrom => 24, + ArchivedHeaderName::ResentBcc => 25, + ArchivedHeaderName::ResentCc => 26, + ArchivedHeaderName::ResentSender => 27, + ArchivedHeaderName::ResentDate => 28, + ArchivedHeaderName::ResentMessageId => 29, + ArchivedHeaderName::ListArchive => 30, + ArchivedHeaderName::ListHelp => 31, + ArchivedHeaderName::ListId => 32, + ArchivedHeaderName::ListOwner => 33, + ArchivedHeaderName::ListPost => 34, + ArchivedHeaderName::ListSubscribe => 35, + ArchivedHeaderName::ListUnsubscribe => 36, + ArchivedHeaderName::Other(_) => 37, + ArchivedHeaderName::ArcAuthenticationResults => 38, + ArchivedHeaderName::ArcMessageSignature => 39, + ArchivedHeaderName::ArcSeal => 40, + ArchivedHeaderName::DkimSignature => 41, + } + } + + pub fn as_str(&self) -> &str { + match self { + ArchivedHeaderName::Subject => "Subject", + ArchivedHeaderName::From => "From", + ArchivedHeaderName::To => "To", + ArchivedHeaderName::Cc => "Cc", + ArchivedHeaderName::Date => "Date", + ArchivedHeaderName::Bcc => "Bcc", + ArchivedHeaderName::ReplyTo => "Reply-To", + ArchivedHeaderName::Sender => "Sender", + ArchivedHeaderName::Comments => "Comments", + ArchivedHeaderName::InReplyTo => "In-Reply-To", + ArchivedHeaderName::Keywords => "Keywords", + ArchivedHeaderName::Received => "Received", + ArchivedHeaderName::MessageId => "Message-ID", + ArchivedHeaderName::References => "References", + ArchivedHeaderName::ReturnPath => "Return-Path", + ArchivedHeaderName::MimeVersion => "MIME-Version", + ArchivedHeaderName::ContentDescription => "Content-Description", + ArchivedHeaderName::ContentId => "Content-ID", + ArchivedHeaderName::ContentLanguage => "Content-Language", + ArchivedHeaderName::ContentLocation => "Content-Location", + ArchivedHeaderName::ContentTransferEncoding => "Content-Transfer-Encoding", + ArchivedHeaderName::ContentType => "Content-Type", + ArchivedHeaderName::ContentDisposition => "Content-Disposition", + ArchivedHeaderName::ResentTo => "Resent-To", + ArchivedHeaderName::ResentFrom => "Resent-From", + ArchivedHeaderName::ResentBcc => "Resent-Bcc", + ArchivedHeaderName::ResentCc => "Resent-Cc", + ArchivedHeaderName::ResentSender => "Resent-Sender", + ArchivedHeaderName::ResentDate => "Resent-Date", + ArchivedHeaderName::ResentMessageId => "Resent-Message-ID", + ArchivedHeaderName::ListArchive => "List-Archive", + ArchivedHeaderName::ListHelp => "List-Help", + ArchivedHeaderName::ListId => "List-ID", + ArchivedHeaderName::ListOwner => "List-Owner", + ArchivedHeaderName::ListPost => "List-Post", + ArchivedHeaderName::ListSubscribe => "List-Subscribe", + ArchivedHeaderName::ListUnsubscribe => "List-Unsubscribe", + ArchivedHeaderName::ArcAuthenticationResults => "ARC-Authentication-Results", + ArchivedHeaderName::ArcMessageSignature => "ARC-Message-Signature", + ArchivedHeaderName::ArcSeal => "ARC-Seal", + ArchivedHeaderName::DkimSignature => "DKIM-Signature", + ArchivedHeaderName::Other(v) => v.as_str(), + } + } + + pub fn is_mime_header(&self) -> bool { + matches!( + self, + ArchivedHeaderName::ContentDescription + | ArchivedHeaderName::ContentId + | ArchivedHeaderName::ContentLanguage + | ArchivedHeaderName::ContentLocation + | ArchivedHeaderName::ContentTransferEncoding + | ArchivedHeaderName::ContentType + | ArchivedHeaderName::ContentDisposition + ) + } +} + +impl Display for ArchivedHeaderName { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.as_str()) + } +} + +impl ArchivedContentType { + pub fn ctype(&self) -> &str { + &self.c_type + } + + pub fn subtype(&self) -> Option<&str> { + self.c_subtype.as_deref() + } + + pub fn attribute(&self, name: &str) -> Option<&str> { + self.attributes + .as_ref()? + .iter() + .find(|k| k.0 == name)? + .1 + .as_ref() + .into() + } +} + +impl ArchivedMessageMetadataContents { + pub fn root_part(&self) -> &ArchivedMessageMetadataPart { + &self.parts[0] + } +} + +impl ArchivedAddress { + pub fn iter(&self) -> Box + '_ + Sync + Send> { + match self { + ArchivedAddress::List(list) => Box::new(list.iter()), + ArchivedAddress::Group(group) => { + Box::new(group.iter().flat_map(|group| group.addresses.iter())) + } + } + } +} + +impl ArchivedAddr { + pub fn name(&self) -> Option<&str> { + self.name.as_deref() + } + + pub fn address(&self) -> Option<&str> { + self.address.as_deref() + } +} + +impl From<&ArchivedHeaderName> for mail_parser::HeaderName<'static> { + fn from(value: &ArchivedHeaderName) -> Self { + match value { + ArchivedHeaderName::Subject => mail_parser::HeaderName::Subject, + ArchivedHeaderName::From => mail_parser::HeaderName::From, + ArchivedHeaderName::To => mail_parser::HeaderName::To, + ArchivedHeaderName::Cc => mail_parser::HeaderName::Cc, + ArchivedHeaderName::Date => mail_parser::HeaderName::Date, + ArchivedHeaderName::Bcc => mail_parser::HeaderName::Bcc, + ArchivedHeaderName::ReplyTo => mail_parser::HeaderName::ReplyTo, + ArchivedHeaderName::Sender => mail_parser::HeaderName::Sender, + ArchivedHeaderName::Comments => mail_parser::HeaderName::Comments, + ArchivedHeaderName::InReplyTo => mail_parser::HeaderName::InReplyTo, + ArchivedHeaderName::Keywords => mail_parser::HeaderName::Keywords, + ArchivedHeaderName::Received => mail_parser::HeaderName::Received, + ArchivedHeaderName::MessageId => mail_parser::HeaderName::MessageId, + ArchivedHeaderName::References => mail_parser::HeaderName::References, + ArchivedHeaderName::ReturnPath => mail_parser::HeaderName::ReturnPath, + ArchivedHeaderName::MimeVersion => mail_parser::HeaderName::MimeVersion, + ArchivedHeaderName::ContentDescription => mail_parser::HeaderName::ContentDescription, + ArchivedHeaderName::ContentId => mail_parser::HeaderName::ContentId, + ArchivedHeaderName::ContentLanguage => mail_parser::HeaderName::ContentLanguage, + ArchivedHeaderName::ContentLocation => mail_parser::HeaderName::ContentLocation, + ArchivedHeaderName::ContentTransferEncoding => { + mail_parser::HeaderName::ContentTransferEncoding + } + ArchivedHeaderName::ContentType => mail_parser::HeaderName::ContentType, + ArchivedHeaderName::ContentDisposition => mail_parser::HeaderName::ContentDisposition, + ArchivedHeaderName::ResentTo => mail_parser::HeaderName::ResentTo, + ArchivedHeaderName::ResentFrom => mail_parser::HeaderName::ResentFrom, + ArchivedHeaderName::ResentBcc => mail_parser::HeaderName::ResentBcc, + ArchivedHeaderName::ResentCc => mail_parser::HeaderName::ResentCc, + ArchivedHeaderName::ResentSender => mail_parser::HeaderName::ResentSender, + ArchivedHeaderName::ResentDate => mail_parser::HeaderName::ResentDate, + ArchivedHeaderName::ResentMessageId => mail_parser::HeaderName::ResentMessageId, + ArchivedHeaderName::ListArchive => mail_parser::HeaderName::ListArchive, + ArchivedHeaderName::ListHelp => mail_parser::HeaderName::ListHelp, + ArchivedHeaderName::ListId => mail_parser::HeaderName::ListId, + ArchivedHeaderName::ListOwner => mail_parser::HeaderName::ListOwner, + ArchivedHeaderName::ListPost => mail_parser::HeaderName::ListPost, + ArchivedHeaderName::ListSubscribe => mail_parser::HeaderName::ListSubscribe, + ArchivedHeaderName::ListUnsubscribe => mail_parser::HeaderName::ListUnsubscribe, + ArchivedHeaderName::Other(other) => { + mail_parser::HeaderName::Other(other.to_string().into()) + } + ArchivedHeaderName::ArcAuthenticationResults => { + mail_parser::HeaderName::ArcAuthenticationResults + } + ArchivedHeaderName::ArcMessageSignature => mail_parser::HeaderName::ArcMessageSignature, + ArchivedHeaderName::ArcSeal => mail_parser::HeaderName::ArcSeal, + ArchivedHeaderName::DkimSignature => mail_parser::HeaderName::DkimSignature, + } + } +} + +impl From<&ArchivedHeaderValue> for mail_parser::HeaderValue<'static> { + fn from(value: &ArchivedHeaderValue) -> Self { + match value { + ArchivedHeaderValue::Text(s) => mail_parser::HeaderValue::Text(s.to_string().into()), + ArchivedHeaderValue::TextList(list) => mail_parser::HeaderValue::TextList( + list.iter().map(|s| s.to_string().into()).collect(), + ), + ArchivedHeaderValue::DateTime(d) => { + mail_parser::HeaderValue::DateTime(DateTime::from_timestamp(i64::from(d))) + } + ArchivedHeaderValue::ContentType(ct) => { + mail_parser::HeaderValue::ContentType(ct.into()) + } + ArchivedHeaderValue::Empty => mail_parser::HeaderValue::Empty, + ArchivedHeaderValue::Address(a) => mail_parser::HeaderValue::Address(a.into()), + } + } +} + +impl From<&ArchivedAddress> for mail_parser::Address<'static> { + fn from(value: &ArchivedAddress) -> Self { + match value { + ArchivedAddress::List(list) => { + mail_parser::Address::List(list.iter().map(Into::into).collect()) + } + ArchivedAddress::Group(groups) => { + mail_parser::Address::Group(groups.iter().map(Into::into).collect()) + } + } + } +} + +impl From<&ArchivedContentType> for mail_parser::ContentType<'static> { + fn from(value: &ArchivedContentType) -> Self { + mail_parser::ContentType { + c_type: value.c_type.to_string().into(), + c_subtype: value.subtype().map(|s| s.to_string().into()), + attributes: value.attributes.as_ref().map(|attrs| { + attrs + .iter() + .map(|a| (a.0.to_string().into(), a.0.to_string().into())) + .collect() + }), + } + } +} + +impl From<&ArchivedGroup> for mail_parser::Group<'static> { + fn from(value: &ArchivedGroup) -> Self { + mail_parser::Group { + name: value.name.as_ref().map(|s| s.to_string().into()), + addresses: value.addresses.iter().map(|a| a.into()).collect(), + } + } +} + +impl From<&ArchivedAddr> for mail_parser::Addr<'static> { + fn from(value: &ArchivedAddr) -> Self { + mail_parser::Addr { + name: value.name().map(|s| s.to_string().into()), + address: value.address().map(|s| s.to_string().into()), + } + } +} + +impl Encoding { + pub fn id(&self) -> u8 { + match self { + Encoding::None => 0, + Encoding::QuotedPrintable => 1, + Encoding::Base64 => 2, + } + } +} + +impl ArchivedEncoding { + pub fn id(&self) -> u8 { + match self { + ArchivedEncoding::None => 0, + ArchivedEncoding::QuotedPrintable => 1, + ArchivedEncoding::Base64 => 2, + } } } diff --git a/crates/email/src/sieve/activate.rs b/crates/email/src/sieve/activate.rs index 04cf759c..479f5a68 100644 --- a/crates/email/src/sieve/activate.rs +++ b/crates/email/src/sieve/activate.rs @@ -9,11 +9,11 @@ use jmap_proto::types::{collection::Collection, property::Property}; use store::{ SerializeInfallible, query::Filter, - write::{ArchivedValue, BatchBuilder, assert::HashedValue}, + write::{Archive, BatchBuilder, assert::HashedValue}, }; use trc::AddContext; -use super::ArchivedSieveScript; +use super::{ArchivedSieveScript, SieveScript}; pub trait SieveScriptActivate: Sync + Send { fn sieve_activate_script( @@ -59,7 +59,7 @@ impl SieveScriptActivate for Server { // Deactivate scripts for document_id in active_ids { if let Some(sieve) = self - .get_property::>>( + .get_property::>( account_id, Collection::SieveScript, document_id, @@ -67,7 +67,9 @@ impl SieveScriptActivate for Server { ) .await? { - let sieve = sieve.into_deserialized().caused_by(trc::location!())?; + let sieve = sieve + .into_deserialized::() + .caused_by(trc::location!())?; let mut new_sieve = sieve.inner.clone(); new_sieve.is_active = false; batch @@ -86,7 +88,7 @@ impl SieveScriptActivate for Server { // Activate script if let Some(document_id) = activate_id { if let Some(sieve) = self - .get_property::>>( + .get_property::>( account_id, Collection::SieveScript, document_id, @@ -94,7 +96,9 @@ impl SieveScriptActivate for Server { ) .await? { - let sieve = sieve.into_deserialized().caused_by(trc::location!())?; + let sieve = sieve + .into_deserialized::() + .caused_by(trc::location!())?; let mut new_sieve = sieve.inner.clone(); new_sieve.is_active = true; batch diff --git a/crates/email/src/sieve/delete.rs b/crates/email/src/sieve/delete.rs index 1bbead79..131ed8b0 100644 --- a/crates/email/src/sieve/delete.rs +++ b/crates/email/src/sieve/delete.rs @@ -6,10 +6,10 @@ use common::{Server, auth::ResourceToken, storage::index::ObjectIndexBuilder}; use jmap_proto::types::{collection::Collection, property::Property}; -use store::write::{ArchivedValue, BatchBuilder, BlobOp, assert::HashedValue}; +use store::write::{Archive, BatchBuilder, BlobOp, assert::HashedValue}; use trc::AddContext; -use super::ArchivedSieveScript; +use super::{ArchivedSieveScript, SieveScript}; pub trait SieveScriptDelete: Sync + Send { fn sieve_script_delete( @@ -30,7 +30,7 @@ impl SieveScriptDelete for Server { // Fetch record let account_id = resource_token.account_id; let obj = self - .get_property::>>( + .get_property::>( account_id, Collection::SieveScript, document_id, @@ -43,7 +43,7 @@ impl SieveScriptDelete for Server { .caused_by(trc::location!()) .document_id(document_id) })? - .into_deserialized() + .into_deserialized::() .caused_by(trc::location!())?; // Make sure the script is not active diff --git a/crates/email/src/sieve/ingest.rs b/crates/email/src/sieve/ingest.rs index 5a8ec1ce..7af3d64e 100644 --- a/crates/email/src/sieve/ingest.rs +++ b/crates/email/src/sieve/ingest.rs @@ -24,14 +24,14 @@ use store::{ Deserialize, Serialize, SerializeInfallible, ahash::AHashSet, query::Filter, - write::{ArchivedValue, BatchBuilder, Bincode, BlobOp, assert::HashedValue, now}, + write::{Archive, Archiver, BatchBuilder, BlobOp, LegacyBincode, assert::HashedValue, now}, }; use trc::{AddContext, SieveEvent}; use utils::config::utils::ParseValue; use std::future::Future; -use super::{ActiveScript, ArchivedSieveScript, SeenIdHash, SeenIds}; +use super::{ActiveScript, ArchivedSeenIds, ArchivedSieveScript, SeenIdHash, SeenIds}; struct SieveMessage<'x> { pub raw_message: Cow<'x, [u8]>, @@ -79,7 +79,7 @@ impl SieveScriptIngest for Server { envelope_from: &str, envelope_to: &str, session_id: u64, - mut active_script: ActiveScript, + active_script: ActiveScript, autogenerated: &mut Vec, ) -> trc::Result { // Parse message @@ -147,6 +147,13 @@ impl SieveScriptIngest for Server { size: raw_message.len(), imap_uids: Vec::new(), }; + let mut seen_ids = if let Some(seen_ids) = active_script.seen_ids { + seen_ids + .deserialize::() + .caused_by(trc::location!())? + } else { + SeenIds::default() + }; while let Some(event) = instance.run(input) { match event { @@ -185,8 +192,10 @@ impl SieveScriptIngest for Server { } else { let mut mailbox_id = u32::MAX; if let Ok(role) = SpecialUse::parse_value(&role) { - if let Ok(Some(mailbox_id_)) = - self.mailbox_get_by_role(account_id, role).await + if let Some(mailbox_id_) = self + .mailbox_get_by_role(account_id, role) + .await + .caused_by(trc::location!())? { mailbox_id = mailbox_id_; } @@ -201,8 +210,8 @@ impl SieveScriptIngest for Server { match mailbox { Mailbox::Name(name) => { if !matches!( - self.mailbox_get_by_name(account_id, &name).await, - Ok(Some(document_id)) if special_use_ids.is_empty() || + self.mailbox_get_by_name(account_id, &name).await.caused_by(trc::location!())?, + Some(document_id) if special_use_ids.is_empty() || special_use_ids.contains(&document_id) ) { result = false; @@ -249,7 +258,7 @@ impl SieveScriptIngest for Server { } Event::DuplicateId { id, expiry, last } => { let id_hash = SeenIdHash::new(&id, expiry + now); - let seen_id = active_script.seen_ids.ids.contains(&id_hash); + let seen_id = seen_ids.ids.contains(&id_hash); if !seen_id || last { new_ids.insert(id_hash); } @@ -522,8 +531,8 @@ impl SieveScriptIngest for Server { } // Save new ids script changes - if !new_ids.is_empty() || active_script.seen_ids.has_changes { - active_script.seen_ids.ids.extend(new_ids); + if !new_ids.is_empty() || seen_ids.has_changes { + seen_ids.ids.extend(new_ids); let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) @@ -531,7 +540,7 @@ impl SieveScriptIngest for Server { .update_document(active_script.document_id) .set( Property::EmailIds, - Bincode::new(active_script.seen_ids) + Archiver::new(seen_ids) .serialize() .caused_by(trc::location!())?, ); @@ -575,15 +584,13 @@ impl SieveScriptIngest for Server { script: Arc::new(script), script_name, seen_ids: self - .get_property::>( + .get_property::( account_id, Collection::SieveScript, document_id, Property::EmailIds, ) - .await? - .map(|seen_ids| seen_ids.inner) - .unwrap_or_default(), + .await?, })) } else { Ok(None) @@ -624,7 +631,7 @@ impl SieveScriptIngest for Server { ) -> trc::Result<(Sieve, String)> { // Obtain script object let script_object = self - .get_property::>>( + .get_property::>( account_id, Collection::SieveScript, document_id, @@ -641,7 +648,7 @@ impl SieveScriptIngest for Server { // Obtain the sieve script length let unarchived_script = script_object .inner - .unarchive() + .unarchive::() .caused_by(trc::location!())?; let script_offset = u32::from(unarchived_script.size) as usize; @@ -663,7 +670,7 @@ impl SieveScriptIngest for Server { // Obtain the precompiled script if let Some(sieve) = script_bytes .get(script_offset..) - .and_then(|bytes| Bincode::::deserialize(bytes).ok()) + .and_then(|bytes| LegacyBincode::::deserialize(bytes).ok()) { Ok((sieve.inner, unarchived_script.name.to_string())) } else { @@ -678,7 +685,7 @@ impl SieveScriptIngest for Server { ) { Ok(sieve) => { // Store updated compiled sieve script - let sieve = Bincode::new(sieve); + let sieve = LegacyBincode::new(sieve); let compiled_bytes = sieve.serialize().caused_by(trc::location!())?; let mut updated_sieve_bytes = Vec::with_capacity(script_offset + compiled_bytes.len()); diff --git a/crates/email/src/sieve/mod.rs b/crates/email/src/sieve/mod.rs index ac40003c..8a26328d 100644 --- a/crates/email/src/sieve/mod.rs +++ b/crates/email/src/sieve/mod.rs @@ -7,7 +7,7 @@ use std::{collections::HashSet, sync::Arc}; use sieve::Sieve; -use store::{ahash::RandomState, blake3}; +use store::{ahash::RandomState, blake3, write::Archive}; use utils::BlobHash; pub mod activate; @@ -21,7 +21,7 @@ pub struct ActiveScript { pub document_id: u32, pub script_name: String, pub script: Arc, - pub seen_ids: SeenIds, + pub seen_ids: Option, } #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone)] diff --git a/crates/imap-proto/src/protocol/mod.rs b/crates/imap-proto/src/protocol/mod.rs index 6f5932ba..7ffb0ffc 100644 --- a/crates/imap-proto/src/protocol/mod.rs +++ b/crates/imap-proto/src/protocol/mod.rs @@ -8,7 +8,7 @@ use std::{cmp::Ordering, fmt::Display}; use ahash::AHashSet; use chrono::{DateTime, Utc}; -use jmap_proto::types::keyword::Keyword; +use jmap_proto::types::keyword::{ArchivedKeyword, Keyword}; use crate::{Command, ResponseCode, ResponseType, StatusResponse}; @@ -282,6 +282,26 @@ impl From for Flag { } } +impl From<&ArchivedKeyword> for Flag { + fn from(value: &ArchivedKeyword) -> Self { + match value { + ArchivedKeyword::Seen => Flag::Seen, + ArchivedKeyword::Draft => Flag::Draft, + ArchivedKeyword::Flagged => Flag::Flagged, + ArchivedKeyword::Answered => Flag::Answered, + ArchivedKeyword::Recent => Flag::Recent, + ArchivedKeyword::Important => Flag::Important, + ArchivedKeyword::Phishing => Flag::Phishing, + ArchivedKeyword::Junk => Flag::Junk, + ArchivedKeyword::NotJunk => Flag::NotJunk, + ArchivedKeyword::Deleted => Flag::Deleted, + ArchivedKeyword::Forwarded => Flag::Forwarded, + ArchivedKeyword::MdnSent => Flag::MDNSent, + ArchivedKeyword::Other(value) => Flag::Keyword(value.to_string()), + } + } +} + impl From for Keyword { fn from(value: Flag) -> Self { match value { diff --git a/crates/imap/src/core/mailbox.rs b/crates/imap/src/core/mailbox.rs index faad4f71..372edae8 100644 --- a/crates/imap/src/core/mailbox.rs +++ b/crates/imap/src/core/mailbox.rs @@ -18,7 +18,7 @@ use jmap_proto::types::{acl::Acl, collection::Collection, id::Id, property::Prop use parking_lot::Mutex; use store::{ query::log::{Change, Query}, - write::ArchivedValue, + write::Archive, }; use trc::AddContext; @@ -155,7 +155,7 @@ impl SessionData { let mut special_uses = AHashMap::new(); for (mailbox_id, mailbox_) in self .server - .get_properties::, _, _>( + .get_properties::( account_id, Collection::Mailbox, &mailbox_ids, @@ -164,7 +164,9 @@ impl SessionData { .await .caused_by(trc::location!())? { - let mailbox = mailbox_.unarchive().caused_by(trc::location!())?; + let mailbox = mailbox_ + .unarchive::() + .caused_by(trc::location!())?; // Map special uses let role = SpecialUse::from(&mailbox.role); if !matches!(mailbox.role, ArchivedSpecialUse::None) { @@ -177,10 +179,7 @@ impl SessionData { parent_id: u32::from(mailbox.parent_id), role, name: mailbox.name.to_string(), - is_subscribed: mailbox - .subscribers - .iter() - .any(|s| u32::from(s) == access_token.primary_id()), + is_subscribed: mailbox.is_subscribed(access_token.primary_id()), }); } @@ -216,8 +215,8 @@ impl SessionData { .map(|k| k.len() + std::mem::size_of::()) .sum::() + (account.mailbox_state.len() - * (std::mem::size_of::>() - + std::mem::size_of::()))) as u64; + * (std::mem::size_of::() + std::mem::size_of::()))) + as u64; loop { while let Some(mailbox) = iter.next() { @@ -621,7 +620,7 @@ impl SessionData { Ok(access_token.is_member(account_id) || self .server - .get_property::>( + .get_property::( account_id, Collection::Mailbox, document_id, @@ -632,7 +631,7 @@ impl SessionData { if let Some(mailbox) = mailbox { Ok(Some( mailbox - .unarchive()? + .unarchive::()? .acls .effective_acl(&access_token) .contains(item), diff --git a/crates/imap/src/core/message.rs b/crates/imap/src/core/message.rs index 7a315e8e..0b3f7d1f 100644 --- a/crates/imap/src/core/message.rs +++ b/crates/imap/src/core/message.rs @@ -8,10 +8,10 @@ use std::{collections::BTreeMap, sync::Arc}; use ahash::AHashMap; use common::{NextMailboxState, listener::SessionStream}; -use email::mailbox::{ArchivedMailbox, UidMailbox}; +use email::mailbox::{ArchivedMailbox, ArchivedUidMailbox}; use imap_proto::protocol::{Sequence, expunge, select::Exists}; use jmap_proto::types::{collection::Collection, property::Property}; -use store::write::{ArchivedValue, assert::HashedValue}; +use store::{rkyv::vec::ArchivedVec, write::Archive}; use trc::AddContext; use crate::core::ImapId; @@ -50,9 +50,9 @@ impl SessionData { // Obtain all message ids let mut uid_map = BTreeMap::new(); - for (message_id, uid_mailbox) in self + for (message_id, uid_mailbox_) in self .server - .get_properties::>, _, _>( + .get_properties::( mailbox.account_id, Collection::Email, &message_ids, @@ -61,14 +61,16 @@ impl SessionData { .await? .into_iter() { + let uid_mailbox = uid_mailbox_ + .unarchive::>() + .caused_by(trc::location!())?; // Make sure the message is still in this mailbox if let Some(item) = uid_mailbox - .inner .iter() .find(|item| item.mailbox_id == mailbox.mailbox_id) { debug_assert!(item.uid != 0, "UID is zero for message {item:?}"); - if uid_map.insert(item.uid, message_id).is_some() { + if uid_map.insert(u32::from(item.uid), message_id).is_some() { trc::event!( Store(trc::StoreEvent::UnexpectedError), AccountId = mailbox.account_id, @@ -225,7 +227,7 @@ impl SessionData { pub async fn get_uid_validity(&self, mailbox: &MailboxId) -> trc::Result { self.server - .get_property::>( + .get_property::( mailbox.account_id, Collection::Mailbox, mailbox.mailbox_id, @@ -240,7 +242,10 @@ impl SessionData { .collection(Collection::Mailbox) .document_id(mailbox.mailbox_id) }) - .and_then(|m| m.unarchive().map(|m| u32::from(m.uid_validity))) + .and_then(|m| { + m.unarchive::() + .map(|m| u32::from(m.uid_validity)) + }) } pub async fn get_uid_next(&self, mailbox: &MailboxId) -> trc::Result { diff --git a/crates/imap/src/op/acl.rs b/crates/imap/src/op/acl.rs index 6325934c..4ca8cb59 100644 --- a/crates/imap/src/op/acl.rs +++ b/crates/imap/src/op/acl.rs @@ -27,7 +27,7 @@ use jmap_proto::types::{ acl::Acl, collection::Collection, property::Property, state::StateChange, type_state::DataType, value::AclGrant, }; -use store::write::{ArchivedValue, BatchBuilder, assert::HashedValue, log::ChangeLogBuilder}; +use store::write::{Archive, BatchBuilder, assert::HashedValue, log::ChangeLogBuilder}; use trc::AddContext; use utils::map::bitmap::Bitmap; @@ -54,7 +54,7 @@ impl Session { .imap_ctx(&arguments.tag, trc::location!())?; let mut permissions = Vec::new(); let mailbox = mailbox_ - .to_unarchived() + .to_unarchived::() .imap_ctx(&arguments.tag, trc::location!())?; for item in mailbox.inner.acls.iter() { @@ -152,7 +152,7 @@ impl Session { .await .imap_ctx(&arguments.tag, trc::location!())?; let mailbox = mailbox_ - .to_unarchived() + .to_unarchived::() .imap_ctx(&arguments.tag, trc::location!())?; let rights = if access_token.is_shared(mailbox_id.account_id) { let acl = mailbox.inner.acls.effective_acl(&access_token); @@ -241,7 +241,7 @@ impl Session { .await .imap_ctx(&arguments.tag, trc::location!())?; let current_mailbox = current_mailbox - .into_deserialized() + .into_deserialized::() .imap_ctx(&arguments.tag, trc::location!())?; // Obtain principal id @@ -436,15 +436,11 @@ impl SessionData { &self, arguments: &Arguments, validate: bool, - ) -> trc::Result<( - MailboxId, - HashedValue>, - Arc, - )> { + ) -> trc::Result<(MailboxId, HashedValue, Arc)> { if let Some(mailbox) = self.get_mailbox_by_name(&arguments.mailbox_name) { if let Some(values) = self .server - .get_property::>>( + .get_property::>( mailbox.account_id, Collection::Mailbox, mailbox.mailbox_id, @@ -458,7 +454,7 @@ impl SessionData { || access_token.is_member(mailbox.account_id) || values .inner - .unarchive() + .unarchive::() .caused_by(trc::location!())? .acls .effective_acl(&access_token) diff --git a/crates/imap/src/op/copy_move.rs b/crates/imap/src/op/copy_move.rs index b0b56c71..d68d53fd 100644 --- a/crates/imap/src/op/copy_move.rs +++ b/crates/imap/src/op/copy_move.rs @@ -8,13 +8,14 @@ use std::{sync::Arc, time::Instant}; use directory::Permission; use email::{ - mailbox::{JUNK_ID, UidMailbox}, + mailbox::{ArchivedUidMailbox, JUNK_ID, UidMailbox}, message::{bayes::EmailBayesTrain, copy::EmailCopy, ingest::EmailIngest}, }; use imap_proto::{ Command, ResponseCode, ResponseType, StatusResponse, protocol::copy_move::Arguments, receiver::Request, }; +use trc::AddContext; use crate::{ core::{SelectedMailbox, Session, SessionData}, @@ -30,8 +31,9 @@ use jmap_proto::{ }; use store::{ SerializeInfallible, + rkyv::vec::ArchivedVec, roaring::RoaringBitmap, - write::{BatchBuilder, ValueClass, assert::HashedValue, log::ChangeLogBuilder}, + write::{Archive, BatchBuilder, ValueClass, assert::HashedValue, log::ChangeLogBuilder}, }; use super::ImapContext; @@ -473,7 +475,7 @@ impl SessionData { // Obtain mailbox tags if let (Some(mailboxes), Some(thread_id)) = ( self.server - .get_property::>>( + .get_property::>( account_id, Collection::Email, id, @@ -484,7 +486,14 @@ impl SessionData { .get_property::(account_id, Collection::Email, id, Property::ThreadId) .await?, ) { - Ok(Some((TagManager::new(mailboxes), thread_id))) + Ok(Some(( + TagManager::new( + mailboxes + .into_deserialized::, Vec>() + .caused_by(trc::location!())?, + ), + thread_id, + ))) } else { trc::event!( Store(trc::StoreEvent::NotFound), diff --git a/crates/imap/src/op/expunge.rs b/crates/imap/src/op/expunge.rs index 904091ae..7ca4575b 100644 --- a/crates/imap/src/op/expunge.rs +++ b/crates/imap/src/op/expunge.rs @@ -8,7 +8,10 @@ use std::{sync::Arc, time::Instant}; use ahash::AHashMap; use directory::Permission; -use email::{mailbox::UidMailbox, message::delete::EmailDeletion}; +use email::{ + mailbox::{ArchivedUidMailbox, UidMailbox}, + message::delete::EmailDeletion, +}; use imap_proto::{ Command, ResponseCode, ResponseType, StatusResponse, parser::parse_sequence_set, @@ -19,13 +22,19 @@ use trc::AddContext; use crate::core::{SavedSearch, SelectedMailbox, Session, SessionData}; use common::{ImapId, listener::SessionStream, storage::tag::TagManager}; use jmap_proto::types::{ - acl::Acl, collection::Collection, id::Id, keyword::Keyword, property::Property, - state::StateChange, type_state::DataType, + acl::Acl, + collection::Collection, + id::Id, + keyword::{ArchivedKeyword, Keyword}, + property::Property, + state::StateChange, + type_state::DataType, }; use store::{ SerializeInfallible, + rkyv::vec::ArchivedVec, roaring::RoaringBitmap, - write::{BatchBuilder, assert::HashedValue, log::ChangeLogBuilder}, + write::{Archive, BatchBuilder, assert::HashedValue, log::ChangeLogBuilder}, }; use super::{ImapContext, ToModSeq}; @@ -194,7 +203,7 @@ impl SessionData { for (id, mailbox_ids) in self .server - .get_properties::>, _, _>( + .get_properties::, _, _>( account_id, Collection::Email, deleted_ids, @@ -203,14 +212,18 @@ impl SessionData { .await .caused_by(trc::location!())? { - let mut mailboxes = TagManager::new(mailbox_ids); + let mut mailboxes = TagManager::new( + mailbox_ids + .into_deserialized::, Vec>() + .caused_by(trc::location!())?, + ); if mailboxes.current().contains(&mailbox_id) { if mailboxes.current().len() > 1 { // Remove deleted flag let (mut keywords, thread_id) = if let (Some(keywords), Some(thread_id)) = ( self.server - .get_property::>>( + .get_property::>( account_id, Collection::Email, id, @@ -228,7 +241,14 @@ impl SessionData { .await .caused_by(trc::location!())?, ) { - (TagManager::new(keywords), thread_id) + ( + TagManager::new( + keywords + .into_deserialized::, Vec>() + .caused_by(trc::location!())?, + ), + thread_id, + ) } else { continue; }; diff --git a/crates/imap/src/op/fetch.rs b/crates/imap/src/op/fetch.rs index e49a9f2f..1f043431 100644 --- a/crates/imap/src/op/fetch.rs +++ b/crates/imap/src/op/fetch.rs @@ -13,7 +13,11 @@ use crate::{ use ahash::AHashMap; use common::listener::SessionStream; use directory::Permission; -use email::message::metadata::MessageMetadata; +use email::message::metadata::{ + ArchivedAddress, ArchivedGetHeader, ArchivedHeaderName, ArchivedHeaderValue, + ArchivedMessageMetadata, ArchivedMessageMetadataContents, ArchivedMetadataPartType, + DecodedParts, +}; use imap_proto::{ Command, ResponseCode, ResponseType, StatusResponse, parser::PushUnique, @@ -28,16 +32,21 @@ use imap_proto::{ receiver::Request, }; use jmap_proto::types::{ - acl::Acl, collection::Collection, id::Id, keyword::Keyword, property::Property, - state::StateChange, type_state::DataType, + acl::Acl, + collection::Collection, + id::Id, + keyword::{ArchivedKeyword, Keyword}, + property::Property, + state::StateChange, + type_state::DataType, }; -use mail_parser::{Address, GetHeader, HeaderName, Message, PartType}; +use mail_parser::DateTime; use store::{ Serialize, SerializeInfallible, query::log::{Change, Query}, - write::{BatchBuilder, Bincode, assert::HashedValue}, + rkyv::{rend::u16_le, vec::ArchivedVec}, + write::{Archive, Archiver, BatchBuilder, assert::HashedValue, serialize::rkyv_deserialize}, }; -use trc::AddContext; use super::{FromModSeq, ImapContext}; @@ -300,9 +309,9 @@ impl SessionData { for (seqnum, uid, id) in ids { // Obtain attributes and keywords - let (email, keywords) = if let (Some(email), Some(keywords)) = ( + let (email_, keywords_) = if let (Some(email), Some(keywords)) = ( self.server - .get_property::>( + .get_property::( account_id, Collection::Email, id, @@ -311,7 +320,7 @@ impl SessionData { .await .imap_ctx(&arguments.tag, trc::location!())?, self.server - .get_property::>>( + .get_property::>( account_id, Collection::Email, id, @@ -320,7 +329,7 @@ impl SessionData { .await .imap_ctx(&arguments.tag, trc::location!())?, ) { - (email.inner, keywords) + (email, keywords) } else { trc::event!( Store(trc::StoreEvent::NotFound), @@ -332,25 +341,32 @@ impl SessionData { ); continue; }; + let email = email_ + .unarchive::() + .imap_ctx(&arguments.tag, trc::location!())?; + let keywords = keywords_ + .inner + .unarchive::>() + .imap_ctx(&arguments.tag, trc::location!())?; // Fetch and parse blob - let raw_message = if needs_blobs { + let raw_message: Cow<[u8]> = if needs_blobs { // Retrieve raw message if needed match self .server .blob_store() - .get_blob(email.blob_hash.as_slice(), 0..usize::MAX) + .get_blob(email.blob_hash.0.as_slice(), 0..usize::MAX) .await .imap_ctx(&arguments.tag, trc::location!())? { - Some(raw_message) => raw_message, + Some(raw_message) => raw_message.into(), None => { trc::event!( Store(trc::StoreEvent::NotFound), AccountId = account_id, DocumentId = id, Collection = Collection::Email, - BlobId = email.blob_hash.to_hex(), + BlobId = email.blob_hash.0.as_slice(), Details = "Blob not found.", CausedBy = trc::location!(), ); @@ -359,14 +375,15 @@ impl SessionData { } } } else { - email.raw_headers + email.raw_headers.as_slice().into() }; - let message = email.contents.into_message(&raw_message); + let message = &email.contents; + let decoded = message.decode_contents(raw_message.as_ref()); // Build response let mut items = Vec::with_capacity(arguments.attributes.len()); let set_seen_flag = - set_seen_flags && !keywords.inner.iter().any(|k| k == &Keyword::Seen); + set_seen_flags && !keywords.iter().any(|k| k == &ArchivedKeyword::Seen); let thread_id = if needs_thread_id || set_seen_flag { if let Some(thread_id) = self .server @@ -389,11 +406,7 @@ impl SessionData { }); } Attribute::Flags => { - let mut flags = keywords - .inner - .iter() - .map(|k| Flag::from(k.clone())) - .collect::>(); + let mut flags = keywords.iter().map(Flag::from).collect::>(); if set_seen_flag { flags.push(Flag::Seen); } @@ -401,7 +414,7 @@ impl SessionData { } Attribute::InternalDate => { items.push(DataItem::InternalDate { - date: email.received_at as i64, + date: u64::from(email.received_at) as i64, }); } Attribute::Preview { .. } => { @@ -414,21 +427,24 @@ impl SessionData { }); } Attribute::Rfc822Size => { - items.push(DataItem::Rfc822Size { size: email.size }); + items.push(DataItem::Rfc822Size { + size: u32::from(email.size) as usize, + }); } Attribute::Uid => { items.push(DataItem::Uid { uid }); } Attribute::Rfc822 => { items.push(DataItem::Rfc822 { - contents: raw_message.as_slice().into(), + contents: raw_message.as_ref().into(), }); } Attribute::Rfc822Header => { - let message = message.root_part(); - if let Some(header) = - raw_message.get(message.offset_header..message.offset_body) - { + let message = email.root_part(); + if let Some(header) = raw_message.get( + u32::from(message.offset_header) as usize + ..u32::from(message.offset_body) as usize, + ) { items.push(DataItem::Rfc822Header { contents: header.into(), }); @@ -436,23 +452,23 @@ impl SessionData { } Attribute::Rfc822Text => { items.push(DataItem::Rfc822Text { - contents: raw_message.as_slice().into(), + contents: raw_message.as_ref().into(), }); } Attribute::Body => { items.push(DataItem::Body { - part: message.body_structure(false), + part: message.body_structure(&decoded, false), }); } Attribute::BodyStructure => { items.push(DataItem::BodyStructure { - part: message.body_structure(true), + part: message.body_structure(&decoded, true), }); } Attribute::BodySection { sections, partial, .. } => { - if let Some(contents) = message.body_section(sections, *partial) { + if let Some(contents) = message.body_section(&decoded, sections, *partial) { items.push(DataItem::BodySection { sections: sections.to_vec(), origin_octet: partial.map(|(start, _)| start), @@ -463,7 +479,7 @@ impl SessionData { Attribute::Binary { sections, partial, .. - } => match message.binary(sections, *partial) { + } => match message.binary(&decoded, sections, *partial) { Ok(Some(contents)) => { items.push(DataItem::Binary { sections: sections.to_vec(), @@ -492,7 +508,7 @@ impl SessionData { _ => (), }, Attribute::BinarySize { sections } => { - if let Some(size) = message.binary_size(sections) { + if let Some(size) = message.binary_size(&decoded, sections) { items.push(DataItem::BinarySize { sections: sections.to_vec(), size, @@ -523,11 +539,7 @@ impl SessionData { // Add flags to the response if the message was unseen if set_seen_flag && !arguments.attributes.contains(&Attribute::Flags) { - let mut flags = keywords - .inner - .iter() - .map(|k| Flag::from(k.clone())) - .collect::>(); + let mut flags = keywords.iter().map(Flag::from).collect::>(); flags.push(Flag::Seen); items.push(DataItem::Flags { flags }); } @@ -539,7 +551,14 @@ impl SessionData { // Add to set flags if set_seen_flag { - set_seen_ids.push((Id::from_parts(thread_id, id), keywords)); + set_seen_ids.push(( + Id::from_parts(thread_id, id), + HashedValue { + hash: keywords_.hash, + inner: rkyv_deserialize::<_, Vec>(keywords) + .imap_ctx(&arguments.tag, trc::location!())?, + }, + )); } } @@ -559,7 +578,9 @@ impl SessionData { .assert_value(Property::Keywords, &keywords) .set( Property::Keywords, - keywords.inner.serialize().caused_by(trc::location!())?, + Archiver::new(keywords.inner) + .serialize() + .imap_ctx(&arguments.tag, trc::location!())?, ) .tag(Property::Keywords, Keyword::Seen) .set(Property::Cid, changelog.change_id.serialize()); @@ -568,7 +589,7 @@ impl SessionData { .store() .write(batch) .await - .caused_by(trc::location!()) + .imap_ctx(&arguments.tag, trc::location!()) { Ok(_) => { changelog.log_update(Collection::Email, id); @@ -625,46 +646,58 @@ impl SessionData { } #[allow(clippy::result_unit_err)] -pub trait AsImapDataItem<'x> { - fn body_structure(&self, is_extended: bool) -> BodyPart; - fn body_section<'z: 'x>( - &'z self, +pub trait AsImapDataItem { + fn body_structure(&self, decoded: &DecodedParts<'_>, is_extended: bool) -> BodyPart; + fn body_section<'x>( + &self, + decoded: &'x DecodedParts<'x>, sections: &[Section], partial: Option<(u32, u32)>, ) -> Option>; - fn binary( + fn binary<'x>( &self, + decoded: &'x DecodedParts<'x>, sections: &[u32], partial: Option<(u32, u32)>, - ) -> Result, ()>; - fn binary_size(&self, sections: &[u32]) -> Option; - fn as_body_part(&self, part_id: usize, is_extended: bool) -> BodyPart; + ) -> Result>, ()>; + fn binary_size(&self, decoded: &DecodedParts<'_>, sections: &[u32]) -> Option; + fn as_body_part( + &self, + decoded: &DecodedParts<'_>, + message_id: usize, + part_id: usize, + is_extended: bool, + ) -> BodyPart; fn envelope(&self) -> Envelope; } -impl<'x> AsImapDataItem<'x> for Message<'x> { - fn body_structure(&self, is_extended: bool) -> BodyPart { +impl AsImapDataItem for ArchivedMessageMetadataContents { + fn body_structure(&self, decoded: &DecodedParts<'_>, is_extended: bool) -> BodyPart { let mut stack = Vec::new(); - let mut parts = [0].iter(); + let base_part = [u16_le::from_native(0)]; + let mut parts = base_part.as_slice().iter(); let mut message = self; let mut root_part = None; + let mut message_id = 0; loop { while let Some(part_id) = parts.next() { - let mut part = message.as_body_part(*part_id, is_extended); + let part_id = u16::from(part_id) as usize; + let mut part = message.as_body_part(decoded, message_id, part_id, is_extended); - match &message.parts[*part_id].body { - PartType::Message(nested_message) => { + match &message.parts[part_id].body { + ArchivedMetadataPartType::Message(nested_message) => { part.set_envelope(nested_message.envelope()); if let Some(root_part) = root_part { - stack.push((root_part, parts, message.into())); + stack.push((root_part, parts, (message, message_id).into())); + message_id += 1; } root_part = part.into(); - parts = [0].iter(); + parts = base_part.as_slice().iter(); message = nested_message; continue; } - PartType::Multipart(subparts) => { + ArchivedMetadataPartType::Multipart(subparts) => { if let Some(root_part) = root_part { stack.push((root_part, parts, None)); } @@ -681,8 +714,9 @@ impl<'x> AsImapDataItem<'x> for Message<'x> { } } if let Some((mut prev_root_part, prev_parts, prev_message)) = stack.pop() { - if let Some(prev_message) = prev_message { + if let Some((prev_message, prev_message_id)) = prev_message { message = prev_message; + message_id = prev_message_id; } prev_root_part.add_part(root_part.unwrap()); @@ -696,17 +730,23 @@ impl<'x> AsImapDataItem<'x> for Message<'x> { root_part.unwrap() } - fn as_body_part(&self, part_id: usize, is_extended: bool) -> BodyPart { + fn as_body_part( + &self, + decoded: &DecodedParts<'_>, + message_id: usize, + part_id: usize, + is_extended: bool, + ) -> BodyPart { let part = &self.parts[part_id]; - let body = self.raw_message.get(part.offset_body..part.offset_end); + let body = decoded.raw_message_section_arch(message_id, part.offset_body, part.offset_end); let (is_multipart, is_text) = match &part.body { - PartType::Text(_) | PartType::Html(_) => (false, true), - PartType::Multipart(_) => (true, false), + ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html => (false, true), + ArchivedMetadataPartType::Multipart(_) => (true, false), _ => (false, false), }; let content_type = part .headers - .header_value(&HeaderName::ContentType) + .header_value(&ArchivedHeaderName::ContentType) .and_then(|ct| ct.as_content_type()); let mut body_md5 = None; @@ -717,7 +757,7 @@ impl<'x> AsImapDataItem<'x> for Message<'x> { fields.body_parameters = content_type.as_ref().and_then(|ct| { ct.attributes.as_ref().map(|at| { at.iter() - .map(|(h, v)| (h.as_ref().into(), v.as_ref().into())) + .map(|k| (k.0.as_ref().into(), k.1.as_ref().into())) .collect::>() }) }) @@ -730,17 +770,17 @@ impl<'x> AsImapDataItem<'x> for Message<'x> { fields.body_id = part .headers - .header_value(&HeaderName::ContentId) + .header_value(&ArchivedHeaderName::ContentId) .and_then(|id| id.as_text().map(|id| format!("<{}>", id).into())); fields.body_description = part .headers - .header_value(&HeaderName::ContentDescription) + .header_value(&ArchivedHeaderName::ContentDescription) .and_then(|ct| ct.as_text().map(|ct| ct.into())); fields.body_encoding = part .headers - .header_value(&HeaderName::ContentTransferEncoding) + .header_value(&ArchivedHeaderName::ContentTransferEncoding) .and_then(|ct| ct.as_text().map(|ct| ct.into())); fields.body_size_octets = body.as_ref().map(|b| b.len()).unwrap_or(0); @@ -767,7 +807,7 @@ impl<'x> AsImapDataItem<'x> for Message<'x> { extension.body_disposition = part .headers - .header_value(&HeaderName::ContentDisposition) + .header_value(&ArchivedHeaderName::ContentDisposition) .and_then(|cd| cd.as_content_type()) .map(|cd| { ( @@ -776,7 +816,7 @@ impl<'x> AsImapDataItem<'x> for Message<'x> { .as_ref() .map(|at| { at.iter() - .map(|(h, v)| (h.as_ref().into(), v.as_ref().into())) + .map(|k| (k.0.as_ref().into(), k.1.as_ref().into())) .collect::>() }) .unwrap_or_default(), @@ -785,7 +825,7 @@ impl<'x> AsImapDataItem<'x> for Message<'x> { extension.body_language = part .headers - .header_value(&HeaderName::ContentLanguage) + .header_value(&ArchivedHeaderName::ContentLanguage) .and_then(|hv| { hv.as_text_list() .map(|list| list.iter().map(|item| item.as_ref().into()).collect()) @@ -793,12 +833,12 @@ impl<'x> AsImapDataItem<'x> for Message<'x> { extension.body_location = part .headers - .header_value(&HeaderName::ContentLocation) + .header_value(&ArchivedHeaderName::ContentLocation) .and_then(|ct| ct.as_text().map(|ct| ct.into())); } match &part.body { - PartType::Multipart(parts) => BodyPart::Multipart { + ArchivedMetadataPartType::Multipart(parts) => BodyPart::Multipart { body_parts: Vec::with_capacity(parts.len()), body_subtype: content_type .as_ref() @@ -807,7 +847,7 @@ impl<'x> AsImapDataItem<'x> for Message<'x> { body_parameters: fields.body_parameters, extension, }, - PartType::Message(_) => BodyPart::Message { + ArchivedMetadataPartType::Message(_) => BodyPart::Message { fields, envelope: None, body: None, @@ -840,8 +880,9 @@ impl<'x> AsImapDataItem<'x> for Message<'x> { } } - fn body_section<'z: 'x>( - &'z self, + fn body_section<'x>( + &self, + decoded: &'x DecodedParts<'x>, sections: &[Section], partial: Option<(u32, u32)>, ) -> Option> { @@ -849,7 +890,7 @@ impl<'x> AsImapDataItem<'x> for Message<'x> { if sections.is_empty() { return Some( get_partial_bytes( - self.raw_message.get(part.offset_header..part.offset_end)?, + decoded.raw_message_section_arch(0, part.offset_header, part.offset_end)?, partial, ) .into(), @@ -857,6 +898,7 @@ impl<'x> AsImapDataItem<'x> for Message<'x> { } let mut message = self; + let mut message_id = 0; let mut sections_iter = sections.iter().enumerate().peekable(); while let Some((section_num, section)) = sections_iter.next() { @@ -865,7 +907,7 @@ impl<'x> AsImapDataItem<'x> for Message<'x> { part = if let Some(sub_part_ids) = part.sub_parts() { sub_part_ids .get((*num).saturating_sub(1) as usize) - .and_then(|pos| message.parts.get(*pos)) + .and_then(|pos| message.parts.get(u16::from(*pos) as usize)) } else if *num == 1 && (section_num == sections.len() - 1 || part.is_message()) { Some(part) @@ -874,7 +916,7 @@ impl<'x> AsImapDataItem<'x> for Message<'x> { }?; if let ( - PartType::Message(nested_message), + ArchivedMetadataPartType::Message(nested_message), Some(( _, Section::Part { .. } @@ -886,32 +928,40 @@ impl<'x> AsImapDataItem<'x> for Message<'x> { { message = nested_message; part = message.root_part(); + message_id += 1; } } Section::Header => { return Some( get_partial_bytes( - message - .raw_message - .get(part.offset_header..part.offset_body)?, + decoded.raw_message_section_arch( + message_id, + part.offset_header, + part.offset_body, + )?, partial, ) .into(), ); } Section::HeaderFields { not, fields } => { - let mut headers = - Vec::with_capacity(part.offset_body.saturating_sub(part.offset_header)); - for header in &part.headers { + let mut headers = Vec::with_capacity( + u32::from(part.offset_body).saturating_sub(u32::from(part.offset_header)) + as usize, + ); + for header in part.headers.iter() { let header_name = header.name.as_str(); if fields.iter().any(|f| header_name.eq_ignore_ascii_case(f)) != *not { headers.extend_from_slice(header_name.as_bytes()); headers.push(b':'); headers.extend_from_slice( - message - .raw_message - .get(header.offset_start..header.offset_end) - .unwrap_or(b""), + decoded + .raw_message_section_arch( + message_id, + header.offset_start, + header.offset_end, + ) + .unwrap_or_default(), ); } } @@ -927,26 +977,35 @@ impl<'x> AsImapDataItem<'x> for Message<'x> { Section::Text => { return Some( get_partial_bytes( - message.raw_message.get(part.offset_body..part.offset_end)?, + decoded.raw_message_section_arch( + message_id, + part.offset_body, + part.offset_end, + )?, partial, ) .into(), ); } Section::Mime => { - let mut headers = - Vec::with_capacity(part.offset_body.saturating_sub(part.offset_header)); - for header in &part.headers { + let mut headers = Vec::with_capacity( + u32::from(part.offset_body).saturating_sub(u32::from(part.offset_header)) + as usize, + ); + for header in part.headers.iter() { if header.name.is_mime_header() || header.name.as_str().starts_with("Content-") { headers.extend_from_slice(header.name.as_str().as_bytes()); headers.extend_from_slice(b":"); headers.extend_from_slice( - message - .raw_message - .get(header.offset_start..header.offset_end) - .unwrap_or(b""), + decoded + .raw_message_section_arch( + message_id, + header.offset_start, + header.offset_end, + ) + .unwrap_or_default(), ); } } @@ -965,73 +1024,96 @@ impl<'x> AsImapDataItem<'x> for Message<'x> { Some( get_partial_bytes( - message.raw_message.get(part.offset_body..part.offset_end)?, + decoded.raw_message_section_arch(message_id, part.offset_body, part.offset_end)?, partial, ) .into(), ) } - fn binary( + fn binary<'x>( &self, + decoded: &'x DecodedParts<'x>, sections: &[u32], partial: Option<(u32, u32)>, - ) -> Result, ()> { + ) -> Result>, ()> { let mut message = self; + let mut message_id = 0; let mut part = self.root_part(); + let mut part_id = 0; let mut sections_iter = sections.iter().enumerate().peekable(); while let Some((section_num, num)) = sections_iter.next() { part = if let Some(sub_part_ids) = part.sub_parts() { + part_id = (*num).saturating_sub(1) as usize; if let Some(part) = sub_part_ids - .get((*num).saturating_sub(1) as usize) - .and_then(|pos| message.parts.get(*pos)) + .get(part_id) + .and_then(|pos| message.parts.get(u16::from(*pos) as usize)) { part } else { return Ok(None); } } else if *num == 1 && (section_num == sections.len() - 1 || part.is_message()) { + part_id = 0; part } else { return Ok(None); }; - if let (PartType::Message(nested_message), Some(_)) = (&part.body, sections_iter.peek()) + if let (ArchivedMetadataPartType::Message(nested_message), Some(_)) = + (&part.body, sections_iter.peek()) { message = nested_message; part = message.root_part(); + message_id += 1; } } if !part.is_encoding_problem { Ok(match &part.body { - PartType::Text(text) | PartType::Html(text) => BodyContents::Text( - String::from_utf8_lossy(get_partial_bytes(text.as_bytes(), partial)), - ) - .into(), - PartType::Binary(bytes) | PartType::InlineBinary(bytes) => { - BodyContents::Bytes(get_partial_bytes(bytes.as_ref(), partial).into()).into() - } - PartType::Message(message) => BodyContents::Bytes( - get_partial_bytes( - message - .raw_message - .get( - message.root_part().raw_header_offset() - ..message.root_part().raw_end_offset(), - ) - .unwrap_or_default(), + ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html => { + BodyContents::Text(String::from_utf8_lossy(get_partial_bytes( + decoded.binary_part(message_id, part_id).unwrap_or_default(), partial, + ))) + .into() + } + ArchivedMetadataPartType::Binary | ArchivedMetadataPartType::InlineBinary => { + BodyContents::Bytes( + get_partial_bytes( + decoded.binary_part(message_id, part_id).unwrap_or_default(), + partial, + ) + .into(), ) - .into(), - ) + .into() + } + ArchivedMetadataPartType::Message(message) => BodyContents::Bytes({ + { + let part = message.root_part(); + get_partial_bytes( + decoded + .raw_message_section_arch( + message_id, + part.offset_header, + part.offset_end, + ) + .unwrap_or_default(), + partial, + ) + .into() + } + }) .into(), - PartType::Multipart(_) => BodyContents::Bytes( + ArchivedMetadataPartType::Multipart(_) => BodyContents::Bytes( get_partial_bytes( - message - .raw_message - .get(part.raw_header_offset()..part.raw_end_offset()) + decoded + .raw_message_section_arch( + message_id, + part.offset_header, + part.offset_end, + ) .unwrap_or_default(), partial, ) @@ -1044,67 +1126,79 @@ impl<'x> AsImapDataItem<'x> for Message<'x> { } } - fn binary_size(&self, sections: &[u32]) -> Option { + fn binary_size(&self, decoded: &DecodedParts<'_>, sections: &[u32]) -> Option { let mut message = self; + let mut message_id = 0; let mut part = self.root_part(); + let mut part_id = 0; let mut sections_iter = sections.iter().enumerate().peekable(); while let Some((section_num, num)) = sections_iter.next() { + part_id = (*num).saturating_sub(1) as usize; part = if let Some(sub_part_ids) = part.sub_parts() { sub_part_ids - .get((*num).saturating_sub(1) as usize) - .and_then(|pos| message.parts.get(*pos)) + .get(part_id) + .and_then(|pos| message.parts.get(u16::from(pos) as usize)) } else if *num == 1 && (section_num == sections.len() - 1 || part.is_message()) { Some(part) } else { None }?; - if let (PartType::Message(nested_message), Some(_)) = (&part.body, sections_iter.peek()) + if let (ArchivedMetadataPartType::Message(nested_message), Some(_)) = + (&part.body, sections_iter.peek()) { message = nested_message; + message_id += 1; part = message.root_part(); + part_id = 0; } } match &part.body { - PartType::Text(text) | PartType::Html(text) => text.len(), - PartType::Binary(bytes) | PartType::InlineBinary(bytes) => bytes.len(), - PartType::Message(message) => message.root_part().raw_len(), - PartType::Multipart(_) => part.raw_len(), + ArchivedMetadataPartType::Text + | ArchivedMetadataPartType::Html + | ArchivedMetadataPartType::Binary + | ArchivedMetadataPartType::InlineBinary => decoded + .part(message_id, part_id) + .map(|p| p.len()) + .unwrap_or_default(), + ArchivedMetadataPartType::Message(message) => message.root_part().raw_len(), + ArchivedMetadataPartType::Multipart(_) => part.raw_len(), } .into() } fn envelope(&self) -> Envelope { + let headers = self.root_part(); Envelope { - date: self.date().cloned(), - subject: self.subject().map(|s| s.into()), - from: self - .header_values(HeaderName::From) + date: headers.date().map(DateTime::from_timestamp), + subject: headers.subject().map(|s| s.into()), + from: headers + .header_values(ArchivedHeaderName::From) .flat_map(|a| a.as_imap_address()) .collect(), - sender: self - .header_values(HeaderName::Sender) + sender: headers + .header_values(ArchivedHeaderName::Sender) .flat_map(|a| a.as_imap_address()) .collect(), - reply_to: self - .header_values(HeaderName::ReplyTo) + reply_to: headers + .header_values(ArchivedHeaderName::ReplyTo) .flat_map(|a| a.as_imap_address()) .collect(), - to: self - .header_values(HeaderName::To) + to: headers + .header_values(ArchivedHeaderName::To) .flat_map(|a| a.as_imap_address()) .collect(), - cc: self - .header_values(HeaderName::Cc) + cc: headers + .header_values(ArchivedHeaderName::Cc) .flat_map(|a| a.as_imap_address()) .collect(), - bcc: self - .header_values(HeaderName::Bcc) + bcc: headers + .header_values(ArchivedHeaderName::Bcc) .flat_map(|a| a.as_imap_address()) .collect(), - in_reply_to: self.in_reply_to().as_text_list().map(|list| { + in_reply_to: headers.in_reply_to().as_text_list().map(|list| { let mut irt = String::with_capacity(list.len() * 10); for (pos, l) in list.iter().enumerate() { if pos > 0 { @@ -1116,7 +1210,7 @@ impl<'x> AsImapDataItem<'x> for Message<'x> { } irt.into() }), - message_id: self.message_id().map(|id| format!("<{}>", id).into()), + message_id: headers.message_id().map(|id| format!("<{}>", id).into()), } } } @@ -1136,14 +1230,14 @@ trait AsImapAddress { fn as_imap_address(&self) -> Vec; } -impl AsImapAddress for mail_parser::HeaderValue<'_> { +impl AsImapAddress for ArchivedHeaderValue { fn as_imap_address(&self) -> Vec { let mut addresses = Vec::new(); match self { - mail_parser::HeaderValue::Address(Address::List(list)) => { - for addr in list { - if let Some(email) = &addr.address { + ArchivedHeaderValue::Address(ArchivedAddress::List(list)) => { + for addr in list.iter() { + if let Some(email) = addr.address.as_ref() { addresses.push(fetch::Address::Single(fetch::EmailAddress { name: addr.name.as_ref().map(|n| n.as_ref().into()), address: email.as_ref().into(), @@ -1151,8 +1245,8 @@ impl AsImapAddress for mail_parser::HeaderValue<'_> { } } } - mail_parser::HeaderValue::Address(Address::Group(list)) => { - for group in list { + ArchivedHeaderValue::Address(ArchivedAddress::Group(list)) => { + for group in list.iter() { addresses.push(fetch::Address::Group(fetch::AddressGroup { name: group.name.as_ref().map(|n| n.as_ref().into()), addresses: group diff --git a/crates/imap/src/op/rename.rs b/crates/imap/src/op/rename.rs index eeda162e..2b7bb922 100644 --- a/crates/imap/src/op/rename.rs +++ b/crates/imap/src/op/rename.rs @@ -19,7 +19,7 @@ use imap_proto::{ use jmap_proto::types::{ acl::Acl, collection::Collection, property::Property, state::StateChange, type_state::DataType, }; -use store::write::{ArchivedValue, BatchBuilder, assert::HashedValue}; +use store::write::{Archive, BatchBuilder, assert::HashedValue}; use trc::AddContext; use super::ImapContext; @@ -89,7 +89,7 @@ impl SessionData { // Obtain mailbox let mailbox = self .server - .get_property::>>( + .get_property::>( params.account_id, Collection::Mailbox, mailbox_id, @@ -105,7 +105,7 @@ impl SessionData { .code(ResponseCode::NonExistent) .id(arguments.tag.clone()) })? - .into_deserialized() + .into_deserialized::() .imap_ctx(&arguments.tag, trc::location!())?; // Validate ACL diff --git a/crates/imap/src/op/status.rs b/crates/imap/src/op/status.rs index 5b8492dc..eb44ead2 100644 --- a/crates/imap/src/op/status.rs +++ b/crates/imap/src/op/status.rs @@ -21,7 +21,7 @@ use imap_proto::{ receiver::Request, }; use jmap_proto::types::{collection::Collection, id::Id, keyword::Keyword, property::Property}; -use store::{Deserialize, U32_LEN, write::ArchivedValue}; +use store::{Deserialize, U32_LEN, write::Archive}; use store::{ IndexKeyPrefix, IterateParams, ValueKey, roaring::RoaringBitmap, @@ -255,7 +255,7 @@ impl SessionData { .caused_by(trc::location!())? as u64, Status::UidValidity => u32::from( self.server - .get_property::>( + .get_property::( mailbox.account_id, Collection::Mailbox, mailbox.mailbox_id, @@ -271,7 +271,7 @@ impl SessionData { .account_id(mailbox.account_id) .document_id(mailbox.mailbox_id) })? - .unarchive() + .unarchive::() .caused_by(trc::location!())? .uid_validity, ) as u64, diff --git a/crates/imap/src/op/store.rs b/crates/imap/src/op/store.rs index 5ccfc506..2d402c37 100644 --- a/crates/imap/src/op/store.rs +++ b/crates/imap/src/op/store.rs @@ -14,7 +14,7 @@ use ahash::AHashSet; use common::{listener::SessionStream, storage::tag::TagManager}; use directory::Permission; use email::{ - mailbox::UidMailbox, + mailbox::ArchivedUidMailbox, message::{bayes::EmailBayesTrain, ingest::EmailIngest}, }; use imap_proto::{ @@ -27,13 +27,19 @@ use imap_proto::{ receiver::Request, }; use jmap_proto::types::{ - acl::Acl, collection::Collection, id::Id, keyword::Keyword, property::Property, - state::StateChange, type_state::DataType, + acl::Acl, + collection::Collection, + id::Id, + keyword::{ArchivedKeyword, Keyword}, + property::Property, + state::StateChange, + type_state::DataType, }; use store::{ SerializeInfallible, query::log::{Change, Query}, - write::{BatchBuilder, ValueClass, assert::HashedValue, log::ChangeLogBuilder}, + rkyv::vec::ArchivedVec, + write::{Archive, BatchBuilder, ValueClass, assert::HashedValue, log::ChangeLogBuilder}, }; use trc::AddContext; @@ -207,7 +213,7 @@ impl SessionData { // Obtain current keywords let (mut keywords, thread_id) = if let (Some(keywords), Some(thread_id)) = ( self.server - .get_property::>>( + .get_property::>( account_id, Collection::Email, *id, @@ -220,7 +226,14 @@ impl SessionData { .await .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())?, ) { - (TagManager::new(keywords), thread_id) + ( + TagManager::new( + keywords + .into_deserialized::, Vec>() + .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())?, + ), + thread_id, + ) } else { continue 'outer; }; @@ -323,7 +336,7 @@ impl SessionData { if seen_changed { if let Some(mailboxes) = self .server - .get_property::>( + .get_property::( account_id, Collection::Email, *id, @@ -332,8 +345,12 @@ impl SessionData { .await .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())? { - for mailbox_id in mailboxes { - changed_mailboxes.insert(mailbox_id.mailbox_id); + for mailbox_id in mailboxes + .unarchive::>() + .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())? + .iter() + { + changed_mailboxes.insert(u32::from(mailbox_id.mailbox_id)); } } } diff --git a/crates/imap/src/op/subscribe.rs b/crates/imap/src/op/subscribe.rs index e03c7127..e14aa0e8 100644 --- a/crates/imap/src/op/subscribe.rs +++ b/crates/imap/src/op/subscribe.rs @@ -17,7 +17,7 @@ use imap_proto::{Command, ResponseCode, StatusResponse, receiver::Request}; use jmap_proto::types::{ collection::Collection, property::Property, state::StateChange, type_state::DataType, }; -use store::write::{ArchivedValue, BatchBuilder, assert::HashedValue}; +use store::write::{Archive, BatchBuilder, assert::HashedValue}; use super::ImapContext; @@ -97,7 +97,7 @@ impl SessionData { // Obtain mailbox let mailbox = self .server - .get_property::>>( + .get_property::>( account_id, Collection::Mailbox, mailbox_id, @@ -113,7 +113,7 @@ impl SessionData { .id(tag.clone()) .caused_by(trc::location!()) })? - .into_deserialized() + .into_deserialized::() .imap_ctx(&tag, trc::location!())?; if (subscribe && !mailbox.inner.is_subscribed(self.account_id)) diff --git a/crates/jmap-proto/src/types/blob.rs b/crates/jmap-proto/src/types/blob.rs index 86b577da..070703d7 100644 --- a/crates/jmap-proto/src/types/blob.rs +++ b/crates/jmap-proto/src/types/blob.rs @@ -6,19 +6,16 @@ use std::borrow::Borrow; -use store::{ - write::{DeserializeFrom, SerializeInto}, - BlobClass, -}; +use store::BlobClass; use utils::{ + BlobHash, codec::{ base32_custom::{Base32Reader, Base32Writer}, leb128::{Leb128Iterator, Leb128Writer}, }, - BlobHash, }; -use crate::parser::{base32::JsonBase32Reader, json::Parser, JsonObjectParser}; +use crate::parser::{JsonObjectParser, base32::JsonBase32Reader, json::Parser}; const B_LINKED: u8 = 0x10; const B_RESERVED: u8 = 0x20; @@ -199,15 +196,3 @@ impl std::fmt::Display for BlobId { f.write_str(&writer.finalize()) } } - -impl SerializeInto for BlobId { - fn serialize_into(&self, buf: &mut Vec) { - self.serialize_as(buf) - } -} - -impl DeserializeFrom for BlobId { - fn deserialize_from(bytes: &mut std::slice::Iter<'_, u8>) -> Option { - BlobId::from_iter(bytes) - } -} diff --git a/crates/jmap-proto/src/types/keyword.rs b/crates/jmap-proto/src/types/keyword.rs index 8ca997c2..0f5972e4 100644 --- a/crates/jmap-proto/src/types/keyword.rs +++ b/crates/jmap-proto/src/types/keyword.rs @@ -8,9 +8,8 @@ use std::fmt::Display; use store::{ Serialize, - write::{DeserializeFrom, MaybeDynamicId, SerializeInto, TagValue}, + write::{MaybeDynamicId, TagValue}, }; -use utils::codec::leb128::{Leb128Iterator, Leb128Vec}; use crate::parser::{JsonObjectParser, json::Parser}; @@ -28,8 +27,19 @@ pub const FORWARDED: usize = 10; pub const MDN_SENT: usize = 11; pub const OTHER: usize = 12; -#[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize)] +#[derive( + rkyv::Serialize, + rkyv::Deserialize, + rkyv::Archive, + Debug, + Clone, + PartialEq, + Eq, + Hash, + serde::Serialize, +)] #[serde(untagged)] +#[rkyv(derive(PartialEq))] pub enum Keyword { #[serde(rename(serialize = "$seen"))] Seen, @@ -168,6 +178,26 @@ impl Display for Keyword { } } +impl Display for ArchivedKeyword { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ArchivedKeyword::Seen => write!(f, "$seen"), + ArchivedKeyword::Draft => write!(f, "$draft"), + ArchivedKeyword::Flagged => write!(f, "$flagged"), + ArchivedKeyword::Answered => write!(f, "$answered"), + ArchivedKeyword::Recent => write!(f, "$recent"), + ArchivedKeyword::Important => write!(f, "$important"), + ArchivedKeyword::Phishing => write!(f, "$phishing"), + ArchivedKeyword::Junk => write!(f, "$junk"), + ArchivedKeyword::NotJunk => write!(f, "$notjunk"), + ArchivedKeyword::Deleted => write!(f, "$deleted"), + ArchivedKeyword::Forwarded => write!(f, "$forwarded"), + ArchivedKeyword::MdnSent => write!(f, "$mdnsent"), + ArchivedKeyword::Other(s) => write!(f, "{}", s), + } + } +} + impl Serialize for Keyword { fn serialize(&self) -> trc::Result> { Ok(match self { @@ -188,58 +218,6 @@ impl Serialize for Keyword { } } -impl SerializeInto for Keyword { - fn serialize_into(&self, buf: &mut Vec) { - match self { - Keyword::Seen => buf.push(SEEN as u8), - Keyword::Draft => buf.push(DRAFT as u8), - Keyword::Flagged => buf.push(FLAGGED as u8), - Keyword::Answered => buf.push(ANSWERED as u8), - Keyword::Recent => buf.push(RECENT as u8), - Keyword::Important => buf.push(IMPORTANT as u8), - Keyword::Phishing => buf.push(PHISHING as u8), - Keyword::Junk => buf.push(JUNK as u8), - Keyword::NotJunk => buf.push(NOTJUNK as u8), - Keyword::Deleted => buf.push(DELETED as u8), - Keyword::Forwarded => buf.push(FORWARDED as u8), - Keyword::MdnSent => buf.push(MDN_SENT as u8), - Keyword::Other(string) => { - buf.push_leb128(OTHER + string.len()); - if !string.is_empty() { - buf.extend_from_slice(string.as_bytes()) - } - } - } - } -} - -impl DeserializeFrom for Keyword { - fn deserialize_from(bytes: &mut std::slice::Iter<'_, u8>) -> Option { - match bytes.next_leb128::()? { - SEEN => Some(Keyword::Seen), - DRAFT => Some(Keyword::Draft), - FLAGGED => Some(Keyword::Flagged), - ANSWERED => Some(Keyword::Answered), - RECENT => Some(Keyword::Recent), - IMPORTANT => Some(Keyword::Important), - PHISHING => Some(Keyword::Phishing), - JUNK => Some(Keyword::Junk), - NOTJUNK => Some(Keyword::NotJunk), - DELETED => Some(Keyword::Deleted), - FORWARDED => Some(Keyword::Forwarded), - MDN_SENT => Some(Keyword::MdnSent), - other => { - let len = other - OTHER; - let mut keyword = Vec::with_capacity(len); - for _ in 0..len { - keyword.push(*bytes.next()?); - } - Some(Keyword::Other(String::from_utf8(keyword).ok()?)) - } - } - } -} - impl Keyword { pub fn id(&self) -> Result { match self { @@ -278,6 +256,26 @@ impl Keyword { } } +impl ArchivedKeyword { + pub fn id(&self) -> Result { + match self { + ArchivedKeyword::Seen => Ok(SEEN as u32), + ArchivedKeyword::Draft => Ok(DRAFT as u32), + ArchivedKeyword::Flagged => Ok(FLAGGED as u32), + ArchivedKeyword::Answered => Ok(ANSWERED as u32), + ArchivedKeyword::Recent => Ok(RECENT as u32), + ArchivedKeyword::Important => Ok(IMPORTANT as u32), + ArchivedKeyword::Phishing => Ok(PHISHING as u32), + ArchivedKeyword::Junk => Ok(JUNK as u32), + ArchivedKeyword::NotJunk => Ok(NOTJUNK as u32), + ArchivedKeyword::Deleted => Ok(DELETED as u32), + ArchivedKeyword::Forwarded => Ok(FORWARDED as u32), + ArchivedKeyword::MdnSent => Ok(MDN_SENT as u32), + ArchivedKeyword::Other(string) => Err(string.to_string()), + } + } +} + impl From for TagValue { fn from(value: Keyword) -> Self { match value.into_id() { @@ -313,3 +311,12 @@ impl From<&Keyword> for TagValue { } } } + +impl From<&ArchivedKeyword> for TagValue { + fn from(value: &ArchivedKeyword) -> Self { + match value.id() { + Ok(id) => TagValue::Id(MaybeDynamicId::Static(id)), + Err(string) => TagValue::Text(string.into_bytes()), + } + } +} diff --git a/crates/jmap-proto/src/types/property.rs b/crates/jmap-proto/src/types/property.rs index ad32b329..d0279463 100644 --- a/crates/jmap-proto/src/types/property.rs +++ b/crates/jmap-proto/src/types/property.rs @@ -8,7 +8,7 @@ use std::fmt::{Display, Formatter}; use mail_parser::HeaderName; use serde::Serialize; -use store::write::{DeserializeFrom, SerializeInto, ValueClass}; +use store::write::ValueClass; use crate::parser::{JsonObjectParser, json::Parser}; @@ -1211,240 +1211,6 @@ impl Property { } } -impl SerializeInto for Property { - fn serialize_into(&self, buf: &mut Vec) { - buf.push(match self { - Property::IsActive => 0, - Property::IsEnabled => 1, - Property::IsSubscribed => 2, - Property::Keys => 3, - Property::Keywords => 4, - Property::Language => 5, - Property::Location => 6, - Property::MailboxIds => 7, - Property::MayDelete => 8, - Property::MdnBlobIds => 9, - Property::Members => 10, - Property::MessageId => 11, - Property::MyRights => 12, - Property::Name => 13, - Property::ParentId => 14, - Property::PartId => 15, - Property::Picture => 16, - Property::Preview => 17, - Property::Quota => 18, - Property::ReceivedAt => 19, - Property::References => 20, - Property::ReplyTo => 21, - Property::Role => 22, - Property::Secret => 23, - Property::SendAt => 24, - Property::Sender => 25, - Property::SentAt => 26, - Property::Size => 27, - Property::SortOrder => 28, - Property::Subject => 29, - Property::SubParts => 30, - Property::TextBody => 31, - Property::TextSignature => 32, - Property::ThreadId => 33, - Property::Timezone => 34, - Property::To => 35, - Property::ToDate => 36, - Property::TotalEmails => 37, - Property::TotalThreads => 38, - Property::Type => 39, - Property::Types => 40, - Property::UndoStatus => 41, - Property::UnreadEmails => 42, - Property::UnreadThreads => 43, - Property::Url => 44, - Property::VerificationCode => 45, - Property::Parameters => 46, - Property::Addresses => 47, - Property::P256dh => 48, - Property::Auth => 49, - Property::Value => 50, - Property::SmtpReply => 51, - Property::Delivered => 52, - Property::Displayed => 53, - Property::MailFrom => 54, - Property::RcptTo => 55, - Property::IsEncodingProblem => 56, - Property::IsTruncated => 57, - Property::MayReadItems => 58, - Property::MayAddItems => 59, - Property::MayRemoveItems => 60, - Property::MaySetSeen => 61, - Property::MaySetKeywords => 62, - Property::MayCreateChild => 63, - Property::MayRename => 64, - Property::MaySubmit => 65, - Property::Acl => 66, - Property::Aliases => 67, - Property::Attachments => 68, - Property::Bcc => 69, - Property::BlobId => 70, - Property::BodyStructure => 71, - Property::BodyValues => 72, - Property::Capabilities => 73, - Property::Cc => 74, - Property::Charset => 75, - Property::Cid => 76, - Property::DeliveryStatus => 77, - Property::Description => 78, - Property::DeviceClientId => 79, - Property::Disposition => 80, - Property::DsnBlobIds => 81, - Property::Email => 82, - Property::EmailId => 83, - Property::EmailIds => 84, - Property::Envelope => 85, - Property::Expires => 86, - Property::From => 87, - Property::FromDate => 88, - Property::HasAttachment => 89, - Property::Header(_) => 90, - Property::Headers => 91, - Property::HtmlBody => 92, - Property::HtmlSignature => 93, - Property::Id => 94, - Property::IdentityId => 95, - Property::InReplyTo => 96, - Property::_T(value) => { - buf.push(97); - value.serialize_into(buf); - return; - } - Property::ResourceType => 98, - Property::Used => 99, - Property::HardLimit => 100, - Property::WarnLimit => 101, - Property::SoftLimit => 102, - Property::Scope => 103, - Property::Digest(_) | Property::Data(_) => { - unreachable!("Property::Digest and Property::Data are not serializable") - } - }); - } -} - -impl DeserializeFrom for Property { - fn deserialize_from(bytes: &mut std::slice::Iter<'_, u8>) -> Option { - match *bytes.next()? { - 0 => Some(Property::IsActive), - 1 => Some(Property::IsEnabled), - 2 => Some(Property::IsSubscribed), - 3 => Some(Property::Keys), - 4 => Some(Property::Keywords), - 5 => Some(Property::Language), - 6 => Some(Property::Location), - 7 => Some(Property::MailboxIds), - 8 => Some(Property::MayDelete), - 9 => Some(Property::MdnBlobIds), - 10 => Some(Property::Members), - 11 => Some(Property::MessageId), - 12 => Some(Property::MyRights), - 13 => Some(Property::Name), - 14 => Some(Property::ParentId), - 15 => Some(Property::PartId), - 16 => Some(Property::Picture), - 17 => Some(Property::Preview), - 18 => Some(Property::Quota), - 19 => Some(Property::ReceivedAt), - 20 => Some(Property::References), - 21 => Some(Property::ReplyTo), - 22 => Some(Property::Role), - 23 => Some(Property::Secret), - 24 => Some(Property::SendAt), - 25 => Some(Property::Sender), - 26 => Some(Property::SentAt), - 27 => Some(Property::Size), - 28 => Some(Property::SortOrder), - 29 => Some(Property::Subject), - 30 => Some(Property::SubParts), - 31 => Some(Property::TextBody), - 32 => Some(Property::TextSignature), - 33 => Some(Property::ThreadId), - 34 => Some(Property::Timezone), - 35 => Some(Property::To), - 36 => Some(Property::ToDate), - 37 => Some(Property::TotalEmails), - 38 => Some(Property::TotalThreads), - 39 => Some(Property::Type), - 40 => Some(Property::Types), - 41 => Some(Property::UndoStatus), - 42 => Some(Property::UnreadEmails), - 43 => Some(Property::UnreadThreads), - 44 => Some(Property::Url), - 45 => Some(Property::VerificationCode), - 46 => Some(Property::Parameters), - 47 => Some(Property::Addresses), - 48 => Some(Property::P256dh), - 49 => Some(Property::Auth), - 50 => Some(Property::Value), - 51 => Some(Property::SmtpReply), - 52 => Some(Property::Delivered), - 53 => Some(Property::Displayed), - 54 => Some(Property::MailFrom), - 55 => Some(Property::RcptTo), - 56 => Some(Property::IsEncodingProblem), - 57 => Some(Property::IsTruncated), - 58 => Some(Property::MayReadItems), - 59 => Some(Property::MayAddItems), - 60 => Some(Property::MayRemoveItems), - 61 => Some(Property::MaySetSeen), - 62 => Some(Property::MaySetKeywords), - 63 => Some(Property::MayCreateChild), - 64 => Some(Property::MayRename), - 65 => Some(Property::MaySubmit), - 66 => Some(Property::Acl), - 67 => Some(Property::Aliases), - 68 => Some(Property::Attachments), - 69 => Some(Property::Bcc), - 70 => Some(Property::BlobId), - 71 => Some(Property::BodyStructure), - 72 => Some(Property::BodyValues), - 73 => Some(Property::Capabilities), - 74 => Some(Property::Cc), - 75 => Some(Property::Charset), - 76 => Some(Property::Cid), - 77 => Some(Property::DeliveryStatus), - 78 => Some(Property::Description), - 79 => Some(Property::DeviceClientId), - 80 => Some(Property::Disposition), - 81 => Some(Property::DsnBlobIds), - 82 => Some(Property::Email), - 83 => Some(Property::EmailId), - 84 => Some(Property::EmailIds), - 85 => Some(Property::Envelope), - 86 => Some(Property::Expires), - 87 => Some(Property::From), - 88 => Some(Property::FromDate), - 89 => Some(Property::HasAttachment), - 90 => Some(Property::Header(HeaderProperty { - form: HeaderForm::Raw, - header: String::new(), - all: false, - })), // Never serialized - 91 => Some(Property::Headers), - 92 => Some(Property::HtmlBody), - 93 => Some(Property::HtmlSignature), - 94 => Some(Property::Id), - 95 => Some(Property::IdentityId), - 96 => Some(Property::InReplyTo), - 97 => String::deserialize_from(bytes).map(Property::_T), - 98 => Some(Property::ResourceType), - 99 => Some(Property::Used), - 100 => Some(Property::HardLimit), - 101 => Some(Property::WarnLimit), - 102 => Some(Property::SoftLimit), - 103 => Some(Property::Scope), - _ => None, - } - } -} - impl Serialize for Property { fn serialize(&self, serializer: S) -> Result where diff --git a/crates/jmap-proto/src/types/type_state.rs b/crates/jmap-proto/src/types/type_state.rs index 19b3af7c..53bd7e80 100644 --- a/crates/jmap-proto/src/types/type_state.rs +++ b/crates/jmap-proto/src/types/type_state.rs @@ -7,7 +7,6 @@ use std::fmt::Display; use serde::Serialize; -use store::write::{DeserializeFrom, SerializeInto}; use utils::map::bitmap::BitmapItem; use crate::parser::{JsonObjectParser, json::Parser}; @@ -206,33 +205,6 @@ impl Display for DataType { } } -impl SerializeInto for DataType { - fn serialize_into(&self, buf: &mut Vec) { - buf.push(*self as u8); - } -} - -impl DeserializeFrom for DataType { - fn deserialize_from(bytes: &mut std::slice::Iter<'_, u8>) -> Option { - match *bytes.next()? { - 0 => Some(DataType::Email), - 1 => Some(DataType::EmailDelivery), - 2 => Some(DataType::EmailSubmission), - 3 => Some(DataType::Mailbox), - 4 => Some(DataType::Thread), - 5 => Some(DataType::Identity), - 6 => Some(DataType::Core), - 7 => Some(DataType::PushSubscription), - 8 => Some(DataType::SearchSnippet), - 9 => Some(DataType::VacationResponse), - 10 => Some(DataType::Mdn), - 11 => Some(DataType::Quota), - 12 => Some(DataType::SieveScript), - _ => None, - } - } -} - impl<'de> serde::Deserialize<'de> for DataType { fn deserialize(deserializer: D) -> Result where diff --git a/crates/jmap/src/api/form.rs b/crates/jmap/src/api/form.rs index ffd209be..eff3d6f9 100644 --- a/crates/jmap/src/api/form.rs +++ b/crates/jmap/src/api/form.rs @@ -182,7 +182,7 @@ impl FormHandler for Server { .unwrap_or_default(); // Reserve and write blob - let message_blob = BlobHash::from(message.as_bytes()); + let message_blob = BlobHash::generate(message.as_bytes()); let message_size = message.len(); let mut batch = BatchBuilder::new(); batch.set( diff --git a/crates/jmap/src/api/management/queue.rs b/crates/jmap/src/api/management/queue.rs index acbced01..e7899ae3 100644 --- a/crates/jmap/src/api/management/queue.rs +++ b/crates/jmap/src/api/management/queue.rs @@ -6,11 +6,11 @@ use std::{future::Future, sync::atomic::Ordering}; -use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; -use common::{auth::AccessToken, ipc::QueueEvent, Server}; +use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD}; +use common::{Server, auth::AccessToken, ipc::QueueEvent}; use directory::{ - backend::internal::{manage::ManageDirectory, PrincipalField}, Permission, Type, + backend::internal::{PrincipalField, manage::ManageDirectory}, }; use hyper::Method; use mail_auth::{ @@ -22,19 +22,19 @@ use mail_parser::DateTime; use serde::{Deserializer, Serializer}; use serde_json::json; use smtp::{ - queue::{self, spool::SmtpSpool, ErrorDetails, HostResponse, QueueId, Status}, + queue::{self, ErrorDetails, HostResponse, QueueId, Status, spool::SmtpSpool}, reporting::{dmarc::DmarcReporting, tls::TlsReporting}, }; use store::{ - write::{key::DeserializeBigEndian, now, Bincode, QueueClass, ReportEvent, ValueClass}, Deserialize, IterateParams, ValueKey, + write::{LegacyBincode, QueueClass, ReportEvent, ValueClass, key::DeserializeBigEndian, now}, }; use trc::AddContext; use utils::url_params::UrlParams; -use crate::api::{http::ToHttpResponse, HttpRequest, HttpResponse, JsonResponse}; +use crate::api::{HttpRequest, HttpResponse, JsonResponse, http::ToHttpResponse}; -use super::{decode_path_element, FutureTimestamp}; +use super::{FutureTimestamp, decode_path_element}; #[derive(Debug, serde::Serialize, serde::Deserialize, PartialEq, Eq)] pub struct Message { @@ -205,7 +205,7 @@ impl QueueManagement for Server { .filter(|message| { tenant_domains .as_ref() - .is_none_or( |domains| message.has_domain(domains)) + .is_none_or(|domains| message.has_domain(domains)) }) { Ok(JsonResponse::new(json!({ @@ -282,7 +282,7 @@ impl QueueManagement for Server { .filter(|message| { tenant_domains .as_ref() - .is_none_or( |domains| message.has_domain(domains)) + .is_none_or(|domains| message.has_domain(domains)) }) { let prev_event = message.next_event().unwrap_or_default(); @@ -294,7 +294,7 @@ impl QueueManagement for Server { Status::Scheduled | Status::TemporaryFailure(_) ) && item .as_ref() - .is_none_or( |item| domain.domain.contains(item)) + .is_none_or(|item| domain.domain.contains(item)) { domain.retry.due = time; if domain.expires > time { @@ -374,7 +374,7 @@ impl QueueManagement for Server { .filter(|message| { tenant_domains .as_ref() - .is_none_or( |domains| message.has_domain(domains)) + .is_none_or(|domains| message.has_domain(domains)) }) { let mut found = false; @@ -475,7 +475,7 @@ impl QueueManagement for Server { QueueClass::DmarcReportHeader(event) if tenant_domains .as_ref() - .is_none_or( |domains| domains.contains(&event.domain)) => + .is_none_or(|domains| domains.contains(&event.domain)) => { let mut rua = Vec::new(); if let Some(report) = self @@ -488,7 +488,7 @@ impl QueueManagement for Server { QueueClass::TlsReportHeader(event) if tenant_domains .as_ref() - .is_none_or( |domains| domains.contains(&event.domain)) => + .is_none_or(|domains| domains.contains(&event.domain)) => { let mut rua = Vec::new(); if let Some(report) = self @@ -548,7 +548,7 @@ impl QueueManagement for Server { QueueClass::DmarcReportHeader(event) if tenant_domains .as_ref() - .is_none_or( |domains| domains.contains(&event.domain)) => + .is_none_or(|domains| domains.contains(&event.domain)) => { self.delete_dmarc_report(event).await; true @@ -556,7 +556,7 @@ impl QueueManagement for Server { QueueClass::TlsReportHeader(event) if tenant_domains .as_ref() - .is_none_or( |domains| domains.contains(&event.domain)) => + .is_none_or(|domains| domains.contains(&event.domain)) => { self.delete_tls_report(vec![event]).await; true @@ -714,12 +714,12 @@ async fn fetch_queued_messages( .iterate( IterateParams::new(from_key, to_key).ascending(), |key, value| { - let message = Bincode::::deserialize(value) + let message = LegacyBincode::::deserialize(value) .add_context(|ctx| ctx.ctx(trc::Key::Key, key))? .inner; let matches = tenant_domains .as_ref() - .is_none_or( |domains| message.has_domain(domains)) + .is_none_or(|domains| message.has_domain(domains)) && (!has_filters || (text .as_ref() @@ -732,8 +732,8 @@ async fn fetch_queued_messages( }) .unwrap_or_else(|| { from.as_ref() - .is_none_or( |from| message.return_path.contains(from)) - && to.as_ref().is_none_or( |to| { + .is_none_or(|from| message.return_path.contains(from)) + && to.as_ref().is_none_or(|to| { message .recipients .iter() @@ -742,10 +742,10 @@ async fn fetch_queued_messages( }) && before .as_ref() - .is_none_or( |before| message.next_delivery_event() < *before) + .is_none_or(|before| message.next_delivery_event() < *before) && after .as_ref() - .is_none_or( |after| message.next_delivery_event() > *after))); + .is_none_or(|after| message.next_delivery_event() > *after))); if matches { if offset == 0 { @@ -825,13 +825,13 @@ async fn fetch_queued_reports( .iterate( IterateParams::new(from_key, to_key).ascending().no_values(), |key, _| { - if type_.is_none_or( |t| t == *key.last().unwrap()) { + if type_.is_none_or(|t| t == *key.last().unwrap()) { let event = ReportEvent::deserialize(key)?; if tenant_domains .as_ref() - .is_none_or( |domains| domains.contains(&event.domain)) + .is_none_or(|domains| domains.contains(&event.domain)) && event.seq_id != 0 - && domain.as_ref().is_none_or( |d| event.domain.contains(d)) + && domain.as_ref().is_none_or(|d| event.domain.contains(d)) { if offset == 0 { if limit == 0 || total_returned < limit { diff --git a/crates/jmap/src/api/management/report.rs b/crates/jmap/src/api/management/report.rs index a6e7ac64..4cc485f0 100644 --- a/crates/jmap/src/api/management/report.rs +++ b/crates/jmap/src/api/management/report.rs @@ -6,26 +6,26 @@ use std::future::Future; -use common::{auth::AccessToken, Server}; +use common::{Server, auth::AccessToken}; use directory::{ - backend::internal::{manage::ManageDirectory, PrincipalField}, Permission, Type, + backend::internal::{PrincipalField, manage::ManageDirectory}, }; use hyper::Method; use mail_auth::report::{ - tlsrpt::{FailureDetails, Policy, TlsReport}, Feedback, + tlsrpt::{FailureDetails, Policy, TlsReport}, }; use serde_json::json; use smtp::reporting::analysis::IncomingReport; use store::{ - write::{key::DeserializeBigEndian, BatchBuilder, Bincode, ReportClass, ValueClass}, - Deserialize, IterateParams, ValueKey, U64_LEN, + Deserialize, IterateParams, U64_LEN, ValueKey, + write::{BatchBuilder, LegacyBincode, ReportClass, ValueClass, key::DeserializeBigEndian}, }; use trc::AddContext; use utils::url_params::UrlParams; -use crate::api::{http::ToHttpResponse, HttpRequest, HttpResponse, JsonResponse}; +use crate::api::{HttpRequest, HttpResponse, JsonResponse, http::ToHttpResponse}; use super::decode_path_element; @@ -121,7 +121,7 @@ impl ManageReports for Server { .core .storage .data - .get_value::>>(ValueKey::from( + .get_value::>>(ValueKey::from( ValueClass::Report(report_id), )) .await? @@ -129,7 +129,7 @@ impl ManageReports for Server { Some(report) if tenant_domains .as_ref() - .is_none_or( |domains| report.inner.has_domain(domains)) => + .is_none_or(|domains| report.inner.has_domain(domains)) => { Ok(JsonResponse::new(json!({ "data": report.inner, @@ -142,7 +142,7 @@ impl ManageReports for Server { .core .storage .data - .get_value::>>( + .get_value::>>( ValueKey::from(ValueClass::Report(report_id)), ) .await? @@ -150,7 +150,7 @@ impl ManageReports for Server { Some(report) if tenant_domains .as_ref() - .is_none_or( |domains| report.inner.has_domain(domains)) => + .is_none_or(|domains| report.inner.has_domain(domains)) => { Ok(JsonResponse::new(json!({ "data": report.inner, @@ -163,7 +163,7 @@ impl ManageReports for Server { .core .storage .data - .get_value::>>(ValueKey::from( + .get_value::>>(ValueKey::from( ValueClass::Report(report_id), )) .await? @@ -171,7 +171,7 @@ impl ManageReports for Server { Some(report) if tenant_domains .as_ref() - .is_none_or( |domains| report.inner.has_domain(domains)) => + .is_none_or(|domains| report.inner.has_domain(domains)) => { Ok(JsonResponse::new(json!({ "data": report.inner, @@ -249,7 +249,7 @@ impl ManageReports for Server { .core .storage .data - .get_value::>>(ValueKey::from( + .get_value::>>(ValueKey::from( ValueClass::Report(report_id.clone()), )) .await? @@ -258,7 +258,7 @@ impl ManageReports for Server { .core .storage .data - .get_value::>>( + .get_value::>>( ValueKey::from(ValueClass::Report(report_id.clone())), ) .await? @@ -268,7 +268,7 @@ impl ManageReports for Server { .core .storage .data - .get_value::>>(ValueKey::from( + .get_value::>>(ValueKey::from( ValueClass::Report(report_id.clone()), )) .await? @@ -382,7 +382,7 @@ async fn fetch_incoming_reports( match typ { ReportType::Dmarc => { let report = - Bincode::>::deserialize( + LegacyBincode::>::deserialize( value, ) .caused_by(trc::location!())? @@ -394,7 +394,7 @@ async fn fetch_incoming_reports( .is_none_or( |domains| report.has_domain(domains)) } ReportType::Tls => { - let report = Bincode::>::deserialize(value) + let report = LegacyBincode::>::deserialize(value) .caused_by(trc::location!())? .inner; @@ -404,7 +404,7 @@ async fn fetch_incoming_reports( .is_none_or( |domains| report.has_domain(domains)) } ReportType::Arf => { - let report = Bincode::>::deserialize(value) + let report = LegacyBincode::>::deserialize(value) .caused_by(trc::location!())? .inner; diff --git a/crates/jmap/src/api/management/stores.rs b/crates/jmap/src/api/management/stores.rs index 99da0d23..47bc9a1f 100644 --- a/crates/jmap/src/api/management/stores.rs +++ b/crates/jmap/src/api/management/stores.rs @@ -17,7 +17,7 @@ use directory::{ backend::internal::manage::{self, ManageDirectory}, }; use email::{ - mailbox::{ArchivedMailbox, UidMailbox}, + mailbox::{ArchivedMailbox, ArchivedUidMailbox, UidMailbox}, message::ingest::EmailIngest, }; use hyper::Method; @@ -25,7 +25,8 @@ use jmap_proto::types::{collection::Collection, property::Property}; use serde_json::json; use store::{ Serialize, - write::{ArchivedValue, BatchBuilder, ValueClass, assert::HashedValue}, + rkyv::vec::ArchivedVec, + write::{Archive, Archiver, BatchBuilder, ValueClass, assert::HashedValue}, }; use trc::AddContext; use utils::url_params::UrlParams; @@ -341,7 +342,7 @@ pub async fn reset_imap_uids(server: &Server, account_id: u32) -> trc::Result<(u .unwrap_or_default() { let mailbox = server - .get_property::>>( + .get_property::>( account_id, Collection::Mailbox, mailbox_id, @@ -350,7 +351,7 @@ pub async fn reset_imap_uids(server: &Server, account_id: u32) -> trc::Result<(u .await .caused_by(trc::location!())? .ok_or_else(|| trc::ImapEvent::Error.into_err().caused_by(trc::location!()))? - .into_deserialized::() + .into_deserialized::() .caused_by(trc::location!())?; let mut new_mailbox = mailbox.inner.clone(); new_mailbox.uid_validity = rand::random::(); @@ -382,7 +383,7 @@ pub async fn reset_imap_uids(server: &Server, account_id: u32) -> trc::Result<(u .unwrap_or_default() { let uids = server - .get_property::>>( + .get_property::>( account_id, Collection::Email, message_id, @@ -390,8 +391,9 @@ pub async fn reset_imap_uids(server: &Server, account_id: u32) -> trc::Result<(u ) .await .caused_by(trc::location!())?; - let mut uids = if let Some(uids) = uids.filter(|uids| !uids.inner.is_empty()) { - uids + let mut uids = if let Some(uids) = uids { + uids.into_deserialized::, Vec>() + .caused_by(trc::location!())? } else { continue; }; @@ -412,7 +414,9 @@ pub async fn reset_imap_uids(server: &Server, account_id: u32) -> trc::Result<(u .assert_value(ValueClass::Property(Property::MailboxIds.into()), &uids) .set( Property::MailboxIds, - uids.inner.serialize().caused_by(trc::location!())?, + Archiver::new(uids.inner) + .serialize() + .caused_by(trc::location!())?, ); server .store() diff --git a/crates/jmap/src/auth/oauth/auth.rs b/crates/jmap/src/auth/oauth/auth.rs index b4ab3c02..b7f2bc75 100644 --- a/crates/jmap/src/auth/oauth/auth.rs +++ b/crates/jmap/src/auth/oauth/auth.rs @@ -21,7 +21,7 @@ use rand::{ use serde::Deserialize; use serde_json::json; use std::future::Future; -use store::{Serialize, dispatch::lookup::KeyValue, write::Bincode}; +use store::{Serialize, dispatch::lookup::KeyValue, write::LegacyBincode}; use trc::AddContext; use crate::{ @@ -107,7 +107,7 @@ impl OAuthApiHandler for Server { .collect::(); // Serialize OAuth code - let value = Bincode::new(OAuthCode { + let value = LegacyBincode::new(OAuthCode { status: OAuthStatus::Authorized, account_id: access_token.primary_id(), client_id, @@ -149,7 +149,7 @@ impl OAuthApiHandler for Server { .core .storage .lookup - .key_get::>(KeyValue::<()>::build_key( + .key_get::>(KeyValue::<()>::build_key( KV_OAUTH, code.as_bytes(), )) @@ -232,7 +232,7 @@ impl OAuthApiHandler for Server { } // Add OAuth status - let oauth_code = Bincode::new(OAuthCode { + let oauth_code = LegacyBincode::new(OAuthCode { status: OAuthStatus::Pending, account_id: u32::MAX, client_id, diff --git a/crates/jmap/src/auth/oauth/token.rs b/crates/jmap/src/auth/oauth/token.rs index 8a9e927f..ea573b44 100644 --- a/crates/jmap/src/auth/oauth/token.rs +++ b/crates/jmap/src/auth/oauth/token.rs @@ -5,25 +5,25 @@ */ use common::{ + KV_OAUTH, Server, auth::{ - oauth::{oidc::StandardClaims, GrantType}, AccessToken, + oauth::{GrantType, oidc::StandardClaims}, }, - Server, KV_OAUTH, }; use hyper::StatusCode; use std::future::Future; -use store::{dispatch::lookup::KeyValue, write::Bincode}; +use store::{dispatch::lookup::KeyValue, write::LegacyBincode}; use trc::AddContext; use crate::api::{ - http::{HttpContext, HttpSessionData, ToHttpResponse}, HttpRequest, HttpResponse, JsonResponse, + http::{HttpContext, HttpSessionData, ToHttpResponse}, }; use super::{ - registration::ClientRegistrationHandler, ErrorType, FormData, OAuthCode, OAuthResponse, - OAuthStatus, TokenResponse, MAX_POST_LEN, + ErrorType, FormData, MAX_POST_LEN, OAuthCode, OAuthResponse, OAuthStatus, TokenResponse, + registration::ClientRegistrationHandler, }; pub trait TokenHandler: Sync + Send { @@ -79,7 +79,7 @@ impl TokenHandler for Server { .core .storage .lookup - .key_get::>(KeyValue::<()>::build_key( + .key_get::>(KeyValue::<()>::build_key( KV_OAUTH, code.as_bytes(), )) @@ -149,7 +149,7 @@ impl TokenHandler for Server { .core .storage .lookup - .key_get::>(KeyValue::<()>::build_key( + .key_get::>(KeyValue::<()>::build_key( KV_OAUTH, device_code.as_bytes(), )) @@ -231,10 +231,11 @@ impl TokenHandler for Server { .caused_by(trc::location!()) })?, Err(err) => { - trc::error!(err - .caused_by(trc::location!()) - .details("Failed to validate refresh token") - .span_id(session.session_id)); + trc::error!( + err.caused_by(trc::location!()) + .details("Failed to validate refresh token") + .span_id(session.session_id) + ); TokenResponse::error(ErrorType::InvalidGrant) } }; diff --git a/crates/jmap/src/blob/get.rs b/crates/jmap/src/blob/get.rs index d9ac3297..8289e55b 100644 --- a/crates/jmap/src/blob/get.rs +++ b/crates/jmap/src/blob/get.rs @@ -5,7 +5,7 @@ */ use common::{Server, auth::AccessToken}; -use email::mailbox::UidMailbox; +use email::mailbox::ArchivedUidMailbox; use jmap_proto::{ method::{ get::{GetRequest, GetResponse}, @@ -24,7 +24,8 @@ use jmap_proto::{ use mail_builder::encoders::base64::base64_encode; use sha1::{Digest, Sha1}; use sha2::{Sha256, Sha512}; -use store::BlobClass; +use store::{BlobClass, rkyv::vec::ArchivedVec, write::Archive}; +use trc::AddContext; use utils::map::vec_map::VecMap; use std::future::Future; @@ -238,7 +239,7 @@ impl BlobOperations for Server { } if include_mailbox { if let Some(mailboxes) = self - .get_property::>( + .get_property::( req_account_id, Collection::Email, *document_id, @@ -249,10 +250,12 @@ impl BlobOperations for Server { matched_ids.append( DataType::Mailbox, mailboxes - .into_iter() + .unarchive::>() + .caused_by(trc::location!())? + .iter() .map(|m| { debug_assert!(m.uid != 0); - Id::from(m.mailbox_id) + Id::from(u32::from(m.mailbox_id)) }) .collect::>(), ); diff --git a/crates/jmap/src/email/body.rs b/crates/jmap/src/email/body.rs index 611240cc..7610077d 100644 --- a/crates/jmap/src/email/body.rs +++ b/crates/jmap/src/email/body.rs @@ -4,7 +4,9 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use email::message::metadata::{MessageMetadataContents, MetadataPartType}; +use email::message::metadata::{ + ArchivedHeaderValue, ArchivedMessageMetadataContents, ArchivedMetadataPartType, +}; use jmap_proto::types::{ blob::BlobId, property::Property, @@ -135,7 +137,7 @@ impl ToBodyPart for Vec> { } } -impl ToBodyPart for MessageMetadataContents<'_> { +impl ToBodyPart for ArchivedMessageMetadataContents { fn to_body_part( &self, part_id: usize, @@ -152,7 +154,7 @@ impl ToBodyPart for MessageMetadataContents<'_> { parts.next().map(|part_id| (part_id, &self.parts[part_id])) { let mut values = Object::with_capacity(properties.len()); - let multipart = if let MetadataPartType::Multipart(parts) = &part.body { + let multipart = if let ArchivedMetadataPartType::Multipart(parts) = &part.body { parts.into() } else { None @@ -166,13 +168,13 @@ impl ToBodyPart for MessageMetadataContents<'_> { BlobId::new_section( blob_id.hash.clone(), blob_id.class.clone(), - part.offset_body + base_offset, - part.offset_end + base_offset, - part.encoding as u8, + u32::from(part.offset_body) as usize + base_offset, + u32::from(part.offset_end) as usize + base_offset, + part.encoding.id(), ) .into() } - Property::Size if multipart.is_none() => part.size.into(), + Property::Size if multipart.is_none() => u32::from(part.size).into(), Property::Name => part.attachment_name().into(), Property::Type => part .content_type() @@ -182,27 +184,31 @@ impl ToBodyPart for MessageMetadataContents<'_> { .unwrap_or_else(|| ct.ctype().to_string()) }) .or_else(|| match &part.body { - MetadataPartType::Text => Some("text/plain".to_string()), - MetadataPartType::Html => Some("text/html".to_string()), - MetadataPartType::Message(_) => Some("message/rfc822".to_string()), - _ => None, - }) - .into(), - Property::Charset => part - .content_type() - .and_then(|ct| ct.attribute("charset")) - .or(match &part.body { - MetadataPartType::Text | MetadataPartType::Html => Some("us-ascii"), + ArchivedMetadataPartType::Text => Some("text/plain".to_string()), + ArchivedMetadataPartType::Html => Some("text/html".to_string()), + ArchivedMetadataPartType::Message(_) => { + Some("message/rfc822".to_string()) + } _ => None, }) .into(), + Property::Charset => { + part.content_type() + .and_then(|ct| ct.attribute("charset")) + .or(match &part.body { + ArchivedMetadataPartType::Text + | ArchivedMetadataPartType::Html => Some("us-ascii"), + _ => None, + }) + .into() + } Property::Disposition => { part.content_disposition().map(|cd| cd.ctype()).into() } Property::Cid => part.content_id().into(), Property::Language => match part.content_language() { - HeaderValue::Text(text) => vec![text.to_string()].into(), - HeaderValue::TextList(list) => list + ArchivedHeaderValue::Text(text) => vec![text.to_string()].into(), + ArchivedHeaderValue::TextList(list) => list .iter() .map(|text| text.to_string().into()) .collect::>() @@ -221,7 +227,10 @@ impl ToBodyPart for MessageMetadataContents<'_> { subparts.push(values); if let Some(multipart) = multipart { - let multipart = multipart.clone(); + let multipart = multipart + .iter() + .map(|id| u16::from(id) as usize) + .collect::>(); parts_stack.push(( parts, std::mem::replace(&mut subparts, Vec::with_capacity(multipart.len())), @@ -247,93 +256,10 @@ pub(super) trait TruncateBody { } impl TruncateBody for PartType<'_> { - fn truncate(&self, mut max_len: usize) -> (bool, String) { + fn truncate(&self, max_len: usize) -> (bool, String) { match self { - PartType::Text(text) => { - if max_len != 0 && text.len() > max_len { - let add_dots = max_len > 6; - if add_dots { - max_len -= 3; - } - let mut result = String::with_capacity(max_len); - for ch in text.chars() { - if ch != '\r' { - if ch.len_utf8() + result.len() > max_len { - break; - } - result.push(ch); - } - } - if add_dots { - result.push_str("..."); - } - (true, result) - } else { - (false, text.replace('\r', "")) - } - } - PartType::Html(html) => { - if max_len != 0 && html.len() > max_len { - let add_dots = max_len > 6; - if add_dots { - max_len -= 3; - } - - let mut result = String::with_capacity(max_len); - let mut in_tag = false; - let mut in_comment = false; - let mut last_tag_end_pos = 0; - let mut cr_count = 0; - for (pos, ch) in html.char_indices() { - let mut set_last_tag = 0; - match ch { - '<' if !in_tag => { - in_tag = true; - if let Some("!--") = html.get(pos + 1..pos + 4) { - in_comment = true; - } - set_last_tag = pos; - } - '>' if in_tag => { - if in_comment { - if let Some("--") = html.get(pos - 2..pos) { - in_comment = false; - in_tag = false; - set_last_tag = pos + 1; - } - } else { - in_tag = false; - set_last_tag = pos + 1; - } - } - '\r' => { - cr_count += 1; - continue; - } - _ => (), - } - if ch.len_utf8() + pos - cr_count > max_len { - result.push_str( - &html[0..if (in_tag || set_last_tag > 0) && last_tag_end_pos > 0 { - last_tag_end_pos - } else { - pos - }] - .replace('\r', ""), - ); - if add_dots { - result.push_str("..."); - } - break; - } else if set_last_tag > 0 { - last_tag_end_pos = set_last_tag; - } - } - (true, result) - } else { - (false, html.replace('\r', "")) - } - } + PartType::Text(text) => truncate_plain(text, max_len), + PartType::Html(html) => truncate_html(html, max_len), PartType::Binary(bytes) | PartType::InlineBinary(bytes) => { PartType::Text(String::from_utf8_lossy(bytes)).truncate(max_len) } @@ -341,3 +267,90 @@ impl TruncateBody for PartType<'_> { } } } + +pub(crate) fn truncate_plain(text: &str, mut max_len: usize) -> (bool, String) { + if max_len != 0 && text.len() > max_len { + let add_dots = max_len > 6; + if add_dots { + max_len -= 3; + } + let mut result = String::with_capacity(max_len); + for ch in text.chars() { + if ch != '\r' { + if ch.len_utf8() + result.len() > max_len { + break; + } + result.push(ch); + } + } + if add_dots { + result.push_str("..."); + } + (true, result) + } else { + (false, text.replace('\r', "")) + } +} + +pub(crate) fn truncate_html(html: &str, mut max_len: usize) -> (bool, String) { + if max_len != 0 && html.len() > max_len { + let add_dots = max_len > 6; + if add_dots { + max_len -= 3; + } + + let mut result = String::with_capacity(max_len); + let mut in_tag = false; + let mut in_comment = false; + let mut last_tag_end_pos = 0; + let mut cr_count = 0; + for (pos, ch) in html.char_indices() { + let mut set_last_tag = 0; + match ch { + '<' if !in_tag => { + in_tag = true; + if let Some("!--") = html.get(pos + 1..pos + 4) { + in_comment = true; + } + set_last_tag = pos; + } + '>' if in_tag => { + if in_comment { + if let Some("--") = html.get(pos - 2..pos) { + in_comment = false; + in_tag = false; + set_last_tag = pos + 1; + } + } else { + in_tag = false; + set_last_tag = pos + 1; + } + } + '\r' => { + cr_count += 1; + continue; + } + _ => (), + } + if ch.len_utf8() + pos - cr_count > max_len { + result.push_str( + &html[0..if (in_tag || set_last_tag > 0) && last_tag_end_pos > 0 { + last_tag_end_pos + } else { + pos + }] + .replace('\r', ""), + ); + if add_dots { + result.push_str("..."); + } + break; + } else if set_last_tag > 0 { + last_tag_end_pos = set_last_tag; + } + } + (true, result) + } else { + (false, html.replace('\r', "")) + } +} diff --git a/crates/jmap/src/email/crypto.rs b/crates/jmap/src/email/crypto.rs index 7db5853d..9656ff8d 100644 --- a/crates/jmap/src/email/crypto.rs +++ b/crates/jmap/src/email/crypto.rs @@ -10,6 +10,7 @@ use crate::api::{HttpResponse, JsonResponse, http::ToHttpResponse}; use common::{Server, auth::AccessToken}; use directory::backend::internal::manage; use email::message::crypto::{ + Algorithm, ArchivedAlgorithm, ArchivedEncryptionMethod, ArchivedEncryptionParams, EncryptMessage, EncryptMessageError, EncryptionMethod, EncryptionParams, EncryptionType, try_parse_certs, }; @@ -19,7 +20,7 @@ use mail_parser::MessageParser; use serde_json::json; use store::{ Serialize, - write::{BatchBuilder, Bincode}, + write::{Archive, Archiver, BatchBuilder, serialize::rkyv_unarchive}, }; use trc::AddContext; @@ -38,34 +39,39 @@ pub trait CryptoHandler: Sync + Send { impl CryptoHandler for Server { async fn handle_crypto_get(&self, access_token: Arc) -> trc::Result { - let params = self - .get_property::( + let ec = if let Some(params_) = self + .get_property::( access_token.primary_id(), Collection::Principal, 0, Property::Parameters, ) - .await?; - let ec = params - .map(|params| { - let method = params.method; - let algo = params.algo; - let mut certs = Vec::new(); - certs.extend_from_slice(b"-----STALWART CERTIFICATE-----\r\n"); - let _ = base64_encode_mime( - &Bincode::new(params).serialize().unwrap_or_default(), - &mut certs, - false, - ); - certs.extend_from_slice(b"\r\n"); - let certs = String::from_utf8(certs).unwrap_or_default(); + .await? + { + let params = params_ + .unarchive::() + .caused_by(trc::location!())?; + let algo = match ¶ms.algo { + ArchivedAlgorithm::Aes128 => Algorithm::Aes128, + ArchivedAlgorithm::Aes256 => Algorithm::Aes256, + }; + let method = match ¶ms.method { + ArchivedEncryptionMethod::PGP => EncryptionMethod::PGP, + ArchivedEncryptionMethod::SMIME => EncryptionMethod::SMIME, + }; + let mut certs = Vec::new(); + certs.extend_from_slice(b"-----STALWART CERTIFICATE-----\r\n"); + let _ = base64_encode_mime(¶ms_.into_inner(), &mut certs, false); + certs.extend_from_slice(b"\r\n"); + let certs = String::from_utf8(certs).unwrap_or_default(); - match method { - EncryptionMethod::PGP => EncryptionType::PGP { algo, certs }, - EncryptionMethod::SMIME => EncryptionType::SMIME { algo, certs }, - } - }) - .unwrap_or(EncryptionType::Disabled); + match method { + EncryptionMethod::PGP => EncryptionType::PGP { algo, certs }, + EncryptionMethod::SMIME => EncryptionType::SMIME { algo, certs }, + } + } else { + EncryptionType::Disabled + }; Ok(JsonResponse::new(json!({ "data": ec, @@ -111,34 +117,34 @@ impl CryptoHandler for Server { } // Parse certificates - let params = EncryptionParams { + let certs = try_parse_certs(method, certs.into_bytes()) + .map_err(|err| manage::error(err, None::))?; + let num_certs = certs.len(); + let params = Archiver::new(EncryptionParams { method, algo, - certs: try_parse_certs(method, certs.into_bytes()) - .map_err(|err| manage::error(err, None::))?, - }; + certs, + }) + .serialize() + .caused_by(trc::location!())?; // Try a test encryption if let Err(EncryptMessageError::Error(message)) = MessageParser::new() .parse("Subject: test\r\ntest\r\n".as_bytes()) .unwrap() - .encrypt(¶ms) + .encrypt(rkyv_unarchive(¶ms)?) .await { return Err(manage::error(message, None::)); } // Save encryption params - let num_certs = params.certs.len(); let mut batch = BatchBuilder::new(); batch .with_account_id(access_token.primary_id()) .with_collection(Collection::Principal) .update_document(0) - .set( - Property::Parameters, - params.serialize().caused_by(trc::location!())?, - ); + .set(Property::Parameters, params); self.core.storage.data.write(batch.build()).await?; Ok(JsonResponse::new(json!({ diff --git a/crates/jmap/src/email/get.rs b/crates/jmap/src/email/get.rs index 238a70bb..f5b22425 100644 --- a/crates/jmap/src/email/get.rs +++ b/crates/jmap/src/email/get.rs @@ -7,8 +7,10 @@ use common::{Server, auth::AccessToken}; use email::{ - mailbox::UidMailbox, - message::metadata::{MessageMetadata, MetadataPartType}, + mailbox::ArchivedUidMailbox, + message::metadata::{ + ArchivedGetHeader, ArchivedHeaderName, ArchivedMessageMetadata, ArchivedMetadataPartType, + }, thread::cache::ThreadCache, }; use jmap_proto::{ @@ -20,22 +22,23 @@ use jmap_proto::{ collection::Collection, date::UTCDate, id::Id, - keyword::Keyword, + keyword::ArchivedKeyword, property::{HeaderForm, Property}, value::{Object, Value}, }, }; -use mail_parser::HeaderName; -use store::{BlobClass, write::Bincode}; + +use store::{BlobClass, rkyv::vec::ArchivedVec, write::Archive}; use trc::{AddContext, StoreEvent}; +use utils::BlobHash; use crate::{ blob::download::BlobDownload, changes::state::StateManager, email::headers::HeaderToValue, }; -use std::future::Future; +use std::{borrow::Cow, future::Future}; use super::{ - body::{ToBodyPart, TruncateBody}, + body::{ToBodyPart, truncate_html, truncate_plain}, headers::IntoForm, }; @@ -153,8 +156,8 @@ impl EmailGet for Server { response.not_found.push(id.into()); continue; } - let mut metadata = match self - .get_property::>( + let metadata_ = match self + .get_property::( account_id, Collection::Email, id.document_id(), @@ -162,25 +165,28 @@ impl EmailGet for Server { ) .await? { - Some(metadata) => metadata.inner, + Some(metadata) => metadata, None => { response.not_found.push(id.into()); continue; } }; + let metadata = metadata_ + .unarchive::() + .caused_by(trc::location!())?; // Retrieve raw message if needed - let raw_message = if needs_body { - if let Some(raw_message) = self.get_blob(&metadata.blob_hash, 0..usize::MAX).await? - { - raw_message + let blob_hash = BlobHash::from(&metadata.blob_hash); + let raw_message: Cow<[u8]> = if needs_body { + if let Some(raw_message) = self.get_blob(&blob_hash, 0..usize::MAX).await? { + raw_message.into() } else { trc::event!( Store(StoreEvent::NotFound), AccountId = account_id, DocumentId = id.document_id(), Collection = Collection::Email, - BlobId = metadata.blob_hash.to_hex(), + BlobId = blob_hash.to_hex(), Details = "Blob not found.", CausedBy = trc::location!(), ); @@ -189,10 +195,10 @@ impl EmailGet for Server { continue; } } else { - metadata.raw_headers + metadata.raw_headers.as_slice().into() }; let blob_id = BlobId { - hash: metadata.blob_hash.clone(), + hash: blob_hash, class: BlobClass::Linked { account_id, collection: Collection::Email.into(), @@ -215,27 +221,28 @@ impl EmailGet for Server { email.append(Property::BlobId, blob_id.clone()); } Property::MailboxIds => { - if let Some(mailboxes) = self - .get_property::>( + if let Some(mailboxes_) = self + .get_property::( account_id, Collection::Email, id.document_id(), &Property::MailboxIds, ) .await? - .map(|ids| { - let mut obj = Object::with_capacity(ids.len()); - for id in ids { - debug_assert!(id.uid != 0); - obj.append( - Property::_T(Id::from(id.mailbox_id).to_string()), - true, - ); - } - Value::Object(obj) - }) { - email.append(property.clone(), mailboxes); + let mailboxes = mailboxes_ + .unarchive::>() + .caused_by(trc::location!())?; + let mut obj = Object::with_capacity(mailboxes.len()); + for id in mailboxes.iter() { + debug_assert!(id.uid != 0); + obj.append( + Property::_T(Id::from(u32::from(id.mailbox_id)).to_string()), + true, + ); + } + + email.append(property.clone(), Value::Object(obj)); } else { trc::event!( Store(StoreEvent::NotFound), @@ -251,23 +258,23 @@ impl EmailGet for Server { } } Property::Keywords => { - if let Some(keywords) = self - .get_property::>( + if let Some(keywords_) = self + .get_property::( account_id, Collection::Email, id.document_id(), &Property::Keywords, ) .await? - .map(|keywords| { - let mut obj = Object::with_capacity(keywords.len()); - for keyword in keywords { - obj.append(Property::_T(keyword.to_string()), true); - } - Value::Object(obj) - }) { - email.append(property.clone(), keywords); + let keywords = keywords_ + .unarchive::>() + .caused_by(trc::location!())?; + let mut obj = Object::with_capacity(keywords.len()); + for keyword in keywords.iter() { + obj.append(Property::_T(keyword.to_string()), true); + } + email.append(property.clone(), Value::Object(obj)); } else { trc::event!( Store(StoreEvent::NotFound), @@ -283,17 +290,19 @@ impl EmailGet for Server { } } Property::Size => { - email.append(Property::Size, metadata.size); + email.append(Property::Size, u32::from(metadata.size)); } Property::ReceivedAt => { email.append( Property::ReceivedAt, - Value::Date(UTCDate::from_timestamp(metadata.received_at as i64)), + Value::Date(UTCDate::from_timestamp( + u64::from(metadata.received_at) as i64 + )), ); } Property::Preview => { if !metadata.preview.is_empty() { - email.append(Property::Preview, std::mem::take(&mut metadata.preview)); + email.append(Property::Preview, metadata.preview.to_string()); } } Property::HasAttachment => { @@ -303,7 +312,8 @@ impl EmailGet for Server { email.append( Property::Subject, metadata.contents.parts[0] - .remove_header(&HeaderName::Subject) + .headers + .convert_header(&ArchivedHeaderName::Subject) .map(|value| value.into_form(&HeaderForm::Text)) .unwrap_or_default(), ); @@ -312,7 +322,8 @@ impl EmailGet for Server { email.append( Property::SentAt, metadata.contents.parts[0] - .remove_header(&HeaderName::Date) + .headers + .convert_header(&ArchivedHeaderName::Date) .map(|value| value.into_form(&HeaderForm::Date)) .unwrap_or_default(), ); @@ -321,10 +332,11 @@ impl EmailGet for Server { email.append( property.clone(), metadata.contents.parts[0] - .remove_header(&match property { - Property::MessageId => HeaderName::MessageId, - Property::InReplyTo => HeaderName::InReplyTo, - Property::References => HeaderName::References, + .headers + .convert_header(&match property { + Property::MessageId => ArchivedHeaderName::MessageId, + Property::InReplyTo => ArchivedHeaderName::InReplyTo, + Property::References => ArchivedHeaderName::References, _ => unreachable!(), }) .map(|value| value.into_form(&HeaderForm::MessageIds)) @@ -341,13 +353,14 @@ impl EmailGet for Server { email.append( property.clone(), metadata.contents.parts[0] - .remove_header(&match property { - Property::Sender => HeaderName::Sender, - Property::From => HeaderName::From, - Property::To => HeaderName::To, - Property::Cc => HeaderName::Cc, - Property::Bcc => HeaderName::Bcc, - Property::ReplyTo => HeaderName::ReplyTo, + .headers + .convert_header(&match property { + Property::Sender => ArchivedHeaderName::Sender, + Property::From => ArchivedHeaderName::From, + Property::To => ArchivedHeaderName::To, + Property::Cc => ArchivedHeaderName::Cc, + Property::Bcc => ArchivedHeaderName::Bcc, + Property::ReplyTo => ArchivedHeaderName::ReplyTo, _ => unreachable!(), }) .map(|value| value.into_form(&HeaderForm::Addresses)) @@ -382,7 +395,7 @@ impl EmailGet for Server { property.clone(), list.map(|part_id| { metadata.contents.to_body_part( - *part_id, + u16::from(part_id) as usize, &body_properties, &raw_message, &blob_id, @@ -405,18 +418,27 @@ impl EmailGet for Server { Property::BodyValues => { let mut body_values = Object::with_capacity(metadata.contents.parts.len()); for (part_id, part) in metadata.contents.parts.iter().enumerate() { - if ((metadata.contents.html_body.contains(&part_id) + if ((metadata.contents.is_html_part(part_id as u16) && (fetch_all_body_values || fetch_html_body_values)) - || (metadata.contents.text_body.contains(&part_id) + || (metadata.contents.is_text_part(part_id as u16) && (fetch_all_body_values || fetch_text_body_values))) && matches!( part.body, - MetadataPartType::Text | MetadataPartType::Html + ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html ) { - let (is_truncated, value) = part - .decode_contents(&raw_message) - .truncate(max_body_value_bytes); + let contents = part.decode_contents(&raw_message); + + let (is_truncated, value) = match &part.body { + ArchivedMetadataPartType::Text => { + truncate_plain(contents.as_str(), max_body_value_bytes) + } + ArchivedMetadataPartType::Html => { + truncate_html(contents.as_str(), max_body_value_bytes) + } + _ => unreachable!(), + }; + body_values.append( Property::_T(part_id.to_string()), Object::with_capacity(3) diff --git a/crates/jmap/src/email/headers.rs b/crates/jmap/src/email/headers.rs index 3e84c0e1..97413908 100644 --- a/crates/jmap/src/email/headers.rs +++ b/crates/jmap/src/email/headers.rs @@ -6,6 +6,7 @@ use std::borrow::Cow; +use email::message::metadata::{ArchivedHeader, ArchivedHeaderValue}; use jmap_proto::types::{ property::{HeaderForm, HeaderProperty, Property}, value::{Object, Value}, @@ -22,6 +23,7 @@ use mail_builder::{ }, }; use mail_parser::{Addr, Header, HeaderName, HeaderValue, parsers::MessageStream}; +use store::rkyv::vec::ArchivedVec; pub trait IntoForm { fn into_form(self, form: &HeaderForm) -> Value; @@ -337,6 +339,96 @@ impl BuildHeader for MessageBuilder<'_> { } } +impl HeaderToValue for ArchivedVec { + fn header_to_value(&self, property: &Property, raw_message: &[u8]) -> Value { + let (header_name, form, all) = match property { + Property::Header(header) => ( + HeaderName::parse(header.header.as_str()) + .unwrap_or_else(|| HeaderName::Other(header.header.as_str().into())), + header.form, + header.all, + ), + Property::Sender => (HeaderName::Sender, HeaderForm::Addresses, false), + Property::From => (HeaderName::From, HeaderForm::Addresses, false), + Property::To => (HeaderName::To, HeaderForm::Addresses, false), + Property::Cc => (HeaderName::Cc, HeaderForm::Addresses, false), + Property::Bcc => (HeaderName::Bcc, HeaderForm::Addresses, false), + Property::ReplyTo => (HeaderName::ReplyTo, HeaderForm::Addresses, false), + Property::Subject => (HeaderName::Subject, HeaderForm::Text, false), + Property::MessageId => (HeaderName::MessageId, HeaderForm::MessageIds, false), + Property::InReplyTo => (HeaderName::InReplyTo, HeaderForm::MessageIds, false), + Property::References => (HeaderName::References, HeaderForm::MessageIds, false), + Property::SentAt => (HeaderName::Date, HeaderForm::Date, false), + _ => return Value::Null, + }; + + let is_raw = matches!(form, HeaderForm::Raw) || matches!(header_name, HeaderName::Other(_)); + let mut headers = Vec::new(); + let header_name = header_name.as_str(); + for header in self.iter().rev() { + if header.name.as_str().eq_ignore_ascii_case(header_name) { + let header_value = if is_raw || matches!(header.value, ArchivedHeaderValue::Empty) { + raw_message + .get( + u32::from(header.offset_start) as usize + ..u32::from(header.offset_end) as usize, + ) + .map_or(HeaderValue::Empty, |bytes| match form { + HeaderForm::Raw => { + HeaderValue::Text(String::from_utf8_lossy(bytes.trim_end())) + } + HeaderForm::Text => MessageStream::new(bytes).parse_unstructured(), + HeaderForm::Addresses + | HeaderForm::GroupedAddresses + | HeaderForm::URLs => MessageStream::new(bytes).parse_address(), + HeaderForm::MessageIds => MessageStream::new(bytes).parse_id(), + HeaderForm::Date => MessageStream::new(bytes).parse_date(), + }) + } else { + HeaderValue::from(&header.value) + }; + headers.push(header_value.into_form(&form)); + if !all { + break; + } + } + } + + if !all { + headers.pop().unwrap_or_default() + } else { + if headers.len() > 1 { + headers.reverse(); + } + Value::List(headers) + } + } + + fn headers_to_value(&self, raw_message: &[u8]) -> Value { + let mut headers = Vec::with_capacity(self.len()); + for header in self.iter() { + headers.push(Value::Object( + Object::with_capacity(2) + .with_property(Property::Name, header.name.to_string()) + .with_property( + Property::Value, + String::from_utf8_lossy( + raw_message + .get( + u32::from(header.offset_start) as usize + ..u32::from(header.offset_end) as usize, + ) + .unwrap_or_default() + .trim_end(), + ) + .into_owned(), + ), + )); + } + headers.into() + } +} + trait ByteTrim { fn trim_end(&self) -> Self; } diff --git a/crates/jmap/src/email/set.rs b/crates/jmap/src/email/set.rs index 081dc6dc..21beff44 100644 --- a/crates/jmap/src/email/set.rs +++ b/crates/jmap/src/email/set.rs @@ -8,7 +8,7 @@ use std::{borrow::Cow, collections::HashMap}; use common::{Server, auth::AccessToken, storage::tag::TagManager}; use email::{ - mailbox::{UidMailbox, manage::MailboxFnc}, + mailbox::{ArchivedUidMailbox, UidMailbox, manage::MailboxFnc}, message::{ delete::EmailDeletion, ingest::{EmailIngest, IngestEmail, IngestSource}, @@ -21,7 +21,7 @@ use jmap_proto::{ types::{ acl::Acl, collection::Collection, - keyword::Keyword, + keyword::{ArchivedKeyword, Keyword}, property::Property, state::{State, StateChange}, type_state::DataType, @@ -40,8 +40,9 @@ use mail_parser::MessageParser; use store::{ SerializeInfallible, ahash::AHashSet, + rkyv::vec::ArchivedVec, roaring::RoaringBitmap, - write::{BatchBuilder, assert::HashedValue, log::ChangeLogBuilder}, + write::{Archive, BatchBuilder, assert::HashedValue, log::ChangeLogBuilder}, }; use trc::AddContext; @@ -764,14 +765,14 @@ impl EmailSet for Server { // Obtain current keywords and mailboxes let document_id = id.document_id(); let (mut mailboxes, mut keywords) = if let (Some(mailboxes), Some(keywords)) = ( - self.get_property::>>( + self.get_property::>( account_id, Collection::Email, document_id, Property::MailboxIds, ) .await?, - self.get_property::>>( + self.get_property::>( account_id, Collection::Email, document_id, @@ -779,7 +780,18 @@ impl EmailSet for Server { ) .await?, ) { - (TagManager::new(mailboxes), TagManager::new(keywords)) + ( + TagManager::new( + mailboxes + .into_deserialized::, Vec>() + .caused_by(trc::location!())?, + ), + TagManager::new( + keywords + .into_deserialized::, Vec>() + .caused_by(trc::location!())?, + ), + ) } else { response.not_updated.append(id, SetError::not_found()); continue 'update; diff --git a/crates/jmap/src/email/snippet.rs b/crates/jmap/src/email/snippet.rs index 4cdff507..97919ce8 100644 --- a/crates/jmap/src/email/snippet.rs +++ b/crates/jmap/src/email/snippet.rs @@ -5,7 +5,10 @@ */ use common::{Server, auth::AccessToken}; -use email::message::metadata::{MessageMetadata, MetadataPartType}; +use email::message::metadata::{ + ArchivedGetHeader, ArchivedHeaderName, ArchivedMessageMetadata, ArchivedMetadataPartType, + DecodedPartContent, +}; use jmap_proto::{ method::{ query::Filter, @@ -13,9 +16,11 @@ use jmap_proto::{ }, types::{acl::Acl, collection::Collection, property::Property}, }; -use mail_parser::{GetHeader, HeaderName, PartType, decoders::html::html_to_text}; +use mail_parser::decoders::html::html_to_text; use nlp::language::{Language, search_snippet::generate_snippet, stemmer::Stemmer}; -use store::{backend::MAX_TOKEN_LENGTH, write::Bincode}; +use store::{backend::MAX_TOKEN_LENGTH, write::Archive}; +use trc::AddContext; +use utils::BlobHash; use crate::blob::download::BlobDownload; @@ -114,8 +119,8 @@ impl EmailSearchSnippet for Server { response.list.push(snippet); continue; } - let metadata = match self - .get_property::>( + let metadata_ = match self + .get_property::( account_id, Collection::Email, document_id, @@ -123,19 +128,22 @@ impl EmailSearchSnippet for Server { ) .await? { - Some(metadata) => metadata.inner, + Some(metadata) => metadata, None => { response.not_found.push(email_id); continue; } }; + let metadata = metadata_ + .unarchive::() + .caused_by(trc::location!())?; // Add subject snippet if let Some(subject) = metadata .contents .root_part() .headers - .header_value(&HeaderName::Subject) + .header_value(&ArchivedHeaderName::Subject) .and_then(|v| v.as_text()) .and_then(|v| generate_snippet(v, &terms, language, is_exact)) { @@ -147,8 +155,9 @@ impl EmailSearchSnippet for Server { snippet.preview = body.into(); } else {*/ // Download message - let raw_message = if let Some(raw_message) = - self.get_blob(&metadata.blob_hash, 0..usize::MAX).await? + let raw_message = if let Some(raw_message) = self + .get_blob(&BlobHash::from(&metadata.blob_hash), 0..usize::MAX) + .await? { raw_message } else { @@ -157,7 +166,7 @@ impl EmailSearchSnippet for Server { AccountId = account_id, DocumentId = email_id.document_id(), Collection = Collection::Email, - BlobId = metadata.blob_hash.to_hex(), + BlobId = metadata.blob_hash.0.as_slice(), Details = "Blob not found.", CausedBy = trc::location!(), ); @@ -167,12 +176,11 @@ impl EmailSearchSnippet for Server { }; // Find a matching part - 'outer: for part in &metadata.contents.parts { + 'outer: for part in metadata.contents.parts.iter() { match &part.body { - MetadataPartType::Text | MetadataPartType::Html => { + ArchivedMetadataPartType::Text => { let text = match part.decode_contents(&raw_message) { - PartType::Text(text) => text, - PartType::Html(html) => html_to_text(&html).into(), + DecodedPartContent::Text(text) => text, _ => unreachable!(), }; @@ -181,12 +189,31 @@ impl EmailSearchSnippet for Server { break; } } - MetadataPartType::Message(message) => { - for part in &message.parts { - if let MetadataPartType::Text | MetadataPartType::Html = part.body { - let text = match part.decode_contents(&raw_message) { - PartType::Text(text) => text, - PartType::Html(html) => html_to_text(&html).into(), + ArchivedMetadataPartType::Html => { + let text = match part.decode_contents(&raw_message) { + DecodedPartContent::Text(html) => html_to_text(&html), + _ => unreachable!(), + }; + + if let Some(body) = generate_snippet(&text, &terms, language, is_exact) { + snippet.preview = body.into(); + break; + } + } + ArchivedMetadataPartType::Message(message) => { + for part in message.parts.iter() { + if let ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html = + part.body + { + let text = match (part.decode_contents(&raw_message), &part.body) { + ( + DecodedPartContent::Text(text), + ArchivedMetadataPartType::Text, + ) => text, + ( + DecodedPartContent::Text(html), + ArchivedMetadataPartType::Html, + ) => html_to_text(&html).into(), _ => unreachable!(), }; diff --git a/crates/jmap/src/identity/get.rs b/crates/jmap/src/identity/get.rs index 81e17555..5c118073 100644 --- a/crates/jmap/src/identity/get.rs +++ b/crates/jmap/src/identity/get.rs @@ -19,7 +19,7 @@ use store::{ Serialize, rkyv::{option::ArchivedOption, vec::ArchivedVec}, roaring::RoaringBitmap, - write::{ArchivedValue, BatchBuilder}, + write::{Archive, BatchBuilder}, }; use trc::AddContext; use utils::sanitize_email; @@ -85,7 +85,7 @@ impl IdentityGet for Server { continue; } let _identity = if let Some(identity) = self - .get_property::>( + .get_property::( account_id, Collection::Identity, document_id, @@ -98,7 +98,9 @@ impl IdentityGet for Server { response.not_found.push(id.into()); continue; }; - let identity = _identity.unarchive().caused_by(trc::location!())?; + let identity = _identity + .unarchive::() + .caused_by(trc::location!())?; let mut result = Object::with_capacity(properties.len()); for property in &properties { match property { diff --git a/crates/jmap/src/identity/set.rs b/crates/jmap/src/identity/set.rs index e9e57c10..eee73596 100644 --- a/crates/jmap/src/identity/set.rs +++ b/crates/jmap/src/identity/set.rs @@ -18,8 +18,8 @@ use jmap_proto::{ }, }; use std::future::Future; -use store::write::{BatchBuilder, log::ChangeLogBuilder}; -use store::{Serialize, write::ArchivedValue}; +use store::Serialize; +use store::write::{Archive, BatchBuilder, log::ChangeLogBuilder}; use trc::AddContext; use utils::sanitize_email; @@ -119,7 +119,7 @@ impl IdentitySet for Server { // Obtain identity let document_id = id.document_id(); let mut identity = if let Some(identity) = self - .get_property::>( + .get_property::( account_id, Collection::Identity, document_id, @@ -127,7 +127,9 @@ impl IdentitySet for Server { ) .await? { - identity.deserialize().caused_by(trc::location!())? + identity + .deserialize::() + .caused_by(trc::location!())? } else { response.not_updated.append(id, SetError::not_found()); continue 'update; diff --git a/crates/jmap/src/mailbox/get.rs b/crates/jmap/src/mailbox/get.rs index 59dc713d..bf9add60 100644 --- a/crates/jmap/src/mailbox/get.rs +++ b/crates/jmap/src/mailbox/get.rs @@ -15,7 +15,7 @@ use jmap_proto::{ value::{Object, Value}, }, }; -use store::write::ArchivedValue; +use store::write::{Archive}; use trc::AddContext; use crate::changes::state::StateManager; @@ -98,7 +98,7 @@ impl MailboxGet for Server { let archived_mailbox_ = if fetch_properties { match self - .get_property::>( + .get_property::( account_id, Collection::Mailbox, document_id, @@ -118,7 +118,7 @@ impl MailboxGet for Server { }; let archived_mailbox = if let Some(archived_mailbox) = &archived_mailbox_ { archived_mailbox - .unarchive() + .unarchive::() .caused_by(trc::location!())? .into() } else { diff --git a/crates/jmap/src/mailbox/query.rs b/crates/jmap/src/mailbox/query.rs index d9fdea7a..50e347a0 100644 --- a/crates/jmap/src/mailbox/query.rs +++ b/crates/jmap/src/mailbox/query.rs @@ -16,7 +16,7 @@ use store::{ ahash::{AHashMap, AHashSet}, query::{self, sort::Pagination}, roaring::RoaringBitmap, - write::ArchivedValue, + write::{Archive}, }; use crate::{JmapMethods, UpdateResults}; @@ -125,7 +125,7 @@ impl MailboxQuery for Server { || (response.total.is_some_and(|total| total > 0) && filter_as_tree)) { for (document_id, value) in self - .get_properties::, _, _>( + .get_properties::( account_id, Collection::Mailbox, &mailbox_ids, @@ -134,7 +134,7 @@ impl MailboxQuery for Server { .await? { let todo = "use index"; - let mailbox = value.unarchive()?; + let mailbox = value.unarchive::()?; let parent_id = u32::from(mailbox.parent_id); hierarchy.insert(document_id + 1, parent_id); tree.entry(parent_id) diff --git a/crates/jmap/src/mailbox/set.rs b/crates/jmap/src/mailbox/set.rs index 4e3fa009..972a6b3c 100644 --- a/crates/jmap/src/mailbox/set.rs +++ b/crates/jmap/src/mailbox/set.rs @@ -30,7 +30,7 @@ use store::{ query::Filter, roaring::RoaringBitmap, write::{ - ArchivedValue, BatchBuilder, + Archive, BatchBuilder, assert::{AssertValue, HashedValue}, log::ChangeLogBuilder, }, @@ -158,7 +158,7 @@ impl MailboxSet for Server { // Obtain mailbox let document_id = id.document_id(); if let Some(mailbox) = self - .get_property::>>( + .get_property::>( account_id, Collection::Mailbox, document_id, @@ -167,7 +167,9 @@ impl MailboxSet for Server { .await? { // Validate ACL - let mailbox = mailbox.into_deserialized().caused_by(trc::location!())?; + let mailbox = mailbox + .into_deserialized::() + .caused_by(trc::location!())?; if ctx.is_shared { let acl = mailbox.inner.acls.effective_acl(access_token); if !acl.contains(Acl::Modify) { @@ -406,7 +408,7 @@ impl MailboxSet for Server { let parent_document_id = mailbox_parent_id - 1; if let Some(mailbox_) = self - .get_property::>( + .get_property::( ctx.account_id, Collection::Mailbox, parent_document_id, @@ -414,7 +416,9 @@ impl MailboxSet for Server { ) .await? { - let mailbox = mailbox_.unarchive().caused_by(trc::location!())?; + let mailbox = mailbox_ + .unarchive::() + .caused_by(trc::location!())?; if depth == 0 && ctx.is_shared && !mailbox diff --git a/crates/jmap/src/push/get.rs b/crates/jmap/src/push/get.rs index 99d636f0..1a9948ae 100644 --- a/crates/jmap/src/push/get.rs +++ b/crates/jmap/src/push/get.rs @@ -21,7 +21,7 @@ use jmap_proto::{ }; use store::{ BitmapKey, ValueKey, - write::{ArchivedValue, ValueClass, now}, + write::{Archive, ValueClass, now}, }; use trc::AddContext; use utils::map::bitmap::Bitmap; @@ -85,7 +85,7 @@ impl PushSubscriptionFetch for Server { continue; } let push_ = if let Some(push) = self - .get_property::>( + .get_property::( account_id, Collection::PushSubscription, document_id, @@ -98,7 +98,9 @@ impl PushSubscriptionFetch for Server { response.not_found.push(id.into()); continue; }; - let push = push_.unarchive().caused_by(trc::location!())?; + let push = push_ + .unarchive::() + .caused_by(trc::location!())?; let mut result = Object::with_capacity(properties.len()); for property in &properties { match property { @@ -166,7 +168,7 @@ impl PushSubscriptionFetch for Server { .core .storage .data - .get_value::>(ValueKey { + .get_value::(ValueKey { account_id, collection: Collection::PushSubscription.into(), document_id, @@ -179,7 +181,7 @@ impl PushSubscriptionFetch for Server { .caused_by(trc::location!()) .document_id(document_id) })? - .deserialize() + .deserialize::() .caused_by(trc::location!())?; if subscription.expires > current_time { diff --git a/crates/jmap/src/push/set.rs b/crates/jmap/src/push/set.rs index 57a44a53..8951e28a 100644 --- a/crates/jmap/src/push/set.rs +++ b/crates/jmap/src/push/set.rs @@ -24,7 +24,7 @@ use std::future::Future; use store::{ Serialize, rand::{Rng, rng}, - write::{ArchivedValue, BatchBuilder, now}, + write::{Archive, BatchBuilder, now}, }; use trc::AddContext; use utils::map::bitmap::Bitmap; @@ -136,7 +136,7 @@ impl PushSubscriptionSet for Server { // Obtain push subscription let document_id = id.document_id(); let mut push = if let Some(push) = self - .get_property::>( + .get_property::( account_id, Collection::PushSubscription, document_id, @@ -144,7 +144,8 @@ impl PushSubscriptionSet for Server { ) .await? { - push.deserialize().caused_by(trc::location!())? + push.deserialize::() + .caused_by(trc::location!())? } else { response.not_updated.append(id, SetError::not_found()); continue 'update; diff --git a/crates/jmap/src/services/index.rs b/crates/jmap/src/services/index.rs index 073d86c8..ad71cbb8 100644 --- a/crates/jmap/src/services/index.rs +++ b/crates/jmap/src/services/index.rs @@ -11,15 +11,18 @@ use directory::{ Type, backend::internal::{PrincipalField, manage::ManageDirectory}, }; -use email::message::{bayes::EmailBayesTrain, index::IndexMessageText, metadata::MessageMetadata}; +use email::message::{ + bayes::EmailBayesTrain, index::IndexMessageText, metadata::ArchivedMessageMetadata, +}; use jmap_proto::types::{collection::Collection, property::Property}; +use mail_parser::MessageParser; use store::{ IterateParams, SerializeInfallible, U32_LEN, U64_LEN, ValueKey, ahash::AHashMap, fts::index::FtsDocument, roaring::RoaringBitmap, write::{ - BatchBuilder, Bincode, BlobOp, MaybeDynamicId, TaskQueueClass, ValueClass, + Archive, BatchBuilder, BlobOp, MaybeDynamicId, TaskQueueClass, ValueClass, key::{DeserializeBigEndian, KeySerializer}, now, }, @@ -147,94 +150,117 @@ impl Indexer for Server { unlock_events.push(event.clone()); } - match self - .get_property::>( - event.account_id, - Collection::Email, - event.document_id, - Property::BodyStructure, - ) - .await - { - Ok(Some(metadata)) - if metadata.inner.blob_hash.as_slice() == event.hash.as_slice() => - { - // Obtain raw message - let raw_message = if let Ok(Some(raw_message)) = self - .get_blob(&metadata.inner.blob_hash, 0..usize::MAX) + // Obtain raw message + let raw_message = + if let Ok(Some(raw_message)) = self.get_blob(&event.hash, 0..usize::MAX).await { + raw_message + } else { + trc::event!( + TaskQueue(TaskQueueEvent::BlobNotFound), + AccountId = event.account_id, + DocumentId = event.document_id, + BlobId = event.hash.as_slice(), + ); + continue; + }; + + match event.action { + EmailTaskAction::Index => { + match self + .get_property::( + event.account_id, + Collection::Email, + event.document_id, + Property::BodyStructure, + ) .await { - raw_message - } else { - trc::event!( - TaskQueue(TaskQueueEvent::BlobNotFound), - AccountId = event.account_id, - DocumentId = event.document_id, - BlobId = metadata.inner.blob_hash.to_hex(), - ); - continue; - }; - let message = metadata.inner.contents.into_message(&raw_message); - - match event.action { - EmailTaskAction::Index => { - // Index message - let document = - FtsDocument::with_default_language(self.core.jmap.default_language) + Ok(Some(metadata_)) => { + match metadata_.unarchive::() { + Ok(metadata) + if metadata.blob_hash.0.as_slice() == event.hash.as_slice() => + { + // Index message + let document = FtsDocument::with_default_language( + self.core.jmap.default_language, + ) .with_account_id(event.account_id) .with_collection(Collection::Email) .with_document_id(event.document_id) - .index_message(&message); - if let Err(err) = self.core.storage.fts.index(document).await { - trc::error!( - err.account_id(event.account_id) - .document_id(event.document_id) - .details("Failed to index email in FTS index") - ); + .index_message(&metadata, &raw_message); + if let Err(err) = self.core.storage.fts.index(document).await { + trc::error!( + err.account_id(event.account_id) + .document_id(event.document_id) + .details("Failed to index email in FTS index") + ); - continue; + continue; + } + + trc::event!( + TaskQueue(TaskQueueEvent::Index), + AccountId = event.account_id, + Collection = Collection::Email, + DocumentId = event.document_id, + Elapsed = op_start.elapsed(), + ); + } + Err(err) => { + trc::error!( + err.account_id(event.account_id) + .document_id(event.document_id) + .details("Failed to unarchive email metadata") + ); + } + + _ => { + // The message was probably deleted or overwritten + trc::event!( + TaskQueue(TaskQueueEvent::MetadataNotFound), + Details = "Blob hash mismatch", + AccountId = event.account_id, + DocumentId = event.document_id, + ); + } } - - trc::event!( - TaskQueue(TaskQueueEvent::Index), - AccountId = event.account_id, - Collection = Collection::Email, - DocumentId = event.document_id, - Elapsed = op_start.elapsed(), - ); } - EmailTaskAction::BayesTrain { learn_spam } => { - // Train bayes classifier for account - self.email_bayes_train(event.account_id, 0, message, learn_spam) - .await; + Err(err) => { + trc::error!( + err.account_id(event.account_id) + .document_id(event.document_id) + .caused_by(trc::location!()) + .details("Failed to retrieve email metadata") + ); + continue; + } + _ => { + // The message was probably deleted or overwritten trc::event!( - TaskQueue(TaskQueueEvent::BayesTrain), + TaskQueue(TaskQueueEvent::MetadataNotFound), AccountId = event.account_id, - Collection = Collection::Email, DocumentId = event.document_id, - Elapsed = op_start.elapsed(), ); } } } + EmailTaskAction::BayesTrain { learn_spam } => { + // Train bayes classifier for account + self.email_bayes_train( + event.account_id, + 0, + MessageParser::new().parse(&raw_message).unwrap_or_default(), + learn_spam, + ) + .await; - Err(err) => { - trc::error!( - err.account_id(event.account_id) - .document_id(event.document_id) - .caused_by(trc::location!()) - .details("Failed to retrieve email metadata") - ); - - continue; - } - _ => { - // The message was probably deleted or overwritten trc::event!( - TaskQueue(TaskQueueEvent::MetadataNotFound), + TaskQueue(TaskQueueEvent::BayesTrain), AccountId = event.account_id, + Collection = Collection::Email, DocumentId = event.document_id, + Elapsed = op_start.elapsed(), ); } } diff --git a/crates/jmap/src/sieve/get.rs b/crates/jmap/src/sieve/get.rs index 81693bd0..4d229afa 100644 --- a/crates/jmap/src/sieve/get.rs +++ b/crates/jmap/src/sieve/get.rs @@ -15,7 +15,10 @@ use jmap_proto::{ value::{Object, Value}, }, }; -use store::{BlobClass, write::ArchivedValue}; +use store::{ + BlobClass, + write::{Archive}, +}; use trc::AddContext; use crate::changes::state::StateManager; @@ -73,7 +76,7 @@ impl SieveScriptGet for Server { continue; } let sieve_ = if let Some(sieve) = self - .get_property::>( + .get_property::( account_id, Collection::SieveScript, document_id, @@ -86,7 +89,9 @@ impl SieveScriptGet for Server { response.not_found.push(id.into()); continue; }; - let sieve = sieve_.unarchive().caused_by(trc::location!())?; + let sieve = sieve_ + .unarchive::() + .caused_by(trc::location!())?; let mut result = Object::with_capacity(properties.len()); for property in &properties { match property { diff --git a/crates/jmap/src/sieve/set.rs b/crates/jmap/src/sieve/set.rs index fd5c9904..201d345b 100644 --- a/crates/jmap/src/sieve/set.rs +++ b/crates/jmap/src/sieve/set.rs @@ -32,7 +32,7 @@ use store::{ BlobClass, query::Filter, rand::{Rng, rng}, - write::{ArchivedValue, BatchBuilder, BlobOp, assert::HashedValue, log::ChangeLogBuilder}, + write::{Archive, BatchBuilder, BlobOp, assert::HashedValue, log::ChangeLogBuilder}, }; use trc::AddContext; @@ -184,7 +184,7 @@ impl SieveScriptSet for Server { // Obtain sieve script let document_id = id.document_id(); if let Some(sieve) = self - .get_property::>>( + .get_property::>( account_id, Collection::SieveScript, document_id, @@ -192,7 +192,9 @@ impl SieveScriptSet for Server { ) .await? { - let sieve = sieve.into_deserialized().caused_by(trc::location!())?; + let sieve = sieve + .into_deserialized::() + .caused_by(trc::location!())?; let prev_blob_hash = sieve.inner.blob_hash.clone(); match self diff --git a/crates/jmap/src/submission/get.rs b/crates/jmap/src/submission/get.rs index a55f590f..e6cdff9a 100644 --- a/crates/jmap/src/submission/get.rs +++ b/crates/jmap/src/submission/get.rs @@ -21,7 +21,10 @@ use jmap_proto::{ }; use smtp::queue::{self, spool::SmtpSpool}; use std::future::Future; -use store::{rkyv::option::ArchivedOption, write::ArchivedValue}; +use store::{ + rkyv::option::ArchivedOption, + write::{Archive}, +}; use trc::AddContext; use utils::map::vec_map::VecMap; @@ -84,7 +87,7 @@ impl EmailSubmissionGet for Server { continue; } let submission_ = if let Some(submission) = self - .get_property::>( + .get_property::( account_id, Collection::EmailSubmission, document_id, @@ -97,7 +100,9 @@ impl EmailSubmissionGet for Server { response.not_found.push(id.into()); continue; }; - let submission = submission_.unarchive().caused_by(trc::location!())?; + let submission = submission_ + .unarchive::() + .caused_by(trc::location!())?; // Obtain queueId let mut delivery_status = submission diff --git a/crates/jmap/src/submission/set.rs b/crates/jmap/src/submission/set.rs index cbc48ae5..28d6b28d 100644 --- a/crates/jmap/src/submission/set.rs +++ b/crates/jmap/src/submission/set.rs @@ -13,7 +13,7 @@ use common::{ }; use email::{ identity::ArchivedIdentity, - message::metadata::MessageMetadata, + message::metadata::{ArchivedHeaderName, ArchivedHeaderValue, ArchivedMessageMetadata}, submission::{ Address, ArchivedEmailSubmission, Delivered, DeliveryStatus, EmailSubmission, UndoStatus, }, @@ -35,17 +35,14 @@ use jmap_proto::{ value::{MaybePatchValue, Object, SetValue, Value}, }, }; -use mail_parser::{HeaderName, HeaderValue}; use smtp::{ core::{Session, SessionData, State}, queue::spool::SmtpSpool, }; use smtp_proto::{MailFrom, RcptTo, request::parser::Rfc5321Parser}; -use store::write::{ - ArchivedValue, BatchBuilder, Bincode, assert::HashedValue, log::ChangeLogBuilder, now, -}; +use store::write::{Archive, BatchBuilder, assert::HashedValue, log::ChangeLogBuilder, now}; use trc::AddContext; -use utils::{map::vec_map::VecMap, sanitize_email}; +use utils::{BlobHash, map::vec_map::VecMap, sanitize_email}; use crate::blob::download::BlobDownload; use std::future::Future; @@ -126,7 +123,7 @@ impl EmailSubmissionSet for Server { // Obtain submission let document_id = id.document_id(); let submission = if let Some(submission) = self - .get_property::>>( + .get_property::>( account_id, Collection::EmailSubmission, document_id, @@ -134,7 +131,9 @@ impl EmailSubmissionSet for Server { ) .await? { - submission.into_deserialized().caused_by(trc::location!())? + submission + .into_deserialized::() + .caused_by(trc::location!())? } else { response.not_updated.append(id, SetError::not_found()); continue 'update; @@ -228,7 +227,7 @@ impl EmailSubmissionSet for Server { for id in will_destroy { let document_id = id.document_id(); if let Some(submission) = self - .get_property::>>( + .get_property::>( account_id, Collection::EmailSubmission, document_id, @@ -244,7 +243,9 @@ impl EmailSubmissionSet for Server { .delete_document(document_id) .custom( ObjectIndexBuilder::new().with_current( - submission.into_deserialized().caused_by(trc::location!())?, + submission + .into_deserialized::() + .caused_by(trc::location!())?, ), ) .caused_by(trc::location!())?; @@ -459,7 +460,7 @@ impl EmailSubmissionSet for Server { // Fetch identity's mailFrom let identity_mail_from = if let Some(identity) = self - .get_property::>( + .get_property::( account_id, Collection::Identity, submission.identity_id, @@ -468,7 +469,7 @@ impl EmailSubmissionSet for Server { .await? { identity - .unarchive() + .unarchive::() .caused_by(trc::location!())? .email .to_string() @@ -499,8 +500,8 @@ impl EmailSubmissionSet for Server { }; // Obtain message metadata - let metadata = if let Some(metadata) = self - .get_property::>( + let metadata_ = if let Some(metadata) = self + .get_property::( account_id, Collection::Email, submission.email_id, @@ -508,25 +509,28 @@ impl EmailSubmissionSet for Server { ) .await? { - metadata.inner + metadata } else { return Ok(Err(SetError::invalid_properties() .with_property(Property::EmailId) .with_description("Email not found."))); }; + let metadata = metadata_ + .unarchive::() + .caused_by(trc::location!())?; // Add recipients to envelope if missing let mut bcc_header = None; if rcpt_to.is_empty() { - for header in &metadata.contents.parts[0].headers { + for header in metadata.contents.parts[0].headers.iter() { if matches!( header.name, - HeaderName::To | HeaderName::Cc | HeaderName::Bcc + ArchivedHeaderName::To | ArchivedHeaderName::Cc | ArchivedHeaderName::Bcc ) { - if matches!(header.name, HeaderName::Bcc) { + if matches!(header.name, ArchivedHeaderName::Bcc) { bcc_header = Some(header); } - if let HeaderValue::Address(addr) = &header.value { + if let ArchivedHeaderValue::Address(addr) = &header.value { for address in addr.iter() { if let Some(address) = address.address().and_then(sanitize_email) { if !rcpt_to.iter().any(|rcpt| rcpt.address == address) { @@ -553,7 +557,7 @@ impl EmailSubmissionSet for Server { bcc_header = metadata.contents.parts[0] .headers .iter() - .find(|header| matches!(header.name, HeaderName::Bcc)); + .find(|header| matches!(header.name, ArchivedHeaderName::Bcc)); } // Update sendAt @@ -566,28 +570,30 @@ impl EmailSubmissionSet for Server { }; // Obtain raw message - let mut message = - if let Some(message) = self.get_blob(&metadata.blob_hash, 0..usize::MAX).await? { - if message.len() > self.core.jmap.mail_max_size { - return Ok(Err(SetError::new(SetErrorType::InvalidEmail) - .with_description(format!( - "Message exceeds maximum size of {} bytes.", - self.core.jmap.mail_max_size - )))); - } + let mut message = if let Some(message) = self + .get_blob(&BlobHash::from(&metadata.blob_hash), 0..usize::MAX) + .await? + { + if message.len() > self.core.jmap.mail_max_size { + return Ok(Err(SetError::new(SetErrorType::InvalidEmail) + .with_description(format!( + "Message exceeds maximum size of {} bytes.", + self.core.jmap.mail_max_size + )))); + } - message - } else { - return Ok(Err(SetError::invalid_properties() - .with_property(Property::EmailId) - .with_description("Blob for email not found."))); - }; + message + } else { + return Ok(Err(SetError::invalid_properties() + .with_property(Property::EmailId) + .with_description("Blob for email not found."))); + }; // Remove BCC header if present if let Some(bcc_header) = bcc_header { let mut new_message = Vec::with_capacity(message.len()); - new_message.extend_from_slice(&message[..bcc_header.offset_field]); - new_message.extend_from_slice(&message[bcc_header.offset_end..]); + new_message.extend_from_slice(&message[..u32::from(bcc_header.offset_field) as usize]); + new_message.extend_from_slice(&message[u32::from(bcc_header.offset_end) as usize..]); message = new_message; } diff --git a/crates/jmap/src/vacation/get.rs b/crates/jmap/src/vacation/get.rs index bccecaf2..cb468cfa 100644 --- a/crates/jmap/src/vacation/get.rs +++ b/crates/jmap/src/vacation/get.rs @@ -19,7 +19,7 @@ use jmap_proto::{ }, }; use std::future::Future; -use store::{query::Filter, write::ArchivedValue}; +use store::{query::Filter, write::Archive}; use trc::AddContext; use crate::{JmapMethods, changes::state::StateManager}; @@ -81,7 +81,7 @@ impl VacationResponseGet for Server { if do_get { if let Some(document_id) = self.get_vacation_sieve_script_id(account_id).await? { if let Some(sieve_) = self - .get_property::>( + .get_property::( account_id, Collection::SieveScript, document_id, @@ -89,7 +89,9 @@ impl VacationResponseGet for Server { ) .await? { - let sieve = sieve_.unarchive().caused_by(trc::location!())?; + let sieve = sieve_ + .unarchive::() + .caused_by(trc::location!())?; let vacation = sieve.vacation_response.as_ref(); let mut result = Object::with_capacity(properties.len()); for property in &properties { diff --git a/crates/jmap/src/vacation/set.rs b/crates/jmap/src/vacation/set.rs index 55999acc..a8fb957c 100644 --- a/crates/jmap/src/vacation/set.rs +++ b/crates/jmap/src/vacation/set.rs @@ -27,7 +27,7 @@ use mail_builder::MessageBuilder; use mail_parser::decoders::html::html_to_text; use std::future::Future; use store::write::{ - ArchivedValue, BatchBuilder, BlobOp, + Archive, BatchBuilder, BlobOp, assert::HashedValue, log::{Changes, LogInsert}, }; @@ -219,7 +219,7 @@ impl VacationResponseSet for Server { let mut obj = if let Some(document_id) = document_id { let prev_sieve = self - .get_property::>>( + .get_property::>( account_id, Collection::SieveScript, document_id, @@ -231,7 +231,9 @@ impl VacationResponseSet for Server { .into_err() .caused_by(trc::location!()) })?; - let prev_sieve = prev_sieve.into_deserialized().caused_by(trc::location!())?; + let prev_sieve = prev_sieve + .into_deserialized::() + .caused_by(trc::location!())?; was_active = prev_sieve.inner.is_active; let mut sieve = prev_sieve.inner.clone(); sieve.vacation_response = vacation.into(); diff --git a/crates/managesieve/src/op/getscript.rs b/crates/managesieve/src/op/getscript.rs index cb7378cc..559eaa09 100644 --- a/crates/managesieve/src/op/getscript.rs +++ b/crates/managesieve/src/op/getscript.rs @@ -11,7 +11,7 @@ use directory::Permission; use email::sieve::ArchivedSieveScript; use imap_proto::receiver::Request; use jmap_proto::types::{blob::BlobSection, collection::Collection, property::Property}; -use store::write::ArchivedValue; +use store::write::Archive; use trc::AddContext; use utils::BlobHash; @@ -37,7 +37,7 @@ impl Session { let document_id = self.get_script_id(account_id, &name).await?; let sieve_ = self .server - .get_property::>( + .get_property::( account_id, Collection::SieveScript, document_id, @@ -51,7 +51,9 @@ impl Session { .details("Script not found") .code(ResponseCode::NonExistent) })?; - let sieve = sieve_.unarchive().caused_by(trc::location!())?; + let sieve = sieve_ + .unarchive::() + .caused_by(trc::location!())?; let blob_size = u32::from(sieve.size) as usize; let script = self .server diff --git a/crates/managesieve/src/op/listscripts.rs b/crates/managesieve/src/op/listscripts.rs index bfe4a483..06638bb7 100644 --- a/crates/managesieve/src/op/listscripts.rs +++ b/crates/managesieve/src/op/listscripts.rs @@ -10,7 +10,7 @@ use common::listener::SessionStream; use directory::Permission; use email::sieve::ArchivedSieveScript; use jmap_proto::types::{collection::Collection, property::Property}; -use store::write::ArchivedValue; +use store::write::Archive; use trc::AddContext; use crate::core::{Session, StatusResponse}; @@ -39,7 +39,7 @@ impl Session { for document_id in document_ids { if let Some(script_) = self .server - .get_property::>( + .get_property::( account_id, Collection::SieveScript, document_id, @@ -48,7 +48,9 @@ impl Session { .await .caused_by(trc::location!())? { - let script = script_.unarchive().caused_by(trc::location!())?; + let script = script_ + .unarchive::() + .caused_by(trc::location!())?; response.push(b'\"'); for ch in script.name.as_bytes() { if [b'\\', b'\"'].contains(ch) { diff --git a/crates/managesieve/src/op/putscript.rs b/crates/managesieve/src/op/putscript.rs index 952b5cb3..f5af603b 100644 --- a/crates/managesieve/src/op/putscript.rs +++ b/crates/managesieve/src/op/putscript.rs @@ -15,7 +15,7 @@ use sieve::compiler::ErrorType; use store::{ BlobClass, query::Filter, - write::{ArchivedValue, BatchBuilder, BlobOp, assert::HashedValue, log::LogInsert}, + write::{Archive, BatchBuilder, BlobOp, assert::HashedValue, log::LogInsert}, }; use trc::AddContext; @@ -101,7 +101,7 @@ impl Session { // Obtain script values let script = self .server - .get_property::>>( + .get_property::>( account_id, Collection::SieveScript, document_id, @@ -115,7 +115,7 @@ impl Session { .details("Script not found") .code(ResponseCode::NonExistent) })? - .into_deserialized() + .into_deserialized::() .caused_by(trc::location!())?; // Write script blob diff --git a/crates/managesieve/src/op/renamescript.rs b/crates/managesieve/src/op/renamescript.rs index 737d6993..9c6e34b4 100644 --- a/crates/managesieve/src/op/renamescript.rs +++ b/crates/managesieve/src/op/renamescript.rs @@ -8,10 +8,10 @@ use std::time::Instant; use common::{listener::SessionStream, storage::index::ObjectIndexBuilder}; use directory::Permission; -use email::sieve::ArchivedSieveScript; +use email::sieve::{ArchivedSieveScript, SieveScript}; use imap_proto::receiver::Request; use jmap_proto::types::{collection::Collection, property::Property}; -use store::write::{ArchivedValue, BatchBuilder, assert::HashedValue, log::ChangeLogBuilder}; +use store::write::{Archive, BatchBuilder, assert::HashedValue, log::ChangeLogBuilder}; use trc::AddContext; use crate::core::{Command, ResponseCode, Session, StatusResponse}; @@ -60,7 +60,7 @@ impl Session { // Obtain script values let script = self .server - .get_property::>>( + .get_property::>( account_id, Collection::SieveScript, document_id, @@ -74,7 +74,7 @@ impl Session { .details("Script not found") .code(ResponseCode::NonExistent) })? - .into_deserialized() + .into_deserialized::() .caused_by(trc::location!())?; // Write record diff --git a/crates/pop3/src/mailbox.rs b/crates/pop3/src/mailbox.rs index b4c79487..188bb8b0 100644 --- a/crates/pop3/src/mailbox.rs +++ b/crates/pop3/src/mailbox.rs @@ -7,12 +7,13 @@ use std::collections::BTreeMap; use common::listener::SessionStream; -use email::mailbox::{ArchivedMailbox, INBOX_ID, UidMailbox, manage::MailboxFnc}; +use email::mailbox::{ArchivedMailbox, ArchivedUidMailbox, INBOX_ID, manage::MailboxFnc}; use jmap_proto::types::{collection::Collection, property::Property}; use store::{ IndexKey, IterateParams, SerializeInfallible, U32_LEN, ahash::AHashMap, - write::{ArchivedValue, key::DeserializeBigEndian}, + rkyv::vec::ArchivedVec, + write::{Archive, key::DeserializeBigEndian}, }; use trc::AddContext; @@ -63,7 +64,7 @@ impl Session { .caused_by(trc::location!())?; let uid_validity = u32::from( self.server - .get_property::>( + .get_property::( account_id, Collection::Mailbox, INBOX_ID, @@ -78,7 +79,7 @@ impl Session { .account_id(account_id) .document_id(INBOX_ID) })? - .unarchive() + .unarchive::() .caused_by(trc::location!())? .uid_validity, ); @@ -124,7 +125,7 @@ impl Session { // Sort by UID for (message_id, uid_mailbox) in self .server - .get_properties::, _, _>( + .get_properties::( account_id, Collection::Email, &message_ids, @@ -135,9 +136,14 @@ impl Session { .into_iter() { // Make sure the message is still in Inbox - if let Some(item) = uid_mailbox.iter().find(|item| item.mailbox_id == INBOX_ID) { + if let Some(item) = uid_mailbox + .unarchive::>() + .caused_by(trc::location!())? + .iter() + .find(|item| item.mailbox_id == INBOX_ID) + { debug_assert!(item.uid != 0, "UID is zero for message {item:?}"); - message_map.insert(item.uid, message_id); + message_map.insert(u32::from(item.uid), message_id); } } diff --git a/crates/pop3/src/op/fetch.rs b/crates/pop3/src/op/fetch.rs index 28743e2f..2bb69014 100644 --- a/crates/pop3/src/op/fetch.rs +++ b/crates/pop3/src/op/fetch.rs @@ -8,9 +8,9 @@ use std::time::Instant; use common::listener::SessionStream; use directory::Permission; -use email::message::metadata::MessageMetadata; +use email::message::metadata::ArchivedMessageMetadata; use jmap_proto::types::{collection::Collection, property::Property}; -use store::write::Bincode; +use store::write::Archive; use trc::AddContext; use crate::{Session, protocol::response::Response}; @@ -25,9 +25,9 @@ impl Session { let op_start = Instant::now(); let mailbox = self.state.mailbox(); if let Some(message) = mailbox.messages.get(msg.saturating_sub(1) as usize) { - if let Some(metadata) = self + if let Some(metadata_) = self .server - .get_property::>( + .get_property::( mailbox.account_id, Collection::Email, message.id, @@ -36,10 +36,13 @@ impl Session { .await .caused_by(trc::location!())? { + let metadata = metadata_ + .unarchive::() + .caused_by(trc::location!())?; if let Some(bytes) = self .server .blob_store() - .get_blob(metadata.inner.blob_hash.as_slice(), 0..usize::MAX) + .get_blob(metadata.blob_hash.0.as_slice(), 0..usize::MAX) .await .caused_by(trc::location!())? { diff --git a/crates/smtp/src/queue/spool.rs b/crates/smtp/src/queue/spool.rs index 97708bed..1bb1ba1e 100644 --- a/crates/smtp/src/queue/spool.rs +++ b/crates/smtp/src/queue/spool.rs @@ -11,7 +11,7 @@ use std::borrow::Cow; use std::future::Future; use std::time::{Duration, SystemTime}; use store::write::key::DeserializeBigEndian; -use store::write::{BatchBuilder, Bincode, BlobOp, QueueClass, ValueClass, now}; +use store::write::{BatchBuilder, BlobOp, LegacyBincode, QueueClass, ValueClass, now}; use store::{IterateParams, Serialize, SerializeInfallible, U64_LEN, ValueKey}; use trc::ServerEvent; use utils::BlobHash; @@ -151,9 +151,9 @@ impl SmtpSpool for Server { async fn read_message(&self, id: QueueId) -> Option { match self .store() - .get_value::>(ValueKey::from(ValueClass::Queue(QueueClass::Message( - id, - )))) + .get_value::>(ValueKey::from(ValueClass::Queue( + QueueClass::Message(id), + ))) .await { Ok(Some(message)) => Some(message.inner), @@ -188,7 +188,7 @@ impl Message { } else { raw_message.into() }; - self.blob_hash = BlobHash::from(message.as_ref()); + self.blob_hash = BlobHash::generate(message.as_ref()); // Generate id if self.size == 0 { @@ -298,7 +298,7 @@ impl Message { ) .set( ValueClass::Queue(QueueClass::Message(self.queue_id)), - match Bincode::new(self).serialize() { + match LegacyBincode::new(self).serialize() { Ok(data) => data, Err(err) => { trc::error!( @@ -430,7 +430,7 @@ impl Message { let span_id = self.span_id; batch.set( ValueClass::Queue(QueueClass::Message(self.queue_id)), - match Bincode::new(self).serialize() { + match LegacyBincode::new(self).serialize() { Ok(data) => data, Err(err) => { trc::error!( diff --git a/crates/smtp/src/reporting/analysis.rs b/crates/smtp/src/reporting/analysis.rs index a9cf415c..a7cccdad 100644 --- a/crates/smtp/src/reporting/analysis.rs +++ b/crates/smtp/src/reporting/analysis.rs @@ -21,7 +21,7 @@ use mail_parser::{Message, MimeHeaders, PartType}; use store::{ Serialize, - write::{BatchBuilder, Bincode, ReportClass, ValueClass, now}, + write::{BatchBuilder, LegacyBincode, ReportClass, ValueClass, now}, }; use trc::IncomingReportEvent; @@ -281,7 +281,7 @@ impl AnalyzeReport for Server { Format::Dmarc(report) => { batch.set( ValueClass::Report(ReportClass::Dmarc { id, expires }), - Bincode::new(IncomingReport { + LegacyBincode::new(IncomingReport { from, to, subject, @@ -294,7 +294,7 @@ impl AnalyzeReport for Server { Format::Tls(report) => { batch.set( ValueClass::Report(ReportClass::Tls { id, expires }), - Bincode::new(IncomingReport { + LegacyBincode::new(IncomingReport { from, to, subject, @@ -307,7 +307,7 @@ impl AnalyzeReport for Server { Format::Arf(report) => { batch.set( ValueClass::Report(ReportClass::Arf { id, expires }), - Bincode::new(IncomingReport { + LegacyBincode::new(IncomingReport { from, to, subject, diff --git a/crates/smtp/src/reporting/dmarc.rs b/crates/smtp/src/reporting/dmarc.rs index 7b589eab..79ab24e2 100644 --- a/crates/smtp/src/reporting/dmarc.rs +++ b/crates/smtp/src/reporting/dmarc.rs @@ -22,7 +22,7 @@ use mail_auth::{ }; use store::{ Deserialize, IterateParams, Serialize, ValueKey, - write::{BatchBuilder, Bincode, QueueClass, ReportEvent, ValueClass, now}, + write::{BatchBuilder, LegacyBincode, QueueClass, ReportEvent, ValueClass, now}, }; use trc::{AddContext, OutgoingReportEvent}; use utils::config::Rate; @@ -467,7 +467,7 @@ impl DmarcReporting for Server { .core .storage .data - .get_value::>(ValueKey::from(ValueClass::Queue( + .get_value::>(ValueKey::from(ValueClass::Queue( QueueClass::DmarcReportHeader(event.clone()), ))) .await? @@ -543,7 +543,7 @@ impl DmarcReporting for Server { .data .iterate( IterateParams::new(from_key, to_key).ascending(), - |_, v| match record_map.entry(Bincode::::deserialize(v)?.inner) { + |_, v| match record_map.entry(LegacyBincode::::deserialize(v)?.inner) { Entry::Occupied(mut e) => { *e.get_mut() += 1; Ok(true) @@ -650,7 +650,7 @@ impl DmarcReporting for Server { // Write report builder.set( ValueClass::Queue(QueueClass::DmarcReportHeader(report_event.clone())), - match Bincode::new(entry).serialize() { + match LegacyBincode::new(entry).serialize() { Ok(data) => data, Err(err) => { trc::error!( @@ -667,7 +667,7 @@ impl DmarcReporting for Server { report_event.seq_id = self.inner.data.queue_id_gen.generate().unwrap_or_else(now); builder.set( ValueClass::Queue(QueueClass::DmarcReportEvent(report_event)), - match Bincode::new(event.report_record).serialize() { + match LegacyBincode::new(event.report_record).serialize() { Ok(data) => data, Err(err) => { trc::error!( diff --git a/crates/smtp/src/reporting/tls.rs b/crates/smtp/src/reporting/tls.rs index ce236f18..cbe2b041 100644 --- a/crates/smtp/src/reporting/tls.rs +++ b/crates/smtp/src/reporting/tls.rs @@ -28,7 +28,7 @@ use reqwest::header::CONTENT_TYPE; use std::fmt::Write; use store::{ Deserialize, IterateParams, Serialize, ValueKey, - write::{BatchBuilder, Bincode, QueueClass, ReportEvent, ValueClass, now}, + write::{BatchBuilder, LegacyBincode, QueueClass, ReportEvent, ValueClass, now}, }; use trc::{AddContext, OutgoingReportEvent}; @@ -299,7 +299,7 @@ impl TlsReporting for Server { .core .storage .data - .get_value::>(ValueKey::from(ValueClass::Queue( + .get_value::>(ValueKey::from(ValueClass::Queue( QueueClass::TlsReportHeader(event.clone()), ))) .await? @@ -338,7 +338,7 @@ impl TlsReporting for Server { .data .iterate(IterateParams::new(from_key, to_key).ascending(), |_, v| { if let Some(failure_details) = - Bincode::>::deserialize(v)?.inner + LegacyBincode::>::deserialize(v)?.inner { match record_map.entry(failure_details) { Entry::Occupied(mut e) => { @@ -491,7 +491,7 @@ impl TlsReporting for Server { // Write report builder.set( ValueClass::Queue(QueueClass::TlsReportHeader(report_event.clone())), - match Bincode::new(entry).serialize() { + match LegacyBincode::new(entry).serialize() { Ok(data) => data, Err(err) => { trc::error!( @@ -508,7 +508,7 @@ impl TlsReporting for Server { report_event.seq_id = self.inner.data.queue_id_gen.generate().unwrap_or_else(now); builder.set( ValueClass::Queue(QueueClass::TlsReportEvent(report_event)), - match Bincode::new(event.failure).serialize() { + match LegacyBincode::new(event.failure).serialize() { Ok(data) => data, Err(err) => { trc::error!( diff --git a/crates/store/src/fts/index.rs b/crates/store/src/fts/index.rs index 57c9e9f9..0b9c9190 100644 --- a/crates/store/src/fts/index.rs +++ b/crates/store/src/fts/index.rs @@ -31,7 +31,7 @@ use super::{Field, postings::Postings}; pub const TERM_INDEX_VERSION: u8 = 1; #[derive(Debug)] -pub(crate) struct Text<'x, T: Into + Display + Clone + std::fmt::Debug> { +pub(crate) struct Text<'x, T: Into + Display + std::fmt::Debug> { pub field: Field, pub text: Cow<'x, str>, pub typ: Type, @@ -45,7 +45,7 @@ pub(crate) enum Type { } #[derive(Debug)] -pub struct FtsDocument<'x, T: Into + Display + Clone + std::fmt::Debug> { +pub struct FtsDocument<'x, T: Into + Display + std::fmt::Debug> { pub(crate) parts: Vec>, pub(crate) default_language: Language, pub(crate) account_id: u32, @@ -53,7 +53,7 @@ pub struct FtsDocument<'x, T: Into + Display + Clone + std::fmt::Debug> { pub(crate) document_id: u32, } -impl<'x, T: Into + Display + Clone + std::fmt::Debug> FtsDocument<'x, T> { +impl<'x, T: Into + Display + std::fmt::Debug> FtsDocument<'x, T> { pub fn with_default_language(default_language: Language) -> FtsDocument<'x, T> { FtsDocument { parts: vec![], @@ -107,7 +107,7 @@ impl<'x, T: Into + Display + Clone + std::fmt::Debug> FtsDocument<'x, T> { } } -impl + Display + Clone + std::fmt::Debug> From> for u8 { +impl + Display + std::fmt::Debug> From> for u8 { fn from(value: Field) -> Self { match value { Field::Body => 0, @@ -119,7 +119,7 @@ impl + Display + Clone + std::fmt::Debug> From> for u8 { } impl Store { - pub async fn fts_index + Display + Clone + std::fmt::Debug>( + pub async fn fts_index + Display + std::fmt::Debug>( &self, document: FtsDocument<'_, T>, ) -> trc::Result<()> { diff --git a/crates/store/src/fts/mod.rs b/crates/store/src/fts/mod.rs index c38767d9..7ca01fec 100644 --- a/crates/store/src/fts/mod.rs +++ b/crates/store/src/fts/mod.rs @@ -13,7 +13,7 @@ pub mod postings; pub mod query; #[derive(Clone, Debug, PartialEq, Eq)] -pub enum Field + Display + Clone + std::fmt::Debug> { +pub enum Field + Display + std::fmt::Debug> { Header(T), Body, Attachment, @@ -21,7 +21,7 @@ pub enum Field + Display + Clone + std::fmt::Debug> { } #[derive(Debug, PartialEq, Eq)] -pub enum FtsFilter + Display + Clone + std::fmt::Debug> { +pub enum FtsFilter + Display + std::fmt::Debug> { Exact { field: Field, text: String, @@ -42,7 +42,7 @@ pub enum FtsFilter + Display + Clone + std::fmt::Debug> { End, } -impl + Display + Clone + std::fmt::Debug> FtsFilter { +impl + Display + std::fmt::Debug> FtsFilter { pub fn has_text_detect( field: Field, text: impl Into, diff --git a/crates/store/src/write/mod.rs b/crates/store/src/write/mod.rs index d5a6aec4..35d99257 100644 --- a/crates/store/src/write/mod.rs +++ b/crates/store/src/write/mod.rs @@ -9,22 +9,15 @@ use std::{ collections::HashSet, fmt::{self, Formatter}, hash::Hash, - slice::Iter, time::{Duration, SystemTime}, }; -use assert::HashedValue; use nlp::tokenizers::word::WordTokenizer; use rand::Rng; use roaring::RoaringBitmap; -use utils::{ - BlobHash, - codec::leb128::{Leb128Iterator, Leb128Vec}, -}; +use utils::BlobHash; -use crate::{ - BlobClass, Deserialize, Serialize, SerializeInfallible, Value, backend::MAX_TOKEN_LENGTH, -}; +use crate::{BlobClass, SerializeInfallible, backend::MAX_TOKEN_LENGTH}; use self::assert::AssertValue; @@ -34,6 +27,30 @@ pub mod blob; pub mod hash; pub mod key; pub mod log; +pub mod serialize; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Archive { + Raw(Vec), + Uncompressed(Vec), +} + +#[repr(transparent)] +pub struct Archiver(pub T) +where + T: rkyv::Archive + + for<'a> rkyv::Serialize< + rkyv::api::high::HighSerializer< + rkyv::util::AlignedVec, + rkyv::ser::allocator::ArenaHandle<'a>, + rkyv::rancor::Error, + >, + >; + +#[derive(Debug)] +pub struct LegacyBincode { + pub inner: T, +} pub trait SerializeWithId: Send + Sync { fn serialize_with_id(&self, ids: &AssignedIds) -> trc::Result>; @@ -303,177 +320,6 @@ impl From<()> for TagValue { } } -impl SerializeInfallible for u32 { - fn serialize(&self) -> Vec { - self.to_be_bytes().to_vec() - } -} - -impl SerializeInfallible for u64 { - fn serialize(&self) -> Vec { - self.to_be_bytes().to_vec() - } -} - -impl SerializeInfallible for i64 { - fn serialize(&self) -> Vec { - self.to_be_bytes().to_vec() - } -} - -impl SerializeInfallible for u16 { - fn serialize(&self) -> Vec { - self.to_be_bytes().to_vec() - } -} - -impl SerializeInfallible for f64 { - fn serialize(&self) -> Vec { - self.to_be_bytes().to_vec() - } -} - -impl SerializeInfallible for &str { - fn serialize(&self) -> Vec { - self.as_bytes().to_vec() - } -} - -impl Deserialize for String { - fn deserialize(bytes: &[u8]) -> trc::Result { - Ok(String::from_utf8_lossy(bytes).into_owned()) - } - - fn deserialize_owned(bytes: Vec) -> trc::Result { - Ok(String::from_utf8(bytes) - .unwrap_or_else(|err| String::from_utf8_lossy(err.as_bytes()).into_owned())) - } -} - -impl Deserialize for u64 { - fn deserialize(bytes: &[u8]) -> trc::Result { - Ok(u64::from_be_bytes(bytes.try_into().map_err(|_| { - trc::StoreEvent::DataCorruption.caused_by(trc::location!()) - })?)) - } -} - -impl Deserialize for i64 { - fn deserialize(bytes: &[u8]) -> trc::Result { - Ok(i64::from_be_bytes(bytes.try_into().map_err(|_| { - trc::StoreEvent::DataCorruption.caused_by(trc::location!()) - })?)) - } -} - -impl Deserialize for u32 { - fn deserialize(bytes: &[u8]) -> trc::Result { - Ok(u32::from_be_bytes(bytes.try_into().map_err(|_| { - trc::StoreEvent::DataCorruption.caused_by(trc::location!()) - })?)) - } -} - -pub trait SerializeInto { - fn serialize_into(&self, buf: &mut Vec); -} - -pub trait DeserializeFrom: Sized { - fn deserialize_from(bytes: &mut Iter<'_, u8>) -> Option; -} - -pub struct ArchivedValue { - inner: Vec, - _phantom: std::marker::PhantomData, -} - -impl Serialize for Vec { - fn serialize(&self) -> trc::Result> { - let mut bytes = Vec::with_capacity(self.len() * 4); - bytes.push_leb128(self.len()); - for item in self.iter() { - item.serialize_into(&mut bytes); - } - Ok(bytes) - } -} - -impl SerializeInto for String { - fn serialize_into(&self, buf: &mut Vec) { - buf.push_leb128(self.len()); - if !self.is_empty() { - buf.extend_from_slice(self.as_bytes()); - } - } -} - -impl SerializeInto for Vec { - fn serialize_into(&self, buf: &mut Vec) { - buf.push_leb128(self.len()); - if !self.is_empty() { - buf.extend_from_slice(self.as_slice()); - } - } -} - -impl SerializeInto for u32 { - fn serialize_into(&self, buf: &mut Vec) { - buf.push_leb128(*self); - } -} - -impl SerializeInto for u64 { - fn serialize_into(&self, buf: &mut Vec) { - buf.push_leb128(*self); - } -} - -impl DeserializeFrom for u32 { - fn deserialize_from(bytes: &mut Iter<'_, u8>) -> Option { - bytes.next_leb128() - } -} - -impl DeserializeFrom for u64 { - fn deserialize_from(bytes: &mut Iter<'_, u8>) -> Option { - bytes.next_leb128() - } -} - -impl DeserializeFrom for String { - fn deserialize_from(bytes: &mut Iter<'_, u8>) -> Option { - >::deserialize_from(bytes).and_then(|s| String::from_utf8(s).ok()) - } -} - -impl DeserializeFrom for Vec { - fn deserialize_from(bytes: &mut Iter<'_, u8>) -> Option { - let len: usize = bytes.next_leb128()?; - let mut buf = Vec::with_capacity(len); - for _ in 0..len { - buf.push(*bytes.next()?); - } - buf.into() - } -} - -impl Deserialize for Vec { - fn deserialize(bytes: &[u8]) -> trc::Result { - let mut bytes = bytes.iter(); - let len: usize = bytes - .next_leb128() - .ok_or_else(|| trc::StoreEvent::DataCorruption.caused_by(trc::location!()))?; - let mut list = Vec::with_capacity(len); - for _ in 0..len { - list.push( - T::deserialize_from(&mut bytes) - .ok_or_else(|| trc::StoreEvent::DataCorruption.caused_by(trc::location!()))?, - ); - } - Ok(list) - } -} - pub trait TokenizeText { fn tokenize_into(&self, tokens: &mut HashSet); fn to_tokens(&self) -> HashSet; @@ -493,18 +339,6 @@ impl TokenizeText for &str { } } -impl Serialize for () { - fn serialize(&self) -> trc::Result> { - Ok(Vec::with_capacity(0)) - } -} - -impl Deserialize for () { - fn deserialize(_bytes: &[u8]) -> trc::Result { - Ok(()) - } -} - pub trait IntoOperations { fn build(self, batch: &mut BatchBuilder) -> trc::Result<()>; } @@ -574,124 +408,6 @@ impl BlobClass { } } -#[derive(Debug)] -pub struct Bincode { - pub inner: T, -} - -impl Bincode { - pub fn new(inner: T) -> Self { - Self { inner } - } -} - -impl From> for Bincode { - fn from(_: Value<'static>) -> Self { - unreachable!("From Value called on Bincode") - } -} - -impl Serialize for Bincode { - fn serialize(&self) -> trc::Result> { - bincode::serialize(&self.inner) - .map(|bytes| lz4_flex::compress_prepend_size(&bytes)) - .map_err(|err| { - trc::StoreEvent::DeserializeError - .caused_by(trc::location!()) - .reason(err) - }) - } -} - -impl Deserialize - for Bincode -{ - fn deserialize(bytes: &[u8]) -> trc::Result { - lz4_flex::decompress_size_prepended(bytes) - .map_err(|err| { - trc::StoreEvent::DecompressError - .ctx(trc::Key::Value, bytes) - .caused_by(trc::location!()) - .reason(err) - }) - .and_then(|result| { - bincode::deserialize(&result).map_err(|err| { - trc::StoreEvent::DataCorruption - .ctx(trc::Key::Value, bytes) - .caused_by(trc::location!()) - .reason(err) - }) - }) - .map(|inner| Self { inner }) - } -} - -impl Deserialize for ArchivedValue { - fn deserialize(bytes: &[u8]) -> trc::Result { - Ok(ArchivedValue { - inner: bytes.to_vec(), - _phantom: std::marker::PhantomData, - }) - } - - fn deserialize_owned(bytes: Vec) -> trc::Result { - Ok(ArchivedValue { - inner: bytes, - _phantom: std::marker::PhantomData, - }) - } -} - -impl ArchivedValue -where - T: rkyv::Portable - + for<'a> rkyv::bytecheck::CheckBytes> - + Sync - + Send, -{ - pub fn unarchive(&self) -> trc::Result<&T> { - rkyv::access::(&self.inner).map_err(Into::into) - } - - pub fn unarchive_unsafe(&self) -> &T { - unsafe { rkyv::access_unchecked::(&self.inner) } - } - - pub fn deserialize(&self) -> trc::Result - where - T: rkyv::Deserialize>, - { - rkyv::access::(&self.inner) - .and_then(|value| rkyv::deserialize::(value)) - .map_err(Into::into) - } -} - -impl HashedValue> -where - T: rkyv::Portable - + for<'a> rkyv::bytecheck::CheckBytes> - + Sync - + Send, -{ - pub fn to_unarchived(&self) -> trc::Result> { - self.inner.unarchive().map(|inner| HashedValue { - hash: self.hash, - inner, - }) - } - - pub fn into_deserialized(self) -> trc::Result> - where - T: rkyv::Deserialize>, - { - self.inner.deserialize().map(|inner| HashedValue { - hash: self.hash, - inner, - }) - } -} - impl AssignedIds { pub fn push_document_id(&mut self, id: u32) { self.document_ids.push(id); diff --git a/crates/store/src/write/serialize.rs b/crates/store/src/write/serialize.rs new file mode 100644 index 00000000..079199dc --- /dev/null +++ b/crates/store/src/write/serialize.rs @@ -0,0 +1,382 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::{Deserialize, Serialize, SerializeInfallible, Value}; + +use super::{Archive, Archiver, LegacyBincode, assert::HashedValue}; + +const MAGIC_MARKER: u8 = 1 << 7; +const LZ4_COMPRESSES: u8 = 1 << 6; +const ARCHIVE_UNCOMPRESSED: u8 = MAGIC_MARKER; +const ARCHIVE_LZ4_COMPRESSED: u8 = MAGIC_MARKER | LZ4_COMPRESSES; +const COMPRESS_WATERMARK: usize = 8192; +const COMPRESS_DATA_OFFSET: usize = std::mem::size_of::() + 1; + +impl Deserialize for Archive { + fn deserialize(bytes: &[u8]) -> trc::Result { + match bytes.first().copied() { + Some(ARCHIVE_UNCOMPRESSED) => Ok(Archive::Raw(bytes.to_vec())), + Some(ARCHIVE_LZ4_COMPRESSED) => { + lz4_flex::decompress_size_prepended(bytes.get(1..).unwrap_or_default()) + .map_err(|err| { + trc::StoreEvent::DecompressError + .ctx(trc::Key::Value, bytes) + .caused_by(trc::location!()) + .reason(err) + }) + .map(Archive::Uncompressed) + } + _ => Err(trc::StoreEvent::DataCorruption + .into_err() + .details("Invalid archive marker.") + .ctx(trc::Key::Value, bytes) + .caused_by(trc::location!())), + } + } + + fn deserialize_owned(bytes: Vec) -> trc::Result { + match bytes.first().copied() { + Some(ARCHIVE_UNCOMPRESSED) => Ok(Archive::Raw(bytes)), + Some(ARCHIVE_LZ4_COMPRESSED) => { + lz4_flex::decompress_size_prepended(bytes.get(1..).unwrap_or_default()) + .map_err(|err| { + trc::StoreEvent::DecompressError + .ctx(trc::Key::Value, bytes) + .caused_by(trc::location!()) + .reason(err) + }) + .map(Archive::Uncompressed) + } + _ => Err(trc::StoreEvent::DataCorruption + .into_err() + .details("Invalid archive marker.") + .ctx(trc::Key::Value, bytes) + .caused_by(trc::location!())), + } + } +} + +impl Serialize for Archiver +where + T: rkyv::Archive + + for<'a> rkyv::Serialize< + rkyv::api::high::HighSerializer< + rkyv::util::AlignedVec, + rkyv::ser::allocator::ArenaHandle<'a>, + rkyv::rancor::Error, + >, + >, +{ + fn serialize(&self) -> trc::Result> { + rkyv::to_bytes::(&self.0) + .map_err(|err| { + trc::StoreEvent::DeserializeError + .caused_by(trc::location!()) + .reason(err) + }) + .map(|input| { + let input = input.as_ref(); + let input_len = input.len(); + if input_len > COMPRESS_WATERMARK { + let mut bytes = vec![ + ARCHIVE_LZ4_COMPRESSED; + lz4_flex::block::get_maximum_output_size(input_len) + + COMPRESS_DATA_OFFSET + ]; + bytes[1..COMPRESS_DATA_OFFSET] + .copy_from_slice(&(input_len as u32).to_le_bytes()); + let bytes_len = + lz4_flex::compress_into(input, &mut bytes[COMPRESS_DATA_OFFSET..]).unwrap() + + COMPRESS_DATA_OFFSET; + if bytes_len < input_len { + bytes.truncate(bytes_len); + } else { + bytes.clear(); + bytes.push(ARCHIVE_UNCOMPRESSED); + bytes.extend_from_slice(input); + } + bytes + } else { + let mut bytes = Vec::with_capacity(input_len + 1); + bytes.push(ARCHIVE_UNCOMPRESSED); + bytes.extend_from_slice(input); + bytes + } + }) + } +} + +impl Archive { + pub fn unarchive(&self) -> trc::Result<&T> + where + T: rkyv::Portable + + for<'a> rkyv::bytecheck::CheckBytes< + rkyv::api::high::HighValidator<'a, rkyv::rancor::Error>, + > + Sync + + Send, + { + match self { + Archive::Raw(bytes) => rkyv::access::(bytes.get(1..).unwrap()) + .map_err(|err| { + trc::StoreEvent::DataCorruption + .caused_by(trc::location!()) + .ctx(trc::Key::Value, bytes.as_slice()) + .reason(err) + }), + Archive::Uncompressed(bytes) => Ok(unsafe { rkyv::access_unchecked::(bytes) }), + } + } + + pub fn deserialize(&self) -> trc::Result + where + T: rkyv::Portable + + for<'a> rkyv::bytecheck::CheckBytes< + rkyv::api::high::HighValidator<'a, rkyv::rancor::Error>, + > + Sync + + Send + + rkyv::Deserialize>, + { + self.unarchive::().and_then(|value| { + rkyv::deserialize::(value).map_err(|err| { + trc::StoreEvent::DeserializeError + .ctx( + trc::Key::Value, + match self { + Archive::Raw(bytes) => bytes, + Archive::Uncompressed(bytes) => bytes, + } + .as_slice(), + ) + .caused_by(trc::location!()) + .reason(err) + }) + }) + } + + pub fn into_inner(self) -> Vec { + match self { + Archive::Raw(bytes) => bytes, + Archive::Uncompressed(bytes) => { + let mut result = Vec::with_capacity(bytes.len() + 1); + result.push(ARCHIVE_UNCOMPRESSED); + result.extend_from_slice(&bytes); + result + } + } + } +} + +impl Archiver +where + T: rkyv::Archive + + for<'a> rkyv::Serialize< + rkyv::api::high::HighSerializer< + rkyv::util::AlignedVec, + rkyv::ser::allocator::ArenaHandle<'a>, + rkyv::rancor::Error, + >, + >, +{ + pub fn new(inner: T) -> Self { + Self(inner) + } + + pub fn into_inner(self) -> T { + self.0 + } +} + +impl HashedValue { + pub fn to_unarchived(&self) -> trc::Result> + where + T: rkyv::Portable + + for<'a> rkyv::bytecheck::CheckBytes< + rkyv::api::high::HighValidator<'a, rkyv::rancor::Error>, + > + Sync + + Send, + { + self.inner.unarchive().map(|inner| HashedValue { + hash: self.hash, + inner, + }) + } + + pub fn into_deserialized(self) -> trc::Result> + where + T: rkyv::Portable + + for<'a> rkyv::bytecheck::CheckBytes< + rkyv::api::high::HighValidator<'a, rkyv::rancor::Error>, + > + Sync + + Send + + rkyv::Deserialize>, + { + self.inner.deserialize::().map(|inner| HashedValue { + hash: self.hash, + inner, + }) + } +} + +#[inline] +pub fn rkyv_deserialize(input: &T) -> trc::Result +where + T: rkyv::Portable + + for<'a> rkyv::bytecheck::CheckBytes> + + Sync + + Send + + rkyv::Deserialize>, +{ + rkyv::deserialize::(input).map_err(|err| { + trc::StoreEvent::DeserializeError + .caused_by(trc::location!()) + .reason(err) + }) +} + +pub fn rkyv_unarchive(input: &[u8]) -> trc::Result<&T> +where + T: rkyv::Portable + + for<'a> rkyv::bytecheck::CheckBytes> + + Sync + + Send, +{ + rkyv::access::(input).map_err(|err| { + trc::StoreEvent::DataCorruption + .caused_by(trc::location!()) + .ctx(trc::Key::Value, input) + .reason(err) + }) +} + +impl SerializeInfallible for u32 { + fn serialize(&self) -> Vec { + self.to_be_bytes().to_vec() + } +} + +impl SerializeInfallible for u64 { + fn serialize(&self) -> Vec { + self.to_be_bytes().to_vec() + } +} + +impl SerializeInfallible for i64 { + fn serialize(&self) -> Vec { + self.to_be_bytes().to_vec() + } +} + +impl SerializeInfallible for u16 { + fn serialize(&self) -> Vec { + self.to_be_bytes().to_vec() + } +} + +impl SerializeInfallible for f64 { + fn serialize(&self) -> Vec { + self.to_be_bytes().to_vec() + } +} + +impl SerializeInfallible for &str { + fn serialize(&self) -> Vec { + self.as_bytes().to_vec() + } +} + +impl Deserialize for String { + fn deserialize(bytes: &[u8]) -> trc::Result { + Ok(String::from_utf8_lossy(bytes).into_owned()) + } + + fn deserialize_owned(bytes: Vec) -> trc::Result { + Ok(String::from_utf8(bytes) + .unwrap_or_else(|err| String::from_utf8_lossy(err.as_bytes()).into_owned())) + } +} + +impl Deserialize for u64 { + fn deserialize(bytes: &[u8]) -> trc::Result { + Ok(u64::from_be_bytes(bytes.try_into().map_err(|_| { + trc::StoreEvent::DataCorruption.caused_by(trc::location!()) + })?)) + } +} + +impl Deserialize for i64 { + fn deserialize(bytes: &[u8]) -> trc::Result { + Ok(i64::from_be_bytes(bytes.try_into().map_err(|_| { + trc::StoreEvent::DataCorruption.caused_by(trc::location!()) + })?)) + } +} + +impl Deserialize for u32 { + fn deserialize(bytes: &[u8]) -> trc::Result { + Ok(u32::from_be_bytes(bytes.try_into().map_err(|_| { + trc::StoreEvent::DataCorruption.caused_by(trc::location!()) + })?)) + } +} + +impl Deserialize for () { + fn deserialize(_bytes: &[u8]) -> trc::Result { + Ok(()) + } +} + +impl LegacyBincode { + pub fn new(inner: T) -> Self { + Self { inner } + } +} + +impl Serialize for LegacyBincode { + fn serialize(&self) -> trc::Result> { + bincode::serialize(&self.inner) + .map(|bytes| lz4_flex::compress_prepend_size(&bytes)) + .map_err(|err| { + trc::StoreEvent::DeserializeError + .caused_by(trc::location!()) + .reason(err) + }) + } +} + +impl Deserialize + for LegacyBincode +{ + fn deserialize(bytes: &[u8]) -> trc::Result { + lz4_flex::decompress_size_prepended(bytes) + .map_err(|err| { + trc::StoreEvent::DecompressError + .ctx(trc::Key::Value, bytes) + .caused_by(trc::location!()) + .reason(err) + }) + .and_then(|result| { + bincode::deserialize(&result).map_err(|err| { + trc::StoreEvent::DataCorruption + .ctx(trc::Key::Value, bytes) + .caused_by(trc::location!()) + .reason(err) + }) + }) + .map(|inner| Self { inner }) + } +} + +impl From> for Archive { + fn from(_: Value<'static>) -> Self { + unimplemented!() + } +} + +impl From> for LegacyBincode { + fn from(_: Value<'static>) -> Self { + unimplemented!() + } +} diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index f3b7f078..0105dfc8 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -48,6 +48,10 @@ impl BlobHash { BlobHash([u8::MAX; BLOB_HASH_LEN]) } + pub fn generate(value: impl AsRef<[u8]>) -> Self { + BlobHash(blake3::hash(value.as_ref()).into()) + } + pub fn try_from_hash_slice(value: &[u8]) -> Result { value.try_into().map(BlobHash) } @@ -65,27 +69,9 @@ impl BlobHash { } } -impl From<&[u8]> for BlobHash { - fn from(value: &[u8]) -> Self { - BlobHash(blake3::hash(value).into()) - } -} - impl From<&ArchivedBlobHash> for BlobHash { fn from(value: &ArchivedBlobHash) -> Self { - value.0.as_slice().into() - } -} - -impl From> for BlobHash { - fn from(value: Vec) -> Self { - value.as_slice().into() - } -} - -impl From<&Vec> for BlobHash { - fn from(value: &Vec) -> Self { - value.as_slice().into() + BlobHash(value.0) } } diff --git a/tests/src/imap/body_structure.rs b/tests/src/imap/body_structure.rs index 3e97f087..83f08d6a 100644 --- a/tests/src/imap/body_structure.rs +++ b/tests/src/imap/body_structure.rs @@ -6,12 +6,17 @@ use std::fs; +use email::message::metadata::{ArchivedMessageMetadata, MessageMetadata}; use imap::op::fetch::AsImapDataItem; use imap_proto::{ - protocol::fetch::{BodyContents, DataItem, Section}, ResponseCode, StatusResponse, + protocol::fetch::{BodyContents, DataItem, Section}, }; use mail_parser::MessageParser; +use store::{ + Deserialize, Serialize, + write::{Archive, Archiver}, +}; use super::resources_dir; @@ -21,19 +26,41 @@ fn body_structure() { for file_name in fs::read_dir(resources_dir()).unwrap() { let mut file_name = file_name.as_ref().unwrap().path(); - if file_name.extension().is_none_or( |e| e != "txt") { + if file_name.extension().is_none_or(|e| e != "txt") { continue; } - let raw_message = fs::read(&file_name).unwrap(); - let message = MessageParser::new().parse(&raw_message).unwrap(); let mut buf = Vec::new(); + let raw_message = fs::read(&file_name).unwrap(); + let message_ = MessageParser::new().parse(&raw_message).unwrap(); + let metadata_ = Archive::deserialize_owned( + Archiver::new(MessageMetadata { + preview: Default::default(), + size: message_.raw_message.len() as u32, + raw_headers: message_ + .raw_message + .as_ref() + .get(message_.root_part().offset_header..message_.root_part().offset_body) + .unwrap_or_default() + .to_vec(), + contents: message_.into(), + received_at: 0, + has_attachments: false, + blob_hash: Default::default(), + }) + .serialize() + .unwrap(), + ) + .unwrap(); + let metadata = metadata_.unarchive::().unwrap(); + let message = &metadata.contents; + let decoded = message.decode_contents(&raw_message); // Serialize body and bodystructure for is_extended in [false, true] { let mut buf_ = Vec::new(); message - .body_structure(is_extended) + .body_structure(&decoded, is_extended) .serialize(&mut buf_, is_extended); if is_extended { buf.extend_from_slice(b"BODYSTRUCTURE "); @@ -92,7 +119,8 @@ fn body_structure() { true }; - if let Some(contents) = message.body_section(&body_sections, None) { + if let Some(contents) = message.body_section(&decoded, &body_sections, None) + { DataItem::BodySection { sections: body_sections, origin_octet: None, @@ -101,7 +129,7 @@ fn body_structure() { .serialize(&mut buf); if is_first { - match message.binary(§ions, None) { + match message.binary(&decoded, §ions, None) { Ok(Some(contents)) => { buf.push(b'\n'); DataItem::Binary { @@ -135,7 +163,7 @@ fn body_structure() { } } - if let Some(size) = message.binary_size(§ions) { + if let Some(size) = message.binary_size(&decoded, §ions) { buf.push(b'\n'); DataItem::BinarySize { sections: sections.clone(), @@ -175,14 +203,16 @@ fn body_structure() { }], ] { DataItem::BodySection { - contents: message.body_section(§ions, None).unwrap(), + contents: message.body_section(&decoded, §ions, None).unwrap(), sections: sections.clone(), origin_octet: None, } .serialize(&mut buf); buf.extend_from_slice(b"\n----------------------------------\n"); DataItem::BodySection { - contents: message.body_section(§ions, (10, 25).into()).unwrap(), + contents: message + .body_section(&decoded, §ions, (10, 25).into()) + .unwrap(), sections, origin_octet: 10.into(), } diff --git a/tests/src/jmap/crypto.rs b/tests/src/jmap/crypto.rs index 35db3c7c..d8f19af2 100644 --- a/tests/src/jmap/crypto.rs +++ b/tests/src/jmap/crypto.rs @@ -11,6 +11,10 @@ use email::message::crypto::{ }; use jmap_proto::types::id::Id; use mail_parser::{MessageParser, MimeHeaders}; +use store::{ + Deserialize, Serialize, + write::{Archive, Archiver}, +}; use crate::{ directory::internal::TestInternalDirectory, @@ -213,7 +217,10 @@ pub async fn import_certs_and_encrypt() { .unwrap(); assert!(!message.is_encrypted()); params.algo = algo; - message.encrypt(¶ms).await.unwrap(); + let arch = + Archive::deserialize_owned(Archiver::new(params.clone()).serialize().unwrap()) + .unwrap(); + message.encrypt(arch.unarchive().unwrap()).await.unwrap(); } } diff --git a/tests/src/jmap/permissions.rs b/tests/src/jmap/permissions.rs index bf68b0de..5ff6304b 100644 --- a/tests/src/jmap/permissions.rs +++ b/tests/src/jmap/permissions.rs @@ -597,7 +597,7 @@ pub async fn test(params: &JMAPTest) { ); // John should not be allowed to receive email - let message_blob = BlobHash::from(TEST_MESSAGE.as_bytes()); + let message_blob = BlobHash::generate(TEST_MESSAGE.as_bytes()); server .blob_store() .put_blob(message_blob.as_ref(), TEST_MESSAGE.as_bytes()) diff --git a/tests/src/jmap/stress_test.rs b/tests/src/jmap/stress_test.rs index eae82a28..4c8e53b7 100644 --- a/tests/src/jmap/stress_test.rs +++ b/tests/src/jmap/stress_test.rs @@ -9,7 +9,7 @@ use std::{sync::Arc, time::Duration}; use crate::jmap::{mailbox::destroy_all_mailboxes_no_wait, wait_for_index}; use common::Server; use directory::backend::internal::manage::ManageDirectory; -use email::mailbox::UidMailbox; +use email::mailbox::{ArchivedUidMailbox, UidMailbox}; use futures::future::join_all; use jmap_client::{ client::Client, @@ -17,7 +17,11 @@ use jmap_client::{ mailbox::{self, Mailbox, Role}, }; use jmap_proto::types::{collection::Collection, id::Id, property::Property}; -use store::rand::{self, Rng}; +use store::{ + rand::{self, Rng}, + rkyv::vec::ArchivedVec, + write::Archive, +}; use super::assert_is_empty; @@ -70,8 +74,7 @@ async fn email_tests(server: Server, client: Arc) { let client = client.clone(); let mailboxes = mailboxes.clone(); futures.push(tokio::spawn(async move { - let mailbox_num = - rand::rng().random_range::(0..mailboxes.len()); + let mailbox_num = rand::rng().random_range::(0..mailboxes.len()); let _message_id = client .email_import( format!( @@ -230,7 +233,7 @@ async fn email_tests(server: Server, client: Arc) { for email_id in &email_ids_in_mailbox { if let Some(mailbox_tags) = server - .get_property::>( + .get_property::( TEST_USER_ID, Collection::Email, email_id, @@ -239,11 +242,14 @@ async fn email_tests(server: Server, client: Arc) { .await .unwrap() { + let mailbox_tags = mailbox_tags + .deserialize::, Vec>() + .unwrap(); if mailbox_tags.len() != 1 { panic!( - "Email ORM has more than one mailbox {:?}! Id {} in mailbox {} with messages {:?}", - mailbox_tags, email_id, mailbox_id, email_ids_in_mailbox - ); + "Email ORM has more than one mailbox {:?}! Id {} in mailbox {} with messages {:?}", + mailbox_tags, email_id, mailbox_id, email_ids_in_mailbox + ); } let mailbox_tag = mailbox_tags[0]; assert!(mailbox_tag.uid != 0); diff --git a/tests/src/smtp/inbound/mod.rs b/tests/src/smtp/inbound/mod.rs index 4506eebf..bd8b76e1 100644 --- a/tests/src/smtp/inbound/mod.rs +++ b/tests/src/smtp/inbound/mod.rs @@ -7,12 +7,12 @@ use std::time::Duration; use common::{ - ipc::{DmarcEvent, QueueEvent, QueueEventStatus, ReportingEvent, TlsEvent}, Server, + ipc::{DmarcEvent, QueueEvent, QueueEventStatus, ReportingEvent, TlsEvent}, }; use store::{ - write::{key::DeserializeBigEndian, Bincode, QueueClass, ReportEvent, ValueClass}, - Deserialize, IterateParams, ValueKey, U64_LEN, + Deserialize, IterateParams, U64_LEN, ValueKey, + write::{LegacyBincode, QueueClass, ReportEvent, ValueClass, key::DeserializeBigEndian}, }; use tokio::sync::mpsc::error::TryRecvError; @@ -186,7 +186,7 @@ impl QueueReceiver { .iterate( IterateParams::new(from_key, to_key).descending(), |key, value| { - let value = Bincode::::deserialize(value)?; + let value = LegacyBincode::::deserialize(value)?; assert_eq!(key.deserialize_be_u64(0)?, value.inner.queue_id); messages.push(value.inner); Ok(true) diff --git a/tests/src/smtp/queue/dsn.rs b/tests/src/smtp/queue/dsn.rs index 75624ba9..b1fa8ae9 100644 --- a/tests/src/smtp/queue/dsn.rs +++ b/tests/src/smtp/queue/dsn.rs @@ -6,13 +6,13 @@ use std::{fs, path::PathBuf, time::SystemTime}; -use smtp_proto::{Response, RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_SUCCESS}; +use smtp_proto::{RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_SUCCESS, Response}; use store::write::now; use utils::BlobHash; -use crate::smtp::{inbound::sign::SIGNATURES, QueueReceiver, TestSMTP}; +use crate::smtp::{QueueReceiver, TestSMTP, inbound::sign::SIGNATURES}; use smtp::queue::{ - dsn::SendDsn, Domain, Error, ErrorDetails, HostResponse, Message, Recipient, Schedule, Status, + Domain, Error, ErrorDetails, HostResponse, Message, Recipient, Schedule, Status, dsn::SendDsn, }; const CONFIG: &str = r#" @@ -87,7 +87,7 @@ async fn generate_dsn() { flags: 0, env_id: None, priority: 0, - blob_hash: BlobHash::from(dsn_original.as_bytes()), + blob_hash: BlobHash::generate(dsn_original.as_bytes()), quota_keys: vec![], }; diff --git a/tests/src/store/blob.rs b/tests/src/store/blob.rs index 082d4419..eb89049e 100644 --- a/tests/src/store/blob.rs +++ b/tests/src/store/blob.rs @@ -35,7 +35,7 @@ pub async fn blob_tests() { let blob_store: BlobStore = store.clone().into(); // Blob hash exists - let hash = BlobHash::from(b"abc".as_slice()); + let hash = BlobHash::generate(b"abc".as_slice()); assert!(!store.blob_exists(&hash).await.unwrap()); // Reserve blob @@ -163,7 +163,7 @@ pub async fn blob_tests() { .into_iter() .enumerate() { - let hash = BlobHash::from(blob.as_slice()); + let hash = BlobHash::generate(blob.as_slice()); let blob_op = if let Some(until) = expiry_times.get(blob) { BlobOp::Reserve { until: *until, @@ -256,7 +256,7 @@ pub async fn blob_tests() { .enumerate() { let ct = pos == 0; - let hash = BlobHash::from(blob.as_slice()); + let hash = BlobHash::generate(blob.as_slice()); assert!(store.blob_has_access(&hash, blob_class).await.unwrap() ^ ct); assert!(store.blob_exists(&hash).await.unwrap() ^ ct); assert!( @@ -273,7 +273,7 @@ pub async fn blob_tests() { assert!( !store .blob_has_access( - BlobHash::from(b"123".as_slice()), + BlobHash::generate(b"123".as_slice()), BlobClass::Linked { account_id: 0, collection: 0, @@ -292,7 +292,7 @@ pub async fn blob_tests() { .with_collection(0) .update_document(2) .clear(BlobOp::Link { - hash: BlobHash::from(b"789".as_slice()), + hash: BlobHash::generate(b"789".as_slice()), }) .build_batch(), ) @@ -345,7 +345,7 @@ pub async fn blob_tests() { .enumerate() { let ct = pos == 0; - let hash = BlobHash::from(blob.as_slice()); + let hash = BlobHash::generate(blob.as_slice()); assert!(store.blob_has_access(&hash, blob_class).await.unwrap() ^ ct); assert!(store.blob_exists(&hash).await.unwrap() ^ ct); assert!( @@ -399,7 +399,7 @@ pub async fn blob_tests() { .enumerate() { let ct = pos == 0; - let hash = BlobHash::from(blob.as_slice()); + let hash = BlobHash::generate(blob.as_slice()); assert!(store.blob_has_access(&hash, blob_class).await.unwrap() ^ ct); assert!(store.blob_exists(&hash).await.unwrap() ^ ct); assert!( @@ -418,7 +418,7 @@ pub async fn blob_tests() { async fn test_store(store: BlobStore) { // Test small blob const DATA: &[u8] = b"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Fusce erat nisl, dignissim a porttitor id, varius nec arcu. Sed mauris."; - let hash = BlobHash::from(DATA); + let hash = BlobHash::generate(DATA); store.put_blob(hash.as_slice(), DATA).await.unwrap(); assert_eq!( @@ -459,7 +459,7 @@ async fn test_store(store: BlobStore) { let marker = format!(" [{}] ", data.len()); data.extend_from_slice(marker.as_bytes()); } - let hash = BlobHash::from(&data); + let hash = BlobHash::generate(&data); store.put_blob(hash.as_slice(), &data).await.unwrap(); assert_eq!( String::from_utf8( diff --git a/tests/src/store/import_export.rs b/tests/src/store/import_export.rs index 8adf6d1c..cca42a83 100644 --- a/tests/src/store/import_export.rs +++ b/tests/src/store/import_export.rs @@ -5,7 +5,7 @@ */ use ahash::AHashSet; -use common::{manager::backup::BackupParams, Core}; +use common::{Core, manager::backup::BackupParams}; use jmap_proto::types::{collection::Collection, property::Property}; use store::{ rand, @@ -35,7 +35,7 @@ pub async fn test(db: Store) { let mut blob_hashes = Vec::new(); for blob_size in [16, 128, 1024, 2056, 102400] { let data = random_bytes(blob_size); - let hash = BlobHash::from(data.as_slice()); + let hash = BlobHash::generate(data.as_slice()); blob_hashes.push(hash.clone()); core.storage .blob