From dded91b93e9a9c269167ee1f1e9eb8115326b9b5 Mon Sep 17 00:00:00 2001 From: mdecimus Date: Sun, 5 Nov 2023 16:01:16 +0100 Subject: [PATCH] Store message metadata, avoid parsing messages on get --- Cargo.lock | 20 + crates/imap/src/op/fetch.rs | 69 ++- crates/jmap/Cargo.toml | 1 + crates/jmap/src/email/body.rs | 115 ++++- crates/jmap/src/email/copy.rs | 42 +- crates/jmap/src/email/get.rs | 248 +++++----- crates/jmap/src/email/headers.rs | 69 ++- crates/jmap/src/email/index.rs | 547 ++++++++++------------- crates/jmap/src/email/ingest.rs | 1 - crates/jmap/src/email/metadata.rs | 313 +++++++++++++ crates/jmap/src/email/mod.rs | 1 + crates/jmap/src/email/parse.rs | 6 +- crates/jmap/src/email/set.rs | 8 +- crates/jmap/src/lib.rs | 24 +- crates/jmap/src/submission/set.rs | 44 +- crates/store/src/backend/sqlite/write.rs | 2 +- tests/src/jmap/mod.rs | 5 +- 17 files changed, 965 insertions(+), 550 deletions(-) create mode 100644 crates/jmap/src/email/metadata.rs diff --git a/Cargo.lock b/Cargo.lock index 80e61090..41628638 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2482,6 +2482,7 @@ dependencies = [ "hyper 1.0.0-rc.4", "hyper-util", "jmap_proto", + "lz4_flex", "mail-builder", "mail-parser", "mail-send", @@ -2771,6 +2772,15 @@ dependencies = [ "libc", ] +[[package]] +name = "lz4_flex" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ea9b256699eda7b0387ffbc776dd625e28bde3918446381781245b7a50349d8" +dependencies = [ + "twox-hash", +] + [[package]] name = "mail-auth" version = "0.3.6" @@ -5757,6 +5767,16 @@ dependencies = [ "cipher 0.4.4", ] +[[package]] +name = "twox-hash" +version = "1.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" +dependencies = [ + "cfg-if", + "static_assertions", +] + [[package]] name = "typenum" version = "1.17.0" diff --git a/crates/imap/src/op/fetch.rs b/crates/imap/src/op/fetch.rs index 98d3a417..bdcb58a6 100644 --- a/crates/imap/src/op/fetch.rs +++ b/crates/imap/src/op/fetch.rs @@ -37,15 +37,15 @@ use imap_proto::{ receiver::Request, Command, ResponseCode, StatusResponse, }; +use jmap::{email::metadata::MessageMetadata, Bincode}; use jmap_proto::{ error::method::MethodError, - object::Object, types::{ acl::Acl, blob::BlobId, collection::Collection, id::Id, keyword::Keyword, - property::Property, state::StateChange, type_state::DataType, value::Value, + property::Property, state::StateChange, type_state::DataType, }, }; -use mail_parser::{Address, GetHeader, HeaderName, Message, MessageParser, PartType}; +use mail_parser::{Address, GetHeader, HeaderName, Message, PartType}; use store::{ query::log::{Change, Query}, write::{assert::HashedValue, BatchBuilder, F_BITMAP, F_VALUE}, @@ -268,7 +268,7 @@ impl SessionData { // Obtain attributes and keywords let (email, keywords) = if let (Ok(Some(email)), Ok(Some(keywords))) = ( self.jmap - .get_property::>( + .get_property::>( account_id, Collection::Email, id, @@ -284,7 +284,7 @@ impl SessionData { ) .await, ) { - (email, keywords) + (email.inner, keywords) } else { tracing::debug!( event = "not-found", @@ -317,23 +317,9 @@ impl SessionData { } else { None }; - - let message = if let Some(raw_message) = &raw_message { - if let Some(message) = MessageParser::new().parse(raw_message) { - message.into() - } else { - tracing::warn!( - event = "parse-error", - account_id = account_id, - collection = ?Collection::Email, - document_id = id, - blob_id = ?BlobId::maildir(account_id, id), - "Failed to parse stored message"); - continue; - } - } else { - None - }; + let message = email + .contents + .into_message(raw_message.as_deref().unwrap_or_default()); // Build response let mut items = Vec::with_capacity(arguments.attributes.len()); @@ -356,7 +342,7 @@ impl SessionData { match attribute { Attribute::Envelope => { items.push(DataItem::Envelope { - envelope: message.as_ref().unwrap().envelope(), + envelope: message.envelope(), }); } Attribute::Flags => { @@ -371,24 +357,21 @@ impl SessionData { items.push(DataItem::Flags { flags }); } Attribute::InternalDate => { - if let Some(date) = email.get(&Property::ReceivedAt).as_date() { - items.push(DataItem::InternalDate { - date: date.timestamp(), - }); - } + items.push(DataItem::InternalDate { + date: email.received_at as i64, + }); } Attribute::Preview { .. } => { items.push(DataItem::Preview { - contents: email - .get(&Property::Preview) - .as_string() - .map(|p| p.as_bytes().into()), + contents: if !email.preview.is_empty() { + Some(email.preview.as_bytes().into()) + } else { + None + }, }); } Attribute::Rfc822Size => { - items.push(DataItem::Rfc822Size { - size: email.get(&Property::Size).as_uint().unwrap_or(0) as usize, - }); + items.push(DataItem::Rfc822Size { size: email.size }); } Attribute::Uid => { items.push(DataItem::Uid { uid }); @@ -399,7 +382,7 @@ impl SessionData { }); } Attribute::Rfc822Header => { - let message = message.as_ref().unwrap().root_part(); + let message = message.root_part(); if let Some(header) = raw_message .as_ref() .unwrap() @@ -411,7 +394,7 @@ impl SessionData { } } Attribute::Rfc822Text => { - let message = message.as_ref().unwrap().root_part(); + let message = message.root_part(); if let Some(text) = raw_message .as_ref() .unwrap() @@ -424,20 +407,18 @@ impl SessionData { } Attribute::Body => { items.push(DataItem::Body { - part: message.as_ref().unwrap().body_structure(false), + part: message.body_structure(false), }); } Attribute::BodyStructure => { items.push(DataItem::BodyStructure { - part: message.as_ref().unwrap().body_structure(true), + part: message.body_structure(true), }); } Attribute::BodySection { sections, partial, .. } => { - if let Some(contents) = - message.as_ref().unwrap().body_section(sections, *partial) - { + if let Some(contents) = message.body_section(sections, *partial) { items.push(DataItem::BodySection { sections: sections.to_vec(), origin_octet: partial.map(|(start, _)| start), @@ -448,7 +429,7 @@ impl SessionData { Attribute::Binary { sections, partial, .. - } => match message.as_ref().unwrap().binary(sections, *partial) { + } => match message.binary(sections, *partial) { Ok(Some(contents)) => { items.push(DataItem::Binary { sections: sections.to_vec(), @@ -476,7 +457,7 @@ impl SessionData { _ => (), }, Attribute::BinarySize { sections } => { - if let Some(size) = message.as_ref().unwrap().binary_size(sections) { + if let Some(size) = message.binary_size(sections) { items.push(DataItem::BinarySize { sections: sections.to_vec(), size, diff --git a/crates/jmap/Cargo.toml b/crates/jmap/Cargo.toml index dd118584..adb88759 100644 --- a/crates/jmap/Cargo.toml +++ b/crates/jmap/Cargo.toml @@ -51,6 +51,7 @@ rasn-cms = "0.10" rasn-pkix = "0.10" rsa = "0.9.2" async-trait = "0.1.68" +lz4_flex = { version = "0.11" } [dev-dependencies] ece = "2.2" diff --git a/crates/jmap/src/email/body.rs b/crates/jmap/src/email/body.rs index cbbfff91..c4dfd306 100644 --- a/crates/jmap/src/email/body.rs +++ b/crates/jmap/src/email/body.rs @@ -27,7 +27,10 @@ use jmap_proto::{ }; use mail_parser::{HeaderValue, MessagePart, MimeHeaders, PartType}; -use super::headers::HeaderToValue; +use super::{ + headers::HeaderToValue, + metadata::{MessageMetadataContents, MetadataPartType}, +}; pub trait ToBodyPart { fn to_body_part( @@ -117,8 +120,114 @@ impl ToBodyPart for Vec> { _ => Value::Null, }, Property::Location => part.content_location().into(), - Property::Header(_) => part.header_to_value(property, raw_message), - Property::Headers => part.headers_to_value(raw_message), + Property::Header(_) => part.headers.header_to_value(property, raw_message), + Property::Headers => part.headers.headers_to_value(raw_message), + Property::SubParts => continue, + _ => Value::Null, + }; + values.append(property.clone(), value); + } + + subparts.push(values); + + if let Some(multipart) = multipart { + let multipart = multipart.clone(); + parts_stack.push(( + parts, + std::mem::replace(&mut subparts, Vec::with_capacity(multipart.len())), + )); + parts = multipart.into_iter(); + } + } else if let Some((prev_parts, mut prev_subparts)) = parts_stack.pop() { + prev_subparts + .last_mut() + .unwrap() + .append(Property::SubParts, subparts); + parts = prev_parts; + subparts = prev_subparts; + } else { + return subparts.pop().map(Into::into).unwrap_or_default(); + } + } + } +} + +impl ToBodyPart for MessageMetadataContents<'_> { + fn to_body_part( + &self, + part_id: usize, + properties: &[Property], + raw_message: &[u8], + blob_id: &BlobId, + ) -> Value { + let mut parts = vec![part_id].into_iter(); + let mut parts_stack = Vec::new(); + let mut subparts = Vec::with_capacity(1); + + loop { + if let Some((part_id, part)) = + parts.next().map(|part_id| (part_id, &self.parts[part_id])) + { + let mut values = Object::with_capacity(properties.len()); + let multipart = if let MetadataPartType::Multipart(parts) = &part.body { + parts.into() + } else { + None + }; + + for property in properties { + let value = match property { + Property::PartId if multipart.is_none() => part_id.to_string().into(), + Property::BlobId if multipart.is_none() => { + let base_offset = blob_id.start_offset(); + BlobId::new_section( + blob_id.kind, + part.offset_body + base_offset, + part.offset_end + base_offset, + part.encoding as u8, + ) + .into() + } + Property::Size if multipart.is_none() => part.size.into(), + Property::Name => part.attachment_name().into(), + Property::Type => part + .content_type() + .map(|ct| { + ct.subtype() + .map(|st| format!("{}/{}", ct.ctype(), st)) + .unwrap_or_else(|| ct.ctype().to_string()) + }) + .or_else(|| match &part.body { + MetadataPartType::Text => Some("text/plain".to_string()), + MetadataPartType::Html => Some("text/html".to_string()), + MetadataPartType::Message(_) => Some("message/rfc822".to_string()), + _ => None, + }) + .into(), + Property::Charset => part + .content_type() + .and_then(|ct| ct.attribute("charset")) + .or(match &part.body { + MetadataPartType::Text | MetadataPartType::Html => Some("us-ascii"), + _ => None, + }) + .into(), + Property::Disposition => { + part.content_disposition().map(|cd| cd.ctype()).into() + } + Property::Cid => part.content_id().into(), + Property::Language => match part.content_language() { + HeaderValue::Text(text) => vec![text.to_string()].into(), + HeaderValue::TextList(list) => list + .iter() + .map(|text| text.to_string().into()) + .collect::>() + .into(), + _ => Value::Null, + }, + Property::Location => part.content_location().into(), + Property::Header(_) => part.headers.header_to_value(property, raw_message), + Property::Headers => part.headers.headers_to_value(raw_message), Property::SubParts => continue, _ => Value::Null, }; diff --git a/crates/jmap/src/email/copy.rs b/crates/jmap/src/email/copy.rs index 65332801..efca2338 100644 --- a/crates/jmap/src/email/copy.rs +++ b/crates/jmap/src/email/copy.rs @@ -27,7 +27,6 @@ use jmap_proto::{ copy::{CopyRequest, CopyResponse, RequestArguments}, set::{self, SetRequest}, }, - object::Object, request::{ method::{MethodFunction, MethodName, MethodObject}, reference::MaybeReference, @@ -47,7 +46,7 @@ use jmap_proto::{ value::{MaybePatchValue, Value}, }, }; -use mail_parser::parsers::fields::thread::thread_name; +use mail_parser::{parsers::fields::thread::thread_name, HeaderName, HeaderValue}; use store::{ fts::term_index::TokenIndex, query::RawValue, @@ -56,11 +55,12 @@ use store::{ }; use utils::map::vec_map::VecMap; -use crate::{auth::AccessToken, JMAP}; +use crate::{auth::AccessToken, Bincode, JMAP}; use super::{ index::{EmailIndexBuilder, TrimTextValue, MAX_SORT_FIELD_LENGTH}, ingest::IngestedEmail, + metadata::MessageMetadata, }; impl JMAP { @@ -295,7 +295,7 @@ impl JMAP { ) -> Result, MethodError> { // Obtain term index and metadata let (mut metadata, token_index) = if let (Some(metadata), Some(token_index)) = ( - self.get_property::>( + self.get_property::>( from_account_id, Collection::Email, from_message_id, @@ -309,7 +309,7 @@ impl JMAP { ) .await?, ) { - (metadata, token_index) + (metadata.inner, token_index) } else { return Ok(Err(SetError::not_found().with_description(format!( "Message not found not found in account {}.", @@ -319,37 +319,35 @@ impl JMAP { // Check quota if account_quota > 0 - && metadata.get(&Property::Size).as_uint().unwrap_or_default() as i64 - + self.get_used_quota(account_id).await? - > account_quota + && metadata.size as i64 + self.get_used_quota(account_id).await? > account_quota { return Ok(Err(SetError::over_quota())); } // Set receivedAt if let Some(received_at) = received_at { - metadata.set(Property::ReceivedAt, Value::Date(received_at)); + metadata.received_at = received_at.timestamp() as u64; } // Obtain threadId let mut references = vec![]; let mut subject = ""; - for (property, value) in &metadata.properties { - match property { - Property::MessageId - | Property::InReplyTo - | Property::References - | Property::EmailIds => match value { - Value::Text(text) => { - references.push(text.as_str()); + for header in &metadata.contents.parts[0].headers { + match header.name { + HeaderName::MessageId + | HeaderName::InReplyTo + | HeaderName::References + | HeaderName::ResentMessageId => match &header.value { + HeaderValue::Text(text) => { + references.push(text.as_ref()); } - Value::List(list) => { - references.extend(list.iter().filter_map(|v| v.as_string())); + HeaderValue::TextList(list) => { + references.extend(list.iter().map(|v| v.as_ref())); } _ => (), }, - Property::Subject => { - if let Some(value) = value.as_string() { + HeaderName::Subject => { + if let HeaderValue::Text(value) = &header.value { subject = thread_name(value).trim_text(MAX_SORT_FIELD_LENGTH); } } @@ -373,7 +371,7 @@ impl JMAP { account_id, document_id: message_id, }), - size: metadata.get(&Property::Size).as_uint().unwrap_or(0) as usize, + size: metadata.size, ..Default::default() }; self.store diff --git a/crates/jmap/src/email/get.rs b/crates/jmap/src/email/get.rs index 40edb02b..4daa60bf 100644 --- a/crates/jmap/src/email/get.rs +++ b/crates/jmap/src/email/get.rs @@ -26,15 +26,25 @@ use jmap_proto::{ method::get::{GetRequest, GetResponse}, object::{email::GetArguments, Object}, types::{ - acl::Acl, blob::BlobId, collection::Collection, id::Id, keyword::Keyword, - property::Property, value::Value, + acl::Acl, + blob::BlobId, + collection::Collection, + date::UTCDate, + id::Id, + keyword::Keyword, + property::{HeaderForm, Property}, + value::Value, }, }; -use mail_parser::MessageParser; +use mail_parser::HeaderName; -use crate::{auth::AccessToken, email::headers::HeaderToValue, JMAP}; +use crate::{auth::AccessToken, email::headers::HeaderToValue, Bincode, JMAP}; -use super::body::{ToBodyPart, TruncateBody}; +use super::{ + body::{ToBodyPart, TruncateBody}, + headers::IntoForm, + metadata::{MessageMetadata, MetadataPartType}, +}; impl JMAP { pub async fn email_get( @@ -147,8 +157,8 @@ impl JMAP { response.not_found.push(id.into()); continue; } - let mut values = match self - .get_property::>( + let mut metadata = match self + .get_property::>( account_id, Collection::Email, id.document_id(), @@ -156,7 +166,7 @@ impl JMAP { ) .await? { - Some(values) => values, + Some(metadata) => metadata.inner, None => { response.not_found.push(id.into()); continue; @@ -191,21 +201,6 @@ impl JMAP { } else { vec![] }; - let message = if !raw_message.is_empty() { - let message = MessageParser::new().parse(&raw_message); - if message.is_none() { - tracing::warn!( - event = "parse-error", - account_id = account_id, - collection = ?Collection::Email, - document_id = id.document_id(), - blob_id = ?blob_id, - "Failed to parse stored message"); - } - message - } else { - None - }; // Prepare response let mut email = Object::with_capacity(properties.len()); @@ -276,101 +271,154 @@ impl JMAP { continue 'outer; } } - Property::Size - | Property::ReceivedAt - | Property::MessageId - | Property::InReplyTo - | Property::References - | Property::Sender + Property::Size => { + email.append(Property::Size, metadata.size); + } + Property::ReceivedAt => { + email.append( + Property::ReceivedAt, + Value::Date(UTCDate::from_timestamp(metadata.received_at as i64)), + ); + } + Property::Preview => { + if !metadata.preview.is_empty() { + email.append(Property::Preview, std::mem::take(&mut metadata.preview)); + } + } + Property::HasAttachment => { + email.append(Property::HasAttachment, metadata.has_attachments); + } + Property::Subject => { + email.append( + Property::Subject, + metadata.contents.parts[0] + .remove_header(&HeaderName::Subject) + .map(|value| value.into_form(&HeaderForm::Text)) + .unwrap_or_default(), + ); + } + Property::SentAt => { + email.append( + Property::SentAt, + metadata.contents.parts[0] + .remove_header(&HeaderName::Date) + .map(|value| value.into_form(&HeaderForm::Date)) + .unwrap_or_default(), + ); + } + Property::MessageId | Property::InReplyTo | Property::References => { + email.append( + property.clone(), + metadata.contents.parts[0] + .remove_header(&match property { + Property::MessageId => HeaderName::MessageId, + Property::InReplyTo => HeaderName::InReplyTo, + Property::References => HeaderName::References, + _ => unreachable!(), + }) + .map(|value| value.into_form(&HeaderForm::MessageIds)) + .unwrap_or_default(), + ); + } + + Property::Sender | Property::From | Property::To | Property::Cc | Property::Bcc - | Property::ReplyTo - | Property::Subject - | Property::SentAt - | Property::HasAttachment - | Property::Preview => { - email.append(property.clone(), values.remove(property)); + | Property::ReplyTo => { + email.append( + property.clone(), + metadata.contents.parts[0] + .remove_header(&match property { + Property::Sender => HeaderName::Sender, + Property::From => HeaderName::From, + Property::To => HeaderName::To, + Property::Cc => HeaderName::Cc, + Property::Bcc => HeaderName::Bcc, + Property::ReplyTo => HeaderName::ReplyTo, + _ => unreachable!(), + }) + .map(|value| value.into_form(&HeaderForm::Addresses)) + .unwrap_or_default(), + ); } Property::Header(_) => { - if let Some(message) = &message { - email.append( - property.clone(), - message.parts[0].header_to_value(property, &raw_message), - ); - } + email.append( + property.clone(), + metadata.contents.parts[0] + .headers + .header_to_value(property, &raw_message), + ); } Property::Headers => { - if let Some(message) = &message { - email.append( - Property::Headers, - message.parts[0].headers_to_value(&raw_message), - ); - } + email.append( + Property::Headers, + metadata.contents.parts[0] + .headers + .headers_to_value(&raw_message), + ); } Property::TextBody | Property::HtmlBody | Property::Attachments => { - if let Some(message) = &message { - let list = match property { - Property::TextBody => &message.text_body, - Property::HtmlBody => &message.html_body, - Property::Attachments => &message.attachments, - _ => unreachable!(), - } - .iter(); - email.append( - property.clone(), - list.map(|part_id| { - message.parts.to_body_part( - *part_id, - &body_properties, - &raw_message, - &blob_id, - ) - }) - .collect::>(), - ); + let list = match property { + Property::TextBody => &metadata.contents.text_body, + Property::HtmlBody => &metadata.contents.html_body, + Property::Attachments => &metadata.contents.attachments, + _ => unreachable!(), } - } - Property::BodyStructure => { - if let Some(message) = &message { - email.append( - Property::BodyStructure, - message.parts.to_body_part( - 0, + .iter(); + email.append( + property.clone(), + list.map(|part_id| { + metadata.contents.to_body_part( + *part_id, &body_properties, &raw_message, &blob_id, - ), - ); - } + ) + }) + .collect::>(), + ); + } + Property::BodyStructure => { + email.append( + Property::BodyStructure, + metadata.contents.to_body_part( + 0, + &body_properties, + &raw_message, + &blob_id, + ), + ); } Property::BodyValues => { - if let Some(message) = &message { - let mut body_values = Object::with_capacity(message.parts.len()); - for (part_id, part) in message.parts.iter().enumerate() { - if ((message.html_body.contains(&part_id) - && (fetch_all_body_values || fetch_html_body_values)) - || (message.text_body.contains(&part_id) - && (fetch_all_body_values || fetch_text_body_values))) - && part.is_text() - { - let (is_truncated, value) = - part.body.truncate(max_body_value_bytes); - body_values.append( - Property::_T(part_id.to_string()), - Object::with_capacity(3) - .with_property( - Property::IsEncodingProblem, - part.is_encoding_problem, - ) - .with_property(Property::IsTruncated, is_truncated) - .with_property(Property::Value, value), - ); - } + let mut body_values = Object::with_capacity(metadata.contents.parts.len()); + for (part_id, part) in metadata.contents.parts.iter().enumerate() { + if ((metadata.contents.html_body.contains(&part_id) + && (fetch_all_body_values || fetch_html_body_values)) + || (metadata.contents.text_body.contains(&part_id) + && (fetch_all_body_values || fetch_text_body_values))) + && matches!( + part.body, + MetadataPartType::Text | MetadataPartType::Html + ) + { + let (is_truncated, value) = part + .decode_contents(&raw_message) + .truncate(max_body_value_bytes); + body_values.append( + Property::_T(part_id.to_string()), + Object::with_capacity(3) + .with_property( + Property::IsEncodingProblem, + part.is_encoding_problem, + ) + .with_property(Property::IsTruncated, is_truncated) + .with_property(Property::Value, value), + ); } - email.append(Property::BodyValues, body_values); } + email.append(Property::BodyValues, body_values); } _ => { diff --git a/crates/jmap/src/email/headers.rs b/crates/jmap/src/email/headers.rs index 23de70d1..095656b2 100644 --- a/crates/jmap/src/email/headers.rs +++ b/crates/jmap/src/email/headers.rs @@ -41,7 +41,7 @@ use mail_builder::{ }, MessageBuilder, }; -use mail_parser::{parsers::MessageStream, Addr, HeaderName, HeaderValue, MessagePart}; +use mail_parser::{parsers::MessageStream, Addr, Header, HeaderName, HeaderValue}; pub trait IntoForm { fn into_form(self, form: &HeaderForm) -> Value; @@ -62,7 +62,7 @@ pub trait BuildHeader: Sized { fn build_header(self, header: HeaderProperty, value: Value) -> Result; } -impl HeaderToValue for MessagePart<'_> { +impl HeaderToValue for Vec> { fn header_to_value(&self, property: &Property, raw_message: &[u8]) -> Value { let (header_name, form, all) = match property { Property::Header(header) => ( @@ -85,44 +85,31 @@ impl HeaderToValue for MessagePart<'_> { _ => return Value::Null, }; + let is_raw = matches!(form, HeaderForm::Raw) || matches!(header_name, HeaderName::Other(_)); let mut headers = Vec::new(); - - match (&header_name, &form) { - (HeaderName::Other(_), _) | (_, HeaderForm::Raw) => { - let header_name = header_name.as_str(); - for header in self.headers().iter().rev() { - if header.name.as_str().eq_ignore_ascii_case(header_name) { - let header_value = raw_message - .get(header.offset_start..header.offset_end) - .map_or(HeaderValue::Empty, |bytes| match form { - HeaderForm::Raw => { - HeaderValue::Text(String::from_utf8_lossy(bytes.trim_end())) - } - HeaderForm::Text => MessageStream::new(bytes).parse_unstructured(), - HeaderForm::Addresses => MessageStream::new(bytes).parse_address(), - HeaderForm::GroupedAddresses => { - MessageStream::new(bytes).parse_address() - } - HeaderForm::MessageIds => MessageStream::new(bytes).parse_id(), - HeaderForm::Date => MessageStream::new(bytes).parse_date(), - HeaderForm::URLs => MessageStream::new(bytes).parse_address(), - }); - headers.push(header_value.into_form(&form)); - if !all { - break; - } - } - } - } - (header_name, _) => { - let header_name = header_name.as_str(); - for header in self.headers().iter().rev() { - if header.name.as_str().eq_ignore_ascii_case(header_name) { - headers.push(header.value.clone().into_form(&form)); - if !all { - break; - } - } + let header_name = header_name.as_str(); + for header in self.iter().rev() { + if header.name.as_str().eq_ignore_ascii_case(header_name) { + let header_value = if is_raw || matches!(header.value, HeaderValue::Empty) { + raw_message + .get(header.offset_start..header.offset_end) + .map_or(HeaderValue::Empty, |bytes| match form { + HeaderForm::Raw => { + HeaderValue::Text(String::from_utf8_lossy(bytes.trim_end())) + } + HeaderForm::Text => MessageStream::new(bytes).parse_unstructured(), + HeaderForm::Addresses + | HeaderForm::GroupedAddresses + | HeaderForm::URLs => MessageStream::new(bytes).parse_address(), + HeaderForm::MessageIds => MessageStream::new(bytes).parse_id(), + HeaderForm::Date => MessageStream::new(bytes).parse_date(), + }) + } else { + header.value.clone() + }; + headers.push(header_value.into_form(&form)); + if !all { + break; } } } @@ -138,8 +125,8 @@ impl HeaderToValue for MessagePart<'_> { } fn headers_to_value(&self, raw_message: &[u8]) -> Value { - let mut headers = Vec::with_capacity(self.headers.len()); - for header in self.headers() { + let mut headers = Vec::with_capacity(self.len()); + for header in self.iter() { headers.push(Value::Object( Object::with_capacity(2) .with_property(Property::Name, header.name().to_string()) diff --git a/crates/jmap/src/email/index.rs b/crates/jmap/src/email/index.rs index fcb5f826..6e38c953 100644 --- a/crates/jmap/src/email/index.rs +++ b/crates/jmap/src/email/index.rs @@ -23,19 +23,12 @@ use std::borrow::Cow; -use jmap_proto::{ - object::Object, - types::{ - date::UTCDate, - keyword::Keyword, - property::{HeaderForm, Property}, - value::Value, - }, -}; +use jmap_proto::types::{keyword::Keyword, property::Property}; use mail_parser::{ decoders::html::html_to_text, parsers::{fields::thread::thread_name, preview::preview_text}, - Addr, Address, GetHeader, Group, HeaderName, HeaderValue, Message, MessagePart, PartType, + Addr, Address, GetHeader, Group, Header, HeaderName, HeaderValue, Message, MessagePart, + PartType, }; use nlp::language::Language; use store::{ @@ -43,7 +36,9 @@ use store::{ write::{BatchBuilder, IntoOperations, F_BITMAP, F_CLEAR, F_INDEX, F_VALUE}, }; -use crate::email::headers::IntoForm; +use crate::Bincode; + +use super::metadata::MessageMetadata; pub const MAX_MESSAGE_PARTS: usize = 1000; pub const MAX_ID_LENGTH: usize = 100; @@ -64,8 +59,13 @@ pub(super) trait IndexMessage { keywords: Vec, mailbox_ids: Vec, received_at: u64, - default_language: Language, ) -> store::Result<&mut Self>; + + fn index_headers(&mut self, headers: &[Header<'_>], options: u32); +} + +pub(super) trait IndexMessageText<'x> { + fn index_message(&mut self, message: &'x Message<'x>); } impl IndexMessage for BatchBuilder { @@ -75,10 +75,7 @@ impl IndexMessage for BatchBuilder { keywords: Vec, mailbox_ids: Vec, received_at: u64, - default_language: Language, ) -> store::Result<&mut Self> { - let mut metadata = Object::with_capacity(15); - // Index keywords self.value(Property::Keywords, keywords, F_VALUE | F_BITMAP); @@ -86,21 +83,14 @@ impl IndexMessage for BatchBuilder { self.value(Property::MailboxIds, mailbox_ids, F_VALUE | F_BITMAP); // Index size - metadata.append(Property::Size, message.raw_message.len()); self.value(Property::Size, message.raw_message.len() as u32, F_INDEX) .quota(message.raw_message.len() as i64); // Index receivedAt - metadata.append( - Property::ReceivedAt, - Value::Date(UTCDate::from_timestamp(received_at as i64)), - ); self.value(Property::ReceivedAt, received_at, F_INDEX); - let mut fts = FtsIndexBuilder::with_default_language(default_language); - let mut seen_headers = [false; 40]; - let mut language = Language::Unknown; let mut has_attachments = false; + let mut preview = None; let preview_part_id = message .text_body .first() @@ -108,23 +98,181 @@ impl IndexMessage for BatchBuilder { .copied() .unwrap_or(usize::MAX); - for (part_id, part) in message - .parts - .into_iter() - .take(MAX_MESSAGE_PARTS) - .enumerate() - { + for (part_id, part) in message.parts.iter().take(MAX_MESSAGE_PARTS).enumerate() { + if part_id == 0 { + self.index_headers(&part.headers, 0); + } + + match &part.body { + PartType::Text(text) => { + if part_id == preview_part_id { + preview = + preview_text(text.replace('\r', "").into(), PREVIEW_LENGTH).into(); + } + + if !message.text_body.contains(&part_id) + && !message.html_body.contains(&part_id) + { + has_attachments = true; + } + } + PartType::Html(html) => { + let text = html_to_text(html); + if part_id == preview_part_id { + preview = + preview_text(text.replace('\r', "").into(), PREVIEW_LENGTH).into(); + } + + if !message.text_body.contains(&part_id) + && !message.html_body.contains(&part_id) + { + has_attachments = true; + } + } + PartType::Binary(_) | PartType::Message(_) if !has_attachments => { + has_attachments = true; + } + _ => {} + } + } + + // Store and index hasAttachment property + if has_attachments { + self.bitmap(Property::HasAttachment, (), 0); + } + + // FTS index + let mut fts = FtsIndexBuilder::with_default_language(Language::English); + fts.index_message(&message); + self.custom(fts); + + // Store message metadata + self.value( + Property::BodyStructure, + Bincode::new(MessageMetadata { + preview: preview.unwrap_or_default().into_owned(), + size: message.raw_message.len(), + contents: message.into(), + received_at, + has_attachments, + }), + F_VALUE, + ); + + Ok(self) + } + + fn index_headers(&mut self, headers: &[Header<'_>], options: u32) { + let mut seen_headers = [false; 40]; + for header in headers.iter().rev() { + if matches!(header.name, HeaderName::Other(_)) { + continue; + } + + match header.name { + HeaderName::MessageId + | HeaderName::InReplyTo + | HeaderName::References + | HeaderName::ResentMessageId => { + header.value.visit_text(|id| { + // Add ids to inverted index + if id.len() < MAX_ID_LENGTH { + self.value(Property::MessageId, id, F_INDEX | options); + } + }); + } + HeaderName::From | HeaderName::To | HeaderName::Cc | HeaderName::Bcc => { + if !seen_headers[header.name.id() as usize] { + let property = Property::from_header(&header.name); + let mut sort_text = SortedAddressBuilder::new(); + let mut found_addr = false; + + header.value.visit_addresses(|element, value| { + if !found_addr { + match element { + AddressElement::Name => { + found_addr = !sort_text.push(value); + } + AddressElement::Address => { + sort_text.push(value); + found_addr = true; + } + AddressElement::GroupName => (), + } + } + }); + + // Add address to inverted index + self.value(u8::from(&property), sort_text.build(), F_INDEX | options); + seen_headers[header.name.id() as usize] = true; + } + } + HeaderName::Date => { + if !seen_headers[header.name.id() as usize] { + if let HeaderValue::DateTime(datetime) = &header.value { + self.value( + Property::SentAt, + datetime.to_timestamp() as u64, + F_INDEX | options, + ); + } + seen_headers[header.name.id() as usize] = true; + } + } + HeaderName::Subject => { + if !seen_headers[header.name.id() as usize] { + // Index subject + let subject = match &header.value { + HeaderValue::Text(text) => text.clone(), + HeaderValue::TextList(list) if !list.is_empty() => { + list.first().unwrap().clone() + } + _ => "".into(), + }; + + // Index thread name + let thread_name = thread_name(&subject); + self.value( + Property::Subject, + if !thread_name.is_empty() { + thread_name.trim_text(MAX_SORT_FIELD_LENGTH) + } else { + "!" + }, + F_INDEX | options, + ); + + seen_headers[header.name.id() as usize] = true; + } + } + + _ => (), + } + } + + // Add subject to index if missing + if !seen_headers[HeaderName::Subject.id() as usize] { + self.value(Property::Subject, "!", F_INDEX | options); + } + } +} + +impl<'x> IndexMessageText<'x> for FtsIndexBuilder<'x> { + fn index_message(&mut self, message: &'x Message<'x>) { + let mut language = Language::Unknown; + + for (part_id, part) in message.parts.iter().take(MAX_MESSAGE_PARTS).enumerate() { let part_language = part.language().unwrap_or(language); if part_id == 0 { language = part_language; - let mut extra_ids = Vec::new(); - for header in part.headers.into_iter().rev() { + + for header in part.headers.iter().rev() { if matches!(header.name, HeaderName::Other(_)) { continue; } // Index hasHeader property let header_num = header.name.id().to_string(); - fts.index_raw_token(Property::Headers, &header_num); + self.index_raw_token(Property::Headers, &header_num); match header.name { HeaderName::MessageId @@ -132,156 +280,43 @@ impl IndexMessage for BatchBuilder { | HeaderName::References | HeaderName::ResentMessageId => { header.value.visit_text(|id| { - // Add ids to inverted index - if id.len() < MAX_ID_LENGTH { - self.value(Property::MessageId, id, F_INDEX); - } - // Index ids without stemming if id.len() < MAX_TOKEN_LENGTH { - fts.index_raw_token( + self.index_raw_token( Property::Headers, format!("{header_num}{id}"), ); } }); - - if matches!( - header.name, - HeaderName::MessageId - | HeaderName::InReplyTo - | HeaderName::References - ) && !seen_headers[header.name.id() as usize] - { - metadata.append( - Property::from_header(&header.name), - header - .value - .trim_text(MAX_STORED_FIELD_LENGTH) - .into_form(&HeaderForm::MessageIds), - ); - seen_headers[header.name.id() as usize] = true; - } else { - header.value.into_visit_text(|id| { - extra_ids.push(Value::Text(id)); - }); - } } - HeaderName::From - | HeaderName::To - | HeaderName::Cc - | HeaderName::Bcc - | HeaderName::ReplyTo - | HeaderName::Sender => { + HeaderName::From | HeaderName::To | HeaderName::Cc | HeaderName::Bcc => { let property = Property::from_header(&header.name); - let seen_header = seen_headers[header.name.id() as usize]; - if matches!( - header.name, - HeaderName::From - | HeaderName::To - | HeaderName::Cc - | HeaderName::Bcc - ) { - let mut sort_text = SortedAddressBuilder::new(); - let mut found_addr = seen_header; - header.value.visit_addresses(|element, value| { - if !found_addr { - match element { - AddressElement::Name => { - found_addr = !sort_text.push(value); - } - AddressElement::Address => { - sort_text.push(value); - found_addr = true; - } - AddressElement::GroupName => (), - } - } - - // Index an address name or email without stemming - fts.index_raw(u8::from(&property), value); - }); - - if !seen_header { - // Add address to inverted index - self.value(u8::from(&property), sort_text.build(), F_INDEX); - } - } - - if !seen_header { - // Add address to metadata - metadata.append( - property, - header - .value - .trim_text(MAX_STORED_FIELD_LENGTH) - .into_form(&HeaderForm::Addresses), - ); - seen_headers[header.name.id() as usize] = true; - } - } - HeaderName::Date => { - if !seen_headers[header.name.id() as usize] { - if let HeaderValue::DateTime(datetime) = &header.value { - self.value( - Property::SentAt, - datetime.to_timestamp() as u64, - F_INDEX, - ); - } - metadata.append( - Property::SentAt, - header.value.into_form(&HeaderForm::Date), - ); - seen_headers[header.name.id() as usize] = true; - } + header.value.visit_addresses(|_, value| { + // Index an address name or email without stemming + self.index_raw(u8::from(&property), value); + }); } HeaderName::Subject => { - // Index subject - let subject = match &header.value { - HeaderValue::Text(text) => text.clone(), - HeaderValue::TextList(list) if !list.is_empty() => { - list.first().unwrap().clone() - } - _ => "".into(), - }; - - if !seen_headers[header.name.id() as usize] { - // Add to metadata - metadata.append( - Property::Subject, - header - .value - .trim_text(MAX_STORED_FIELD_LENGTH) - .into_form(&HeaderForm::Text), - ); - - // Index thread name - let thread_name = thread_name(&subject); - self.value( - Property::Subject, - if !thread_name.is_empty() { - thread_name.trim_text(MAX_SORT_FIELD_LENGTH) - } else { - "!" - }, - F_INDEX, - ); - - seen_headers[header.name.id() as usize] = true; - } - // Index subject for FTS - fts.index(Property::Subject, subject, language); + self.index( + Property::Subject, + match &header.value { + HeaderValue::Text(text) => text.clone(), + HeaderValue::TextList(list) if !list.is_empty() => { + list.first().unwrap().clone() + } + _ => "".into(), + }, + language, + ); } - HeaderName::Comments | HeaderName::Keywords | HeaderName::ListId => { // Index headers header.value.visit_text(|text| { for token in text.split_ascii_whitespace() { if token.len() < MAX_TOKEN_LENGTH { - fts.index_raw_token( + self.index_raw_token( Property::Headers, format!("{header_num}{}", token.to_lowercase()), ); @@ -292,123 +327,83 @@ impl IndexMessage for BatchBuilder { _ => (), } } - - // Add any extra Ids to metadata - if !extra_ids.is_empty() { - metadata.append(Property::EmailIds, Value::List(extra_ids)); - } } - // Add subject to index if missing - if !seen_headers[HeaderName::Subject.id() as usize] { - self.value(Property::Subject, "!", F_INDEX); - } - - match part.body { + match &part.body { PartType::Text(text) => { - if part_id == preview_part_id { - metadata.append( - Property::Preview, - preview_text(text.replace('\r', "").into(), PREVIEW_LENGTH), - ); - } - if message.text_body.contains(&part_id) || message.html_body.contains(&part_id) { - fts.index(Property::TextBody, text, part_language); + self.index(Property::TextBody, text.as_ref(), part_language); } else { - fts.index(Property::Attachments, text, part_language); - has_attachments = true; + self.index(Property::Attachments, text.as_ref(), part_language); } } PartType::Html(html) => { - let text = html_to_text(&html); - if part_id == preview_part_id { - metadata.append( - Property::Preview, - preview_text(text.replace('\r', "").into(), PREVIEW_LENGTH), - ); - } + let text = html_to_text(html); if message.text_body.contains(&part_id) || message.html_body.contains(&part_id) { - fts.index(Property::TextBody, text, part_language); + self.index(Property::TextBody, text, part_language); } else { - fts.index(Property::Attachments, text, part_language); - has_attachments = true; + self.index(Property::Attachments, text, part_language); } } - PartType::Binary(_) if !has_attachments => { - has_attachments = true; - } - PartType::Message(mut nested_message) => { + PartType::Message(nested_message) => { let nested_message_language = nested_message .root_part() .language() .unwrap_or(Language::Unknown); if let Some(HeaderValue::Text(subject)) = - nested_message.remove_header(HeaderName::Subject) + nested_message.header(HeaderName::Subject) { - fts.index( + self.index( Property::Attachments, - subject.into_owned(), + subject.as_ref(), nested_message_language, ); } - for sub_part in nested_message.parts.into_iter().take(MAX_MESSAGE_PARTS) { + for sub_part in nested_message.parts.iter().take(MAX_MESSAGE_PARTS) { let language = sub_part.language().unwrap_or(nested_message_language); - match sub_part.body { + match &sub_part.body { PartType::Text(text) => { - fts.index(Property::Attachments, text, language); + self.index(Property::Attachments, text.as_ref(), language); } PartType::Html(html) => { - fts.index(Property::Attachments, html_to_text(&html), language); + self.index(Property::Attachments, html_to_text(html), language); } _ => (), } } - - if !has_attachments { - has_attachments = true; - } } _ => {} } } - - // Store and index hasAttachment property - metadata.append(Property::HasAttachment, has_attachments); - if has_attachments { - self.bitmap(Property::HasAttachment, (), 0); - } - - // Store properties - self.value(Property::BodyStructure, metadata, F_VALUE); - - // Store full text index - self.custom(fts); - - Ok(self) } } -pub struct EmailIndexBuilder { - inner: Object, +pub struct EmailIndexBuilder<'x> { + inner: Bincode>, set: bool, } -impl EmailIndexBuilder { - pub fn set(inner: Object) -> Self { - Self { inner, set: true } +impl<'x> EmailIndexBuilder<'x> { + pub fn set(inner: MessageMetadata<'x>) -> Self { + Self { + inner: Bincode { inner }, + set: true, + } } - pub fn clear(inner: Object) -> Self { - Self { inner, set: false } + pub fn clear(inner: MessageMetadata<'x>) -> Self { + Self { + inner: Bincode { inner }, + set: false, + } } } -impl IntoOperations for EmailIndexBuilder { +impl<'x> IntoOperations for EmailIndexBuilder<'x> { fn build(self, batch: &mut BatchBuilder) { let options = if self.set { // Serialize metadata @@ -419,81 +414,25 @@ impl IntoOperations for EmailIndexBuilder { batch.value(Property::BodyStructure, (), F_VALUE | F_CLEAR); F_CLEAR }; + let metadata = &self.inner.inner; - // Remove properties from index - let mut has_subject = false; - for (property, value) in self.inner.properties { - match (&property, value) { - (Property::Size, Value::UnsignedInt(size)) => { - batch - .value(Property::Size, size as u32, F_INDEX | options) - .quota(if self.set { - size as i64 - } else { - -(size as i64) - }); - } - (Property::ReceivedAt | Property::SentAt, Value::Date(date)) => { - batch.value(property, date.timestamp() as u64, F_INDEX | options); - } - ( - Property::MessageId - | Property::InReplyTo - | Property::References - | Property::EmailIds, - Value::List(ids), - ) => { - // Remove messageIds from index - for id in ids { - match id { - Value::Text(id) if id.len() < MAX_ID_LENGTH => { - batch.value(Property::MessageId, id, F_INDEX | options); - } - _ => {} - } - } - } - ( - Property::From | Property::To | Property::Cc | Property::Bcc, - Value::List(addresses), - ) => { - let mut sort_text = SortedAddressBuilder::new(); - 'outer: for addr in addresses { - if let Some(addr) = addr.try_unwrap_object() { - for part in [Property::Name, Property::Email] { - if let Some(Value::Text(value)) = addr.properties.get(&part) { - if !sort_text.push(value) || part == Property::Email { - break 'outer; - } - } - } - } - } - batch.value(property, sort_text.build(), F_INDEX | options); - } - (Property::Subject, Value::Text(value)) => { - let thread_name = thread_name(&value); - batch.value( - Property::Subject, - if !thread_name.is_empty() { - thread_name.trim_text(MAX_SORT_FIELD_LENGTH) - } else { - "!" - }, - F_INDEX | options, - ); - has_subject = true; - } - (Property::HasAttachment, Value::Bool(true)) => { - batch.bitmap(Property::HasAttachment, (), options); - } - _ => {} - } - } - - if !has_subject { - batch.value(Property::Subject, "!", F_INDEX | options); + // Index properties + batch + .value(Property::Size, metadata.size as u32, F_INDEX | options) + .quota(if self.set { + metadata.size as i64 + } else { + -(metadata.size as i64) + }); + batch.value( + Property::ReceivedAt, + metadata.received_at, + F_INDEX | options, + ); + if metadata.has_attachments { + batch.bitmap(Property::HasAttachment, (), options); } + batch.index_headers(&metadata.contents.parts[0].headers, options); } } diff --git a/crates/jmap/src/email/ingest.rs b/crates/jmap/src/email/ingest.rs index 34add63a..0e44a5d8 100644 --- a/crates/jmap/src/email/ingest.rs +++ b/crates/jmap/src/email/ingest.rs @@ -303,7 +303,6 @@ impl JMAP { params.keywords, params.mailbox_ids, params.received_at.unwrap_or_else(now), - self.config.default_language, ) .map_err(|err| { tracing::error!( diff --git a/crates/jmap/src/email/metadata.rs b/crates/jmap/src/email/metadata.rs new file mode 100644 index 00000000..22f707af --- /dev/null +++ b/crates/jmap/src/email/metadata.rs @@ -0,0 +1,313 @@ +/* + * Copyright (c) 2023 Stalwart Labs Ltd. + * + * This file is part of Stalwart Mail Server. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of + * the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * in the LICENSE file at the top-level directory of this distribution. + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + * You can be released from the requirements of the AGPLv3 license by + * purchasing a commercial license. Please contact licensing@stalw.art + * for more details. +*/ + +use std::borrow::Cow; + +use mail_parser::{ + decoders::{ + base64::base64_decode, charsets::map::charset_decoder, + quoted_printable::quoted_printable_decode, + }, + ContentType, Encoding, GetHeader, Header, HeaderName, HeaderValue, Message, MessagePart, + MessagePartId, MimeHeaders, PartType, +}; +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize)] +pub struct MessageMetadata<'x> { + pub contents: MessageMetadataContents<'x>, + pub size: usize, + pub received_at: u64, + pub preview: String, + pub has_attachments: bool, +} + +#[derive(Serialize, Deserialize)] +pub struct MessageMetadataContents<'x> { + pub html_body: Vec, + pub text_body: Vec, + pub attachments: Vec, + pub parts: Vec>, +} + +#[derive(Serialize, Deserialize)] +pub struct MessageMetadataPart<'x> { + pub headers: Vec>, + pub is_encoding_problem: bool, + pub body: MetadataPartType<'x>, + pub encoding: Encoding, + pub size: usize, + pub offset_header: usize, + pub offset_body: usize, + pub offset_end: usize, +} + +#[derive(Serialize, Deserialize)] +pub enum MetadataPartType<'x> { + Text, + Html, + Binary, + InlineBinary, + Message(MessageMetadataContents<'x>), + Multipart(Vec), +} + +impl<'x> MessageMetadataContents<'x> { + pub fn into_message(self, raw_message: &'x [u8]) -> Message<'x> { + Message { + html_body: self.html_body, + text_body: self.text_body, + attachments: self.attachments, + parts: self + .parts + .into_iter() + .map(|part| MessagePart { + body: match part.body { + MetadataPartType::Text + | MetadataPartType::Html + | MetadataPartType::Binary + | MetadataPartType::InlineBinary + if !raw_message.is_empty() => + { + part.decode_contents(raw_message) + } + MetadataPartType::Message(_) if !raw_message.is_empty() => { + match part.contents(raw_message) { + Cow::Borrowed(_) => PartType::Message( + part.body.unwrap_message().into_message(raw_message), + ), + Cow::Owned(raw_message) => PartType::Message( + part.body + .unwrap_message() + .into_message(&raw_message) + .into_owned(), + ), + } + } + MetadataPartType::Multipart(parts) => PartType::Multipart(parts), + _ => PartType::Binary(Cow::Borrowed(&[])), + }, + headers: part.headers, + is_encoding_problem: part.is_encoding_problem, + encoding: part.encoding, + offset_header: part.offset_header, + offset_body: part.offset_body, + offset_end: part.offset_end, + }) + .collect(), + raw_message: raw_message.into(), + } + } + + pub fn root_part(&self) -> &MessageMetadataPart<'x> { + &self.parts[0] + } +} + +impl<'x> MessageMetadataPart<'x> { + pub fn contents<'y>(&self, raw_message: &'y [u8]) -> Cow<'y, [u8]> { + let bytes = raw_message + .get(self.offset_body..self.offset_end) + .unwrap_or_default(); + match self.encoding { + Encoding::None => bytes.into(), + Encoding::QuotedPrintable => quoted_printable_decode(bytes).unwrap_or_default().into(), + Encoding::Base64 => base64_decode(bytes).unwrap_or_default().into(), + } + } + + pub fn decode_contents<'y>(&self, raw_message: &'y [u8]) -> PartType<'y> { + let bytes = self.contents(raw_message); + + match self.body { + MetadataPartType::Text | MetadataPartType::Html => { + let text = match ( + bytes, + self.headers + .header_value(&HeaderName::ContentType) + .and_then(|c| c.as_content_type()) + .and_then(|ct| { + ct.attribute("charset") + .and_then(|c| charset_decoder(c.as_bytes())) + }), + ) { + (Cow::Owned(vec), Some(charset_decoder)) => charset_decoder(&vec).into(), + (Cow::Owned(vec), None) => String::from_utf8(vec) + .unwrap_or_else(|e| String::from_utf8_lossy(e.as_bytes()).into_owned()) + .into(), + (Cow::Borrowed(bytes), Some(charset_decoder)) => charset_decoder(bytes).into(), + (Cow::Borrowed(bytes), None) => String::from_utf8_lossy(bytes), + }; + + if matches!(self.body, MetadataPartType::Text) { + PartType::Text(text) + } else { + PartType::Html(text) + } + } + MetadataPartType::Binary => PartType::Binary(bytes), + MetadataPartType::InlineBinary => PartType::InlineBinary(bytes), + MetadataPartType::Message(_) | MetadataPartType::Multipart(_) => unreachable!(), + } + } + + pub fn remove_header(&mut self, header_name: &HeaderName) -> Option> { + for header in self.headers.iter_mut().rev() { + if header.name == *header_name { + return Some(std::mem::take(&mut header.value)); + } + } + None + } +} + +impl<'x> From> for MessageMetadataContents<'x> { + fn from(value: Message<'x>) -> Self { + MessageMetadataContents { + html_body: value.html_body, + text_body: value.text_body, + attachments: value.attachments, + parts: value + .parts + .into_iter() + .map(|part| { + let (size, body) = match part.body { + PartType::Text(contents) => (contents.len(), MetadataPartType::Text), + PartType::Html(contents) => (contents.len(), MetadataPartType::Html), + PartType::Binary(contents) => (contents.len(), MetadataPartType::Binary), + PartType::InlineBinary(contents) => { + (contents.len(), MetadataPartType::InlineBinary) + } + PartType::Message(message) => ( + message.root_part().raw_len(), + MetadataPartType::Message(message.into()), + ), + PartType::Multipart(parts) => (0, MetadataPartType::Multipart(parts)), + }; + + MessageMetadataPart { + headers: part + .headers + .into_iter() + .map(|hdr| Header { + value: if matches!( + &hdr.name, + HeaderName::Subject + | HeaderName::From + | HeaderName::To + | HeaderName::Cc + | HeaderName::Date + | HeaderName::Bcc + | HeaderName::ReplyTo + | HeaderName::Sender + | HeaderName::Comments + | HeaderName::InReplyTo + | HeaderName::Keywords + | HeaderName::MessageId + | HeaderName::References + | HeaderName::ResentMessageId + | HeaderName::ContentDescription + | HeaderName::ContentId + | HeaderName::ContentLanguage + | HeaderName::ContentLocation + | HeaderName::ContentTransferEncoding + | HeaderName::ContentType + | HeaderName::ContentDisposition + | HeaderName::ListId + ) { + hdr.value + } else { + HeaderValue::Empty + }, + name: hdr.name, + offset_field: hdr.offset_field, + offset_start: hdr.offset_start, + offset_end: hdr.offset_end, + }) + .collect(), + is_encoding_problem: part.is_encoding_problem, + encoding: part.encoding, + body, + size, + offset_header: part.offset_header, + offset_body: part.offset_body, + offset_end: part.offset_end, + } + }) + .collect(), + } + } +} + +impl<'x> MetadataPartType<'x> { + fn unwrap_message(self) -> MessageMetadataContents<'x> { + match self { + MetadataPartType::Message(message) => message, + _ => panic!("unwrap_message called on non-message part"), + } + } +} + +impl<'x> MimeHeaders<'x> for MessageMetadataPart<'x> { + fn content_description(&self) -> Option<&str> { + self.headers + .header_value(&HeaderName::ContentDescription) + .and_then(|header| header.as_text()) + } + + fn content_disposition(&self) -> Option<&ContentType> { + self.headers + .header_value(&HeaderName::ContentDisposition) + .and_then(|header| header.as_content_type()) + } + + fn content_id(&self) -> Option<&str> { + self.headers + .header_value(&HeaderName::ContentId) + .and_then(|header| header.as_text()) + } + + fn content_transfer_encoding(&self) -> Option<&str> { + self.headers + .header_value(&HeaderName::ContentTransferEncoding) + .and_then(|header| header.as_text()) + } + + fn content_type(&self) -> Option<&ContentType> { + self.headers + .header_value(&HeaderName::ContentType) + .and_then(|header| header.as_content_type()) + } + + fn content_language(&self) -> &HeaderValue { + self.headers + .header_value(&HeaderName::ContentLanguage) + .unwrap_or(&HeaderValue::Empty) + } + + fn content_location(&self) -> Option<&str> { + self.headers + .header_value(&HeaderName::ContentLocation) + .and_then(|header| header.as_text()) + } +} diff --git a/crates/jmap/src/email/mod.rs b/crates/jmap/src/email/mod.rs index e0f6a0b8..4c534a67 100644 --- a/crates/jmap/src/email/mod.rs +++ b/crates/jmap/src/email/mod.rs @@ -29,6 +29,7 @@ pub mod headers; pub mod import; pub mod index; pub mod ingest; +pub mod metadata; pub mod parse; pub mod query; pub mod set; diff --git a/crates/jmap/src/email/parse.rs b/crates/jmap/src/email/parse.rs index 1d6370a3..fd8920ea 100644 --- a/crates/jmap/src/email/parse.rs +++ b/crates/jmap/src/email/parse.rs @@ -178,13 +178,15 @@ impl JMAP { | Property::Header(_) => { email.append( property.clone(), - message.parts[0].header_to_value(property, &raw_message), + message.parts[0] + .headers + .header_to_value(property, &raw_message), ); } Property::Headers => { email.append( Property::Headers, - message.parts[0].headers_to_value(&raw_message), + message.parts[0].headers.headers_to_value(&raw_message), ); } Property::TextBody | Property::HtmlBody | Property::Attachments => { diff --git a/crates/jmap/src/email/set.rs b/crates/jmap/src/email/set.rs index e31c80ff..9e4c1073 100644 --- a/crates/jmap/src/email/set.rs +++ b/crates/jmap/src/email/set.rs @@ -29,7 +29,6 @@ use jmap_proto::{ set::{SetError, SetErrorType}, }, method::set::{RequestArguments, SetRequest, SetResponse}, - object::Object, response::references::EvalObjectReferences, types::{ acl::Acl, @@ -61,12 +60,13 @@ use store::{ BlobKind, Serialize, ValueKey, }; -use crate::{auth::AccessToken, IngestError, JMAP}; +use crate::{auth::AccessToken, Bincode, IngestError, JMAP}; use super::{ headers::{BuildHeader, ValueToHeader}, index::EmailIndexBuilder, ingest::IngestEmail, + metadata::MessageMetadata, }; impl JMAP { @@ -1182,7 +1182,7 @@ impl JMAP { // Remove message metadata if let Some(metadata) = self - .get_property::>( + .get_property::>( account_id, Collection::Email, document_id, @@ -1190,7 +1190,7 @@ impl JMAP { ) .await? { - batch.custom(EmailIndexBuilder::clear(metadata)); + batch.custom(EmailIndexBuilder::clear(metadata.inner)); } else { tracing::debug!( event = "error", diff --git a/crates/jmap/src/lib.rs b/crates/jmap/src/lib.rs index aa8564ec..33a466aa 100644 --- a/crates/jmap/src/lib.rs +++ b/crates/jmap/src/lib.rs @@ -736,13 +736,13 @@ impl Bincode { impl Serialize for &Bincode { fn serialize(self) -> Vec { - bincode::serialize(&self.inner).unwrap_or_default() + lz4_flex::compress_prepend_size(&bincode::serialize(&self.inner).unwrap_or_default()) } } impl Serialize for Bincode { fn serialize(self) -> Vec { - bincode::serialize(&self.inner).unwrap_or_default() + lz4_flex::compress_prepend_size(&bincode::serialize(&self.inner).unwrap_or_default()) } } @@ -750,11 +750,19 @@ impl De for Bincode { fn deserialize(bytes: &[u8]) -> store::Result { - bincode::deserialize(bytes) - .map(|inner| Self { inner }) + lz4_flex::decompress_size_prepended(bytes) .map_err(|err| { - store::Error::InternalError(format!("Bincode deserialization failed: {err}")) + store::Error::InternalError(format!("Bincode decompression failed: {err:?}")) }) + .and_then(|result| { + bincode::deserialize(&result).map_err(|err| { + store::Error::InternalError(format!( + "Bincode deserialization failed (len {}): {err:?}", + result.len() + )) + }) + }) + .map(|inner| Self { inner }) } } @@ -764,6 +772,12 @@ impl ToBitmaps for Bincode } } +impl ToBitmaps for &Bincode { + fn to_bitmaps(&self, _ops: &mut Vec, _field: u8, _set: bool) { + unreachable!() + } +} + trait UpdateResults: Sized { fn update_results(&mut self, sorted_results: SortedResultSet) -> Result<(), MethodError>; } diff --git a/crates/jmap/src/submission/set.rs b/crates/jmap/src/submission/set.rs index 1567a6ac..3e9b28f3 100644 --- a/crates/jmap/src/submission/set.rs +++ b/crates/jmap/src/submission/set.rs @@ -47,6 +47,7 @@ use jmap_proto::{ value::{MaybePatchValue, SetValue, Value}, }, }; +use mail_parser::{HeaderName, HeaderValue}; use smtp::{ core::{management::QueueRequest, NullIo, Session, SessionData, State}, queue, @@ -59,7 +60,7 @@ use store::{ use tokio::sync::oneshot; use utils::{listener::ServerInstance, map::vec_map::VecMap}; -use crate::{identity::set::sanitize_email, JMAP}; +use crate::{email::metadata::MessageMetadata, identity::set::sanitize_email, Bincode, JMAP}; pub static SCHEMA: &[IndexProperty] = &[ IndexProperty::new(Property::UndoStatus).index_as(IndexAs::Text { @@ -505,8 +506,8 @@ impl JMAP { // Add recipients to envelope if missing if rcpt_to.is_empty() { - if let Some(obj) = self - .get_property::>( + if let Some(metadata) = self + .get_property::>( account_id, Collection::Email, email_id, @@ -515,29 +516,30 @@ impl JMAP { .await? { let mut envelope_values = Vec::new(); - for property in &[Property::To, Property::Cc, Property::Bcc] { - if let Some(Value::List(addresses)) = obj.properties.get(property) { - for address in addresses { - if let Some(address) = address - .as_obj() - .and_then(|obj| obj.properties.get(&Property::Email)) - .and_then(|value| value.as_string()) - .and_then(sanitize_email) - { - if !rcpt_to.iter().any(|rcpt| rcpt.address == address) { - envelope_values.push(Value::Object( - Object::with_capacity(1) - .with_property(Property::Email, address.clone()), - )); - rcpt_to.push(RcptTo { - address, - ..Default::default() - }); + for header in &metadata.inner.contents.parts[0].headers { + if matches!( + header.name, + HeaderName::To | HeaderName::Cc | HeaderName::Bcc + ) { + if let HeaderValue::Address(addr) = &header.value { + for address in addr.iter() { + if let Some(address) = address.address().and_then(sanitize_email) { + if !rcpt_to.iter().any(|rcpt| rcpt.address == address) { + envelope_values.push(Value::Object( + Object::with_capacity(1) + .with_property(Property::Email, address.clone()), + )); + rcpt_to.push(RcptTo { + address, + ..Default::default() + }); + } } } } } } + if !rcpt_to.is_empty() { submission .properties diff --git a/crates/store/src/backend/sqlite/write.rs b/crates/store/src/backend/sqlite/write.rs index 4e180f5b..a64f0cf0 100644 --- a/crates/store/src/backend/sqlite/write.rs +++ b/crates/store/src/backend/sqlite/write.rs @@ -159,7 +159,7 @@ impl Store { .serialize(); if *set { - trx.prepare_cached("INSERT OR REPLACE INTO i (k) VALUES (?)")? + trx.prepare_cached("INSERT OR IGNORE INTO i (k) VALUES (?)")? .execute([&key])?; } else { trx.prepare_cached("DELETE FROM i WHERE k = ?")? diff --git a/tests/src/jmap/mod.rs b/tests/src/jmap/mod.rs index e980dfa7..87eb95c2 100644 --- a/tests/src/jmap/mod.rs +++ b/tests/src/jmap/mod.rs @@ -222,12 +222,13 @@ refresh-token-renew = "2s" #[tokio::test] pub async fn jmap_tests() { - /*tracing::subscriber::set_global_default( + let coco = 1; + tracing::subscriber::set_global_default( tracing_subscriber::FmtSubscriber::builder() .with_max_level(tracing::Level::WARN) .finish(), ) - .unwrap();*/ + .unwrap(); let delete = true; let mut params = init_jmap_tests(delete).await;