diff --git a/crates/jmap-proto/src/method/changes.rs b/crates/jmap-proto/src/method/changes.rs index dd202816..3a638f65 100644 --- a/crates/jmap-proto/src/method/changes.rs +++ b/crates/jmap-proto/src/method/changes.rs @@ -100,3 +100,9 @@ impl JsonObjectParser for ChangesRequest { Ok(request) } } + +impl ChangesResponse { + pub fn has_changes(&self) -> bool { + !self.created.is_empty() || !self.updated.is_empty() || !self.destroyed.is_empty() + } +} diff --git a/crates/jmap-proto/src/method/parse.rs b/crates/jmap-proto/src/method/parse.rs index ee38f6f3..d24a6e84 100644 --- a/crates/jmap-proto/src/method/parse.rs +++ b/crates/jmap-proto/src/method/parse.rs @@ -10,31 +10,31 @@ use crate::{ #[derive(Debug, Clone)] pub struct ParseEmailRequest { pub account_id: Id, - blob_ids: Vec, - properties: Option>, - body_properties: Option>, - fetch_text_body_values: Option, - fetch_html_body_values: Option, - fetch_all_body_values: Option, - max_body_value_bytes: Option, + pub blob_ids: Vec, + pub properties: Option>, + pub body_properties: Option>, + pub fetch_text_body_values: Option, + pub fetch_html_body_values: Option, + pub fetch_all_body_values: Option, + pub max_body_value_bytes: Option, } #[derive(Debug, Clone, serde::Serialize)] pub struct ParseEmailResponse { #[serde(rename = "accountId")] - account_id: Id, + pub account_id: Id, #[serde(rename = "parsed")] #[serde(skip_serializing_if = "VecMap::is_empty")] - parsed: VecMap>, + pub parsed: VecMap>, #[serde(rename = "notParsable")] #[serde(skip_serializing_if = "Vec::is_empty")] - not_parsable: Vec, + pub not_parsable: Vec, #[serde(rename = "notFound")] #[serde(skip_serializing_if = "Vec::is_empty")] - not_found: Vec, + pub not_found: Vec, } impl JsonObjectParser for ParseEmailRequest { diff --git a/crates/jmap-proto/src/method/query.rs b/crates/jmap-proto/src/method/query.rs index 1459bd86..66d17646 100644 --- a/crates/jmap-proto/src/method/query.rs +++ b/crates/jmap-proto/src/method/query.rs @@ -655,6 +655,30 @@ impl RequestPropertyParser for RequestArguments { } } +impl Filter { + pub fn is_immutable(&self) -> bool { + matches!( + self, + Filter::Before(_) + | Filter::After(_) + | Filter::MinSize(_) + | Filter::MaxSize(_) + | Filter::Text(_) + | Filter::HasAttachment(_) + | Filter::From(_) + | Filter::To(_) + | Filter::Cc(_) + | Filter::Bcc(_) + | Filter::Subject(_) + | Filter::Body(_) + | Filter::Header(_) + | Filter::Id(_) + | Filter::SentBefore(_) + | Filter::SentAfter(_) + ) + } +} + impl Comparator { pub fn descending(property: SortProperty) -> Self { Self { @@ -664,6 +688,7 @@ impl Comparator { keyword: None, } } + pub fn ascending(property: SortProperty) -> Self { Self { property, @@ -672,14 +697,29 @@ impl Comparator { keyword: None, } } + + pub fn is_immutable(&self) -> bool { + matches!( + &self.property, + SortProperty::SentAt + | SortProperty::ReceivedAt + | SortProperty::Size + | SortProperty::From + | SortProperty::To + | SortProperty::Subject + | SortProperty::Cc + ) + } } impl QueryRequest { pub fn take_arguments(&mut self) -> RequestArguments { std::mem::replace(&mut self.arguments, RequestArguments::Principal) } +} - pub fn with_arguments(self, arguments: T) -> QueryRequest { +impl QueryRequest { + pub fn with_arguments(self, arguments: A) -> QueryRequest { QueryRequest { arguments, account_id: self.account_id, diff --git a/crates/jmap-proto/src/method/search_snippet.rs b/crates/jmap-proto/src/method/search_snippet.rs index 49a0a5a5..481894df 100644 --- a/crates/jmap-proto/src/method/search_snippet.rs +++ b/crates/jmap-proto/src/method/search_snippet.rs @@ -25,8 +25,8 @@ pub struct GetSearchSnippetResponse { pub list: Vec, #[serde(rename = "notFound")] - #[serde(skip_serializing_if = "Option::is_none")] - pub not_found: Option>, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub not_found: Vec, } #[derive(serde::Serialize, Clone, Debug)] diff --git a/crates/jmap-proto/src/request/parser.rs b/crates/jmap-proto/src/request/parser.rs index 895e4680..cfe71b02 100644 --- a/crates/jmap-proto/src/request/parser.rs +++ b/crates/jmap-proto/src/request/parser.rs @@ -13,6 +13,7 @@ use crate::{ parse::ParseEmailRequest, query::QueryRequest, query_changes::QueryChangesRequest, + search_snippet::GetSearchSnippetRequest, set::SetRequest, validate::ValidateSieveScriptRequest, }, @@ -95,7 +96,12 @@ impl Request { let method = match (&method_name.fnc, &method_name.obj) { (MethodFunction::Get, _) => { - GetRequest::parse(&mut parser).map(RequestMethod::Get) + if method_name.obj != MethodObject::SearchSnippet { + GetRequest::parse(&mut parser).map(RequestMethod::Get) + } else { + GetSearchSnippetRequest::parse(&mut parser) + .map(RequestMethod::SearchSnippet) + } } (MethodFunction::Query, _) => { QueryRequest::parse(&mut parser).map(RequestMethod::Query) diff --git a/crates/jmap/src/api/config.rs b/crates/jmap/src/api/config.rs index c87f7413..f6a4ff06 100644 --- a/crates/jmap/src/api/config.rs +++ b/crates/jmap/src/api/config.rs @@ -46,6 +46,9 @@ impl crate::Config { mail_attachments_max_size: settings .property("jmap.email.max-attachment-size")? .unwrap_or(50000000), + mail_parse_max_items: settings + .property("jmap.email.parse.max-items")? + .unwrap_or(50000000), sieve_max_script_name: settings .property("jmap.sieve.max-name-length")? .unwrap_or(512), diff --git a/crates/jmap/src/api/request.rs b/crates/jmap/src/api/request.rs index d5521a1d..187fa1bf 100644 --- a/crates/jmap/src/api/request.rs +++ b/crates/jmap/src/api/request.rs @@ -68,9 +68,9 @@ impl JMAP { RequestMethod::Copy(_) => todo!(), RequestMethod::CopyBlob(_) => todo!(), RequestMethod::ImportEmail(req) => self.email_import(req).await.into(), - RequestMethod::ParseEmail(_) => todo!(), - RequestMethod::QueryChanges(_) => todo!(), - RequestMethod::SearchSnippet(_) => todo!(), + RequestMethod::ParseEmail(req) => self.email_parse(req).await.into(), + RequestMethod::QueryChanges(req) => self.query_changes(req).await.into(), + RequestMethod::SearchSnippet(req) => self.email_search_snippet(req).await.into(), RequestMethod::ValidateScript(_) => todo!(), RequestMethod::Echo(req) => req.into(), RequestMethod::Error(error) => error.into(), diff --git a/crates/jmap/src/changes/get.rs b/crates/jmap/src/changes/get.rs index 2cceb7c6..a44af4b3 100644 --- a/crates/jmap/src/changes/get.rs +++ b/crates/jmap/src/changes/get.rs @@ -38,7 +38,7 @@ impl JMAP { let (items_sent, mut changelog) = match &request.since_state { State::Initial => { let changelog = self - .query_changes(account_id, collection, Query::All) + .changes_(account_id, collection, Query::All) .await? .unwrap(); if changelog.changes.is_empty() && changelog.from_change_id == 0 { @@ -49,7 +49,7 @@ impl JMAP { } State::Exact(change_id) => ( 0, - self.query_changes(account_id, collection, Query::Since(*change_id)) + self.changes_(account_id, collection, Query::Since(*change_id)) .await? .ok_or_else(|| { MethodError::InvalidArguments( @@ -59,7 +59,7 @@ impl JMAP { ), State::Intermediate(intermediate_state) => { let mut changelog = self - .query_changes( + .changes_( account_id, collection, Query::RangeInclusive(intermediate_state.from_id, intermediate_state.to_id), @@ -73,7 +73,7 @@ impl JMAP { if intermediate_state.items_sent >= changelog.changes.len() { ( 0, - self.query_changes( + self.changes_( account_id, collection, Query::Since(intermediate_state.to_id), @@ -141,7 +141,7 @@ impl JMAP { Ok(response) } - async fn query_changes( + async fn changes_( &self, account_id: u32, collection: Collection, diff --git a/crates/jmap/src/changes/mod.rs b/crates/jmap/src/changes/mod.rs index 969c05d0..b4a5d2aa 100644 --- a/crates/jmap/src/changes/mod.rs +++ b/crates/jmap/src/changes/mod.rs @@ -1,3 +1,4 @@ pub mod get; +pub mod query; pub mod state; pub mod write; diff --git a/crates/jmap/src/changes/query.rs b/crates/jmap/src/changes/query.rs new file mode 100644 index 00000000..c28eb70f --- /dev/null +++ b/crates/jmap/src/changes/query.rs @@ -0,0 +1,105 @@ +use jmap_proto::{ + error::method::MethodError, + method::{ + changes::{self, ChangesRequest}, + query::{self, QueryRequest}, + query_changes::{AddedItem, QueryChangesRequest, QueryChangesResponse}, + }, +}; + +use crate::JMAP; + +impl JMAP { + pub async fn query_changes( + &self, + request: QueryChangesRequest, + ) -> Result { + // Query changes + let changes = self + .changes(ChangesRequest { + account_id: request.account_id, + since_state: request.since_query_state.clone(), + max_changes: request.max_changes, + arguments: match &request.arguments { + query::RequestArguments::Email(_) => changes::RequestArguments::Email, + query::RequestArguments::Mailbox(_) => changes::RequestArguments::Mailbox, + query::RequestArguments::EmailSubmission => { + changes::RequestArguments::EmailSubmission + } + _ => return Err(MethodError::UnknownMethod("Unknown method".to_string())), + }, + }) + .await?; + let calculate_total = request.calculate_total.unwrap_or(false); + let has_changes = changes.has_changes(); + let mut response = QueryChangesResponse { + account_id: request.account_id, + old_query_state: changes.old_state, + new_query_state: changes.new_state, + total: None, + removed: vec![], + added: vec![], + }; + + if has_changes || calculate_total { + let query = QueryRequest { + account_id: request.account_id, + filter: request.filter, + sort: request.sort, + position: None, + anchor: None, + anchor_offset: None, + limit: None, + calculate_total: request.calculate_total, + arguments: (), + }; + let is_mutable = query.filter.iter().any(|f| !f.is_immutable()) + || query + .sort + .as_ref() + .map_or(false, |sort| sort.iter().any(|s| !s.is_immutable())); + let results = match request.arguments { + query::RequestArguments::Email(arguments) => { + self.email_query(query.with_arguments(arguments)).await? + } + query::RequestArguments::Mailbox(arguments) => { + self.mailbox_query(query.with_arguments(arguments)).await? + } + query::RequestArguments::EmailSubmission => { + let implement = "true"; + todo!() + } + _ => unreachable!(), + }; + + if has_changes { + if is_mutable { + for (index, id) in results.ids.into_iter().enumerate() { + if matches!(request.up_to_id, Some(up_to_id) if up_to_id == id) { + break; + } else if changes.created.contains(&id) || changes.updated.contains(&id) { + response.added.push(AddedItem::new(id, index)); + } + } + + response.removed = changes.updated; + } else { + for (index, id) in results.ids.into_iter().enumerate() { + if matches!(request.up_to_id, Some(up_to_id) if up_to_id == id) { + break; + } else if changes.created.contains(&id) { + response.added.push(AddedItem::new(id, index)); + } + } + } + + if !changes.destroyed.is_empty() { + response.removed.extend(changes.destroyed); + } + } + response.total = results.total; + } + + Ok(response) + } +} diff --git a/crates/jmap/src/email/body.rs b/crates/jmap/src/email/body.rs index 13de771f..729e143c 100644 --- a/crates/jmap/src/email/body.rs +++ b/crates/jmap/src/email/body.rs @@ -167,6 +167,7 @@ impl TruncateBody for PartType<'_> { let mut in_tag = false; let mut in_comment = false; let mut last_tag_end_pos = 0; + let mut cr_count = 0; for (pos, ch) in html.char_indices() { let mut set_last_tag = 0; match ch { @@ -189,16 +190,20 @@ impl TruncateBody for PartType<'_> { set_last_tag = pos + 1; } } - '\r' => continue, + '\r' => { + cr_count += 1; + continue; + } _ => (), } - if ch.len_utf8() + pos > max_len { + if ch.len_utf8() + pos - cr_count > max_len { result.push_str( &html[0..if (in_tag || set_last_tag > 0) && last_tag_end_pos > 0 { last_tag_end_pos } else { pos - }], + }] + .replace('\r', ""), ); if add_dots { result.push_str("..."); diff --git a/crates/jmap/src/email/get.rs b/crates/jmap/src/email/get.rs index f3984259..dacc9201 100644 --- a/crates/jmap/src/email/get.rs +++ b/crates/jmap/src/email/get.rs @@ -305,10 +305,11 @@ impl JMAP { if let Some(message) = &message { let mut body_values = Object::with_capacity(message.parts.len()); for (part_id, part) in message.parts.iter().enumerate() { - if (message.html_body.contains(&part_id) + if ((message.html_body.contains(&part_id) && (fetch_all_body_values || fetch_html_body_values)) || (message.text_body.contains(&part_id) - && (fetch_all_body_values || fetch_text_body_values)) + && (fetch_all_body_values || fetch_text_body_values))) + && part.is_text() { let (is_truncated, value) = part.body.truncate(max_body_value_bytes); diff --git a/crates/jmap/src/email/index.rs b/crates/jmap/src/email/index.rs index 27355dc1..5e730fbb 100644 --- a/crates/jmap/src/email/index.rs +++ b/crates/jmap/src/email/index.rs @@ -32,7 +32,7 @@ pub const PREVIEW_LENGTH: usize = 256; pub struct SortedAddressBuilder { last_is_space: bool, - buf: String, + pub buf: String, } pub(super) trait IndexMessage { @@ -164,7 +164,7 @@ impl IndexMessage for BatchBuilder { if !found_addr { match element { AddressElement::Name => { - found_addr = sort_text.push(value); + found_addr = !sort_text.push(value); } AddressElement::Address => { sort_text.push(value); diff --git a/crates/jmap/src/email/mod.rs b/crates/jmap/src/email/mod.rs index 1fef3c22..0b643f28 100644 --- a/crates/jmap/src/email/mod.rs +++ b/crates/jmap/src/email/mod.rs @@ -4,5 +4,7 @@ pub mod headers; pub mod import; pub mod index; pub mod ingest; +pub mod parse; pub mod query; pub mod set; +pub mod snippet; diff --git a/crates/jmap/src/email/parse.rs b/crates/jmap/src/email/parse.rs new file mode 100644 index 00000000..d3d24255 --- /dev/null +++ b/crates/jmap/src/email/parse.rs @@ -0,0 +1,250 @@ +use jmap_proto::{ + error::method::MethodError, + method::parse::{ParseEmailRequest, ParseEmailResponse}, + object::Object, + types::{property::Property, value::Value}, +}; +use mail_parser::{ + decoders::html::html_to_text, parsers::preview::preview_text, Message, PartType, +}; +use utils::map::vec_map::VecMap; + +use crate::JMAP; + +use super::{ + body::{ToBodyPart, TruncateBody}, + headers::HeaderToValue, + index::PREVIEW_LENGTH, +}; + +impl JMAP { + pub async fn email_parse( + &self, + request: ParseEmailRequest, + ) -> Result { + if request.blob_ids.len() > self.config.mail_parse_max_items { + return Err(MethodError::RequestTooLarge); + } + let account_id = request.account_id.document_id(); + let properties = request.properties.unwrap_or_else(|| { + vec![ + Property::BlobId, + Property::Size, + Property::ReceivedAt, + Property::MessageId, + Property::InReplyTo, + Property::References, + Property::Sender, + Property::From, + Property::To, + Property::Cc, + Property::Bcc, + Property::ReplyTo, + Property::Subject, + Property::SentAt, + Property::HasAttachment, + Property::Preview, + Property::BodyValues, + Property::TextBody, + Property::HtmlBody, + Property::Attachments, + ] + }); + let body_properties = request.body_properties.unwrap_or_else(|| { + vec![ + Property::PartId, + Property::BlobId, + Property::Size, + Property::Name, + Property::Type, + Property::Charset, + Property::Disposition, + Property::Cid, + Property::Language, + Property::Location, + ] + }); + let fetch_text_body_values = request.fetch_text_body_values.unwrap_or(false); + let fetch_html_body_values = request.fetch_html_body_values.unwrap_or(false); + let fetch_all_body_values = request.fetch_all_body_values.unwrap_or(false); + let max_body_value_bytes = request.max_body_value_bytes.unwrap_or(0); + + let mut response = ParseEmailResponse { + account_id: request.account_id, + parsed: VecMap::with_capacity(request.blob_ids.len()), + not_parsable: vec![], + not_found: vec![], + }; + + for blob_id in request.blob_ids { + // Fetch raw message to parse + let raw_message = match self.blob_download(&blob_id, account_id).await { + Ok(Some(raw_message)) => raw_message, + Ok(None) => { + response.not_found.push(blob_id); + continue; + } + Err(err) => { + tracing::error!(event = "error", + context = "store", + account_id = account_id, + blob_id = ?blob_id, + error = ?err, + "Failed to retrieve blob"); + return Err(MethodError::ServerPartialFail); + } + }; + let message = if let Some(message) = Message::parse(&raw_message) { + message + } else { + response.not_parsable.push(blob_id); + continue; + }; + + // Prepare response + let mut email = Object::with_capacity(properties.len()); + for property in &properties { + match property { + Property::BlobId => { + email.append(Property::BlobId, blob_id.clone()); + } + + Property::Size => { + email.append(Property::Size, Value::UnsignedInt(raw_message.len() as u64)); + } + Property::HasAttachment => { + email.append( + Property::HasAttachment, + Value::Bool(message.parts.iter().enumerate().any(|(part_id, part)| { + match &part.body { + PartType::Html(_) | PartType::Text(_) => { + !message.text_body.contains(&part_id) + && !message.html_body.contains(&part_id) + } + PartType::Binary(_) | PartType::Message(_) => true, + _ => false, + } + })), + ); + } + Property::Preview => { + email.append( + Property::Preview, + match message + .text_body + .first() + .or_else(|| message.html_body.first()) + .and_then(|idx| message.parts.get(*idx)) + .map(|part| &part.body) + { + Some(PartType::Text(text)) => { + preview_text(text.replace('\r', "").into(), PREVIEW_LENGTH) + .into() + } + Some(PartType::Html(html)) => preview_text( + html_to_text(html).replace('\r', "").into(), + PREVIEW_LENGTH, + ) + .into(), + _ => Value::Null, + }, + ); + } + Property::MessageId + | Property::InReplyTo + | Property::References + | Property::Sender + | Property::From + | Property::To + | Property::Cc + | Property::Bcc + | Property::ReplyTo + | Property::Subject + | Property::SentAt + | Property::Header(_) => { + email.append( + property.clone(), + message.parts[0].header_to_value(property, &raw_message), + ); + } + Property::Headers => { + email.append( + Property::Headers, + message.parts[0].headers_to_value(&raw_message), + ); + } + Property::TextBody | Property::HtmlBody | Property::Attachments => { + let list = match property { + Property::TextBody => &message.text_body, + Property::HtmlBody => &message.html_body, + Property::Attachments => &message.attachments, + _ => unreachable!(), + } + .iter(); + email.append( + property.clone(), + list.map(|part_id| { + message.parts.to_body_part( + *part_id, + &body_properties, + &raw_message, + &blob_id, + ) + }) + .collect::>(), + ); + } + Property::BodyStructure => { + email.append( + Property::BodyStructure, + message + .parts + .to_body_part(0, &body_properties, &raw_message, &blob_id), + ); + } + Property::BodyValues => { + let mut body_values = Object::with_capacity(message.parts.len()); + for (part_id, part) in message.parts.iter().enumerate() { + if ((message.html_body.contains(&part_id) + && (fetch_all_body_values || fetch_html_body_values)) + || (message.text_body.contains(&part_id) + && (fetch_all_body_values || fetch_text_body_values))) + && part.is_text() + { + let (is_truncated, value) = + part.body.truncate(max_body_value_bytes); + body_values.append( + Property::_T(part_id.to_string()), + Object::with_capacity(3) + .with_property( + Property::IsEncodingProblem, + part.is_encoding_problem, + ) + .with_property(Property::IsTruncated, is_truncated) + .with_property(Property::Value, value), + ); + } + } + email.append(Property::BodyValues, body_values); + } + Property::Id + | Property::ThreadId + | Property::Keywords + | Property::MailboxIds + | Property::ReceivedAt => { + email.append(property.clone(), Value::Null); + } + + _ => { + return Err(MethodError::InvalidArguments(format!( + "Invalid property {property:?}" + ))); + } + } + } + response.parsed.append(blob_id, email); + } + + Ok(response) + } +} diff --git a/crates/jmap/src/email/set.rs b/crates/jmap/src/email/set.rs index 39f67061..e17f3847 100644 --- a/crates/jmap/src/email/set.rs +++ b/crates/jmap/src/email/set.rs @@ -1088,7 +1088,10 @@ impl JMAP { } } } - (Property::From | Property::To | Property::Bcc, Value::List(addresses)) => { + ( + Property::From | Property::To | Property::Cc | Property::Bcc, + Value::List(addresses), + ) => { let mut sort_text = SortedAddressBuilder::new(); 'outer: for addr in addresses { if let Some(addr) = addr.try_unwrap_object() { diff --git a/crates/jmap/src/email/snippet.rs b/crates/jmap/src/email/snippet.rs new file mode 100644 index 00000000..f2bf342b --- /dev/null +++ b/crates/jmap/src/email/snippet.rs @@ -0,0 +1,237 @@ +use jmap_proto::{ + error::method::MethodError, + method::{ + query::Filter, + search_snippet::{GetSearchSnippetRequest, GetSearchSnippetResponse, SearchSnippet}, + }, + types::collection::Collection, +}; +use mail_parser::{decoders::html::html_to_text, Message, PartType}; +use store::{ + fts::{ + builder::MAX_TOKEN_LENGTH, + search_snippet::generate_snippet, + stemmer::Stemmer, + term_index::{self}, + tokenizers::Tokenizer, + Language, + }, + BlobKind, +}; + +use crate::JMAP; + +use super::index::MAX_MESSAGE_PARTS; + +impl JMAP { + pub async fn email_search_snippet( + &self, + request: GetSearchSnippetRequest, + ) -> Result { + let mut filter_stack = vec![]; + let mut include_term = true; + let mut terms = vec![]; + let mut match_phrase = false; + + for cond in request.filter { + match cond { + Filter::Text(text) | Filter::Subject(text) | Filter::Body(text) => { + if include_term { + let (text, language) = Language::detect(text, self.config.default_language); + if (text.starts_with('"') && text.ends_with('"')) + || (text.starts_with('\'') && text.ends_with('\'')) + { + terms.push( + Tokenizer::new(&text, language, MAX_TOKEN_LENGTH) + .map(|token| (token.word.into_owned(), None)) + .collect::>(), + ); + match_phrase = true; + } else { + terms.push( + Stemmer::new(&text, language, MAX_TOKEN_LENGTH) + .map(|token| { + ( + token.word.into_owned(), + token.stemmed_word.map(|w| w.into_owned()), + ) + }) + .collect::>(), + ); + } + } + } + Filter::And | Filter::Or => { + filter_stack.push(cond); + } + Filter::Not => { + filter_stack.push(cond); + include_term = !include_term; + } + Filter::Close => { + if matches!(filter_stack.pop(), Some(Filter::Not)) { + include_term = !include_term; + } + } + _ => (), + } + } + let account_id = request.account_id.document_id(); + let todo = "acls"; + let document_ids = self + .get_document_ids(account_id, Collection::Email) + .await? + .unwrap_or_default(); + let email_ids = request.email_ids.unwrap(); + let mut response = GetSearchSnippetResponse { + account_id: request.account_id, + list: Vec::with_capacity(email_ids.len()), + not_found: vec![], + }; + + if email_ids.len() > self.config.get_max_objects { + return Err(MethodError::RequestTooLarge); + } + + for email_id in email_ids { + let document_id = email_id.document_id(); + let mut snippet = SearchSnippet { + email_id, + subject: None, + preview: None, + }; + if !document_ids.contains(document_id) { + response.not_found.push(email_id); + continue; + } else if terms.is_empty() { + response.list.push(snippet); + continue; + } + + // Obtain the term index and raw message + let (term_index, raw_message) = if let (Some(term_index), Some(raw_message)) = ( + self.get_term_index(account_id, Collection::Email, document_id) + .await?, + self.get_blob( + &BlobKind::LinkedMaildir { + account_id, + document_id, + }, + 0..u32::MAX, + ) + .await?, + ) { + (term_index, raw_message) + } else { + response.not_found.push(email_id); + continue; + }; + + // Parse message + let message = if let Some(message) = Message::parse(&raw_message) { + message + } else { + response.not_found.push(email_id); + continue; + }; + + // Build the match terms + let mut match_terms = Vec::new(); + for term in &terms { + for (word, stemmed_word) in term { + match_terms.push(term_index.get_match_term(word, stemmed_word.as_deref())); + } + } + + 'outer: for term_group in term_index + .match_terms(&match_terms, None, match_phrase, true, true) + .map_err(|err| match err { + term_index::Error::InvalidArgument => { + MethodError::UnsupportedFilter("Too many search terms.".to_string()) + } + err => { + tracing::error!( + account_id = account_id, + document_id = document_id, + reason = ?err, + "Failed to generate search snippet."); + MethodError::UnsupportedFilter( + "Failed to generate search snippet.".to_string(), + ) + } + })? + .unwrap_or_default() + { + if term_group.part_id == 0 { + // Generate subject snippent + snippet.subject = + generate_snippet(&term_group.terms, message.subject().unwrap_or_default()); + } else { + let mut part_num = 1; + for part in &message.parts { + match &part.body { + PartType::Text(text) => { + if part_num == term_group.part_id { + snippet.preview = generate_snippet(&term_group.terms, text); + break 'outer; + } else { + part_num += 1; + } + } + PartType::Html(html) => { + if part_num == term_group.part_id { + snippet.preview = + generate_snippet(&term_group.terms, &html_to_text(html)); + break 'outer; + } else { + part_num += 1; + } + } + PartType::Message(message) => { + if let Some(subject) = message.subject() { + if part_num == term_group.part_id { + snippet.preview = + generate_snippet(&term_group.terms, subject); + break 'outer; + } else { + part_num += 1; + } + } + for sub_part in message.parts.iter().take(MAX_MESSAGE_PARTS) { + match &sub_part.body { + PartType::Text(text) => { + if part_num == term_group.part_id { + snippet.preview = + generate_snippet(&term_group.terms, text); + break 'outer; + } else { + part_num += 1; + } + } + PartType::Html(html) => { + if part_num == term_group.part_id { + snippet.preview = generate_snippet( + &term_group.terms, + &html_to_text(html), + ); + break 'outer; + } else { + part_num += 1; + } + } + _ => (), + } + } + } + _ => (), + } + } + } + } + + response.list.push(snippet); + } + + Ok(response) + } +} diff --git a/crates/jmap/src/lib.rs b/crates/jmap/src/lib.rs index f68bd97a..f549c4fe 100644 --- a/crates/jmap/src/lib.rs +++ b/crates/jmap/src/lib.rs @@ -10,7 +10,7 @@ use jmap_proto::{ }; use store::{ ahash::AHashMap, - fts::Language, + fts::{term_index::TermIndex, Language}, query::{sort::Pagination, Comparator, Filter, ResultSet, SortedResultSet}, roaring::RoaringBitmap, write::BitmapFamily, @@ -49,6 +49,7 @@ pub struct Config { pub mailbox_max_depth: usize, pub mailbox_name_max_len: usize, pub mail_attachments_max_size: usize, + pub mail_parse_max_items: usize, pub sieve_max_script_name: usize, pub sieve_max_scripts: usize, @@ -152,6 +153,37 @@ impl JMAP { } } + pub async fn get_term_index( + &self, + account_id: u32, + collection: Collection, + document_id: u32, + ) -> Result, MethodError> { + match self + .store + .get_value::(ValueKey { + account_id, + collection: collection.into(), + document_id, + family: u8::MAX, + field: u8::MAX, + }) + .await + { + Ok(value) => Ok(value), + Err(err) => { + tracing::error!(event = "error", + context = "store", + account_id = account_id, + collection = ?collection, + document_id = document_id, + error = ?err, + "Failed to retrieve term index"); + Err(MethodError::ServerPartialFail) + } + } + } + pub async fn get_document_ids( &self, account_id: u32, diff --git a/crates/store/src/backend/sqlite/main.rs b/crates/store/src/backend/sqlite/main.rs index 6f3fbcc7..45f6602c 100644 --- a/crates/store/src/backend/sqlite/main.rs +++ b/crates/store/src/backend/sqlite/main.rs @@ -7,8 +7,8 @@ use tokio::sync::oneshot; use utils::{config::Config, UnwrapFailure}; use crate::{ - blob::BlobStore, Store, SUBSPACE_ACLS, SUBSPACE_BITMAPS, SUBSPACE_BLOBS, SUBSPACE_INDEXES, - SUBSPACE_LOGS, SUBSPACE_VALUES, + blob::BlobStore, Store, SUBSPACE_ACLS, SUBSPACE_BITMAPS, SUBSPACE_INDEXES, SUBSPACE_LOGS, + SUBSPACE_VALUES, }; use super::pool::SqliteConnectionManager; @@ -46,12 +46,7 @@ impl Store { pub(super) fn create_tables(&self) -> crate::Result<()> { let conn = self.conn_pool.get()?; - for table in [ - SUBSPACE_VALUES, - SUBSPACE_LOGS, - SUBSPACE_BLOBS, - SUBSPACE_ACLS, - ] { + for table in [SUBSPACE_VALUES, SUBSPACE_LOGS, SUBSPACE_ACLS] { let table = char::from(table); conn.execute( &format!( diff --git a/crates/store/src/backend/sqlite/read.rs b/crates/store/src/backend/sqlite/read.rs index bf2d06a1..5713f692 100644 --- a/crates/store/src/backend/sqlite/read.rs +++ b/crates/store/src/backend/sqlite/read.rs @@ -36,7 +36,7 @@ impl ReadTransaction<'_> { mut key: BitmapKey, bm: &mut RoaringBitmap, ) -> crate::Result<()> { - let begin = key.serialize(); + let begin = (&key).serialize(); key.block_num = u32::MAX; let key_len = begin.len(); let end = key.serialize(); @@ -316,4 +316,51 @@ impl Store { _p: std::marker::PhantomData, }) } + + #[cfg(feature = "test_mode")] + pub async fn assert_is_empty(&self) { + let conn = self.read_transaction().unwrap(); + let mut query = conn.conn.prepare_cached("SELECT k, v FROM v").unwrap(); + let mut rows = query.query([]).unwrap(); + + while let Some(row) = rows.next().unwrap() { + let key = row.get_ref(0).unwrap().as_bytes().unwrap(); + let value = row.get_ref(1).unwrap().as_bytes().unwrap(); + + panic!("Table values is not empty: {key:?} {value:?}"); + } + + let mut query = conn.conn.prepare_cached("SELECT k FROM i").unwrap(); + let mut rows = query.query([]).unwrap(); + + while let Some(row) = rows.next().unwrap() { + let key = row.get_ref(0).unwrap().as_bytes().unwrap(); + + panic!( + "Table index is not empty, account {}, collection {}, document {}, property {}, value {:?}: {:?}", + u32::from_be_bytes(key[0..4].try_into().unwrap()), + key[4], + u32::from_be_bytes(key[key.len()-4..].try_into().unwrap()), + key[5], + String::from_utf8_lossy(&key[6..key.len()-4]), + key + ); + } + + let mut query = conn + .conn + .prepare_cached("SELECT z, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, p FROM b") + .unwrap(); + let mut rows = query.query([]).unwrap(); + + while let Some(row) = rows.next().unwrap() { + let key = row.get_ref(0).unwrap().as_bytes().unwrap(); + for bit_pos in 1..=16 { + let bit_value = row.get::<_, i64>(bit_pos).unwrap() as u64; + if bit_value != 0 { + panic!("Table bitmaps is not empty: {key:?} {bit_pos} {bit_value}"); + } + } + } + } } diff --git a/crates/store/src/backend/sqlite/write.rs b/crates/store/src/backend/sqlite/write.rs index 109840ee..71dee1c4 100644 --- a/crates/store/src/backend/sqlite/write.rs +++ b/crates/store/src/backend/sqlite/write.rs @@ -233,15 +233,13 @@ impl Store { #[cfg(feature = "test_mode")] pub async fn destroy(&self) { use crate::{ - SUBSPACE_ACLS, SUBSPACE_BITMAPS, SUBSPACE_BLOBS, SUBSPACE_INDEXES, SUBSPACE_LOGS, - SUBSPACE_VALUES, + SUBSPACE_ACLS, SUBSPACE_BITMAPS, SUBSPACE_INDEXES, SUBSPACE_LOGS, SUBSPACE_VALUES, }; let conn = self.conn_pool.get().unwrap(); for table in [ SUBSPACE_VALUES, SUBSPACE_LOGS, - SUBSPACE_BLOBS, SUBSPACE_ACLS, SUBSPACE_BITMAPS, SUBSPACE_INDEXES, diff --git a/crates/store/src/fts/mod.rs b/crates/store/src/fts/mod.rs index 78817d13..e8bdb41e 100644 --- a/crates/store/src/fts/mod.rs +++ b/crates/store/src/fts/mod.rs @@ -26,7 +26,7 @@ use crate::{ BitmapKey, Serialize, BM_HASH, }; -use self::{bloom::hash_token, builder::MAX_TOKEN_MASK}; +use self::{bloom::hash_token, builder::MAX_TOKEN_MASK, lang::LanguageDetector}; pub mod lang; //pub mod pdf; @@ -34,7 +34,7 @@ pub mod bloom; pub mod builder; pub mod ngram; pub mod query; -//pub mod search_snippet; +pub mod search_snippet; pub mod stemmer; pub mod term_index; pub mod tokenizers; @@ -209,3 +209,19 @@ impl Operation { } } } + +impl Language { + pub fn detect(text: String, default: Language) -> (String, Language) { + if let Some((l, t)) = text + .split_once(':') + .and_then(|(l, t)| (Language::from_iso_639(l)?, t).into()) + { + (t.to_string(), l) + } else { + let l = LanguageDetector::detect_single(&text) + .and_then(|(l, c)| if c > 0.3 { Some(l) } else { None }) + .unwrap_or(default); + (text, l) + } + } +} diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index ad83dc84..91ac350f 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -177,6 +177,5 @@ pub const TAG_STATIC: u8 = 1 << 1; pub const SUBSPACE_BITMAPS: u8 = b'b'; pub const SUBSPACE_VALUES: u8 = b'v'; pub const SUBSPACE_LOGS: u8 = b'l'; -pub const SUBSPACE_BLOBS: u8 = b'o'; pub const SUBSPACE_INDEXES: u8 = b'i'; pub const SUBSPACE_ACLS: u8 = b'c'; diff --git a/crates/store/src/query/log.rs b/crates/store/src/query/log.rs index 2e117597..7e1ba4e0 100644 --- a/crates/store/src/query/log.rs +++ b/crates/store/src/query/log.rs @@ -10,6 +10,7 @@ pub enum Change { Delete(u64), } +#[derive(Debug)] pub struct Changes { pub changes: Vec, pub from_change_id: u64, diff --git a/crates/store/src/query/mod.rs b/crates/store/src/query/mod.rs index 1901f841..7896c8af 100644 --- a/crates/store/src/query/mod.rs +++ b/crates/store/src/query/mod.rs @@ -5,11 +5,7 @@ pub mod sort; use roaring::RoaringBitmap; -use crate::{ - fts::{lang::LanguageDetector, Language}, - write::BitmapFamily, - BitmapKey, Serialize, Store, BM_DOCUMENT_IDS, -}; +use crate::{fts::Language, write::BitmapFamily, BitmapKey, Serialize, BM_DOCUMENT_IDS}; #[derive(Debug, Clone, Copy)] pub enum Operator { @@ -135,18 +131,7 @@ impl Filter { text: impl Into, default_language: Language, ) -> Self { - let mut text = text.into(); - let language = if let Some((l, t)) = text - .split_once(':') - .and_then(|(l, t)| (Language::from_iso_639(l)?, t.to_string()).into()) - { - text = t; - l - } else { - LanguageDetector::detect_single(&text) - .and_then(|(l, c)| if c > 0.3 { Some(l) } else { None }) - .unwrap_or(default_language) - }; + let (text, language) = Language::detect(text.into(), default_language); Self::has_text(field, text, language) } @@ -235,10 +220,3 @@ impl BitmapKey<&'static [u8]> { } } } - -#[cfg(feature = "test_mode")] -impl Store { - pub async fn assert_is_empty(&self) { - todo!() - } -} diff --git a/crates/store/src/write/batch.rs b/crates/store/src/write/batch.rs index 373ddd46..a7ebc8bb 100644 --- a/crates/store/src/write/batch.rs +++ b/crates/store/src/write/batch.rs @@ -124,8 +124,9 @@ impl BatchBuilder { self } - pub fn custom(&mut self, value: impl IntoOperations) { - value.build(self) + pub fn custom(&mut self, value: impl IntoOperations) -> &mut Self { + value.build(self); + self } pub fn build(self) -> Batch { diff --git a/crates/store/src/write/key.rs b/crates/store/src/write/key.rs index 11ed0027..cdce6ded 100644 --- a/crates/store/src/write/key.rs +++ b/crates/store/src/write/key.rs @@ -2,7 +2,8 @@ use std::convert::TryInto; use utils::codec::leb128::Leb128_; use crate::{ - AclKey, BitmapKey, IndexKey, IndexKeyPrefix, Key, LogKey, Serialize, ValueKey, SUBSPACE_LOGS, + AclKey, BitmapKey, IndexKey, IndexKeyPrefix, Key, LogKey, Serialize, ValueKey, + SUBSPACE_BITMAPS, SUBSPACE_INDEXES, SUBSPACE_LOGS, SUBSPACE_VALUES, }; pub struct KeySerializer { @@ -298,3 +299,39 @@ impl Key for LogKey { SUBSPACE_LOGS } } + +impl Key for ValueKey { + fn subspace(&self) -> u8 { + SUBSPACE_VALUES + } +} + +impl + Sync + Send + 'static> Key for IndexKey { + fn subspace(&self) -> u8 { + SUBSPACE_INDEXES + } +} + +impl + Sync + Send + 'static> Key for BitmapKey { + fn subspace(&self) -> u8 { + SUBSPACE_BITMAPS + } +} + +impl Serialize for ValueKey { + fn serialize(self) -> Vec { + (&self).serialize() + } +} + +impl> Serialize for IndexKey { + fn serialize(self) -> Vec { + (&self).serialize() + } +} + +impl> Serialize for BitmapKey { + fn serialize(self) -> Vec { + (&self).serialize() + } +} diff --git a/crates/store/src/write/log.rs b/crates/store/src/write/log.rs index e5c15069..a7a9d6bd 100644 --- a/crates/store/src/write/log.rs +++ b/crates/store/src/write/log.rs @@ -73,6 +73,31 @@ impl ChangeLogBuilder { change.inserts.insert(new_jmap_id.into()); } + pub fn with_log_insert(mut self, collection: impl Into, jmap_id: impl Into) -> Self { + self.log_insert(collection, jmap_id); + self + } + + pub fn with_log_move( + mut self, + collection: impl Into, + old_jmap_id: impl Into, + new_jmap_id: impl Into, + ) -> Self { + self.log_move(collection, old_jmap_id, new_jmap_id); + self + } + + pub fn with_log_update(mut self, collection: impl Into, jmap_id: impl Into) -> Self { + self.log_update(collection, jmap_id); + self + } + + pub fn with_log_delete(mut self, collection: impl Into, jmap_id: impl Into) -> Self { + self.log_delete(collection, jmap_id); + self + } + pub fn merge(&mut self, changes: ChangeLogBuilder) { for (collection, other) in changes.changes { let this = self.changes.get_mut_or_insert(collection); diff --git a/tests/resources/jmap_mail_parse/attachment.json b/tests/resources/jmap_mail_parse/attachment.json index 6b4670ca..f85374c1 100644 --- a/tests/resources/jmap_mail_parse/attachment.json +++ b/tests/resources/jmap_mail_parse/attachment.json @@ -20,6 +20,42 @@ "sentAt": "1998-08-13T07:42:41Z", "bodyStructure": { "headers": [ + { + "name": "Return-Path", + "value": " " + }, + { + "name": "Received", + "value": " from mailhost.whitehouse.gov ([192.168.51.200])\n by heartbeat.whitehouse.gov (8.8.8/8.8.8) with ESMTP id SAA22453\n for ;\n Mon, 13 Aug 1998 l8:14:23 +1000" + }, + { + "name": "Received", + "value": " from the_big_box.whitehouse.gov ([192.168.51.50])\n by mailhost.whitehouse.gov (8.8.8/8.8.7) with ESMTP id RAA20366\n for vice-president@whitehouse.gov; Mon, 13 Aug 1998 17:42:41 +1000" + }, + { + "name": "Date", + "value": " Mon, 13 Aug 1998 17:42:41 +1000" + }, + { + "name": "Message-ID", + "value": " <199804130742.RAA20366@mai1host.whitehouse.gov>" + }, + { + "name": "From", + "value": " Bill Clinton " + }, + { + "name": "To", + "value": " A1 (The Enforcer) Gore " + }, + { + "name": "Subject", + "value": " Map of Argentina with Description" + }, + { + "name": "MIME-Version", + "value": " 1.0" + }, { "name": "Content-Type", "value": " multipart/mixed;\n boundary=\"DC8------------DC8638F443D87A7F0726DEF7\"" diff --git a/tests/resources/jmap_mail_parse/attachment_b64.json b/tests/resources/jmap_mail_parse/attachment_b64.json index 3748c395..a65cb046 100644 --- a/tests/resources/jmap_mail_parse/attachment_b64.json +++ b/tests/resources/jmap_mail_parse/attachment_b64.json @@ -20,6 +20,30 @@ "sentAt": "2021-12-14T10:48:25Z", "bodyStructure": { "headers": [ + { + "name": "To", + "value": " \"email@example.com\" " + }, + { + "name": "From", + "value": " Name " + }, + { + "name": "Subject", + "value": " HTML test" + }, + { + "name": "Message-ID", + "value": " " + }, + { + "name": "Date", + "value": " Tue, 14 Dec 2021 11:48:25 +0100" + }, + { + "name": "MIME-Version", + "value": " 1.0" + }, { "name": "Content-Type", "value": " multipart/alternative;\r\n boundary=\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"" diff --git a/tests/src/jmap/email_changes.rs b/tests/src/jmap/email_changes.rs new file mode 100644 index 00000000..454c80aa --- /dev/null +++ b/tests/src/jmap/email_changes.rs @@ -0,0 +1,294 @@ +use std::sync::Arc; + +use jmap::JMAP; +use jmap_client::client::Client; +use jmap_proto::{ + parser::{json::Parser, JsonObjectParser}, + types::{collection::Collection, id::Id, state::State}, +}; +use store::{ + ahash::AHashSet, + write::{log::ChangeLogBuilder, BatchBuilder}, +}; + +pub async fn test(server: Arc, client: &mut Client) { + println!("Running Email Changes tests..."); + + let mut states = vec![State::Initial]; + + for (change_id, (changes, expected_changelog)) in [ + ( + vec![ + LogAction::Insert(0), + LogAction::Insert(1), + LogAction::Insert(2), + ], + vec![vec![vec![0, 1, 2], vec![], vec![]]], + ), + ( + vec![ + LogAction::Move(0, 3), + LogAction::Insert(4), + LogAction::Insert(5), + LogAction::Update(1), + LogAction::Update(2), + ], + vec![ + vec![vec![1, 2, 3, 4, 5], vec![], vec![]], + vec![vec![3, 4, 5], vec![1, 2], vec![0]], + ], + ), + ( + vec![ + LogAction::Delete(1), + LogAction::Insert(6), + LogAction::Insert(7), + LogAction::Update(2), + ], + vec![ + vec![vec![2, 3, 4, 5, 6, 7], vec![], vec![]], + vec![vec![3, 4, 5, 6, 7], vec![2], vec![0, 1]], + vec![vec![6, 7], vec![2], vec![1]], + ], + ), + ( + vec![ + LogAction::Update(4), + LogAction::Update(5), + LogAction::Update(6), + LogAction::Update(7), + ], + vec![ + vec![vec![2, 3, 4, 5, 6, 7], vec![], vec![]], + vec![vec![3, 4, 5, 6, 7], vec![2], vec![0, 1]], + vec![vec![6, 7], vec![2, 4, 5], vec![1]], + vec![vec![], vec![4, 5, 6, 7], vec![]], + ], + ), + ( + vec![ + LogAction::Delete(4), + LogAction::Delete(5), + LogAction::Delete(6), + LogAction::Delete(7), + ], + vec![ + vec![vec![2, 3], vec![], vec![]], + vec![vec![3], vec![2], vec![0, 1]], + vec![vec![], vec![2], vec![1, 4, 5]], + vec![vec![], vec![], vec![4, 5, 6, 7]], + vec![vec![], vec![], vec![4, 5, 6, 7]], + ], + ), + ( + vec![ + LogAction::Insert(8), + LogAction::Insert(9), + LogAction::Insert(10), + LogAction::Update(3), + ], + vec![ + vec![vec![2, 3, 8, 9, 10], vec![], vec![]], + vec![vec![3, 8, 9, 10], vec![2], vec![0, 1]], + vec![vec![8, 9, 10], vec![2, 3], vec![1, 4, 5]], + vec![vec![8, 9, 10], vec![3], vec![4, 5, 6, 7]], + vec![vec![8, 9, 10], vec![3], vec![4, 5, 6, 7]], + vec![vec![8, 9, 10], vec![3], vec![]], + ], + ), + ( + vec![LogAction::Update(2), LogAction::Update(8)], + vec![ + vec![vec![2, 3, 8, 9, 10], vec![], vec![]], + vec![vec![3, 8, 9, 10], vec![2], vec![0, 1]], + vec![vec![8, 9, 10], vec![2, 3], vec![1, 4, 5]], + vec![vec![8, 9, 10], vec![2, 3], vec![4, 5, 6, 7]], + vec![vec![8, 9, 10], vec![2, 3], vec![4, 5, 6, 7]], + vec![vec![8, 9, 10], vec![2, 3], vec![]], + vec![vec![], vec![2, 8], vec![]], + ], + ), + ( + vec![ + LogAction::Move(9, 11), + LogAction::Move(10, 12), + LogAction::Delete(8), + ], + vec![ + vec![vec![2, 3, 11, 12], vec![], vec![]], + vec![vec![3, 11, 12], vec![2], vec![0, 1]], + vec![vec![11, 12], vec![2, 3], vec![1, 4, 5]], + vec![vec![11, 12], vec![2, 3], vec![4, 5, 6, 7]], + vec![vec![11, 12], vec![2, 3], vec![4, 5, 6, 7]], + vec![vec![11, 12], vec![2, 3], vec![]], + vec![vec![11, 12], vec![2], vec![8, 9, 10]], + vec![vec![11, 12], vec![], vec![8, 9, 10]], + ], + ), + ] + .into_iter() + .enumerate() + { + let mut changelog = ChangeLogBuilder::with_change_id(change_id as u64); + + for change in changes { + match change { + LogAction::Insert(id) => changelog.log_insert(Collection::Email, id), + LogAction::Update(id) => changelog.log_update(Collection::Email, id), + LogAction::Delete(id) => changelog.log_delete(Collection::Email, id), + LogAction::UpdateChild(id) => changelog.log_child_update(Collection::Email, id), + LogAction::Move(old_id, new_id) => { + changelog.log_move(Collection::Email, old_id, new_id) + } + } + } + + server + .store + .write( + BatchBuilder::new() + .with_account_id(1) + .with_collection(Collection::Email) + .custom(changelog) + .build_batch(), + ) + .await + .unwrap(); + + let mut new_state = State::Initial; + for (test_num, state) in (states).iter().enumerate() { + let changes = client.email_changes(state.to_string(), None).await.unwrap(); + + assert_eq!( + expected_changelog[test_num], + [changes.created(), changes.updated(), changes.destroyed()] + .into_iter() + .map(|list| { + let mut list = list + .iter() + .map(|i| Id::from_bytes(i.as_bytes()).unwrap().into()) + .collect::>(); + list.sort_unstable(); + list + }) + .collect::>>(), + "test_num: {}, state: {:?}", + test_num, + state + ); + + if let State::Initial = state { + new_state = State::parse_str(changes.new_state()).unwrap(); + } + + for max_changes in 1..=8 { + let mut insertions = expected_changelog[test_num][0] + .iter() + .copied() + .collect::>(); + let mut updates = expected_changelog[test_num][1] + .iter() + .copied() + .collect::>(); + let mut deletions = expected_changelog[test_num][2] + .iter() + .copied() + .collect::>(); + + let mut int_state = state.clone(); + + for _ in 0..100 { + let changes = client + .email_changes(int_state.to_string(), max_changes.into()) + .await + .unwrap(); + + assert!( + changes.created().len() + + changes.updated().len() + + changes.destroyed().len() + <= max_changes, + "{} > {}", + changes.created().len() + + changes.updated().len() + + changes.destroyed().len(), + max_changes + ); + + changes.created().iter().for_each(|id| { + assert!( + insertions.remove(&Id::from_bytes(id.as_bytes()).unwrap()), + "{:?} != {}", + insertions, + Id::from_bytes(id.as_bytes()).unwrap() + ); + }); + changes.updated().iter().for_each(|id| { + assert!( + updates.remove(&Id::from_bytes(id.as_bytes()).unwrap()), + "{:?} != {}", + updates, + Id::from_bytes(id.as_bytes()).unwrap() + ); + }); + changes.destroyed().iter().for_each(|id| { + assert!( + deletions.remove(&Id::from_bytes(id.as_bytes()).unwrap()), + "{:?} != {}", + deletions, + Id::from_bytes(id.as_bytes()).unwrap() + ); + }); + + int_state = State::parse_str(changes.new_state()).unwrap(); + + if !changes.has_more_changes() { + break; + } + } + + assert_eq!(insertions.len(), 0); + assert_eq!(updates.len(), 0); + assert_eq!(deletions.len(), 0); + } + } + + states.push(new_state); + } + + let changes = client + .email_changes(State::Initial.to_string(), 0.into()) + .await + .unwrap(); + let mut created = changes + .created() + .iter() + .map(|i| Id::from_bytes(i.as_bytes()).unwrap().into()) + .collect::>(); + created.sort_unstable(); + + assert_eq!(created, vec![2, 3, 11, 12]); + assert_eq!(changes.updated(), Vec::::new()); + assert_eq!(changes.destroyed(), Vec::::new()); +} + +#[derive(Debug, Clone, Copy)] +pub enum LogAction { + Insert(u64), + Update(u64), + Delete(u64), + UpdateChild(u64), + Move(u64, u64), +} + +pub trait ParseState: Sized { + fn parse_str(state: &str) -> Option; +} + +impl ParseState for State { + fn parse_str(state: &str) -> Option { + let state = format!("{state}\""); + let mut parser = Parser::new(state.as_bytes()); + State::parse(&mut parser).ok() + } +} diff --git a/tests/src/jmap/email_parse.rs b/tests/src/jmap/email_parse.rs new file mode 100644 index 00000000..339ed103 --- /dev/null +++ b/tests/src/jmap/email_parse.rs @@ -0,0 +1,224 @@ +use std::{fs, path::PathBuf, sync::Arc}; + +use jmap::JMAP; +use jmap_client::{ + client::Client, + email::{self, Header, HeaderForm}, + mailbox::Role, +}; +use jmap_proto::types::id::Id; + +use crate::jmap::{email_get::all_headers, replace_blob_ids}; + +pub async fn test(server: Arc, client: &mut Client) { + println!("Running Email Parse tests..."); + + let mut test_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + test_dir.push("resources"); + test_dir.push("jmap_mail_parse"); + + let mailbox_id = client + .set_default_account_id(Id::new(1).to_string()) + .mailbox_create("JMAP Parse", None::, Role::None) + .await + .unwrap() + .take_id(); + + // Test parsing an email attachment + for test_name in ["attachment.eml", "attachment_b64.eml"] { + let mut test_file = test_dir.clone(); + test_file.push(test_name); + + let email = client + .email_import( + fs::read(&test_file).unwrap(), + [mailbox_id.clone()], + None::>, + None, + ) + .await + .unwrap(); + + let blob_id = client + .email_get(email.id().unwrap(), Some([email::Property::Attachments])) + .await + .unwrap() + .unwrap() + .attachments() + .unwrap() + .first() + .unwrap() + .blob_id() + .unwrap() + .to_string(); + + let email = client + .email_parse( + &blob_id, + [ + email::Property::Id, + email::Property::BlobId, + email::Property::ThreadId, + email::Property::MailboxIds, + email::Property::Keywords, + email::Property::Size, + email::Property::ReceivedAt, + email::Property::MessageId, + email::Property::InReplyTo, + email::Property::References, + email::Property::Sender, + email::Property::From, + email::Property::To, + email::Property::Cc, + email::Property::Bcc, + email::Property::ReplyTo, + email::Property::Subject, + email::Property::SentAt, + email::Property::HasAttachment, + email::Property::Preview, + email::Property::BodyValues, + email::Property::TextBody, + email::Property::HtmlBody, + email::Property::Attachments, + email::Property::BodyStructure, + ] + .into(), + [ + email::BodyProperty::PartId, + email::BodyProperty::BlobId, + email::BodyProperty::Size, + email::BodyProperty::Name, + email::BodyProperty::Type, + email::BodyProperty::Charset, + email::BodyProperty::Headers, + email::BodyProperty::Disposition, + email::BodyProperty::Cid, + email::BodyProperty::Language, + email::BodyProperty::Location, + ] + .into(), + 100.into(), + ) + .await + .unwrap(); + + if !test_name.contains("_b64") { + for parts in [ + email.text_body().unwrap(), + email.html_body().unwrap(), + email.attachments().unwrap(), + ] { + for part in parts { + let blob_id = part.blob_id().unwrap(); + + let inner_blob = client.download(blob_id).await.unwrap(); + + test_file.set_extension(format!("part{}", part.part_id().unwrap())); + + //fs::write(&test_file, inner_blob).unwrap(); + let expected_inner_blob = fs::read(&test_file).unwrap(); + + assert_eq!( + inner_blob, + expected_inner_blob, + "file: {}", + test_file.display() + ); + } + } + } + + test_file.set_extension("json"); + + let result = replace_blob_ids(serde_json::to_string_pretty(&email.into_test()).unwrap()); + + if fs::read(&test_file).unwrap() != result.as_bytes() { + test_file.set_extension("failed"); + fs::write(&test_file, result.as_bytes()).unwrap(); + panic!("Test failed, output saved to {}", test_file.display()); + } + } + + // Test header parsing on a temporary blob + let mut test_file = test_dir; + test_file.push("headers.eml"); + let blob_id = client + .upload(None, fs::read(&test_file).unwrap(), None) + .await + .unwrap() + .take_blob_id(); + + let mut email = client + .email_parse( + &blob_id, + [ + email::Property::Id, + email::Property::MessageId, + email::Property::InReplyTo, + email::Property::References, + email::Property::Sender, + email::Property::From, + email::Property::To, + email::Property::Cc, + email::Property::Bcc, + email::Property::ReplyTo, + email::Property::Subject, + email::Property::SentAt, + email::Property::Preview, + email::Property::TextBody, + email::Property::HtmlBody, + email::Property::Attachments, + ] + .into(), + [ + email::BodyProperty::Size, + email::BodyProperty::Name, + email::BodyProperty::Type, + email::BodyProperty::Charset, + email::BodyProperty::Disposition, + email::BodyProperty::Cid, + email::BodyProperty::Language, + email::BodyProperty::Location, + email::BodyProperty::Header(Header { + name: "X-Custom-Header".into(), + form: HeaderForm::Raw, + all: false, + }), + email::BodyProperty::Header(Header { + name: "X-Custom-Header-2".into(), + form: HeaderForm::Raw, + all: false, + }), + ] + .into(), + 100.into(), + ) + .await + .unwrap() + .into_test(); + + for property in all_headers() { + email.headers.extend( + client + .email_parse(&blob_id, [property].into(), [].into(), None) + .await + .unwrap() + .into_test() + .headers, + ); + } + + test_file.set_extension("json"); + + let result = replace_blob_ids(serde_json::to_string_pretty(&email).unwrap()); + + if fs::read(&test_file).unwrap() != result.as_bytes() { + test_file.set_extension("failed"); + fs::write(&test_file, result.as_bytes()).unwrap(); + panic!("Test failed, output saved to {}", test_file.display()); + } + + client.mailbox_destroy(&mailbox_id, true).await.unwrap(); + + server.store.assert_is_empty().await; +} diff --git a/tests/src/jmap/email_query_changes.rs b/tests/src/jmap/email_query_changes.rs new file mode 100644 index 00000000..7389d498 --- /dev/null +++ b/tests/src/jmap/email_query_changes.rs @@ -0,0 +1,276 @@ +use std::sync::Arc; + +use jmap::JMAP; +use jmap_client::{ + client::Client, + core::query::{Comparator, Filter}, + email, + mailbox::Role, +}; +use jmap_proto::types::{collection::Collection, id::Id, property::Property, state::State}; +use store::{ + ahash::{AHashMap, AHashSet}, + write::{log::ChangeLogBuilder, BatchBuilder, F_BITMAP, F_CLEAR, F_VALUE}, +}; + +use crate::jmap::email_changes::{LogAction, ParseState}; + +pub async fn test(server: Arc, client: &mut Client) { + println!("Running Email QueryChanges tests..."); + + let mailbox1_id = client + .set_default_account_id(Id::new(1).to_string()) + .mailbox_create("JMAP Changes 1", None::, Role::None) + .await + .unwrap() + .take_id(); + let mailbox2_id = client + .mailbox_create("JMAP Changes 2", None::, Role::None) + .await + .unwrap() + .take_id(); + + let mut states = vec![State::Initial]; + let mut id_map = AHashMap::default(); + + let mut updated_ids = AHashSet::default(); + let mut removed_ids = AHashSet::default(); + let mut type1_ids = AHashSet::default(); + + let mut thread_id = 100; + + for (change_num, change) in [ + LogAction::Insert(0), + LogAction::Insert(1), + LogAction::Insert(2), + LogAction::Move(0, 3), + LogAction::Insert(4), + LogAction::Insert(5), + LogAction::Update(1), + LogAction::Update(2), + LogAction::Delete(1), + LogAction::Insert(6), + LogAction::Insert(7), + LogAction::Update(2), + LogAction::Update(4), + LogAction::Update(5), + LogAction::Update(6), + LogAction::Update(7), + LogAction::Delete(4), + LogAction::Delete(5), + LogAction::Delete(6), + LogAction::Insert(8), + LogAction::Insert(9), + LogAction::Insert(10), + LogAction::Update(3), + LogAction::Update(2), + LogAction::Update(8), + LogAction::Move(9, 11), + LogAction::Move(10, 12), + LogAction::Delete(8), + ] + .iter() + .enumerate() + { + match &change { + LogAction::Insert(id) => { + let jmap_id = Id::from_bytes( + client + .email_import( + format!( + "From: test_{}\nSubject: test_{}\n\ntest", + if change_num % 2 == 0 { 1 } else { 2 }, + *id + ) + .into_bytes(), + [if change_num % 2 == 0 { + &mailbox1_id + } else { + &mailbox2_id + }], + [if change_num % 2 == 0 { "1" } else { "2" }].into(), + Some(*id as i64), + ) + .await + .unwrap() + .id() + .unwrap() + .as_bytes(), + ) + .unwrap(); + + id_map.insert(*id, jmap_id); + if change_num % 2 == 0 { + type1_ids.insert(jmap_id); + } + } + LogAction::Update(id) => { + let id = *id_map.get(id).unwrap(); + let mut changelog = ChangeLogBuilder::new(); + changelog.log_update(Collection::Email, id); + server.commit_changes(1, changelog).await.unwrap(); + updated_ids.insert(id); + } + LogAction::Delete(id) => { + let id = *id_map.get(id).unwrap(); + client.email_destroy(&id.to_string()).await.unwrap(); + + // Delete virtual threadId created during tests (so assert_empty_store succeeds) + server + .store + .write( + BatchBuilder::new() + .with_account_id(1) + .with_collection(Collection::Email) + .update_document(id.document_id()) + .bitmap(Property::ThreadId, id.prefix_id(), F_CLEAR) + .build_batch(), + ) + .await + .unwrap(); + removed_ids.insert(id); + } + LogAction::Move(from, to) => { + let id = *id_map.get(from).unwrap(); + let new_id = Id::from_parts(thread_id, id.document_id()); + + server + .store + .write( + BatchBuilder::new() + .with_account_id(1) + .with_collection(Collection::Email) + .update_document(id.document_id()) + .value(Property::ThreadId, id.prefix_id(), F_BITMAP | F_CLEAR) + .value(Property::ThreadId, thread_id, F_VALUE | F_BITMAP) + .custom(server.begin_changes(1).await.unwrap().with_log_move( + Collection::Email, + id, + new_id, + )) + .build_batch(), + ) + .await + .unwrap(); + + id_map.insert(*to, new_id); + if type1_ids.contains(&id) { + type1_ids.insert(new_id); + } + removed_ids.insert(id); + thread_id += 1; + } + LogAction::UpdateChild(_) => unreachable!(), + } + + let mut new_state = State::Initial; + for state in &states { + for (test_num, query) in vec![ + QueryChanges { + filter: None, + sort: vec![email::query::Comparator::received_at()], + since_query_state: state.clone(), + max_changes: 0, + up_to_id: None, + }, + QueryChanges { + filter: Some(email::query::Filter::from("test_1").into()), + sort: vec![email::query::Comparator::received_at()], + since_query_state: state.clone(), + max_changes: 0, + up_to_id: None, + }, + QueryChanges { + filter: Some(email::query::Filter::in_mailbox(&mailbox1_id).into()), + sort: vec![email::query::Comparator::received_at()], + since_query_state: state.clone(), + max_changes: 0, + up_to_id: None, + }, + QueryChanges { + filter: None, + sort: vec![email::query::Comparator::received_at()], + since_query_state: state.clone(), + max_changes: 0, + up_to_id: id_map + .get(&7) + .map(|id| id.to_string().into()) + .unwrap_or(None), + }, + ] + .into_iter() + .enumerate() + { + if test_num == 3 && query.up_to_id.is_none() { + continue; + } + let mut request = client.build(); + let query_request = request + .query_email_changes(query.since_query_state.to_string()) + .sort(query.sort); + + if let Some(filter) = query.filter { + query_request.filter(filter); + } + + if let Some(up_to_id) = query.up_to_id { + query_request.up_to_id(up_to_id); + } + + let changes = request.send_query_email_changes().await.unwrap(); + + if test_num == 0 || test_num == 1 { + // Immutable filters should not return modified ids, only deletions. + for id in changes.removed() { + let id = Id::from_bytes(id.as_bytes()).unwrap(); + assert!( + removed_ids.contains(&id), + "{:?} (id: {})", + changes, + id_map.iter().find(|(_, v)| **v == id).unwrap().0 + ); + } + } + if test_num == 1 || test_num == 2 { + // Only type 1 results should be added to the list. + for item in changes.added() { + let id = Id::from_bytes(item.id().as_bytes()).unwrap(); + assert!( + type1_ids.contains(&id), + "{:?} (id: {})", + changes, + id_map.iter().find(|(_, v)| **v == id).unwrap().0 + ); + } + } + if test_num == 3 { + // Only ids up to 7 should be added to the list. + for item in changes.added() { + let item_id = Id::from_bytes(item.id().as_bytes()).unwrap(); + let id = id_map.iter().find(|(_, v)| **v == item_id).unwrap().0; + assert!(id < &7, "{:?} (id: {})", changes, id); + } + } + + if let State::Initial = state { + new_state = State::parse_str(changes.new_query_state()).unwrap(); + } + } + } + states.push(new_state); + } + + client.mailbox_destroy(&mailbox1_id, true).await.unwrap(); + client.mailbox_destroy(&mailbox2_id, true).await.unwrap(); + + server.store.assert_is_empty().await; +} + +#[derive(Debug, Clone)] +pub struct QueryChanges { + pub filter: Option>, + pub sort: Vec>, + pub since_query_state: State, + pub max_changes: usize, + pub up_to_id: Option, +} diff --git a/tests/src/jmap/email_search_snippet.rs b/tests/src/jmap/email_search_snippet.rs new file mode 100644 index 00000000..c851d852 --- /dev/null +++ b/tests/src/jmap/email_search_snippet.rs @@ -0,0 +1,163 @@ +use std::{fs, path::PathBuf, sync::Arc}; + +use jmap::JMAP; +use jmap_client::{client::Client, core::query, email::query::Filter, mailbox::Role}; +use jmap_proto::types::id::Id; +use store::ahash::AHashMap; + +pub async fn test(server: Arc, client: &mut Client) { + println!("Running SearchSnippet tests..."); + + let mailbox_id = client + .set_default_account_id(Id::new(1).to_string()) + .mailbox_create("JMAP SearchSnippet", None::, Role::None) + .await + .unwrap() + .take_id(); + + let mut email_ids = AHashMap::default(); + + let mut test_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + test_dir.push("resources"); + test_dir.push("jmap_mail_snippet"); + + // Import test messages + for email_name in [ + "html", + "subpart", + "mixed", + "text_plain", + "text_plain_chinese", + ] { + let mut file_name = test_dir.clone(); + file_name.push(format!("{}.eml", email_name)); + let email_id = client + .email_import( + fs::read(&file_name).unwrap(), + [&mailbox_id], + None::>, + None, + ) + .await + .unwrap() + .take_id(); + email_ids.insert(email_name, email_id); + } + + // Run tests + for (filter, email_name, snippet_subject, snippet_preview) in [ + ( + query::Filter::or(vec![ + query::Filter::or(vec![Filter::subject("friend"), Filter::subject("help")]), + query::Filter::or(vec![Filter::body("secret"), Filter::body("call")]), + ]), + "text_plain", + Some("Help a friend from Abidjan Côte d'Ivoire"), + Some(concat!( + "d'Ivoire. He secretly called me on his bedside ", + "and told me that he has a sum of $7.5M (Seven Million five Hundred Thousand", + " Dollars) left in a suspense account in a local bank here in Abidjan Côte ", + "d'Ivoire, that he used my name a" + )), + ), + ( + Filter::text("côte").into(), + "text_plain", + Some("Help a friend from Abidjan Côte d'Ivoire"), + Some(concat!( + "in Abidjan Côte d'Ivoire. He secretly called me on ", + "his bedside and told me that he has a sum of $7.5M (Seven ", + "Million five Hundred Thousand Dollars) left in a suspense ", + "account in a local bank here in Abidjan Côte d'Ivoire, that " + )), + ), + ( + Filter::text("\"your country\"").into(), + "text_plain", + None, + Some(concat!( + "over to your country to further my education and ", + "to secure a residential permit for me in your country", + ". Moreover, I am willing to offer you 30 percent of the total sum as ", + "compensation for your effort inp", + )), + ), + ( + Filter::text("overseas").into(), + "text_plain", + None, + Some("nominated account overseas. "), + ), + ( + Filter::text("孫子兵法").into(), + "text_plain_chinese", + Some("孫子兵法"), + Some(concat!( + "<"孫子兵法："> ", + "孫子曰：兵者，國之大事，死生之地，存亡之道，", + "不可不察也。 孫子曰：凡用兵之法，馳車千駟" + )), + ), + ( + Filter::text("cia").into(), + "subpart", + None, + Some("shouldn't the CIA have something like that? Bill"), + ), + ( + Filter::text("frösche").into(), + "html", + Some("Die Hasen und die Frösche"), + Some(concat!( + "und die Frösche Die Hasen klagten einst über ihre mißliche Lage; ", + ""wir leben", sprach ein Redner, "in steter Furcht vor Menschen und ", + "Tieren, eine Beute der Hunde, der Adler, ja fast aller Raubtiere! ", + "Unsere stete Angst ist är")), + ), + ( + Filter::text("es:galería vasto biblioteca").into(), + "mixed", + Some("Biblioteca de Babel"), + Some(concat!( + "llaman la *Biblioteca*) se compone de un número indefinido, y tal ", + "vez infinito, de galerías hexagonales, con vastos ", + "pozos de ventilación en el medio, cercados por barandas bajísimas. Desde ", + "cualquier hexágono se " + )), + ), + ] { + let mut request = client.build(); + let result_ref = request + .query_email() + .filter(filter.clone()) + .result_reference(); + request + .get_search_snippet() + .filter(filter) + .email_ids_ref(result_ref); + let response = request + .send() + .await + .unwrap() + .unwrap_method_responses() + .pop() + .unwrap() + .unwrap_get_search_snippet() + .unwrap(); + let snippet = response + .snippet(email_ids.get(email_name).unwrap()) + .unwrap_or_else(|| panic!("No snippet for {}", email_name)); + assert_eq!(snippet_subject, snippet.subject()); + assert_eq!(snippet_preview, snippet.preview()); + assert!( + snippet.preview().map_or(0, |p| p.len()) <= 255, + "len: {}", + snippet.preview().map_or(0, |p| p.len()) + ); + } + + // Destroy test data + client.mailbox_destroy(&mailbox_id, true).await.unwrap(); + + server.store.assert_is_empty().await; +} diff --git a/tests/src/jmap/mod.rs b/tests/src/jmap/mod.rs index db51cef0..40553f57 100644 --- a/tests/src/jmap/mod.rs +++ b/tests/src/jmap/mod.rs @@ -7,8 +7,12 @@ use tokio::sync::watch; use crate::{add_test_certs, store::TempDir}; +pub mod email_changes; pub mod email_get; +pub mod email_parse; pub mod email_query; +pub mod email_query_changes; +pub mod email_search_snippet; pub mod email_set; pub mod mailbox; pub mod thread_get; @@ -38,6 +42,10 @@ blob.path = '{TMP}' [certificate.default] cert = 'file://{CERT}' private-key = 'file://{PK}' + +[jmap.protocol] +set.max-objects = 100000 + "; #[tokio::test] @@ -51,12 +59,16 @@ pub async fn jmap_tests() { let delete = true; let mut params = init_jmap_tests(delete).await; + //email_query::test(params.server.clone(), &mut params.client, delete).await; //email_get::test(params.server.clone(), &mut params.client).await; //email_set::test(params.server.clone(), &mut params.client).await; - //email_query::test(params.server.clone(), &mut params.client, delete).await; + //email_parse::test(params.server.clone(), &mut params.client).await; + //email_search_snippet::test(params.server.clone(), &mut params.client).await; + //email_changes::test(params.server.clone(), &mut params.client).await; + email_query_changes::test(params.server.clone(), &mut params.client).await; //thread_get::test(params.server.clone(), &mut params.client).await; //thread_merge::test(params.server.clone(), &mut params.client).await; - mailbox::test(params.server.clone(), &mut params.client).await; + //mailbox::test(params.server.clone(), &mut params.client).await; if delete { params.temp_dir.delete(); }