diff --git a/crates/core/src/email/body.rs b/crates/core/src/email/body.rs index cd13d46c..05797ae7 100644 --- a/crates/core/src/email/body.rs +++ b/crates/core/src/email/body.rs @@ -1,61 +1,229 @@ -use mail_builder::mime::MimePart; -use mail_parser::{MessagePart, PartType}; +use mail_parser::{HeaderValue, MessagePart, MimeHeaders, PartType}; use protocol::{ object::Object, - types::{ - blob::{BlobId, BlobSection}, - property::Property, - value::Value, - }, + types::{blob::BlobId, property::Property, value::Value}, }; -use store::BlobHash; + +use super::headers::HeaderToValue; pub trait ToBodyPart { fn to_body_part( &self, part_id: usize, properties: &[Property], - message_raw: &[u8], + raw_message: &[u8], blob_id: &BlobId, ) -> Value; } -impl ToBodyPart for MessagePart<'_> { +impl ToBodyPart for Vec> { fn to_body_part( &self, part_id: usize, properties: &[Property], - message_raw: &[u8], + raw_message: &[u8], blob_id: &BlobId, ) -> Value { - let mut values = Object::with_capacity(properties.len()); - let has_body = !matches!(self.body, PartType::Multipart(_)); + let mut parts = vec![part_id].into_iter(); + let mut parts_stack = Vec::new(); + let mut subparts = Vec::with_capacity(1); - for property in properties { - let value = match property { - Property::PartId => part_id.to_string().into(), - Property::BlobId if has_body => { - let base_offset = blob_id.start_offset(); - BlobId::new_section( - blob_id.hash, - self.offset_body + base_offset, - self.offset_end + base_offset, - self.encoding as u8, - ) - .into() + loop { + if let Some((part_id, part)) = parts.next().map(|part_id| (part_id, &self[part_id])) { + let mut values = Object::with_capacity(properties.len()); + let multipart = if let PartType::Multipart(parts) = &part.body { + parts.into() + } else { + None + }; + let mut add_subparts = false; + + for property in properties { + let value = match property { + Property::PartId => part_id.to_string().into(), + Property::BlobId if multipart.is_none() => { + let base_offset = blob_id.start_offset(); + BlobId::new_section( + blob_id.hash, + part.offset_body + base_offset, + part.offset_end + base_offset, + part.encoding as u8, + ) + .into() + } + Property::Size if multipart.is_none() => match &part.body { + PartType::Text(text) | PartType::Html(text) => text.len(), + PartType::Binary(bin) | PartType::InlineBinary(bin) => bin.len(), + PartType::Message(message) => message.root_part().raw_len(), + PartType::Multipart(_) => 0, + } + .into(), + Property::Name => part.attachment_name().into(), + Property::Type => part + .content_type() + .map(|ct| { + ct.subtype() + .map(|st| format!("{}/{}", ct.ctype(), st)) + .unwrap_or_else(|| ct.ctype().to_string()) + }) + .or_else(|| match &part.body { + PartType::Text(_) => Some("text/plain".to_string()), + PartType::Html(_) => Some("text/html".to_string()), + PartType::Message(_) => Some("message/rfc822".to_string()), + _ => None, + }) + .into(), + Property::Charset => part + .content_type() + .and_then(|ct| ct.attribute("charset")) + .or(match &part.body { + PartType::Text(_) | PartType::Html(_) => Some("utf-8"), + _ => None, + }) + .into(), + Property::Disposition => { + part.content_disposition().map(|cd| cd.ctype()).into() + } + Property::Cid => part.content_id().into(), + Property::Language => match part.content_language() { + HeaderValue::Text(text) => vec![text.to_string()].into(), + HeaderValue::TextList(list) => list + .iter() + .map(|text| text.to_string().into()) + .collect::>() + .into(), + _ => Value::Null, + }, + Property::Location => part.content_location().into(), + Property::Header(_) => part.header_to_value(property, raw_message), + Property::Headers => part.headers_to_value(raw_message), + Property::SubParts => match multipart { + Some(multipart) if !multipart.is_empty() => { + add_subparts = true; + continue; + } + _ => Vec::::new().into(), + }, + _ => Value::Null, + }; + values.append(property.clone(), value); } - Property::Size if has_body => match &self.body { - PartType::Text(text) | PartType::Html(text) => text.len(), - PartType::Binary(bin) | PartType::InlineBinary(bin) => bin.len(), - PartType::Message(message) => message.root_part().raw_len(), - PartType::Multipart(_) => 0, + + subparts.push(values); + + if add_subparts { + let multipart = multipart.unwrap().clone(); + parts_stack.push(( + parts, + std::mem::replace(&mut subparts, Vec::with_capacity(multipart.len())), + )); + parts = multipart.into_iter(); } - .into(), - _ => Value::Null, - }; - values.append(property.clone(), value); + } else if let Some((prev_parts, mut prev_subparts)) = parts_stack.pop() { + prev_subparts + .last_mut() + .unwrap() + .append(Property::SubParts, subparts); + parts = prev_parts; + subparts = prev_subparts; + } else { + return subparts.pop().map(Into::into).unwrap_or_default(); + } + } + } +} + +pub(super) trait TruncateBody { + fn truncate(&self, max_len: usize) -> (bool, String); +} + +impl TruncateBody for PartType<'_> { + fn truncate(&self, mut max_len: usize) -> (bool, String) { + match self { + PartType::Text(text) => { + if max_len != 0 && text.len() > max_len { + let add_dots = max_len > 6; + if add_dots { + max_len -= 3; + } + let mut result = String::with_capacity(max_len); + for ch in text.chars() { + if ch != '\r' { + if ch.len_utf8() + result.len() > max_len { + break; + } + result.push(ch); + } + } + if add_dots { + result.push_str("..."); + } + (true, result) + } else { + (false, text.replace('\r', "")) + } + } + PartType::Html(html) => { + if max_len != 0 && html.len() > max_len { + let add_dots = max_len > 6; + if add_dots { + max_len -= 3; + } + + let mut result = String::with_capacity(max_len); + let mut in_tag = false; + let mut in_comment = false; + let mut last_tag_end_pos = 0; + for (pos, ch) in html.char_indices() { + let mut set_last_tag = 0; + match ch { + '<' if !in_tag => { + in_tag = true; + if let Some("!--") = html.get(pos + 1..pos + 4) { + in_comment = true; + } + set_last_tag = pos; + } + '>' if in_tag => { + if in_comment { + if let Some("--") = html.get(pos - 2..pos) { + in_comment = false; + in_tag = false; + set_last_tag = pos + 1; + } + } else { + in_tag = false; + set_last_tag = pos + 1; + } + } + '\r' => continue, + _ => (), + } + if ch.len_utf8() + pos > max_len { + result.push_str( + &html[0..if (in_tag || set_last_tag > 0) && last_tag_end_pos > 0 { + last_tag_end_pos + } else { + pos + }], + ); + if add_dots { + result.push_str("..."); + } + break; + } else if set_last_tag > 0 { + last_tag_end_pos = set_last_tag; + } + } + (true, result) + } else { + (false, html.replace('\r', "")) + } + } + PartType::Binary(bytes) | PartType::InlineBinary(bytes) => { + PartType::Text(String::from_utf8_lossy(bytes)).truncate(max_len) + } + _ => (false, "".into()), } - - values.into() } } diff --git a/crates/core/src/email/get.rs b/crates/core/src/email/get.rs index 4bb1483f..d4439e15 100644 --- a/crates/core/src/email/get.rs +++ b/crates/core/src/email/get.rs @@ -3,12 +3,14 @@ use protocol::{ error::method::MethodError, method::get::GetResponse, object::{email::GetArguments, Object}, - types::{collection::Collection, id::Id, property::Property, value::Value}, + types::{blob::BlobId, collection::Collection, id::Id, property::Property, value::Value}, }; use store::ValueKey; use crate::{email::headers::HeaderToValue, JMAP}; +use super::body::{ToBodyPart, TruncateBody}; + impl JMAP { pub async fn email_get( &self, @@ -59,6 +61,11 @@ impl JMAP { Property::Location, ] }); + let fetch_text_body_values = arguments.fetch_text_body_values.unwrap_or(false); + let fetch_html_body_values = arguments.fetch_html_body_values.unwrap_or(false); + let fetch_all_body_values = arguments.fetch_all_body_values.unwrap_or(false); + let max_body_value_bytes = arguments.max_body_value_bytes.unwrap_or(0); + let mut response = GetResponse { account_id: Some(account_id.into()), state: self @@ -143,7 +150,7 @@ impl JMAP { } else { None }; - let blob_hash = blob_id.hash; + let blob_id = BlobId::new(blob_id.hash); // Prepare response let mut email = Object::with_capacity(properties.len()); @@ -156,7 +163,7 @@ impl JMAP { email.append(Property::ThreadId, id.prefix_id()); } Property::BlobId => { - email.append(Property::BlobId, blob_hash); + email.append(Property::BlobId, blob_id.clone()); } Property::MailboxIds | Property::Keywords => { email.append( @@ -205,6 +212,69 @@ impl JMAP { ); } } + Property::TextBody | Property::HtmlBody | Property::Attachments => { + if let Some(message) = &message { + let list = match property { + Property::TextBody => &message.text_body, + Property::HtmlBody => &message.html_body, + Property::Attachments => &message.attachments, + _ => unreachable!(), + } + .iter(); + email.append( + property.clone(), + list.map(|part_id| { + message.parts.to_body_part( + *part_id, + &body_properties, + &raw_message, + &blob_id, + ) + }) + .collect::>(), + ); + } + } + Property::BodyStructure => { + if let Some(message) = &message { + email.append( + Property::BodyStructure, + message.parts.to_body_part( + 0, + &body_properties, + &raw_message, + &blob_id, + ), + ); + } + } + Property::BodyValues => { + if let Some(message) = &message { + let mut body_values = Object::with_capacity(message.parts.len()); + for (part_id, part) in message.parts.iter().enumerate() { + if (message.html_body.contains(&part_id) + && (fetch_all_body_values || fetch_html_body_values)) + || (message.text_body.contains(&part_id) + && (fetch_all_body_values || fetch_text_body_values)) + { + let (is_truncated, value) = + part.body.truncate(max_body_value_bytes); + body_values.append( + Property::_T(part_id.to_string()), + Object::with_capacity(3) + .with_property( + Property::IsEncodingProblem, + part.is_encoding_problem, + ) + .with_property(Property::IsTruncated, is_truncated) + .with_property(Property::Value, value), + ); + } + } + email.append(Property::BodyValues, body_values); + } + } + _ => { return Err(MethodError::InvalidArguments(format!( "Invalid property {property:?}" diff --git a/crates/core/src/email/import.rs b/crates/core/src/email/import.rs index 9c36ab5d..86154942 100644 --- a/crates/core/src/email/import.rs +++ b/crates/core/src/email/import.rs @@ -1,3 +1,15 @@ +use protocol::{ + error::method::MethodError, + method::import::{ImportEmailRequest, ImportEmailResponse}, +}; + use crate::JMAP; -impl JMAP {} +impl JMAP { + pub async fn email_import( + &self, + request: ImportEmailRequest, + ) -> Result { + todo!() + } +} diff --git a/crates/core/src/email/mod.rs b/crates/core/src/email/mod.rs index 41991fcd..5e9f53f4 100644 --- a/crates/core/src/email/mod.rs +++ b/crates/core/src/email/mod.rs @@ -4,3 +4,4 @@ pub mod headers; pub mod import; pub mod index; pub mod ingest; +pub mod query; diff --git a/crates/core/src/email/query.rs b/crates/core/src/email/query.rs new file mode 100644 index 00000000..72f2304a --- /dev/null +++ b/crates/core/src/email/query.rs @@ -0,0 +1,348 @@ +use protocol::{ + error::method::MethodError, + method::query::{Comparator, Filter, QueryRequest, QueryResponse, SortProperty}, + object::email::QueryArguments, + types::{collection::Collection, keyword::Keyword, property::Property}, +}; +use store::{ + fts::Language, + query::{self, sort::Pagination}, + roaring::RoaringBitmap, + BitmapKey, ValueKey, +}; + +use crate::JMAP; + +impl JMAP { + pub async fn email_query( + &self, + request: QueryRequest, + arguments: QueryArguments, + ) -> Result { + let account_id = request.account_id.document_id(); + let mut filters = Vec::with_capacity(request.filter.len()); + + for cond in request.filter { + match cond { + Filter::InMailbox(mailbox) => { + filters.push(query::Filter::is_in_bitmap(Property::MailboxIds, mailbox)) + } + Filter::InMailboxOtherThan(mailboxes) => { + filters.push(query::Filter::Not); + filters.push(query::Filter::Or); + for mailbox in mailboxes { + filters.push(query::Filter::is_in_bitmap(Property::MailboxIds, mailbox)); + } + filters.push(query::Filter::End); + filters.push(query::Filter::End); + } + Filter::Before(date) => filters.push(query::Filter::lt(Property::ReceivedAt, date)), + Filter::After(date) => filters.push(query::Filter::gt(Property::ReceivedAt, date)), + Filter::MinSize(size) => filters.push(query::Filter::ge(Property::Size, size)), + Filter::MaxSize(size) => filters.push(query::Filter::lt(Property::Size, size)), + Filter::AllInThreadHaveKeyword(keyword) => filters.push(query::Filter::is_in_set( + self.thread_keywords(account_id, keyword, true).await?, + )), + Filter::SomeInThreadHaveKeyword(keyword) => filters.push(query::Filter::is_in_set( + self.thread_keywords(account_id, keyword, false).await?, + )), + Filter::NoneInThreadHaveKeyword(keyword) => { + filters.push(query::Filter::Not); + filters.push(query::Filter::is_in_set( + self.thread_keywords(account_id, keyword, false).await?, + )); + filters.push(query::Filter::End); + } + Filter::HasKeyword(keyword) => { + filters.push(query::Filter::is_in_bitmap(Property::Keywords, keyword)) + } + Filter::NotKeyword(keyword) => { + filters.push(query::Filter::Not); + filters.push(query::Filter::is_in_bitmap(Property::Keywords, keyword)); + filters.push(query::Filter::End); + } + Filter::HasAttachment(has_attach) => { + if !has_attach { + filters.push(query::Filter::Not); + } + filters.push(query::Filter::is_in_bitmap(Property::HasAttachment, ())); + if !has_attach { + filters.push(query::Filter::End); + } + } + Filter::Text(text) => { + filters.push(query::Filter::Or); + filters.push(query::Filter::has_text( + Property::From, + &text, + Language::None, + )); + filters.push(query::Filter::has_text(Property::To, &text, Language::None)); + filters.push(query::Filter::has_text(Property::Cc, &text, Language::None)); + filters.push(query::Filter::has_text( + Property::Bcc, + &text, + Language::None, + )); + filters.push(query::Filter::has_text( + Property::Subject, + &text, + Language::Unknown, + )); + filters.push(query::Filter::has_text( + Property::TextBody, + &text, + Language::Unknown, + )); + filters.push(query::Filter::has_text( + Property::Attachments, + text, + Language::Unknown, + )); + filters.push(query::Filter::End); + } + Filter::From(text) => filters.push(query::Filter::has_text( + Property::From, + text, + Language::None, + )), + Filter::To(text) => { + filters.push(query::Filter::has_text(Property::To, text, Language::None)) + } + Filter::Cc(text) => { + filters.push(query::Filter::has_text(Property::Cc, text, Language::None)) + } + Filter::Bcc(text) => { + filters.push(query::Filter::has_text(Property::Bcc, text, Language::None)) + } + Filter::Subject(text) => filters.push(query::Filter::has_text( + Property::Subject, + text, + Language::Unknown, + )), + Filter::Body(text) => filters.push(query::Filter::has_text( + Property::TextBody, + text, + Language::Unknown, + )), + Filter::Header(header) => { + return Err(MethodError::InvalidArguments(format!( + "Querying headers '{}' is not supported.", + header.join(":") + ))); + } + + // Non-standard + Filter::Id(ids) => { + let mut set = RoaringBitmap::new(); + for id in ids { + set.insert(id.document_id()); + } + filters.push(query::Filter::is_in_set(set)); + } + Filter::SentBefore(date) => filters.push(query::Filter::lt(Property::SentAt, date)), + Filter::SentAfter(date) => filters.push(query::Filter::gt(Property::SentAt, date)), + Filter::InThread(id) => { + filters.push(query::Filter::is_in_bitmap(Property::ThreadId, id)) + } + + other => return Err(MethodError::UnsupportedFilter(other.to_string())), + } + } + + let result_set = self + .store + .filter(account_id, Collection::Email, filters) + .await?; + let total = result_set.results.len() as usize; + let (limit_total, limit) = if let Some(limit) = request.limit { + if limit > 0 { + let limit = std::cmp::min(limit, self.config.query_max_results); + (std::cmp::min(limit, total), limit) + } else { + (0, 0) + } + } else { + ( + std::cmp::min(self.config.query_max_results, total), + self.config.query_max_results, + ) + }; + let mut response = QueryResponse { + account_id: request.account_id, + query_state: self + .store + .get_last_change_id(account_id, Collection::Email) + .await? + .into(), + can_calculate_changes: true, + position: 0, + ids: vec![], + total: if request.calculate_total.unwrap_or(false) { + Some(total) + } else { + None + }, + limit: if total > limit { Some(limit) } else { None }, + }; + + if limit_total > 0 { + // Parse sort criteria + let mut comparators = Vec::with_capacity(request.sort.as_ref().map_or(1, |s| s.len())); + for comparator in request + .sort + .and_then(|s| if !s.is_empty() { s.into() } else { None }) + .unwrap_or_else(|| vec![Comparator::descending(SortProperty::ReceivedAt)]) + { + comparators.push(match comparator.property { + SortProperty::ReceivedAt => { + query::Comparator::field(Property::ReceivedAt, comparator.is_ascending) + } + SortProperty::Size => { + query::Comparator::field(Property::Size, comparator.is_ascending) + } + SortProperty::From => { + query::Comparator::field(Property::From, comparator.is_ascending) + } + SortProperty::To => { + query::Comparator::field(Property::To, comparator.is_ascending) + } + SortProperty::Subject => { + query::Comparator::field(Property::Subject, comparator.is_ascending) + } + SortProperty::SentAt => { + query::Comparator::field(Property::SentAt, comparator.is_ascending) + } + SortProperty::HasKeyword => query::Comparator::set( + self.store + .get_bitmap(BitmapKey::value( + account_id, + Collection::Email, + Property::Keywords, + comparator.keyword.unwrap_or(Keyword::Seen), + )) + .await? + .unwrap_or_default(), + comparator.is_ascending, + ), + SortProperty::AllInThreadHaveKeyword => query::Comparator::set( + self.thread_keywords( + account_id, + comparator.keyword.unwrap_or(Keyword::Seen), + true, + ) + .await?, + comparator.is_ascending, + ), + SortProperty::SomeInThreadHaveKeyword => query::Comparator::set( + self.thread_keywords( + account_id, + comparator.keyword.unwrap_or(Keyword::Seen), + false, + ) + .await?, + comparator.is_ascending, + ), + // Non-standard + SortProperty::Cc => { + query::Comparator::field(Property::Cc, comparator.is_ascending) + } + + other => return Err(MethodError::UnsupportedSort(other.to_string())), + }); + } + + // Sort results + let result = self + .store + .sort( + result_set, + comparators, + Pagination::new( + limit_total, + request.position.unwrap_or(0), + request.anchor.map(|a| a.document_id()), + request.anchor_offset.unwrap_or(0), + ValueKey::new(account_id, Collection::Email, 0, Property::ThreadId).into(), + arguments.collapse_threads.unwrap_or(false), + ), + ) + .await?; + + // Prepare response + if result.found_anchor { + response.position = result.position; + response.ids = result + .ids + .into_iter() + .map(|id| id.into()) + .collect::>(); + } else { + return Err(MethodError::AnchorNotFound); + } + } + + Ok(response) + } + + async fn thread_keywords( + &self, + account_id: u32, + keyword: Keyword, + match_all: bool, + ) -> Result { + let keyword_doc_ids = self + .store + .get_bitmap(BitmapKey::value( + account_id, + Collection::Email, + Property::Keywords, + keyword, + )) + .await? + .unwrap_or_default(); + + let mut not_matched_ids = RoaringBitmap::new(); + let mut matched_ids = RoaringBitmap::new(); + + for keyword_doc_id in &keyword_doc_ids { + if matched_ids.contains(keyword_doc_id) || not_matched_ids.contains(keyword_doc_id) { + continue; + } + if let Some(thread_id) = self + .store + .get_value::(ValueKey::new( + account_id, + Collection::Email, + keyword_doc_id, + Property::ThreadId, + )) + .await? + { + if let Some(thread_doc_ids) = self + .store + .get_bitmap(BitmapKey::value( + account_id, + Collection::Email, + Property::ThreadId, + thread_id, + )) + .await? + { + let mut thread_tag_intersection = thread_doc_ids.clone(); + thread_tag_intersection &= &keyword_doc_ids; + + if (match_all && thread_tag_intersection == thread_doc_ids) + || (!match_all && !thread_tag_intersection.is_empty()) + { + matched_ids |= &thread_doc_ids; + } else if !thread_tag_intersection.is_empty() { + not_matched_ids |= &thread_tag_intersection; + } + } + } + } + + Ok(matched_ids) + } +} diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 6ed3b79b..571ad123 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -9,6 +9,7 @@ pub struct JMAP { pub struct Config { pub default_language: Language, + pub query_max_results: usize, } pub enum MaybeError { diff --git a/crates/protocol/src/lib.rs b/crates/protocol/src/lib.rs index 309f956d..b34bcb30 100644 --- a/crates/protocol/src/lib.rs +++ b/crates/protocol/src/lib.rs @@ -3,6 +3,7 @@ pub mod method; pub mod object; pub mod parser; pub mod request; +pub mod response; pub mod types; /* diff --git a/crates/protocol/src/method/changes.rs b/crates/protocol/src/method/changes.rs index 52ce3fbd..8580e1c4 100644 --- a/crates/protocol/src/method/changes.rs +++ b/crates/protocol/src/method/changes.rs @@ -35,7 +35,7 @@ pub struct ChangesResponse { #[serde(rename = "updatedProperties")] #[serde(skip_serializing_if = "Option::is_none")] - updated_properties: Option>, + pub updated_properties: Option>, } #[derive(Debug, Clone, serde::Serialize)] diff --git a/crates/protocol/src/method/import.rs b/crates/protocol/src/method/import.rs index 308baead..b0cb93f1 100644 --- a/crates/protocol/src/method/import.rs +++ b/crates/protocol/src/method/import.rs @@ -22,11 +22,11 @@ use crate::{ pub struct ImportEmailRequest { pub account_id: Id, pub if_in_state: Option, - pub emails: VecMap, + pub emails: VecMap, } #[derive(Debug, Clone)] -pub struct EmailImport { +pub struct ImportEmail { pub blob_id: BlobId, pub mailbox_ids: Option>, ResultReference>>, pub keywords: Vec, @@ -34,7 +34,7 @@ pub struct EmailImport { } #[derive(Debug, Clone, serde::Serialize)] -pub struct EmailImportResponse { +pub struct ImportEmailResponse { #[serde(rename = "accountId")] pub account_id: Id, @@ -81,7 +81,7 @@ impl JsonObjectParser for ImportEmailRequest { .unwrap_string_or_null("ifInState")?; } 0x736c_6961_6d65 if !property.is_ref => { - request.emails = >::parse(parser)?; + request.emails = >::parse(parser)?; } _ => { parser.skip_token(parser.depth_array, parser.depth_dict)?; @@ -95,12 +95,12 @@ impl JsonObjectParser for ImportEmailRequest { } } -impl JsonObjectParser for EmailImport { +impl JsonObjectParser for ImportEmail { fn parse(parser: &mut Parser<'_>) -> crate::parser::Result where Self: Sized, { - let mut request = EmailImport { + let mut request = ImportEmail { blob_id: BlobId::default(), mailbox_ids: None, keywords: vec![], diff --git a/crates/protocol/src/method/parse.rs b/crates/protocol/src/method/parse.rs index aa1959dd..0232178f 100644 --- a/crates/protocol/src/method/parse.rs +++ b/crates/protocol/src/method/parse.rs @@ -20,7 +20,7 @@ pub struct ParseEmailRequest { } #[derive(Debug, Clone, serde::Serialize)] -pub struct EmailParseResponse { +pub struct ParseEmailResponse { #[serde(rename = "accountId")] account_id: Id, diff --git a/crates/protocol/src/method/query.rs b/crates/protocol/src/method/query.rs index ba541a94..9003fb20 100644 --- a/crates/protocol/src/method/query.rs +++ b/crates/protocol/src/method/query.rs @@ -13,9 +13,9 @@ pub struct QueryRequest { pub account_id: Id, pub filter: Vec, pub sort: Option>, - pub position: Option, + pub position: Option, pub anchor: Option, - pub anchor_offset: Option, + pub anchor_offset: Option, pub limit: Option, pub calculate_total: Option, pub arguments: RequestArguments, @@ -56,8 +56,8 @@ pub enum Filter { Type(String), Timezone(String), Members(Id), - QuotaLt(u64), - QuotaGt(u64), + QuotaLt(u32), + QuotaGt(u32), IdentityIds(Vec), EmailIds(Vec), ThreadIds(Vec), @@ -66,8 +66,8 @@ pub enum Filter { After(UTCDate), InMailbox(Id), InMailboxOtherThan(Vec), - MinSize(u64), - MaxSize(u64), + MinSize(u32), + MaxSize(u32), AllInThreadHaveKeyword(Keyword), SomeInThreadHaveKeyword(Keyword), NoneInThreadHaveKeyword(Keyword), @@ -192,7 +192,7 @@ impl JsonObjectParser for QueryRequest { 0x6e6f_6974_6973_6f70 => { request.position = parser .next_token::()? - .unwrap_int_or_null("position")?; + .unwrap_ints_or_null("position")?; } 0x726f_6863_6e61 => { request.anchor = parser.next_token::()?.unwrap_string_or_null("anchor")?; @@ -200,7 +200,7 @@ impl JsonObjectParser for QueryRequest { 0x7465_7366_664f_726f_6863_6e61 => { request.anchor_offset = parser .next_token::()? - .unwrap_int_or_null("anchorOffset")? + .unwrap_ints_or_null("anchorOffset")?; } 0x7469_6d69_6c => { request.limit = parser @@ -274,13 +274,13 @@ pub fn parse_filter(parser: &mut Parser) -> crate::parser::Result> { parser .next_token::()? .unwrap_uint_or_null("quotaLowerThan")? - .unwrap_or_default(), + .unwrap_or_default() as u32, ), (0x6e61_6854_7265_7461_6572_4761_746f_7571, _) => Filter::QuotaGt( parser .next_token::()? .unwrap_uint_or_null("quotaGreaterThan")? - .unwrap_or_default(), + .unwrap_or_default() as u32, ), (0x7364_4979_7469_746e_6564_69, _) => { Filter::IdentityIds(>::parse(parser)?) @@ -308,13 +308,13 @@ pub fn parse_filter(parser: &mut Parser) -> crate::parser::Result> { parser .next_token::()? .unwrap_uint_or_null("minSize")? - .unwrap_or_default(), + .unwrap_or_default() as u32, ), (0x657a_6953_7861_6d, _) => Filter::MaxSize( parser .next_token::()? .unwrap_uint_or_null("maxSize")? - .unwrap_or_default(), + .unwrap_or_default() as u32, ), (0x4b65_7661_4864_6165_7268_546e_496c_6c61, 0x6472_6f77_7965) => { Filter::AllInThreadHaveKeyword( @@ -555,6 +555,59 @@ impl JsonObjectParser for SortProperty { } } +impl Display for Filter { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Filter::Email(_) => "email", + Filter::Name(_) => "name", + Filter::DomainName(_) => "domainName", + Filter::Text(_) => "text", + Filter::Type(_) => "type", + Filter::Timezone(_) => "timezone", + Filter::Members(_) => "members", + Filter::QuotaLt(_) => "quotaLt", + Filter::QuotaGt(_) => "quotaGt", + Filter::IdentityIds(_) => "identityIds", + Filter::EmailIds(_) => "emailIds", + Filter::ThreadIds(_) => "threadIds", + Filter::UndoStatus(_) => "undoStatus", + Filter::Before(_) => "before", + Filter::After(_) => "after", + Filter::InMailbox(_) => "inMailbox", + Filter::InMailboxOtherThan(_) => "inMailboxOtherThan", + Filter::MinSize(_) => "minSize", + Filter::MaxSize(_) => "maxSize", + Filter::AllInThreadHaveKeyword(_) => "allInThreadHaveKeyword", + Filter::SomeInThreadHaveKeyword(_) => "someInThreadHaveKeyword", + Filter::NoneInThreadHaveKeyword(_) => "noneInThreadHaveKeyword", + Filter::HasKeyword(_) => "hasKeyword", + Filter::NotKeyword(_) => "notKeyword", + Filter::HasAttachment(_) => "hasAttachment", + Filter::From(_) => "from", + Filter::To(_) => "to", + Filter::Cc(_) => "cc", + Filter::Bcc(_) => "bcc", + Filter::Subject(_) => "subject", + Filter::Body(_) => "body", + Filter::Header(_) => "header", + Filter::Id(_) => "id", + Filter::SentBefore(_) => "sentBefore", + Filter::SentAfter(_) => "sentAfter", + Filter::InThread(_) => "inThread", + Filter::ParentId(_) => "parentId", + Filter::Role(_) => "role", + Filter::HasAnyRole(_) => "hasAnyRole", + Filter::IsSubscribed(_) => "isSubscribed", + Filter::IsActive(_) => "isActive", + Filter::_T(v) => v.as_str(), + Filter::And => "and", + Filter::Or => "or", + Filter::Not => "not", + Filter::Close => "close", + }) + } +} + impl Display for SortProperty { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(match self { @@ -594,3 +647,22 @@ impl RequestPropertyParser for RequestArguments { } } } + +impl Comparator { + pub fn descending(property: SortProperty) -> Self { + Self { + property, + is_ascending: false, + collation: None, + keyword: None, + } + } + pub fn ascending(property: SortProperty) -> Self { + Self { + property, + is_ascending: true, + collation: None, + keyword: None, + } + } +} diff --git a/crates/protocol/src/method/query_changes.rs b/crates/protocol/src/method/query_changes.rs index 55149419..5bc13234 100644 --- a/crates/protocol/src/method/query_changes.rs +++ b/crates/protocol/src/method/query_changes.rs @@ -43,8 +43,8 @@ pub struct QueryChangesResponse { #[derive(Debug, Clone, serde::Serialize)] pub struct AddedItem { - id: Id, - index: usize, + pub id: Id, + pub index: usize, } impl AddedItem { diff --git a/crates/protocol/src/method/set.rs b/crates/protocol/src/method/set.rs index 120a941d..ae82e891 100644 --- a/crates/protocol/src/method/set.rs +++ b/crates/protocol/src/method/set.rs @@ -243,10 +243,7 @@ impl JsonObjectParser for Object { .unwrap_uint_or_null("")? .map(|uint| SetValue::Value(Value::UnsignedInt(uint))) .unwrap_or(SetValue::Value(Value::Null)), - Property::ParentId - | Property::EmailId - | Property::IdentityId - | Property::UndoStatus => parser + Property::ParentId | Property::EmailId | Property::IdentityId => parser .next_token::>()? .unwrap_string_or_null("")? .map(|id| match id { @@ -304,7 +301,8 @@ impl JsonObjectParser for Object { | Property::ReplyTo | Property::Sender | Property::SubParts - | Property::To => { + | Property::To + | Property::UndoStatus => { SetValue::Value(Value::parse::(parser)?) } Property::Members => { diff --git a/crates/protocol/src/object/email.rs b/crates/protocol/src/object/email.rs index 777a5c3f..914760da 100644 --- a/crates/protocol/src/object/email.rs +++ b/crates/protocol/src/object/email.rs @@ -15,7 +15,7 @@ pub struct GetArguments { #[derive(Debug, Clone, Default)] pub struct QueryArguments { - collapse_threads: Option, + pub collapse_threads: Option, } impl RequestPropertyParser for GetArguments { diff --git a/crates/protocol/src/parser/mod.rs b/crates/protocol/src/parser/mod.rs index 8ff4f6d0..77fa2818 100644 --- a/crates/protocol/src/parser/mod.rs +++ b/crates/protocol/src/parser/mod.rs @@ -100,6 +100,15 @@ impl Token { } } + pub fn unwrap_ints_or_null(self, property: &str) -> Result> { + match self { + Token::Integer(v) => Ok(Some(v as i32)), + Token::Float(v) => Ok(Some(v as i32)), + Token::Null => Ok(None), + token => Err(token.error(property, "unsigned integer")), + } + } + pub fn assert(self, token: Token) -> Result<()> { if self == token { Ok(()) diff --git a/crates/protocol/src/request/echo.rs b/crates/protocol/src/request/echo.rs index 1310866f..e18da058 100644 --- a/crates/protocol/src/request/echo.rs +++ b/crates/protocol/src/request/echo.rs @@ -3,7 +3,7 @@ use std::fmt::Write; use crate::parser::{json::Parser, JsonObjectParser, Token}; -#[derive(Debug)] +#[derive(Debug, serde::Serialize)] pub struct Echo { pub payload: Box, } diff --git a/crates/protocol/src/request/mod.rs b/crates/protocol/src/request/mod.rs index 666c81f7..8842b6da 100644 --- a/crates/protocol/src/request/mod.rs +++ b/crates/protocol/src/request/mod.rs @@ -36,7 +36,7 @@ pub struct Request { pub created_ids: Option>, } -#[derive(Debug)] +#[derive(Debug, serde::Serialize)] pub struct Call { pub id: String, pub method: T, diff --git a/crates/protocol/src/response/mod.rs b/crates/protocol/src/response/mod.rs new file mode 100644 index 00000000..bd59a65c --- /dev/null +++ b/crates/protocol/src/response/mod.rs @@ -0,0 +1,160 @@ +pub mod references; + +use ahash::AHashMap; +use serde::Serialize; + +use crate::{ + error::method::MethodError, + method::{ + ahash_is_empty, + changes::ChangesResponse, + copy::{CopyBlobResponse, CopyResponse}, + get::GetResponse, + import::ImportEmailResponse, + parse::ParseEmailResponse, + query::QueryResponse, + query_changes::QueryChangesResponse, + search_snippet::GetSearchSnippetResponse, + set::SetResponse, + validate::ValidateSieveScriptResponse, + }, + request::{echo::Echo, Call}, + types::id::Id, +}; + +#[derive(Debug, serde::Serialize)] +pub enum ResponseMethod { + Get(GetResponse), + Set(SetResponse), + Changes(ChangesResponse), + Copy(CopyResponse), + CopyBlob(CopyBlobResponse), + ImportEmail(ImportEmailResponse), + ParseEmail(ParseEmailResponse), + QueryChanges(QueryChangesResponse), + Query(QueryResponse), + SearchSnippet(GetSearchSnippetResponse), + ValidateScript(ValidateSieveScriptResponse), + Echo(Echo), + Error(MethodError), +} + +#[derive(Debug, serde::Serialize)] +pub struct Response { + #[serde(rename = "methodResponses")] + pub method_responses: Vec>, + + #[serde(rename = "sessionState")] + #[serde(serialize_with = "serialize_hex")] + pub session_state: u32, + + #[serde(rename(deserialize = "createdIds"))] + #[serde(skip_serializing_if = "ahash_is_empty")] + pub created_ids: AHashMap, +} + +impl Response { + pub fn new(session_state: u32, created_ids: AHashMap, capacity: usize) -> Self { + Response { + session_state, + created_ids, + method_responses: Vec::with_capacity(capacity), + } + } + + pub fn push_response(&mut self, id: String, method: impl Into) { + self.method_responses.push(Call { + id, + method: method.into(), + }); + } + + pub fn push_created_id(&mut self, create_id: String, id: Id) { + self.created_ids.insert(create_id, id); + } +} + +pub fn serialize_hex(value: &u32, serializer: S) -> Result +where + S: serde::Serializer, +{ + format!("{:x}", value).serialize(serializer) +} + +impl From for ResponseMethod { + fn from(error: MethodError) -> Self { + ResponseMethod::Error(error) + } +} + +impl From for ResponseMethod { + fn from(echo: Echo) -> Self { + ResponseMethod::Echo(echo) + } +} + +impl From for ResponseMethod { + fn from(get: GetResponse) -> Self { + ResponseMethod::Get(get) + } +} + +impl From for ResponseMethod { + fn from(set: SetResponse) -> Self { + ResponseMethod::Set(set) + } +} + +impl From for ResponseMethod { + fn from(changes: ChangesResponse) -> Self { + ResponseMethod::Changes(changes) + } +} + +impl From for ResponseMethod { + fn from(copy: CopyResponse) -> Self { + ResponseMethod::Copy(copy) + } +} + +impl From for ResponseMethod { + fn from(copy_blob: CopyBlobResponse) -> Self { + ResponseMethod::CopyBlob(copy_blob) + } +} + +impl From for ResponseMethod { + fn from(import_email: ImportEmailResponse) -> Self { + ResponseMethod::ImportEmail(import_email) + } +} + +impl From for ResponseMethod { + fn from(parse_email: ParseEmailResponse) -> Self { + ResponseMethod::ParseEmail(parse_email) + } +} + +impl From for ResponseMethod { + fn from(query_changes: QueryChangesResponse) -> Self { + ResponseMethod::QueryChanges(query_changes) + } +} + +impl From for ResponseMethod { + fn from(query: QueryResponse) -> Self { + ResponseMethod::Query(query) + } +} + +impl From for ResponseMethod { + fn from(search_snippet: GetSearchSnippetResponse) -> Self { + ResponseMethod::SearchSnippet(search_snippet) + } +} + +impl From for ResponseMethod { + fn from(validate_script: ValidateSieveScriptResponse) -> Self { + ResponseMethod::ValidateScript(validate_script) + } +} diff --git a/crates/protocol/src/response/references.rs b/crates/protocol/src/response/references.rs new file mode 100644 index 00000000..21c94a0b --- /dev/null +++ b/crates/protocol/src/response/references.rs @@ -0,0 +1,185 @@ +use crate::{ + error::method::MethodError, + request::{ + method::MethodFunction, + reference::{MaybeReference, ResultReference}, + Request, RequestMethod, + }, + types::{id::Id, pointer::JSONPointer, property::Property, value::Value}, +}; + +use super::{Response, ResponseMethod}; + +enum EvalResult { + Properties(Vec), + Values(Vec), + Failed, +} + +impl Response { + pub fn resolve_references(&self, request: &mut RequestMethod) -> Result<(), MethodError> { + match request { + RequestMethod::Get(request) => { + // Resolve id references + if let Some(MaybeReference::Reference(reference)) = &request.ids { + request.ids = Some(MaybeReference::Value( + self.eval_result_references(reference) + .unwrap_ids(reference)?, + )); + } + + // Resolve properties references + if let Some(MaybeReference::Reference(reference)) = &request.properties { + request.properties = Some(MaybeReference::Value( + self.eval_result_references(reference) + .unwrap_properties(reference)?, + )); + } + } + RequestMethod::Set(request) => { + // Resolve destroy references + if let Some(MaybeReference::Reference(reference)) = &request.destroy { + request.destroy = Some(MaybeReference::Value( + self.eval_result_references(reference) + .unwrap_ids(reference)?, + )); + } + } + RequestMethod::Changes(_) => todo!(), + RequestMethod::Copy(_) => todo!(), + RequestMethod::CopyBlob(_) => todo!(), + RequestMethod::ImportEmail(_) => todo!(), + RequestMethod::ParseEmail(_) => todo!(), + RequestMethod::QueryChanges(_) => todo!(), + RequestMethod::Query(_) => todo!(), + RequestMethod::SearchSnippet(request) => { + // Resolve emailIds references + if let MaybeReference::Reference(reference) = &request.email_ids { + request.email_ids = MaybeReference::Value( + self.eval_result_references(reference) + .unwrap_ids(reference)?, + ); + } + } + RequestMethod::ValidateScript(_) => todo!(), + RequestMethod::Echo(_) => todo!(), + RequestMethod::Error(_) => todo!(), + } + + Ok(()) + } + + fn eval_result_references(&self, rr: &ResultReference) -> EvalResult { + for response in &self.method_responses { + if response.id == rr.result_of { + match (&rr.name.fnc, &response.method) { + (MethodFunction::Get, ResponseMethod::Get(response)) => { + return match rr.path.item_subquery() { + Some((root, property)) if root == "list" => { + let property = Property::parse(property); + + EvalResult::Values( + response + .list + .iter() + .filter_map(|obj| obj.properties.get(&property).cloned()) + .collect(), + ) + } + _ => EvalResult::Failed, + }; + } + (MethodFunction::Changes, ResponseMethod::Changes(response)) => { + return match rr.path.item_query() { + Some("created") => EvalResult::Values( + response + .created + .clone() + .into_iter() + .map(Into::into) + .collect(), + ), + Some("updated") => EvalResult::Values( + response + .updated + .clone() + .into_iter() + .map(Into::into) + .collect(), + ), + Some("updatedProperties") => EvalResult::Properties( + response.updated_properties.clone().unwrap_or_default(), + ), + _ => EvalResult::Failed, + }; + } + (MethodFunction::Query, ResponseMethod::Query(response)) => { + return if rr.path.item_query() == Some("ids") { + EvalResult::Values( + response.ids.iter().copied().map(Into::into).collect(), + ) + } else { + EvalResult::Failed + }; + } + (MethodFunction::QueryChanges, ResponseMethod::QueryChanges(response)) => { + return if rr.path.item_subquery() == Some(("added", "id")) { + EvalResult::Values( + response.added.iter().map(|item| item.id.into()).collect(), + ) + } else { + EvalResult::Failed + }; + } + _ => (), + } + } + } + + EvalResult::Failed + } +} + +impl EvalResult { + pub fn unwrap_ids(self, rr: &ResultReference) -> Result, MethodError> { + if let EvalResult::Values(values) = self { + let mut ids = Vec::with_capacity(values.len()); + for value in values { + match value { + Value::Id(id) => ids.push(id), + Value::List(list) => { + for value in list { + if let Value::Id(id) = value { + ids.push(id); + } else { + return Err(MethodError::InvalidResultReference(format!( + "Failed to evaluate {rr} result reference." + ))); + } + } + } + _ => { + return Err(MethodError::InvalidResultReference(format!( + "Failed to evaluate {rr} result reference." + ))) + } + } + } + Ok(ids) + } else { + Err(MethodError::InvalidResultReference(format!( + "Failed to evaluate {rr} result reference." + ))) + } + } + + pub fn unwrap_properties(self, rr: &ResultReference) -> Result, MethodError> { + if let EvalResult::Properties(properties) = self { + Ok(properties) + } else { + Err(MethodError::InvalidResultReference(format!( + "Failed to evaluate {rr} result reference." + ))) + } + } +} diff --git a/crates/protocol/src/types/date.rs b/crates/protocol/src/types/date.rs index ae5120e2..f2a91949 100644 --- a/crates/protocol/src/types/date.rs +++ b/crates/protocol/src/types/date.rs @@ -23,6 +23,8 @@ use std::fmt::Display; +use store::Serialize; + use crate::parser::{json::Parser, JsonObjectParser}; #[derive(Debug, Default, Clone, PartialEq, Eq, Hash)] @@ -235,6 +237,12 @@ impl serde::Serialize for UTCDate { } } +impl Serialize for UTCDate { + fn serialize(self) -> Vec { + (self.timestamp() as u64).serialize() + } +} + #[cfg(test)] mod tests { use crate::{parser::json::Parser, types::date::UTCDate}; diff --git a/crates/protocol/src/types/id.rs b/crates/protocol/src/types/id.rs index 81382e48..231b784c 100644 --- a/crates/protocol/src/types/id.rs +++ b/crates/protocol/src/types/id.rs @@ -23,6 +23,7 @@ use std::ops::Deref; +use store::{write::IntoBitmap, Serialize, BM_TAG, TAG_ID}; use utils::codec::base32_custom::{BASE32_ALPHABET, BASE32_INVERSE}; use crate::{ @@ -236,6 +237,12 @@ impl From for String { } } +impl IntoBitmap for Id { + fn into_bitmap(self) -> (Vec, u8) { + (self.serialize(), BM_TAG | TAG_ID) + } +} + impl serde::Serialize for Id { fn serialize(&self, serializer: S) -> Result where diff --git a/crates/protocol/src/types/keyword.rs b/crates/protocol/src/types/keyword.rs index e320d2d5..0126483d 100644 --- a/crates/protocol/src/types/keyword.rs +++ b/crates/protocol/src/types/keyword.rs @@ -142,6 +142,26 @@ impl IntoBitmap for &Keyword { } } +impl IntoBitmap for Keyword { + fn into_bitmap(self) -> (Vec, u8) { + match self { + Keyword::Seen => (vec![SEEN], BM_TAG | TAG_STATIC), + Keyword::Draft => (vec![DRAFT], BM_TAG | TAG_STATIC), + Keyword::Flagged => (vec![FLAGGED], BM_TAG | TAG_STATIC), + Keyword::Answered => (vec![ANSWERED], BM_TAG | TAG_STATIC), + Keyword::Recent => (vec![RECENT], BM_TAG | TAG_STATIC), + Keyword::Important => (vec![IMPORTANT], BM_TAG | TAG_STATIC), + Keyword::Phishing => (vec![PHISHING], BM_TAG | TAG_STATIC), + Keyword::Junk => (vec![JUNK], BM_TAG | TAG_STATIC), + Keyword::NotJunk => (vec![NOTJUNK], BM_TAG | TAG_STATIC), + Keyword::Deleted => (vec![DELETED], BM_TAG | TAG_STATIC), + Keyword::Forwarded => (vec![FORWARDED], BM_TAG | TAG_STATIC), + Keyword::MdnSent => (vec![MDN_SENT], BM_TAG | TAG_STATIC), + Keyword::Other(string) => (string.into_bytes(), BM_TAG | TAG_TEXT), + } + } +} + impl SerializeValue for Keyword { fn serialize_value(self, buf: &mut Vec) { match self { diff --git a/crates/protocol/src/types/pointer.rs b/crates/protocol/src/types/pointer.rs index fda2c900..1860fd9d 100644 --- a/crates/protocol/src/types/pointer.rs +++ b/crates/protocol/src/types/pointer.rs @@ -153,19 +153,35 @@ impl JSONPointer { } } - pub fn is_item_query(&self, name: &str) -> bool { + pub fn item_query(&self) -> Option<&str> { match self { - JSONPointer::String(property) => property == name, + JSONPointer::String(property) => property.as_str().into(), JSONPointer::Path(path) if path.len() == 2 => { if let (Some(JSONPointer::String(property)), Some(JSONPointer::Wildcard)) = (path.get(0), path.get(1)) { - property == name + property.as_str().into() } else { - false + None } } - _ => false, + _ => None, + } + } + + pub fn item_subquery(&self) -> Option<(&str, &str)> { + match self { + JSONPointer::Path(path) if path.len() == 3 => { + match (path.get(0), path.get(1), path.get(2)) { + ( + Some(JSONPointer::String(root)), + Some(JSONPointer::Wildcard), + Some(JSONPointer::String(property)), + ) => Some((root.as_str(), property.as_str())), + _ => None, + } + } + _ => None, } } } diff --git a/crates/protocol/src/types/property.rs b/crates/protocol/src/types/property.rs index 2f1a6bd2..f416e869 100644 --- a/crates/protocol/src/types/property.rs +++ b/crates/protocol/src/types/property.rs @@ -151,7 +151,11 @@ impl JsonObjectParser for Property { } } - parse_property(parser, first_char, hash) + if let Some(property) = parse_property(first_char, hash) { + Ok(property) + } else { + parser.invalid_property() + } } } @@ -204,7 +208,11 @@ impl JsonObjectParser for SetProperty { } } - let mut property = parse_property(parser, first_char, hash)?; + let mut property = if let Some(property) = parse_property(first_char, hash) { + property + } else { + parser.invalid_property()? + }; let mut patch = Vec::new(); if is_patch { @@ -291,31 +299,27 @@ impl JsonObjectParser for SetProperty { } } -fn parse_property( - parser: &mut Parser, - first_char: u8, - hash: u128, -) -> crate::parser::Result { - Ok(match first_char { +fn parse_property(first_char: u8, hash: u128) -> Option { + Some(match first_char { b'a' => match hash { 0x6c63 => Property::Acl, 0x7365_7361_696c => Property::Aliases, 0x7374_6e65_6d68_6361_7474 => Property::Attachments, - _ => parser.invalid_property()?, + _ => return None, }, b'b' => match hash { 0x6363 => Property::Bcc, 0x6449_626f_6c => Property::BlobId, 0x6572_7574_6375_7274_5379_646f => Property::BodyStructure, 0x7365_756c_6156_7964_6f => Property::BodyValues, - _ => parser.invalid_property()?, + _ => return None, }, b'c' => match hash { 0x7365_6974_696c_6962_6170_61 => Property::Capabilities, 0x63 => Property::Cc, 0x7465_7372_6168 => Property::Charset, 0x6469 => Property::Cid, - _ => parser.invalid_property()?, + _ => return None, }, b'd' => match hash { 0x7375_7461_7453_7972_6576_696c_65 => Property::DeliveryStatus, @@ -323,7 +327,7 @@ fn parse_property( 0x6449_746e_6569_6c43_6563_6976_65 => Property::DeviceClientId, 0x6e6f_6974_6973_6f70_7369 => Property::Disposition, 0x7364_4962_6f6c_426e_73 => Property::DsnBlobIds, - _ => parser.invalid_property()?, + _ => return None, }, b'e' => match hash { 0x6c69_616d => Property::Email, @@ -331,19 +335,19 @@ fn parse_property( 0x7364_496c_6961_6d => Property::EmailIds, 0x6570_6f6c_6576_6e => Property::Envelope, 0x7365_7269_7078 => Property::Expires, - _ => parser.invalid_property()?, + _ => return None, }, b'f' => match hash { 0x6d6f_72 => Property::From, 0x6574_6144_6d6f_72 => Property::FromDate, - _ => parser.invalid_property()?, + _ => return None, }, b'h' => match hash { 0x746e_656d_6863_6174_7441_7361 => Property::HasAttachment, 0x7372_6564_6165 => Property::Headers, 0x7964_6f42_6c6d_74 => Property::HtmlBody, 0x6572_7574_616e_6769_536c_6d74 => Property::HtmlSignature, - _ => parser.invalid_property()?, + _ => return None, }, b'i' => match hash { 0x64 => Property::Id, @@ -352,17 +356,17 @@ fn parse_property( 0x6576_6974_6341_73 => Property::IsActive, 0x6465_6c62_616e_4573 => Property::IsEnabled, 0x6465_6269_7263_7362_7553_73 => Property::IsSubscribed, - _ => parser.invalid_property()?, + _ => return None, }, b'k' => match hash { 0x7379_65 => Property::Keys, 0x7364_726f_7779_65 => Property::Keywords, - _ => parser.invalid_property()?, + _ => return None, }, b'l' => match hash { 0x6567_6175_676e_61 => Property::Language, 0x6e6f_6974_6163_6f => Property::Location, - _ => parser.invalid_property()?, + _ => return None, }, b'm' => match hash { 0x7364_4978_6f62_6c69_61 => Property::MailboxIds, @@ -371,29 +375,29 @@ fn parse_property( 0x7372_6562_6d65 => Property::Members, 0x6449_6567_6173_7365 => Property::MessageId, 0x7374_6867_6952_79 => Property::MyRights, - _ => parser.invalid_property()?, + _ => return None, }, b'n' => match hash { 0x656d_61 => Property::Name, - _ => parser.invalid_property()?, + _ => return None, }, b'p' => match hash { 0x6449_746e_6572_61 => Property::ParentId, 0x6449_7472_61 => Property::PartId, 0x6572_7574_6369 => Property::Picture, 0x7765_6976_6572 => Property::Preview, - _ => parser.invalid_property()?, + _ => return None, }, b'q' => match hash { 0x6174_6f75 => Property::Quota, - _ => parser.invalid_property()?, + _ => return None, }, b'r' => match hash { 0x7441_6465_7669_6563_65 => Property::ReceivedAt, 0x7365_636e_6572_6566_65 => Property::References, 0x6f54_796c_7065 => Property::ReplyTo, 0x656c_6f => Property::Role, - _ => parser.invalid_property()?, + _ => return None, }, b's' => match hash { 0x7465_7263_65 => Property::Secret, @@ -404,7 +408,7 @@ fn parse_property( 0x7265_6472_4f74_726f => Property::SortOrder, 0x7463_656a_6275 => Property::Subject, 0x7374_7261_5062_7573 => Property::SubParts, - _ => parser.invalid_property()?, + _ => return None, }, b't' => match hash { 0x7964_6f42_7478_65 => Property::TextBody, @@ -417,20 +421,20 @@ fn parse_property( 0x7364_6165_7268_546c_6174_6f => Property::TotalThreads, 0x6570_79 => Property::Type, 0x7365_7079 => Property::Types, - _ => parser.invalid_property()?, + _ => return None, }, b'u' => match hash { 0x7375_7461_7453_6f64_6e => Property::UndoStatus, 0x736c_6961_6d45_6461_6572_6e => Property::UnreadEmails, 0x7364_6165_7268_5464_6165_726e => Property::UnreadThreads, 0x6c72 => Property::Url, - _ => parser.invalid_property()?, + _ => return None, }, b'v' => match hash { 0x6564_6f43_6e6f_6974_6163_6966_6972_65 => Property::VerificationCode, - _ => parser.invalid_property()?, + _ => return None, }, - _ => parser.invalid_property()?, + _ => return None, }) } @@ -637,6 +641,37 @@ impl<'x> Parser<'x> { } } +impl Property { + pub fn parse(value: &str) -> Property { + let mut first_char = 0; + let mut hash = 0; + let mut shift = 0; + + for &ch in value.as_bytes() { + if ch.is_ascii_alphabetic() { + if first_char != 0 { + if shift < 128 { + hash |= (ch as u128) << shift; + shift += 8; + } else { + return Property::_T(value.to_string()); + } + } else { + first_char = ch; + } + } else { + return Property::_T(value.to_string()); + } + } + + if let Some(property) = parse_property(first_char, hash) { + property + } else { + Property::_T(value.to_string()) + } + } +} + impl Display for Property { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { diff --git a/crates/protocol/src/types/value.rs b/crates/protocol/src/types/value.rs index e0717602..e272f64c 100644 --- a/crates/protocol/src/types/value.rs +++ b/crates/protocol/src/types/value.rs @@ -243,6 +243,12 @@ impl From for Value { } } +impl From<&str> for Value { + fn from(value: &str) -> Self { + Value::Text(value.to_string()) + } +} + impl From for Value { fn from(value: bool) -> Self { Value::Bool(value) diff --git a/crates/store/src/backend/rocksdb/read.rs b/crates/store/src/backend/rocksdb/read.rs index 125650f4..f9d3861d 100644 --- a/crates/store/src/backend/rocksdb/read.rs +++ b/crates/store/src/backend/rocksdb/read.rs @@ -60,20 +60,6 @@ impl Store { Ok(results) } - pub fn get_document_ids( - &self, - account_id: u32, - collection: u8, - ) -> crate::Result> { - self.get_bitmap(BitmapKey { - account_id, - collection, - family: BM_DOCUMENT_IDS, - field: u8::MAX, - key: b"", - }) - } - #[inline(always)] pub fn get_bitmap>( &self, diff --git a/crates/store/src/backend/sqlite/id_assign.rs b/crates/store/src/backend/sqlite/id_assign.rs index 8ccb723f..73a7a209 100644 --- a/crates/store/src/backend/sqlite/id_assign.rs +++ b/crates/store/src/backend/sqlite/id_assign.rs @@ -118,7 +118,7 @@ impl Store { // Obtain used ids let used_ids = - conn.get_bitmap(BitmapKey::new_document_ids(key.account_id, key.collection))?; + conn.get_bitmap(BitmapKey::document_ids(key.account_id, key.collection))?; let next_change_id = conn .get_last_change_id(key.account_id, key.collection)? .map(|id| id + 1) diff --git a/crates/store/src/fts/mod.rs b/crates/store/src/fts/mod.rs index 6ffa9312..e0be97b1 100644 --- a/crates/store/src/fts/mod.rs +++ b/crates/store/src/fts/mod.rs @@ -21,7 +21,10 @@ * for more details. */ -use crate::{write::Operation, BitmapKey, BM_HASH}; +use crate::{ + write::{IntoBitmap, Operation}, + BitmapKey, BM_HASH, +}; use self::{bloom::hash_token, builder::MAX_TOKEN_MASK}; @@ -178,6 +181,23 @@ impl BitmapKey> { key: hash_token(word), } } + + pub fn value( + account_id: u32, + collection: impl Into, + field: impl Into, + value: impl IntoBitmap, + ) -> Self { + let (key, family) = value.into_bitmap(); + BitmapKey { + account_id, + collection: collection.into(), + family, + field: field.into(), + block_num: 0, + key, + } + } } impl Operation { diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index e2335437..48a2dc04 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -9,6 +9,7 @@ pub mod query; pub mod write; pub use ahash; +pub use roaring; #[cfg(feature = "rocks")] pub struct Store { diff --git a/crates/store/src/query/filter.rs b/crates/store/src/query/filter.rs index 5c3049f1..5f044423 100644 --- a/crates/store/src/query/filter.rs +++ b/crates/store/src/query/filter.rs @@ -5,10 +5,10 @@ use roaring::RoaringBitmap; use crate::{ fts::{builder::MAX_TOKEN_LENGTH, tokenizers::space::SpaceTokenizer}, - BitmapKey, ReadTransaction, Store, BM_KEYWORD, + BitmapKey, ReadTransaction, Store, }; -use super::{Filter, ResultSet}; +use super::{Filter, ResultSet, TextMatch}; struct State { op: Filter, @@ -30,7 +30,7 @@ impl ReadTransaction<'_> { account_id, collection, results: self - .get_bitmap(BitmapKey::new_document_ids(account_id, collection)) + .get_bitmap(BitmapKey::document_ids(account_id, collection)) .await? .unwrap_or_else(RoaringBitmap::new), }); @@ -44,40 +44,32 @@ impl ReadTransaction<'_> { self.refresh_if_old().await?; let result = match filter { - Filter::HasKeyword { field, value } => { - self.get_bitmap(BitmapKey { - account_id, - collection, - family: BM_KEYWORD, - field, - key: value.as_bytes(), - block_num: 0, - }) - .await? - } - Filter::HasKeywords { field, value } => { - self.get_bitmaps_intersection( - SpaceTokenizer::new(&value, MAX_TOKEN_LENGTH) - .collect::>() - .into_iter() - .map(|word| BitmapKey::hash(&word, account_id, collection, 0, field)) - .collect(), - ) - .await? - } Filter::MatchValue { field, op, value } => { self.range_to_bitmap(account_id, collection, field, value, op) .await? } - Filter::HasText { - field, - text, - language, - match_phrase, - } => { - self.fts_query(account_id, collection, field, &text, language, match_phrase) + Filter::HasText { field, text, op } => match op { + TextMatch::Exact(language) => { + self.fts_query(account_id, collection, field, &text, language, true) + .await? + } + TextMatch::Stemmed(language) => { + self.fts_query(account_id, collection, field, &text, language, false) + .await? + } + TextMatch::Tokenized => { + self.get_bitmaps_intersection( + SpaceTokenizer::new(&text, MAX_TOKEN_LENGTH) + .collect::>() + .into_iter() + .map(|word| { + BitmapKey::hash(&word, account_id, collection, 0, field) + }) + .collect(), + ) .await? - } + } + }, Filter::InBitmap { family, field, key } => { self.get_bitmap(BitmapKey { account_id, @@ -108,7 +100,7 @@ impl ReadTransaction<'_> { if matches!(state.op, Filter::Not) && !not_fetch { not_mask = self - .get_bitmap(BitmapKey::new_document_ids(account_id, collection)) + .get_bitmap(BitmapKey::document_ids(account_id, collection)) .await? .unwrap_or_else(RoaringBitmap::new); not_fetch = true; diff --git a/crates/store/src/query/get.rs b/crates/store/src/query/get.rs index cab11bc2..6ccc296b 100644 --- a/crates/store/src/query/get.rs +++ b/crates/store/src/query/get.rs @@ -1,4 +1,6 @@ -use crate::{Deserialize, Key, Store, ValueKey}; +use roaring::RoaringBitmap; + +use crate::{BitmapKey, Deserialize, Key, Store, ValueKey}; impl Store { pub async fn get_value(&self, key: ValueKey) -> crate::Result> @@ -72,6 +74,22 @@ impl Store { } } + pub async fn get_bitmap + Send + Sync + 'static>( + &self, + key: BitmapKey, + ) -> crate::Result> { + #[cfg(feature = "is_async")] + { + self.read_transaction().await?.get_bitmap(key).await + } + + #[cfg(feature = "is_sync")] + { + let trx = self.read_transaction()?; + self.spawn_worker(move || trx.get_bitmap(key)).await + } + } + pub async fn iterate( &self, acc: T, diff --git a/crates/store/src/query/mod.rs b/crates/store/src/query/mod.rs index b6a7ef65..ed8e68d2 100644 --- a/crates/store/src/query/mod.rs +++ b/crates/store/src/query/mod.rs @@ -7,7 +7,8 @@ use roaring::RoaringBitmap; use crate::{ fts::{lang::LanguageDetector, Language}, - BitmapKey, Serialize, BM_DOCUMENT_IDS, + write::IntoBitmap, + BitmapKey, Serialize, BM_DOCUMENT_IDS, BM_KEYWORD, }; #[derive(Debug, Clone, Copy)] @@ -21,14 +22,6 @@ pub enum Operator { #[derive(Debug)] pub enum Filter { - HasKeyword { - field: u8, - value: String, - }, - HasKeywords { - field: u8, - value: String, - }, MatchValue { field: u8, op: Operator, @@ -37,8 +30,7 @@ pub enum Filter { HasText { field: u8, text: String, - language: Language, - match_phrase: bool, + op: TextMatch, }, InBitmap { family: u8, @@ -52,6 +44,13 @@ pub enum Filter { End, } +#[derive(Debug)] +pub enum TextMatch { + Exact(Language), + Stemmed(Language), + Tokenized, +} + #[derive(Debug)] pub enum Comparator { Field { field: u8, ascending: bool }, @@ -65,9 +64,9 @@ pub struct ResultSet { pub results: RoaringBitmap, } -pub struct SortedResultRet { +pub struct SortedResultSet { pub position: i32, - pub ids: Vec, + pub ids: Vec, pub found_anchor: bool, } @@ -120,57 +119,80 @@ impl Filter { } } - pub fn has_keyword(field: impl Into, value: impl Into) -> Self { - Filter::HasKeyword { + pub fn has_keyword(field: impl Into, value: impl Serialize) -> Self { + Filter::InBitmap { + family: BM_KEYWORD, field: field.into(), - value: value.into(), + key: value.serialize(), } } - pub fn has_keywords(field: impl Into, value: impl Into) -> Self { - Filter::HasKeywords { - field: field.into(), - value: value.into(), - } - } - - pub fn match_text( - field: impl Into, - text: impl Into, - mut language: Language, - ) -> Self { + pub fn has_text(field: impl Into, text: impl Into, mut language: Language) -> Self { let mut text = text.into(); - let match_phrase = (text.starts_with('"') && text.ends_with('"')) - || (text.starts_with('\'') && text.ends_with('\'')); + let op = if !matches!(language, Language::None) { + let match_phrase = (text.starts_with('"') && text.ends_with('"')) + || (text.starts_with('\'') && text.ends_with('\'')); - if !match_phrase && language == Language::Unknown { - language = if let Some((l, t)) = text - .split_once(':') - .and_then(|(l, t)| (Language::from_iso_639(l)?, t.to_string()).into()) - { - text = t; - l + if !match_phrase && language == Language::Unknown { + language = if let Some((l, t)) = text + .split_once(':') + .and_then(|(l, t)| (Language::from_iso_639(l)?, t.to_string()).into()) + { + text = t; + l + } else { + LanguageDetector::detect_single(&text) + .and_then(|(l, c)| if c > 0.3 { Some(l) } else { None }) + .unwrap_or(Language::Unknown) + }; + } + + if match_phrase { + TextMatch::Exact(language) } else { - LanguageDetector::detect_single(&text) - .and_then(|(l, c)| if c > 0.3 { Some(l) } else { None }) - .unwrap_or(Language::Unknown) - }; - } + TextMatch::Stemmed(language) + } + } else { + TextMatch::Tokenized + }; Filter::HasText { field: field.into(), text, - language, - match_phrase, + op, } } - pub fn match_english(field: impl Into, text: impl Into) -> Self { - Self::match_text(field, text, Language::English) + pub fn has_english_text(field: impl Into, text: impl Into) -> Self { + Self::has_text(field, text, Language::English) + } + + pub fn is_in_bitmap(field: impl Into, value: impl IntoBitmap) -> Self { + let (key, family) = value.into_bitmap(); + Self::InBitmap { + family, + field: field.into(), + key, + } + } + + pub fn is_in_set(set: RoaringBitmap) -> Self { + Filter::DocumentSet(set) } } impl Comparator { + pub fn field(field: impl Into, ascending: bool) -> Self { + Self::Field { + field: field.into(), + ascending, + } + } + + pub fn set(set: RoaringBitmap, ascending: bool) -> Self { + Self::DocumentSet { set, ascending } + } + pub fn ascending(field: impl Into) -> Self { Self::Field { field: field.into(), @@ -187,10 +209,10 @@ impl Comparator { } impl BitmapKey<&'static [u8]> { - pub fn new_document_ids(account_id: u32, collection: u8) -> Self { + pub fn document_ids(account_id: u32, collection: impl Into) -> Self { BitmapKey { account_id, - collection, + collection: collection.into(), family: BM_DOCUMENT_IDS, field: u8::MAX, key: b"", diff --git a/crates/store/src/query/sort.rs b/crates/store/src/query/sort.rs index 2eb13cd0..c3e4dbb8 100644 --- a/crates/store/src/query/sort.rs +++ b/crates/store/src/query/sort.rs @@ -1,8 +1,8 @@ -use ahash::AHashMap; +use ahash::{AHashMap, AHashSet}; -use crate::{ReadTransaction, Store}; +use crate::{ReadTransaction, Store, ValueKey}; -use super::{Comparator, ResultSet, SortedResultRet}; +use super::{Comparator, ResultSet, SortedResultSet}; pub struct Pagination { requested_position: i32, @@ -12,7 +12,9 @@ pub struct Pagination { anchor_offset: i32, has_anchor: bool, anchor_found: bool, - ids: Vec, + ids: Vec, + prefix_key: Option, + prefix_unique: bool, } impl ReadTransaction<'_> { @@ -21,14 +23,9 @@ impl ReadTransaction<'_> { &mut self, result_set: ResultSet, mut comparators: Vec, - limit: usize, - position: i32, - anchor: Option, - anchor_offset: i32, - ) -> crate::Result { - let mut paginate = Pagination::new(limit, position, anchor, anchor_offset); - - if comparators.len() == 1 { + mut paginate: Pagination, + ) -> crate::Result { + if comparators.len() == 1 && !paginate.prefix_unique { match comparators.pop().unwrap() { Comparator::Field { field, ascending } => { let mut results = result_set.results; @@ -38,14 +35,16 @@ impl ReadTransaction<'_> { result_set.collection, field, ascending, - |_, document_id| !results.remove(document_id) || paginate.add(document_id), + |_, document_id| { + !results.remove(document_id) || paginate.add(0, document_id) + }, ) .await?; // Add remaining items not present in the index if !results.is_empty() && !paginate.is_full() { for document_id in results { - if !paginate.add(document_id) { + if !paginate.add(0, document_id) { break; } } @@ -61,13 +60,28 @@ impl ReadTransaction<'_> { }; 'outer: for set in sets { for document_id in set { - if !paginate.add(document_id) { + if !paginate.add(0, document_id) { break 'outer; } } } } } + + // Obtain prefixes + let prefix_key = paginate.prefix_key.take(); + let mut sorted_results = paginate.build(); + if let Some(prefix_key) = prefix_key { + for id in sorted_results.ids.iter_mut() { + if let Some(prefix_id) = + self.get_value::(prefix_key.with_document_id(*id as u32))? + { + *id |= (prefix_id as u64) << 32; + } + } + } + + Ok(sorted_results) } else { let mut sorted_ids = AHashMap::with_capacity(paginate.limit); @@ -138,16 +152,35 @@ impl ReadTransaction<'_> { } } + let mut seen_prefixes = AHashSet::new(); let mut sorted_ids = sorted_ids.into_iter().collect::>(); sorted_ids.sort_by(|a, b| a.1.cmp(&b.1)); for (document_id, _) in sorted_ids { - if !paginate.add(document_id) { + // Obtain document prefixId + let prefix_id = if let Some(prefix_key) = &paginate.prefix_key { + if let Some(prefix_id) = + self.get_value(prefix_key.with_document_id(document_id))? + { + if paginate.prefix_unique && !seen_prefixes.insert(prefix_id) { + continue; + } + prefix_id + } else { + // Document no longer exists? + continue; + } + } else { + 0 + }; + + // Add document to results + if !paginate.add(prefix_id, document_id) { break; } } - } - Ok(paginate.build()) + Ok(paginate.build()) + } } } @@ -156,15 +189,12 @@ impl Store { &self, result_set: ResultSet, comparators: Vec, - limit: usize, - position: i32, - anchor: Option, - anchor_offset: i32, - ) -> crate::Result { - let limit = match (result_set.results.len(), limit) { + mut paginate: Pagination, + ) -> crate::Result { + paginate.limit = match (result_set.results.len(), paginate.limit) { (0, _) => { - return Ok(SortedResultRet { - position, + return Ok(SortedResultSet { + position: paginate.position, ids: vec![], found_anchor: true, }); @@ -177,37 +207,28 @@ impl Store { { self.read_transaction() .await? - .sort( - result_set, - comparators, - limit, - position, - anchor, - anchor_offset, - ) + .sort(result_set, comparators, paginate) .await } #[cfg(feature = "is_sync")] { let mut trx = self.read_transaction()?; - self.spawn_worker(move || { - trx.sort( - result_set, - comparators, - limit, - position, - anchor, - anchor_offset, - ) - }) - .await + self.spawn_worker(move || trx.sort(result_set, comparators, paginate)) + .await } } } impl Pagination { - pub fn new(limit: usize, position: i32, anchor: Option, anchor_offset: i32) -> Self { + pub fn new( + limit: usize, + position: i32, + anchor: Option, + anchor_offset: i32, + prefix_key: Option, + prefix_unique: bool, + ) -> Self { let (has_anchor, anchor) = anchor.map(|anchor| (true, anchor)).unwrap_or((false, 0)); Self { @@ -219,16 +240,20 @@ impl Pagination { has_anchor, anchor_found: false, ids: Vec::with_capacity(limit), + prefix_key, + prefix_unique, } } - pub fn add(&mut self, document_id: u32) -> bool { + pub fn add(&mut self, prefix_id: u32, document_id: u32) -> bool { + let id = ((prefix_id as u64) << 32) | document_id as u64; + // Pagination if !self.has_anchor { if self.position > 0 { self.position -= 1; } else { - self.ids.push(document_id); + self.ids.push(id); if self.ids.len() == self.limit { return false; } @@ -244,14 +269,14 @@ impl Pagination { if self.anchor_offset > 0 { self.anchor_offset -= 1; } else { - self.ids.push(document_id); + self.ids.push(id); if self.ids.len() == self.limit { return false; } } } else { self.anchor_found = document_id == self.anchor; - self.ids.push(document_id); + self.ids.push(id); if self.anchor_found { self.position = self.anchor_offset; @@ -266,8 +291,8 @@ impl Pagination { self.ids.len() == self.limit } - pub fn build(self) -> SortedResultRet { - let mut result = SortedResultRet { + pub fn build(self) -> SortedResultSet { + let mut result = SortedResultSet { ids: self.ids, position: 0, found_anchor: !self.has_anchor || self.anchor_found, diff --git a/crates/store/src/write/key.rs b/crates/store/src/write/key.rs index 6bb5e1af..9d36e75e 100644 --- a/crates/store/src/write/key.rs +++ b/crates/store/src/write/key.rs @@ -137,4 +137,11 @@ impl ValueKey { field: field.into(), } } + + pub fn with_document_id(self, document_id: u32) -> Self { + Self { + document_id, + ..self + } + } }