diff --git a/.gitignore b/.gitignore index 5b70a898..d936efaa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /target /Cargo.lock .vscode +*.failed diff --git a/crates/jmap-proto/src/method/changes.rs b/crates/jmap-proto/src/method/changes.rs index 8580e1c4..80f5f872 100644 --- a/crates/jmap-proto/src/method/changes.rs +++ b/crates/jmap-proto/src/method/changes.rs @@ -75,9 +75,8 @@ impl JsonObjectParser for ChangesRequest { .next_token::()? .assert_jmap(Token::DictStart)?; - while { - let property = parser.next_dict_key::()?; - match &property.hash[0] { + while let Some(key) = parser.next_dict_key::()? { + match &key.hash[0] { 0x6449_746e_756f_6363_61 => { request.account_id = parser.next_token::()?.unwrap_string("accountId")?; } @@ -96,9 +95,7 @@ impl JsonObjectParser for ChangesRequest { parser.skip_token(parser.depth_array, parser.depth_dict)?; } } - - !parser.is_dict_end()? - } {} + } Ok(request) } diff --git a/crates/jmap-proto/src/method/copy.rs b/crates/jmap-proto/src/method/copy.rs index d103c8d3..edc75160 100644 --- a/crates/jmap-proto/src/method/copy.rs +++ b/crates/jmap-proto/src/method/copy.rs @@ -106,9 +106,8 @@ impl JsonObjectParser for CopyRequest { .next_token::()? .assert_jmap(Token::DictStart)?; - while { - let property = parser.next_dict_key::()?; - match &property.hash[0] { + while let Some(key) = parser.next_dict_key::()? { + match &key.hash[0] { 0x6449_746e_756f_6363_61 => { request.account_id = parser.next_token::()?.unwrap_string("accountId")?; } @@ -144,9 +143,7 @@ impl JsonObjectParser for CopyRequest { parser.skip_token(parser.depth_array, parser.depth_dict)?; } } - - !parser.is_dict_end()? - } {} + } Ok(request) } @@ -167,9 +164,8 @@ impl JsonObjectParser for CopyBlobRequest { .next_token::()? .assert_jmap(Token::DictStart)?; - while { - let property = parser.next_dict_key::()?; - match &property.hash[0] { + while let Some(key) = parser.next_dict_key::()? { + match &key.hash[0] { 0x6449_746e_756f_6363_61 => { request.account_id = parser.next_token::()?.unwrap_string("accountId")?; } @@ -186,9 +182,7 @@ impl JsonObjectParser for CopyBlobRequest { parser.skip_token(parser.depth_array, parser.depth_dict)?; } } - - !parser.is_dict_end()? - } {} + } Ok(request) } diff --git a/crates/jmap-proto/src/method/get.rs b/crates/jmap-proto/src/method/get.rs index ed3c4b41..ab79e392 100644 --- a/crates/jmap-proto/src/method/get.rs +++ b/crates/jmap-proto/src/method/get.rs @@ -77,35 +77,32 @@ impl JsonObjectParser for GetRequest { .next_token::()? .assert_jmap(Token::DictStart)?; - while { - let property = parser.next_dict_key::()?; - match &property.hash[0] { - 0x6449_746e_756f_6363_61 if !property.is_ref => { + while let Some(key) = parser.next_dict_key::()? { + match &key.hash[0] { + 0x6449_746e_756f_6363_61 if !key.is_ref => { request.account_id = parser.next_token::()?.unwrap_string("accountId")?; } 0x7364_69 => { - request.ids = if !property.is_ref { + request.ids = if !key.is_ref { >>::parse(parser)?.map(MaybeReference::Value) } else { Some(MaybeReference::Reference(ResultReference::parse(parser)?)) }; } 0x7365_6974_7265_706f_7270 => { - request.properties = if !property.is_ref { + request.properties = if !key.is_ref { >>::parse(parser)?.map(MaybeReference::Value) } else { Some(MaybeReference::Reference(ResultReference::parse(parser)?)) }; } _ => { - if !request.arguments.parse(parser, property)? { + if !request.arguments.parse(parser, key)? { parser.skip_token(parser.depth_array, parser.depth_dict)?; } } } - - !parser.is_dict_end()? - } {} + } Ok(request) } @@ -139,3 +136,30 @@ impl GetRequest { } } } + +impl GetRequest { + pub fn unwrap_properties(&mut self) -> Option> { + let mut properties = self.properties.take()?.unwrap(); + // Add Id Property + if !properties.contains(&Property::Id) { + properties.push(Property::Id); + } + Some(properties) + } + + pub fn unwrap_ids( + &mut self, + max_objects_in_get: usize, + ) -> Result>, MethodError> { + if let Some(ids) = self.ids.take() { + let ids = ids.unwrap(); + if ids.len() <= max_objects_in_get { + Ok(Some(ids)) + } else { + Err(MethodError::RequestTooLarge) + } + } else { + Ok(None) + } + } +} diff --git a/crates/jmap-proto/src/method/import.rs b/crates/jmap-proto/src/method/import.rs index 3e7af966..f9cc99d9 100644 --- a/crates/jmap-proto/src/method/import.rs +++ b/crates/jmap-proto/src/method/import.rs @@ -69,27 +69,24 @@ impl JsonObjectParser for ImportEmailRequest { .next_token::()? .assert_jmap(Token::DictStart)?; - while { - let property = parser.next_dict_key::()?; - match &property.hash[0] { - 0x6449_746e_756f_6363_61 if !property.is_ref => { + while let Some(key) = parser.next_dict_key::()? { + match &key.hash[0] { + 0x6449_746e_756f_6363_61 if !key.is_ref => { request.account_id = parser.next_token::()?.unwrap_string("accountId")?; } - 0x6574_6174_536e_4966_69 if !property.is_ref => { + 0x6574_6174_536e_4966_69 if !key.is_ref => { request.if_in_state = parser .next_token::()? .unwrap_string_or_null("ifInState")?; } - 0x736c_6961_6d65 if !property.is_ref => { + 0x736c_6961_6d65 if !key.is_ref => { request.emails = >::parse(parser)?; } _ => { parser.skip_token(parser.depth_array, parser.depth_dict)?; } } - - !parser.is_dict_end()? - } {} + } Ok(request) } @@ -111,14 +108,13 @@ impl JsonObjectParser for ImportEmail { .next_token::()? .assert_jmap(Token::DictStart)?; - while { - let property = parser.next_dict_key::()?; - match &property.hash[0] { - 0x6449_626f_6c62 if !property.is_ref => { + while let Some(key) = parser.next_dict_key::()? { + match &key.hash[0] { + 0x6449_626f_6c62 if !key.is_ref => { request.blob_id = parser.next_token::()?.unwrap_string("blobId")?; } 0x7364_4978_6f62_6c69_616d => { - request.mailbox_ids = if !property.is_ref { + request.mailbox_ids = if !key.is_ref { MaybeReference::Value( >>::parse(parser)?.values, ) @@ -126,10 +122,10 @@ impl JsonObjectParser for ImportEmail { MaybeReference::Reference(ResultReference::parse(parser)?) }; } - 0x7364_726f_7779_656b if !property.is_ref => { + 0x7364_726f_7779_656b if !key.is_ref => { request.keywords = >::parse(parser)?.values; } - 0x7441_6465_7669_6563_6572 if !property.is_ref => { + 0x7441_6465_7669_6563_6572 if !key.is_ref => { request.received_at = parser .next_token::()? .unwrap_string_or_null("receivedAt")?; @@ -138,9 +134,7 @@ impl JsonObjectParser for ImportEmail { parser.skip_token(parser.depth_array, parser.depth_dict)?; } } - - !parser.is_dict_end()? - } {} + } Ok(request) } diff --git a/crates/jmap-proto/src/method/parse.rs b/crates/jmap-proto/src/method/parse.rs index 0232178f..7d12d667 100644 --- a/crates/jmap-proto/src/method/parse.rs +++ b/crates/jmap-proto/src/method/parse.rs @@ -57,10 +57,9 @@ impl JsonObjectParser for ParseEmailRequest { .next_token::()? .assert_jmap(Token::DictStart)?; - while { - let property = parser.next_dict_key::()?; - match (&property.hash[0], &property.hash[1]) { - (0x6449_746e_756f_6363_61, _) if !property.is_ref => { + while let Some(key) = parser.next_dict_key::()? { + match (&key.hash[0], &key.hash[1]) { + (0x6449_746e_756f_6363_61, _) if !key.is_ref => { request.account_id = parser.next_token::()?.unwrap_string("accountId")?; } (0x7364_4962_6f6c_62, _) => { @@ -96,9 +95,7 @@ impl JsonObjectParser for ParseEmailRequest { parser.skip_token(parser.depth_array, parser.depth_dict)?; } } - - !parser.is_dict_end()? - } {} + } Ok(request) } diff --git a/crates/jmap-proto/src/method/query.rs b/crates/jmap-proto/src/method/query.rs index 8f1c0b59..1fd097b5 100644 --- a/crates/jmap-proto/src/method/query.rs +++ b/crates/jmap-proto/src/method/query.rs @@ -171,9 +171,8 @@ impl JsonObjectParser for QueryRequest { .next_token::()? .assert_jmap(Token::DictStart)?; - while { - let property = parser.next_dict_key::()?; - match &property.hash[0] { + while let Some(key) = parser.next_dict_key::()? { + match &key.hash[0] { 0x6449_746e_756f_6363_61 => { request.account_id = parser.next_token::()?.unwrap_string("accountId")?; } @@ -214,14 +213,12 @@ impl JsonObjectParser for QueryRequest { } _ => { - if !request.arguments.parse(parser, property)? { + if !request.arguments.parse(parser, key)? { parser.skip_token(parser.depth_array, parser.depth_dict)?; } } } - - !parser.is_dict_end()? - } {} + } Ok(request) } @@ -463,8 +460,8 @@ impl JsonObjectParser for Comparator { .next_token::()? .assert_jmap(Token::DictStart)?; - while { - match parser.next_dict_key::()? { + while let Some(key) = parser.next_dict_key::()? { + match key { 0x676e_6964_6e65_6373_4173_69 => { comp.is_ascending = parser .next_token::()? @@ -490,9 +487,7 @@ impl JsonObjectParser for Comparator { parser.skip_token(parser.depth_array, parser.depth_dict)?; } } - - !parser.is_dict_end()? - } {} + } Ok(comp) } diff --git a/crates/jmap-proto/src/method/query_changes.rs b/crates/jmap-proto/src/method/query_changes.rs index 5bc13234..9c7576e9 100644 --- a/crates/jmap-proto/src/method/query_changes.rs +++ b/crates/jmap-proto/src/method/query_changes.rs @@ -83,9 +83,8 @@ impl JsonObjectParser for QueryChangesRequest { .next_token::()? .assert_jmap(Token::DictStart)?; - while { - let property = parser.next_dict_key::()?; - match &property.hash[0] { + while let Some(key) = parser.next_dict_key::()? { + match &key.hash[0] { 0x6449_746e_756f_6363_61 => { request.account_id = parser.next_token::()?.unwrap_string("accountId")?; } @@ -122,14 +121,12 @@ impl JsonObjectParser for QueryChangesRequest { } _ => { - if !request.arguments.parse(parser, property)? { + if !request.arguments.parse(parser, key)? { parser.skip_token(parser.depth_array, parser.depth_dict)?; } } } - - !parser.is_dict_end()? - } {} + } Ok(request) } diff --git a/crates/jmap-proto/src/method/search_snippet.rs b/crates/jmap-proto/src/method/search_snippet.rs index 91bba07c..6fbec0fe 100644 --- a/crates/jmap-proto/src/method/search_snippet.rs +++ b/crates/jmap-proto/src/method/search_snippet.rs @@ -56,13 +56,12 @@ impl JsonObjectParser for GetSearchSnippetRequest { .next_token::()? .assert_jmap(Token::DictStart)?; - while { - let property = parser.next_dict_key::()?; - match &property.hash[0] { - 0x6449_746e_756f_6363_61 if !property.is_ref => { + while let Some(key) = parser.next_dict_key::()? { + match &key.hash[0] { + 0x6449_746e_756f_6363_61 if !key.is_ref => { request.account_id = parser.next_token::()?.unwrap_string("accountId")?; } - 0x7265_746c_6966 if !property.is_ref => match parser.next_token::()? { + 0x7265_746c_6966 if !key.is_ref => match parser.next_token::()? { Token::DictStart => { request.filter = parse_filter(parser)?; } @@ -72,7 +71,7 @@ impl JsonObjectParser for GetSearchSnippetRequest { } }, 0x7364_496c_6961_6d65 => { - request.email_ids = if !property.is_ref { + request.email_ids = if !key.is_ref { MaybeReference::Value(>::parse(parser)?) } else { MaybeReference::Reference(ResultReference::parse(parser)?) @@ -82,9 +81,7 @@ impl JsonObjectParser for GetSearchSnippetRequest { parser.skip_token(parser.depth_array, parser.depth_dict)?; } } - - !parser.is_dict_end()? - } {} + } Ok(request) } diff --git a/crates/jmap-proto/src/method/set.rs b/crates/jmap-proto/src/method/set.rs index 40c25a8a..226545c0 100644 --- a/crates/jmap-proto/src/method/set.rs +++ b/crates/jmap-proto/src/method/set.rs @@ -121,39 +121,36 @@ impl JsonObjectParser for SetRequest { .next_token::()? .assert_jmap(Token::DictStart)?; - while { - let property = parser.next_dict_key::()?; - match &property.hash[0] { - 0x6449_746e_756f_6363_61 if !property.is_ref => { + while let Some(key) = parser.next_dict_key::()? { + match &key.hash[0] { + 0x6449_746e_756f_6363_61 if !key.is_ref => { request.account_id = parser.next_token::()?.unwrap_string("accountId")?; } - 0x6574_6165_7263 if !property.is_ref => { + 0x6574_6165_7263 if !key.is_ref => { request.create = >>>::parse(parser)?; } - 0x6574_6164_7075 if !property.is_ref => { + 0x6574_6164_7075 if !key.is_ref => { request.update = >>>::parse(parser)?; } 0x0079_6f72_7473_6564 => { - request.destroy = if !property.is_ref { + request.destroy = if !key.is_ref { >>::parse(parser)?.map(MaybeReference::Value) } else { Some(MaybeReference::Reference(ResultReference::parse(parser)?)) }; } - 0x6574_6174_536e_4966_69 if !property.is_ref => { + 0x6574_6174_536e_4966_69 if !key.is_ref => { request.if_in_state = parser .next_token::()? .unwrap_string_or_null("ifInState")?; } _ => { - if !request.arguments.parse(parser, property)? { + if !request.arguments.parse(parser, key)? { parser.skip_token(parser.depth_array, parser.depth_dict)?; } } } - - !parser.is_dict_end()? - } {} + } Ok(request) } @@ -172,10 +169,9 @@ impl JsonObjectParser for Object { .next_token::()? .assert_jmap(Token::DictStart)?; - while { - let mut property = parser.next_dict_key::()?; - let value = if !property.is_ref { - match &property.property { + while let Some(mut key) = parser.next_dict_key::()? { + let value = if !key.is_ref { + match &key.property { Property::Id | Property::ThreadId => parser .next_token::()? .unwrap_string_or_null("")? @@ -221,7 +217,10 @@ impl JsonObjectParser for Object { .unwrap_or(SetValue::Value(Value::Null)), Property::TextBody | Property::HtmlBody => { if let MethodObject::Email = &parser.ctx { - SetValue::Value(Value::parse::(parser)?) + SetValue::Value(Value::parse::( + parser.next_token()?, + parser, + )?) } else { parser .next_token::()? @@ -249,17 +248,17 @@ impl JsonObjectParser for Object { .map(SetValue::IdReference) .unwrap_or(SetValue::Value(Value::Null)), Property::MailboxIds => { - if property.patch.is_empty() { + if key.patch.is_empty() { SetValue::IdReferences( >>::parse(parser)?.values, ) } else { - property.patch.push(Value::Bool(bool::parse(parser)?)); - SetValue::Patch(property.patch) + key.patch.push(Value::Bool(bool::parse(parser)?)); + SetValue::Patch(key.patch) } } Property::Keywords => { - if property.patch.is_empty() { + if key.patch.is_empty() { SetValue::Value(Value::List( >::parse(parser)? .values @@ -268,12 +267,14 @@ impl JsonObjectParser for Object { .collect(), )) } else { - property.patch.push(Value::Bool(bool::parse(parser)?)); - SetValue::Patch(property.patch) + key.patch.push(Value::Bool(bool::parse(parser)?)); + SetValue::Patch(key.patch) } } - Property::Acl => SetValue::Value(Value::parse::(parser)?), + Property::Acl => { + SetValue::Value(Value::parse::(parser.next_token()?, parser)?) + } Property::Aliases | Property::Attachments | Property::Bcc @@ -293,19 +294,24 @@ impl JsonObjectParser for Object { | Property::SubParts | Property::To | Property::UndoStatus => { - SetValue::Value(Value::parse::(parser)?) - } - Property::Members => { - SetValue::Value(Value::parse::(parser)?) + SetValue::Value(Value::parse::( + parser.next_token()?, + parser, + )?) } + Property::Members => SetValue::Value(Value::parse::( + parser.next_token()?, + parser, + )?), Property::Header(h) => SetValue::Value(if matches!(h.form, HeaderForm::Date) { - Value::parse::(parser) + Value::parse::(parser.next_token()?, parser) } else { - Value::parse::(parser) + Value::parse::(parser.next_token()?, parser) }?), - Property::Types => { - SetValue::Value(Value::parse::(parser)?) - } + Property::Types => SetValue::Value(Value::parse::( + parser.next_token()?, + parser, + )?), _ => { parser.skip_token(parser.depth_array, parser.depth_dict)?; SetValue::Value(Value::Null) @@ -315,10 +321,8 @@ impl JsonObjectParser for Object { SetValue::ResultReference(ResultReference::parse(parser)?) }; - obj.properties.append(property.property, value); - - !parser.is_dict_end()? - } {} + obj.properties.append(key.property, value); + } Ok(obj) } diff --git a/crates/jmap-proto/src/method/validate.rs b/crates/jmap-proto/src/method/validate.rs index 880fb7a4..892d68c5 100644 --- a/crates/jmap-proto/src/method/validate.rs +++ b/crates/jmap-proto/src/method/validate.rs @@ -34,22 +34,19 @@ impl JsonObjectParser for ValidateSieveScriptRequest { .next_token::()? .assert_jmap(Token::DictStart)?; - while { - let property = parser.next_dict_key::()?; - match &property.hash[0] { - 0x6449_746e_756f_6363_61 if !property.is_ref => { + while let Some(key) = parser.next_dict_key::()? { + match &key.hash[0] { + 0x6449_746e_756f_6363_61 if !key.is_ref => { request.account_id = parser.next_token::()?.unwrap_string("accountId")?; } - 0x6449_626f_6c62 if !property.is_ref => { + 0x6449_626f_6c62 if !key.is_ref => { request.blob_id = parser.next_token::()?.unwrap_string("blobId")?; } _ => { parser.skip_token(parser.depth_array, parser.depth_dict)?; } } - - !parser.is_dict_end()? - } {} + } Ok(request) } diff --git a/crates/jmap-proto/src/parser/impls.rs b/crates/jmap-proto/src/parser/impls.rs index 86558669..4e10e4ea 100644 --- a/crates/jmap-proto/src/parser/impls.rs +++ b/crates/jmap-proto/src/parser/impls.rs @@ -175,12 +175,14 @@ impl JsonObjectParser for Vec { let mut vec = Vec::new(); parser.next_token::()?.assert(Token::ArrayStart)?; - while { - vec.push(parser.next_token::()?.unwrap_string("")?); - - !parser.is_array_end()? - } {} - + loop { + match parser.next_token::()? { + Token::String(item) => vec.push(item), + Token::Comma => (), + Token::ArrayEnd => break, + token => return Err(token.error("", &token.to_string())), + } + } Ok(vec) } } @@ -193,12 +195,14 @@ impl JsonObjectParser for Option> { match parser.next_token::()? { Token::ArrayStart => { let mut vec = Vec::new(); - while { - vec.push(parser.next_token::()?.unwrap_string("")?); - - !parser.is_array_end()? - } {} - + loop { + match parser.next_token::()? { + Token::String(item) => vec.push(item), + Token::Comma => (), + Token::ArrayEnd => break, + token => return Err(token.error("", &token.to_string())), + } + } Ok(Some(vec)) } Token::Null => Ok(None), @@ -215,10 +219,9 @@ impl JsonObjectParser f let mut map = VecMap::new(); parser.next_token::()?.assert(Token::DictStart)?; - while { - map.append(parser.next_dict_key()?, V::parse(parser)?); - !parser.is_dict_end()? - } {} + while let Some(key) = parser.next_dict_key()? { + map.append(key, V::parse(parser)?); + } Ok(map) } @@ -235,10 +238,9 @@ impl JsonObjectParser Token::DictStart => { let mut map = VecMap::new(); - while { - map.append(parser.next_dict_key()?, V::parse(parser)?); - !parser.is_dict_end()? - } {} + while let Some(key) = parser.next_dict_key()? { + map.append(key, V::parse(parser)?); + } Ok(Some(map)) } diff --git a/crates/jmap-proto/src/parser/json.rs b/crates/jmap-proto/src/parser/json.rs index cd9ffa67..a6edfe33 100644 --- a/crates/jmap-proto/src/parser/json.rs +++ b/crates/jmap-proto/src/parser/json.rs @@ -252,15 +252,25 @@ impl<'x> Parser<'x> { Err(self.error("Unexpected EOF")) } - pub fn next_dict_key(&mut self) -> super::Result { - self.next_token::().and_then(|k| { - let k = k.unwrap_string("")?; - self.next_token::()?.assert(Token::Colon)?; - Ok(k) - }) + pub fn next_dict_key( + &mut self, + ) -> super::Result> { + loop { + match self.next_token::()? { + Token::String(k) => { + self.next_token::()?.assert(Token::Colon)?; + return Ok(Some(k)); + } + Token::Comma => (), + Token::DictEnd => return Ok(None), + token => { + return Err(self.error(&format!("Expected object property, found {}", token))) + } + } + } } - pub fn is_dict_end(&mut self) -> super::Result { + /*pub fn is_dict_end(&mut self) -> super::Result { match self.next_token::()? { Token::Comma => Ok(false), Token::DictEnd => Ok(true), @@ -274,7 +284,7 @@ impl<'x> Parser<'x> { Token::ArrayEnd => Ok(true), token => Err(self.error(&format!("Expected ',' or ']', found {}", token))), } - } + }*/ pub fn skip_token( &mut self, diff --git a/crates/jmap-proto/src/request/mod.rs b/crates/jmap-proto/src/request/mod.rs index fb7d9d34..b2f66b25 100644 --- a/crates/jmap-proto/src/request/mod.rs +++ b/crates/jmap-proto/src/request/mod.rs @@ -77,16 +77,15 @@ impl JsonObjectParser for RequestProperty { 'outer: for hash in hash.iter_mut() { while let Some(ch) = parser.next_unescaped()? { - if shift < 128 { - if ch != b'#' || parser.pos > parser.pos_marker + 1 { - *hash |= (ch as u128) << shift; - shift += 8; - } else { - is_ref = true; + if ch != b'#' || parser.pos > parser.pos_marker + 1 { + *hash |= (ch as u128) << shift; + shift += 8; + if shift == 128 { + shift = 0; + continue 'outer; } } else { - shift = 0; - continue 'outer; + is_ref = true; } } break; diff --git a/crates/jmap-proto/src/request/parser.rs b/crates/jmap-proto/src/request/parser.rs index 0ee0ac18..acfe17fc 100644 --- a/crates/jmap-proto/src/request/parser.rs +++ b/crates/jmap-proto/src/request/parser.rs @@ -38,16 +38,25 @@ impl Request { let mut found_valid_keys = false; let mut parser = Parser::new(json); parser.next_token::()?.assert(Token::DictStart)?; - while { - match parser.next_dict_key::()? { + while let Some(key) = parser.next_dict_key::()? { + match key { 0x676e_6973_75 => { found_valid_keys = true; parser.next_token::()?.assert(Token::ArrayStart)?; - while { - request.using |= - parser.next_token::()?.unwrap_string("using")? as u32; - !parser.is_array_end()? - } {} + loop { + match parser.next_token::()? { + Token::String(capability) => { + request.using |= capability as u32; + } + Token::Comma => (), + Token::ArrayEnd => break, + token => { + return Err(token + .error("capability", &token.to_string()) + .into()) + } + } + } } 0x736c_6c61_4364_6f68_7465_6d => { found_valid_keys = true; @@ -55,11 +64,16 @@ impl Request { parser .next_token::()? .assert_jmap(Token::ArrayStart)?; - while { + loop { + match parser.next_token::()? { + Token::ArrayStart => (), + Token::Comma => continue, + Token::ArrayEnd => break, + token => { + return Err(RequestError::not_request("Invalid JMAP request")); + } + }; if request.method_calls.len() < max_calls { - parser - .next_token::()? - .assert_jmap(Token::ArrayStart)?; let method_name = match parser.next_token::() { Ok(Token::String(method)) => method, Ok(_) => { @@ -148,29 +162,25 @@ impl Request { } else { return Err(RequestError::limit(RequestLimitError::CallsIn)); } - !parser.is_array_end()? - } {} + } } 0x7364_4964_6574_6165_7263 => { found_valid_keys = true; let mut created_ids = HashMap::new(); parser.next_token::()?.assert(Token::DictStart)?; - while { + while let Some(key) = parser.next_dict_key::()? { created_ids.insert( - parser.next_dict_key::()?, + key, parser.next_token::()?.unwrap_string("createdIds")?, ); - !parser.is_dict_end()? - } {} + } request.created_ids = Some(created_ids); } _ => { parser.skip_token(parser.depth_array, parser.depth_dict)?; } } - - !parser.is_dict_end()? - } {} + } if found_valid_keys { Ok(request) diff --git a/crates/jmap-proto/src/request/reference.rs b/crates/jmap-proto/src/request/reference.rs index 95e16c40..390b02da 100644 --- a/crates/jmap-proto/src/request/reference.rs +++ b/crates/jmap-proto/src/request/reference.rs @@ -44,8 +44,8 @@ impl JsonObjectParser for ResultReference { .next_token::()? .assert_jmap(Token::DictStart)?; - while { - match parser.next_dict_key::()? { + while let Some(key) = parser.next_dict_key::()? { + match key { 0x664f_746c_7573_6572 => { result_of = Some(parser.next_token::()?.unwrap_string("resultOf")?); } @@ -59,9 +59,7 @@ impl JsonObjectParser for ResultReference { parser.skip_token(parser.depth_array, parser.depth_dict)?; } } - - !parser.is_dict_end()? - } {} + } if let (Some(result_of), Some(name), Some(path)) = (result_of, name, path) { Ok(Self { diff --git a/crates/jmap-proto/src/types/value.rs b/crates/jmap-proto/src/types/value.rs index 09f2becf..e3690068 100644 --- a/crates/jmap-proto/src/types/value.rs +++ b/crates/jmap-proto/src/types/value.rs @@ -58,26 +58,31 @@ pub trait IntoValue: Eq { impl Value { pub fn parse( + token: Token, parser: &mut Parser<'_>, ) -> crate::parser::Result { - Ok(match parser.next_token::()? { + Ok(match token { Token::String(v) => v.into_value(), Token::DictStart => { let mut properties = Object::with_capacity(4); - while { - let property = parser.next_dict_key::()?.into_property(); + while let Some(key) = parser.next_dict_key::()? { + let property = key.into_property(); let value = Value::from_property(parser, &property)?; properties.append(property, value); - !parser.is_dict_end()? - } {} + } Value::Object(properties) } Token::ArrayStart => { let mut values = Vec::with_capacity(4); - while { - values.push(Value::parse::(parser)?); - !parser.is_array_end()? - } {} + loop { + match parser.next_token::()? { + Token::Comma => (), + Token::ArrayEnd => break, + token => { + values.push(Value::parse::(token, parser)?); + } + } + } Value::List(values) } Token::Integer(v) => Value::UnsignedInt(std::cmp::max(v, 0) as u64), @@ -124,9 +129,9 @@ impl Value { Property::Header(h) => { if matches!(h.form, HeaderForm::Date) { - Value::parse::(parser) + Value::parse::(parser.next_token()?, parser) } else { - Value::parse::(parser) + Value::parse::(parser.next_token()?, parser) } } @@ -134,8 +139,12 @@ impl Value { | Property::Addresses | Property::MailFrom | Property::RcptTo - | Property::SubParts => Value::parse::(parser), - Property::Language | Property::Parameters => Value::parse::(parser), + | Property::SubParts => { + Value::parse::(parser.next_token()?, parser) + } + Property::Language | Property::Parameters => { + Value::parse::(parser.next_token()?, parser) + } Property::IsEncodingProblem | Property::IsTruncated @@ -152,7 +161,7 @@ impl Value { .unwrap_bool_or_null("")? .map(Value::Bool) .unwrap_or(Value::Null)), - _ => Value::parse::(parser), + _ => Value::parse::(parser.next_token()?, parser), } } } @@ -165,13 +174,11 @@ impl JsonObjectParser for SetValueMap { let mut values = Vec::new(); match parser.next_token::()? { Token::DictStart => { - while { - let value = parser.next_dict_key::()?; + while let Some(value) = parser.next_dict_key()? { if bool::parse(parser)? { values.push(value); } - !parser.is_dict_end()? - } {} + } } Token::Null => (), token => return Err(token.error("", &token.to_string())), @@ -334,7 +341,22 @@ impl From> for Value { Value::Object( Object::with_capacity(2) .with_property(Property::Name, group.name) - .with_property(Property::Addresses, group.addresses), + .with_property( + Property::Addresses, + Value::List( + group + .addresses + .into_iter() + .filter_map(|addr| { + if addr.address.as_ref()?.contains('@') { + Some(addr.into()) + } else { + None + } + }) + .collect::>(), + ), + ), ) } } diff --git a/crates/jmap/src/api/request.rs b/crates/jmap/src/api/request.rs index c0283f26..eccbc3ab 100644 --- a/crates/jmap/src/api/request.rs +++ b/crates/jmap/src/api/request.rs @@ -9,6 +9,7 @@ use crate::JMAP; impl JMAP { pub async fn handle_request(&self, bytes: &[u8]) -> Result { + println!("<- {}", String::from_utf8_lossy(bytes)); let request = Request::parse( bytes, self.config.request_max_calls, @@ -32,7 +33,7 @@ impl JMAP { self.email_get(call.with_arguments(arguments)).await.into() } get::RequestArguments::Mailbox => todo!(), - get::RequestArguments::Thread => todo!(), + get::RequestArguments::Thread => self.thread_get(call).await.into(), get::RequestArguments::Identity => todo!(), get::RequestArguments::EmailSubmission => todo!(), get::RequestArguments::PushSubscription => todo!(), diff --git a/crates/jmap/src/email/body.rs b/crates/jmap/src/email/body.rs index b86fe659..13de771f 100644 --- a/crates/jmap/src/email/body.rs +++ b/crates/jmap/src/email/body.rs @@ -36,11 +36,10 @@ impl ToBodyPart for Vec> { } else { None }; - let mut add_subparts = false; for property in properties { let value = match property { - Property::PartId => part_id.to_string().into(), + Property::PartId if multipart.is_none() => part_id.to_string().into(), Property::BlobId if multipart.is_none() => { let base_offset = blob_id.start_offset(); BlobId::new_section( @@ -77,7 +76,7 @@ impl ToBodyPart for Vec> { .content_type() .and_then(|ct| ct.attribute("charset")) .or(match &part.body { - PartType::Text(_) | PartType::Html(_) => Some("utf-8"), + PartType::Text(_) | PartType::Html(_) => Some("us-ascii"), _ => None, }) .into(), @@ -97,13 +96,7 @@ impl ToBodyPart for Vec> { Property::Location => part.content_location().into(), Property::Header(_) => part.header_to_value(property, raw_message), Property::Headers => part.headers_to_value(raw_message), - Property::SubParts => match multipart { - Some(multipart) if !multipart.is_empty() => { - add_subparts = true; - continue; - } - _ => Vec::::new().into(), - }, + Property::SubParts => continue, _ => Value::Null, }; values.append(property.clone(), value); @@ -111,8 +104,8 @@ impl ToBodyPart for Vec> { subparts.push(values); - if add_subparts { - let multipart = multipart.unwrap().clone(); + if let Some(multipart) = multipart { + let multipart = multipart.clone(); parts_stack.push(( parts, std::mem::replace(&mut subparts, Vec::with_capacity(multipart.len())), diff --git a/crates/jmap/src/email/get.rs b/crates/jmap/src/email/get.rs index 7a41c384..6835dc99 100644 --- a/crates/jmap/src/email/get.rs +++ b/crates/jmap/src/email/get.rs @@ -16,9 +16,10 @@ use super::body::{ToBodyPart, TruncateBody}; impl JMAP { pub async fn email_get( &self, - request: GetRequest, + mut request: GetRequest, ) -> Result { - let properties = request.properties.map(|v| v.unwrap()).unwrap_or_else(|| { + let ids = request.unwrap_ids(self.config.get_max_objects)?; + let properties = request.unwrap_properties().unwrap_or_else(|| { vec![ Property::Id, Property::BlobId, @@ -65,13 +66,29 @@ impl JMAP { let fetch_all_body_values = request.arguments.fetch_all_body_values.unwrap_or(false); let max_body_value_bytes = request.arguments.max_body_value_bytes.unwrap_or(0); - let ids = if let Some(ids) = request.ids.map(|v| v.unwrap()) { + let account_id = request.account_id.document_id(); + let ids = if let Some(ids) = ids { ids } else { - let implement = ""; - todo!() + let document_ids = self + .get_document_ids(account_id, Collection::Email) + .await? + .unwrap_or_default() + .into_iter() + .take(self.config.get_max_objects) + .collect::>(); + self.get_properties::( + account_id, + Collection::Email, + &document_ids, + Property::ThreadId, + ) + .await? + .into_iter() + .zip(document_ids) + .filter_map(|(thread_id, document_id)| Id::from_parts(thread_id?, document_id).into()) + .collect() }; - let account_id = request.account_id.document_id(); let mut response = GetResponse { account_id: Some(request.account_id), state: self.get_state(account_id, Collection::Email).await?, diff --git a/crates/jmap/src/email/headers.rs b/crates/jmap/src/email/headers.rs index 03334d4f..c7fc30c9 100644 --- a/crates/jmap/src/email/headers.rs +++ b/crates/jmap/src/email/headers.rs @@ -177,7 +177,13 @@ impl IntoForm for HeaderValue<'_> { grouplist .into_iter() .flat_map(|group| group.addresses) - .map(Into::into) + .filter_map(|addr| { + if addr.address.as_ref()?.contains('@') { + Some(addr.into()) + } else { + None + } + }) .collect(), ), (HeaderValue::Address(addr), HeaderForm::GroupedAddresses) => { diff --git a/crates/jmap/src/lib.rs b/crates/jmap/src/lib.rs index 462ae60c..53657e07 100644 --- a/crates/jmap/src/lib.rs +++ b/crates/jmap/src/lib.rs @@ -13,6 +13,7 @@ pub mod api; pub mod blob; pub mod changes; pub mod email; +pub mod thread; pub struct JMAP { pub store: Store, @@ -62,11 +63,12 @@ impl JMAP { account_id: u32, collection: Collection, document_id: u32, - property: &Property, + property: impl AsRef, ) -> Result, MethodError> where U: Deserialize + 'static, { + let property = property.as_ref(); match self .store .get_value::(ValueKey::new(account_id, collection, document_id, property)) @@ -87,6 +89,44 @@ impl JMAP { } } + pub async fn get_properties( + &self, + account_id: u32, + collection: Collection, + document_ids: &[u32], + property: impl AsRef, + ) -> Result>, MethodError> + where + U: Deserialize + 'static, + { + let property = property.as_ref(); + match self + .store + .get_values::( + document_ids + .iter() + .map(|document_id| { + ValueKey::new(account_id, collection, *document_id, property) + }) + .collect(), + ) + .await + { + Ok(value) => Ok(value), + Err(err) => { + tracing::error!(event = "error", + context = "store", + account_id = account_id, + collection = ?collection, + document_ids = ?document_ids, + property = ?property, + error = ?err, + "Failed to retrieve properties"); + Err(MethodError::ServerPartialFail) + } + } + } + pub async fn get_document_ids( &self, account_id: u32, diff --git a/crates/jmap/src/thread/get.rs b/crates/jmap/src/thread/get.rs new file mode 100644 index 00000000..db1777e8 --- /dev/null +++ b/crates/jmap/src/thread/get.rs @@ -0,0 +1,85 @@ +use jmap_proto::{ + error::method::MethodError, + method::get::{GetRequest, GetResponse, RequestArguments}, + object::Object, + types::{collection::Collection, id::Id, property::Property}, +}; +use store::query::{sort::Pagination, Comparator, ResultSet}; + +use crate::JMAP; + +impl JMAP { + pub async fn thread_get( + &self, + mut request: GetRequest, + ) -> Result { + let account_id = request.account_id.document_id(); + let ids = if let Some(ids) = request.unwrap_ids(self.config.get_max_objects)? { + ids + } else { + self.get_document_ids(account_id, Collection::Thread) + .await? + .unwrap_or_default() + .into_iter() + .take(self.config.get_max_objects) + .map(Into::into) + .collect() + }; + let add_email_ids = request + .unwrap_properties() + .map_or(true, |properties| properties.contains(&Property::EmailIds)); + let mut response = GetResponse { + account_id: Some(request.account_id), + state: self.get_state(account_id, Collection::Thread).await?, + list: Vec::with_capacity(ids.len()), + not_found: vec![], + }; + + for id in ids { + let thread_id = id.document_id(); + if let Some(document_ids) = self + .get_tag(account_id, Collection::Email, Property::ThreadId, thread_id) + .await? + { + let mut thread = Object::with_capacity(2).with_property(Property::Id, id); + if add_email_ids { + thread.append( + Property::EmailIds, + self.store + .sort( + ResultSet::new(account_id, Collection::Email, document_ids.clone()), + vec![Comparator::ascending(Property::ReceivedAt)], + Pagination::new( + document_ids.len() as usize, + 0, + None, + 0, + None, + false, + ), + ) + .await + .map_err(|err| { + tracing::error!(event = "error", + context = "store", + account_id = account_id, + collection = "email", + error = ?err, + "Thread emailIds sort failed"); + MethodError::ServerPartialFail + })? + .ids + .into_iter() + .map(|id| Id::from_parts(thread_id, id as u32)) + .collect::>(), + ); + } + response.list.push(thread); + } else { + response.not_found.push(id); + } + } + + Ok(response) + } +} diff --git a/crates/jmap/src/thread/mod.rs b/crates/jmap/src/thread/mod.rs new file mode 100644 index 00000000..125ca70d --- /dev/null +++ b/crates/jmap/src/thread/mod.rs @@ -0,0 +1 @@ +pub mod get; diff --git a/crates/store/src/query/mod.rs b/crates/store/src/query/mod.rs index d37cbbce..2f2cce6f 100644 --- a/crates/store/src/query/mod.rs +++ b/crates/store/src/query/mod.rs @@ -70,6 +70,16 @@ pub struct SortedResultSet { pub found_anchor: bool, } +impl ResultSet { + pub fn new(account_id: u32, collection: impl Into, results: RoaringBitmap) -> Self { + ResultSet { + account_id, + collection: collection.into(), + results, + } + } +} + impl Filter { pub fn cond(field: impl Into, op: Operator, value: impl Serialize) -> Self { Filter::MatchValue { diff --git a/tests/resources/jmap_mail_get/headers.json b/tests/resources/jmap_mail_get/headers.json index f8248125..63fb683b 100644 --- a/tests/resources/jmap_mail_get/headers.json +++ b/tests/resources/jmap_mail_get/headers.json @@ -73,7 +73,6 @@ "subject": "Why not both importing AND exporting? ☺", "sentAt": "2003-07-01T08:52:37Z", "bodyStructure": { - "size": 0, "type": "multipart/mixed", "subParts": [ { diff --git a/tests/resources/jmap_mail_get/message_attachment.json b/tests/resources/jmap_mail_get/message_attachment.json index b2f8ad79..175be198 100644 --- a/tests/resources/jmap_mail_get/message_attachment.json +++ b/tests/resources/jmap_mail_get/message_attachment.json @@ -21,7 +21,6 @@ ], "subject": "[Fwd: Map of Argentina with Description]", "bodyStructure": { - "size": 0, "headers": [ { "name": "From", diff --git a/tests/resources/jmap_mail_get/multipart_alternative.json b/tests/resources/jmap_mail_get/multipart_alternative.json index 7e6a0392..d3bea997 100644 --- a/tests/resources/jmap_mail_get/multipart_alternative.json +++ b/tests/resources/jmap_mail_get/multipart_alternative.json @@ -21,7 +21,6 @@ ], "subject": "Multipart Email Example", "bodyStructure": { - "size": 0, "headers": [ { "name": "From", diff --git a/tests/resources/jmap_mail_get/multipart_cid.json b/tests/resources/jmap_mail_get/multipart_cid.json index 1bcfa5e4..53e8d057 100644 --- a/tests/resources/jmap_mail_get/multipart_cid.json +++ b/tests/resources/jmap_mail_get/multipart_cid.json @@ -25,7 +25,6 @@ "subject": "Test message from Microsoft Outlook 00", "sentAt": "2000-05-17T23:44:45Z", "bodyStructure": { - "size": 0, "headers": [ { "name": "From", @@ -79,7 +78,6 @@ "type": "multipart/related", "subParts": [ { - "size": 0, "headers": [ { "name": "Content-Type", diff --git a/tests/resources/jmap_mail_get/multipart_mixed.json b/tests/resources/jmap_mail_get/multipart_mixed.json index 559763bd..4cd5591c 100644 --- a/tests/resources/jmap_mail_get/multipart_mixed.json +++ b/tests/resources/jmap_mail_get/multipart_mixed.json @@ -22,7 +22,6 @@ "subject": "A multipart example", "sentAt": "1994-10-07T23:15:05Z", "bodyStructure": { - "size": 0, "headers": [ { "name": "MIME-Version", @@ -55,6 +54,7 @@ "partId": "1", "blobId": "blob_0", "size": 262, + "headers": [], "type": "text/plain", "charset": "us-ascii" }, @@ -72,7 +72,6 @@ "charset": "US-ASCII" }, { - "size": 0, "headers": [ { "name": "Content-Type", @@ -161,6 +160,7 @@ "partId": "1", "blobId": "blob_0", "size": 262, + "headers": [], "type": "text/plain", "charset": "us-ascii" }, @@ -183,6 +183,7 @@ "partId": "1", "blobId": "blob_0", "size": 262, + "headers": [], "type": "text/plain", "charset": "us-ascii" }, diff --git a/tests/resources/jmap_mail_get/multipart_related.json b/tests/resources/jmap_mail_get/multipart_related.json index 2d8168ae..51b8e6b2 100644 --- a/tests/resources/jmap_mail_get/multipart_related.json +++ b/tests/resources/jmap_mail_get/multipart_related.json @@ -25,7 +25,6 @@ "subject": "Die Hasen und die Frosche", "sentAt": "2000-05-19T04:36:58Z", "bodyStructure": { - "size": 0, "headers": [ { "name": "Message-ID", @@ -71,7 +70,6 @@ "type": "multipart/mixed", "subParts": [ { - "size": 0, "headers": [ { "name": "Content-Type", @@ -81,7 +79,6 @@ "type": "multipart/related", "subParts": [ { - "size": 0, "headers": [ { "name": "Content-Type", diff --git a/tests/resources/jmap_mail_get/rfc8621.json b/tests/resources/jmap_mail_get/rfc8621.json index 549984ed..dd8fd0b6 100644 --- a/tests/resources/jmap_mail_get/rfc8621.json +++ b/tests/resources/jmap_mail_get/rfc8621.json @@ -9,7 +9,6 @@ "receivedAt": "1997-07-27T10:40:00Z", "subject": "RFC 8621 Section 4.1.4 test", "bodyStructure": { - "size": 0, "headers": [ { "name": "Subject", @@ -41,7 +40,6 @@ "disposition": "inline" }, { - "size": 0, "headers": [ { "name": "Content-Type", @@ -51,7 +49,6 @@ "type": "multipart/mixed", "subParts": [ { - "size": 0, "headers": [ { "name": "Content-Type", @@ -61,7 +58,6 @@ "type": "multipart/alternative", "subParts": [ { - "size": 0, "headers": [ { "name": "Content-Type", @@ -126,7 +122,6 @@ ] }, { - "size": 0, "headers": [ { "name": "Content-Type", diff --git a/tests/resources/jmap_mail_get/text_body_missing_multipart.json b/tests/resources/jmap_mail_get/text_body_missing_multipart.json index 081a02a2..e1b6fa70 100644 --- a/tests/resources/jmap_mail_get/text_body_missing_multipart.json +++ b/tests/resources/jmap_mail_get/text_body_missing_multipart.json @@ -25,7 +25,6 @@ "subject": "Test message from Microsoft Outlook 00", "sentAt": "2000-05-17T23:36:13Z", "bodyStructure": { - "size": 0, "headers": [ { "name": "From", diff --git a/tests/resources/jmap_mail_parse/attachment.json b/tests/resources/jmap_mail_parse/attachment.json index 52f3ed0d..6b4670ca 100644 --- a/tests/resources/jmap_mail_parse/attachment.json +++ b/tests/resources/jmap_mail_parse/attachment.json @@ -19,7 +19,6 @@ "subject": "Map of Argentina with Description", "sentAt": "1998-08-13T07:42:41Z", "bodyStructure": { - "size": 0, "headers": [ { "name": "Content-Type", diff --git a/tests/resources/jmap_mail_parse/attachment_b64.json b/tests/resources/jmap_mail_parse/attachment_b64.json index 777209c4..3748c395 100644 --- a/tests/resources/jmap_mail_parse/attachment_b64.json +++ b/tests/resources/jmap_mail_parse/attachment_b64.json @@ -19,7 +19,6 @@ "subject": "HTML test", "sentAt": "2021-12-14T10:48:25Z", "bodyStructure": { - "size": 0, "headers": [ { "name": "Content-Type", diff --git a/tests/resources/jmap_mail_set/headers.jmap b/tests/resources/jmap_mail_set/headers.jmap index d0697e12..ebc2f3ea 100644 --- a/tests/resources/jmap_mail_set/headers.jmap +++ b/tests/resources/jmap_mail_set/headers.jmap @@ -69,7 +69,6 @@ "subject": "Headers test", "sentAt": "2018-07-10T01:03:11Z", "bodyStructure": { - "size": 0, "headers": [ { "name": "Bcc", diff --git a/tests/resources/jmap_mail_set/mixed.jmap b/tests/resources/jmap_mail_set/mixed.jmap index 7e810e1c..79dc8939 100644 --- a/tests/resources/jmap_mail_set/mixed.jmap +++ b/tests/resources/jmap_mail_set/mixed.jmap @@ -34,7 +34,6 @@ "subject": "Why not both importing AND exporting? ☺", "sentAt": "2021-11-20T22:22:01Z", "bodyStructure": { - "size": 0, "headers": [ { "name": "Date", @@ -64,7 +63,6 @@ "type": "multipart/mixed", "subParts": [ { - "size": 0, "headers": [ { "name": "Content-Type", diff --git a/tests/resources/jmap_mail_set/nested_body.jmap b/tests/resources/jmap_mail_set/nested_body.jmap index f3f8ffda..464f6093 100644 --- a/tests/resources/jmap_mail_set/nested_body.jmap +++ b/tests/resources/jmap_mail_set/nested_body.jmap @@ -19,7 +19,6 @@ "subject": "RFC 8621 Section 4.1.4 test", "sentAt": "2018-07-10T01:03:11Z", "bodyStructure": { - "size": 0, "headers": [ { "name": "Date", @@ -67,7 +66,6 @@ "disposition": "inline" }, { - "size": 0, "headers": [ { "name": "Content-Type", @@ -77,7 +75,6 @@ "type": "multipart/mixed", "subParts": [ { - "size": 0, "headers": [ { "name": "Content-Type", @@ -87,7 +84,6 @@ "type": "multipart/alternative", "subParts": [ { - "size": 0, "headers": [ { "name": "Content-Type", @@ -164,7 +160,6 @@ ] }, { - "size": 0, "headers": [ { "name": "Content-Type", diff --git a/tests/resources/jmap_mail_set/rfc8621_2.jmap b/tests/resources/jmap_mail_set/rfc8621_2.jmap index cd347662..49e7bc77 100644 --- a/tests/resources/jmap_mail_set/rfc8621_2.jmap +++ b/tests/resources/jmap_mail_set/rfc8621_2.jmap @@ -25,7 +25,6 @@ "subject": "World domination", "sentAt": "2018-07-10T01:05:08Z", "bodyStructure": { - "size": 0, "headers": [ { "name": "Date", diff --git a/tests/src/jmap/email_get.rs b/tests/src/jmap/email_get.rs index 83196611..1bfbc288 100644 --- a/tests/src/jmap/email_get.rs +++ b/tests/src/jmap/email_get.rs @@ -21,7 +21,7 @@ * for more details. */ -use std::{fs, path::PathBuf, sync::Arc}; +use std::{fs, path::PathBuf, sync::Arc, time::Instant}; use jmap::JMAP; use jmap_client::{ diff --git a/tests/src/jmap/email_query.rs b/tests/src/jmap/email_query.rs new file mode 100644 index 00000000..2e405cf6 --- /dev/null +++ b/tests/src/jmap/email_query.rs @@ -0,0 +1,766 @@ +use std::{collections::hash_map::Entry, sync::Arc, time::Instant}; + +use jmap::JMAP; +use jmap_client::{ + client::Client, + core::query::{Comparator, Filter}, + email, +}; +use jmap_proto::types::{collection::Collection, id::Id}; +use mail_parser::RfcHeader; +use store::{ahash::AHashMap, write::BatchBuilder}; + +use crate::store::{deflate_artwork_data, query::FIELDS}; + +const MAX_THREADS: usize = 100; +const MAX_MESSAGES: usize = 1000; +const MAX_MESSAGES_PER_THREAD: usize = 100; + +pub async fn test(server: Arc, client: &mut Client) { + println!("Running Email Query tests..."); + + // Add some "virtual" mailbox ids so create doesn't fail + let mut batch = BatchBuilder::new(); + let account_id = Id::from_bytes(client.default_account_id().as_bytes()) + .unwrap() + .document_id(); + batch + .with_account_id(account_id) + .with_collection(Collection::Mailbox); + for mailbox_id in 0..99999 { + batch.create_document(mailbox_id); + } + server.store.write(batch.build()).await.unwrap(); + + // Create test messages + println!("Inserting JMAP Mail query test messages..."); + create(client).await; + + // Remove mailboxes + let mut batch = BatchBuilder::new(); + batch + .with_account_id(account_id) + .with_collection(Collection::Mailbox); + for mailbox_id in 0..99999 { + batch.delete_document(mailbox_id); + } + server.store.write(batch.build()).await.unwrap(); + + for thread_id in 0..MAX_THREADS { + assert!( + client + .thread_get(&Id::new(thread_id as u64).to_string()) + .await + .unwrap() + .is_some(), + "thread {} not found", + thread_id + ); + } + + assert!( + client + .thread_get(&Id::new(MAX_THREADS as u64).to_string()) + .await + .unwrap() + .is_none(), + "thread {} found", + MAX_THREADS + ); + + println!("Running JMAP Mail query tests..."); + query(client).await; + + println!("Running JMAP Mail query options tests..."); + query_options(client).await; + + println!("Deleting all messages..."); + let implement = "fds"; + /*let mut request = client.build(); + let result_ref = request.query_email().result_reference(); + request.set_email().destroy_ref(result_ref); + let response = request.send().await.unwrap(); + response + .unwrap_method_responses() + .pop() + .unwrap() + .unwrap_set_email() + .unwrap(); + + server.store.assert_is_empty();*/ +} + +pub async fn query(client: &mut Client) { + for (filter, sort, expected_results) in [ + ( + Filter::and(vec![ + (email::query::Filter::after(1850)), + (email::query::Filter::from("george")), + ]), + vec![email::query::Comparator::subject()], + vec![ + "N01389", "T10115", "N00618", "N03500", "T01587", "T00397", "N01561", "N05250", + "N03973", "N04973", "N04057", "N01940", "N01539", "N01612", "N04484", "N01954", + "N05998", "T02053", "AR00171", "AR00172", "AR00176", + ], + ), + ( + Filter::and(vec![ + (email::query::Filter::in_mailbox(Id::new(1768u64).to_string())), + (email::query::Filter::cc("canvas")), + ]), + vec![email::query::Comparator::from()], + vec!["T01882", "N04689", "T00925", "N00121"], + ), + ( + Filter::and(vec![ + (email::query::Filter::subject("study")), + (email::query::Filter::in_mailbox_other_than(vec![ + Id::new(1991).to_string(), + Id::new(1870).to_string(), + Id::new(2011).to_string(), + Id::new(1951).to_string(), + Id::new(1902).to_string(), + Id::new(1808).to_string(), + Id::new(1963).to_string(), + ])), + ]), + vec![email::query::Comparator::subject()], + vec![ + "T10330", "N01744", "N01743", "N04885", "N02688", "N02122", "A00059", "A00058", + "N02123", "T00651", "T09439", "N05001", "T05848", "T05508", + ], + ), + ( + Filter::and(vec![ + (email::query::Filter::has_keyword("N0")).into(), + Filter::not(vec![(email::query::Filter::from("collins"))]), + (email::query::Filter::body("bequeathed")).into(), + ]), + vec![email::query::Comparator::subject()], + vec![ + "N02640", "A01020", "N01250", "T03430", "N01800", "N00620", "N05250", "N04630", + "A01040", + ], + ), + ( + email::query::Filter::not_keyword("artist").into(), + vec![email::query::Comparator::subject()], + vec!["T08626", "T09334", "T09455", "N01737", "T10965"], + ), + ( + Filter::and(vec![ + (email::query::Filter::after(1970)), + (email::query::Filter::before(1972)), + (email::query::Filter::text("colour")), + ]), + vec![email::query::Comparator::from()], + vec!["T01745", "P01436", "P01437"], + ), + ( + Filter::and(vec![(email::query::Filter::text("'cats and dogs'"))]), + vec![email::query::Comparator::from()], + vec!["P77623"], + ), + ( + Filter::and(vec![ + (email::query::Filter::header(RfcHeader::Comments.to_string(), Some("attributed"))), + (email::query::Filter::from("john")), + (email::query::Filter::cc("oil")), + ]), + vec![email::query::Comparator::from()], + vec!["T10965"], + ), + ( + Filter::and(vec![ + (email::query::Filter::all_in_thread_have_keyword("N")), + (email::query::Filter::before(1800)), + ]), + vec![email::query::Comparator::from()], + vec![ + "N01496", "N05916", "N01046", "N00675", "N01320", "N01321", "N00273", "N01453", + "N02984", + ], + ), + ( + Filter::and(vec![ + (email::query::Filter::none_in_thread_have_keyword("N")), + (email::query::Filter::after(1995)), + ]), + vec![email::query::Comparator::from()], + vec![ + "AR00163", "AR00164", "AR00472", "P11481", "AR00066", "AR00178", "P77895", + "P77896", "P77897", + ], + ), + ( + Filter::and(vec![ + (email::query::Filter::some_in_thread_have_keyword("Bronze")), + (email::query::Filter::before(1878)), + ]), + vec![email::query::Comparator::from()], + vec![ + "N04326", "N01610", "N02920", "N01587", "T00167", "T00168", "N01554", "N01535", + "N01536", "N01622", "N01754", "N01594", + ], + ), + // Sorting tests + ( + email::query::Filter::before(1800).into(), + vec![ + email::query::Comparator::all_in_thread_have_keyword("N"), + email::query::Comparator::from(), + ], + vec![ + "N01496", "N05916", "N01046", "N00675", "N01320", "N01321", "N00273", "N01453", + "N02984", "T09417", "T01882", "T08820", "N04689", "T08891", "T00986", "N00316", + "N03544", "N04296", "N04297", "T08234", "N00112", "T00211", "N01497", "N02639", + "N02640", "T00925", "T11683", "T08269", "D00001", "D00002", "D00046", "N00121", + "N00126", "T08626", + ], + ), + ( + email::query::Filter::before(1800).into(), + vec![ + email::query::Comparator::all_in_thread_have_keyword("N").descending(), + email::query::Comparator::from(), + ], + vec![ + "T09417", "T01882", "T08820", "N04689", "T08891", "T00986", "N00316", "N03544", + "N04296", "N04297", "T08234", "N00112", "T00211", "N01497", "N02639", "N02640", + "T00925", "T11683", "T08269", "D00001", "D00002", "D00046", "N00121", "N00126", + "T08626", "N01496", "N05916", "N01046", "N00675", "N01320", "N01321", "N00273", + "N01453", "N02984", + ], + ), + ( + Filter::and(vec![ + (email::query::Filter::after(1875)), + (email::query::Filter::before(1878)), + ]), + vec![ + email::query::Comparator::some_in_thread_have_keyword("Bronze"), + email::query::Comparator::from(), + ], + vec![ + "N04326", "N01610", "N02920", "N01587", "T00167", "T00168", "N01554", "N01535", + "N01536", "N01622", "N01754", "N01594", "N01559", "N02123", "N01940", "N03594", + "N01494", "N04271", + ], + ), + ( + Filter::and(vec![ + (email::query::Filter::after(1875)), + (email::query::Filter::before(1878)), + ]), + vec![ + email::query::Comparator::some_in_thread_have_keyword("Bronze").descending(), + email::query::Comparator::from(), + ], + vec![ + "N01559", "N02123", "N01940", "N03594", "N01494", "N04271", "N04326", "N01610", + "N02920", "N01587", "T00167", "T00168", "N01554", "N01535", "N01536", "N01622", + "N01754", "N01594", + ], + ), + ( + Filter::and(vec![ + (email::query::Filter::after(1786)), + (email::query::Filter::before(1840)), + (email::query::Filter::has_keyword("T")), + ]), + vec![ + email::query::Comparator::has_keyword("attributed to"), + email::query::Comparator::from(), + ], + vec![ + "T09455", "T09334", "T10965", "T08626", "T09417", "T08951", "T01851", "T01852", + "T08761", "T08123", "T08756", "T10561", "T10562", "T10563", "T00986", "T03424", + "T03427", "T08234", "T08133", "T06866", "T08897", "T00996", "T00997", "T01095", + "T03393", "T09456", "T00188", "T02362", "T09065", "T09547", "T10330", "T09187", + "T03433", "T08635", "T02366", "T03436", "T09150", "T01861", "T09759", "T11683", + "T02368", "T02369", "T08269", "T01018", "T10066", "T01710", "T01711", "T05764", + ], + ), + ( + Filter::and(vec![ + (email::query::Filter::after(1786)), + (email::query::Filter::before(1840)), + (email::query::Filter::has_keyword("T")), + ]), + vec![ + email::query::Comparator::has_keyword("attributed to").descending(), + email::query::Comparator::from(), + ], + vec![ + "T09417", "T08951", "T01851", "T01852", "T08761", "T08123", "T08756", "T10561", + "T10562", "T10563", "T00986", "T03424", "T03427", "T08234", "T08133", "T06866", + "T08897", "T00996", "T00997", "T01095", "T03393", "T09456", "T00188", "T02362", + "T09065", "T09547", "T10330", "T09187", "T03433", "T08635", "T02366", "T03436", + "T09150", "T01861", "T09759", "T11683", "T02368", "T02369", "T08269", "T01018", + "T10066", "T01710", "T01711", "T05764", "T09455", "T09334", "T10965", "T08626", + ], + ), + ] { + let mut request = client.build(); + let query_request = request + .query_email() + .filter(filter) + .sort(sort) + .calculate_total(true); + query_request.arguments().collapse_threads(false); + let query_result_ref = query_request.result_reference(); + request + .get_email() + .ids_ref(query_result_ref) + .properties([email::Property::MessageId]); + + assert_eq!( + request + .send() + .await + .unwrap() + .unwrap_method_responses() + .pop() + .unwrap() + .unwrap_get_email() + .unwrap() + .take_list() + .into_iter() + .map(|e| e.message_id().unwrap().first().unwrap().to_string()) + .collect::>(), + expected_results + ); + } +} + +pub async fn query_options(client: &mut Client) { + for (query, expected_results, expected_results_collapsed) in [ + ( + EmailQuery { + filter: None, + sort: vec![ + email::query::Comparator::subject(), + email::query::Comparator::from(), + ], + position: 0, + anchor: None, + anchor_offset: 0, + limit: 10, + }, + vec![ + "N01496", "N01320", "N01321", "N05916", "N00273", "N01453", "N02984", "T08820", + "N00112", "T00211", + ], + vec![ + "N01496", "N01320", "N05916", "N01453", "T08820", "N01046", "N00675", "T08891", + "T01882", "N04296", + ], + ), + ( + EmailQuery { + filter: None, + sort: vec![ + email::query::Comparator::subject(), + email::query::Comparator::from(), + ], + position: 10, + anchor: None, + anchor_offset: 0, + limit: 10, + }, + vec![ + "N01046", "N00675", "T08891", "N00126", "T01882", "N04689", "T00925", "N00121", + "N04296", "N04297", + ], + vec![ + "T08234", "T09417", "N01110", "T08123", "N01039", "T09456", "T08951", "N01273", + "N00373", "T09547", + ], + ), + ( + EmailQuery { + filter: None, + sort: vec![ + email::query::Comparator::subject(), + email::query::Comparator::from(), + ], + position: -10, + anchor: None, + anchor_offset: 0, + limit: 0, + }, + vec![ + "T07236", "P11481", "AR00066", "P77895", "P77896", "P77897", "AR00163", "AR00164", + "AR00472", "AR00178", + ], + vec![ + "P07639", "P07522", "AR00089", "P02949", "T05820", "P11441", "T06971", "P11481", + "AR00163", "AR00164", + ], + ), + ( + EmailQuery { + filter: None, + sort: vec![ + email::query::Comparator::subject(), + email::query::Comparator::from(), + ], + position: -20, + anchor: None, + anchor_offset: 0, + limit: 10, + }, + vec![ + "P20079", "AR00024", "AR00182", "P20048", "P20044", "P20045", "P20046", "T06971", + "AR00177", "P77935", + ], + vec![ + "T00300", "P06033", "T02310", "T02135", "P04006", "P03166", "P01358", "P07133", + "P03138", "T03562", + ], + ), + ( + EmailQuery { + filter: None, + sort: vec![ + email::query::Comparator::subject(), + email::query::Comparator::from(), + ], + position: -100000, + anchor: None, + anchor_offset: 0, + limit: 1, + }, + vec!["N01496"], + vec!["N01496"], + ), + ( + EmailQuery { + filter: None, + sort: vec![ + email::query::Comparator::subject(), + email::query::Comparator::from(), + ], + position: -1, + anchor: None, + anchor_offset: 0, + limit: 100000, + }, + vec!["AR00178"], + vec!["AR00164"], + ), + ( + EmailQuery { + filter: None, + sort: vec![ + email::query::Comparator::subject(), + email::query::Comparator::from(), + ], + position: 0, + anchor: get_anchor(client, "N01205").await, + anchor_offset: 0, + limit: 10, + }, + vec![ + "N01205", "N01976", "T01139", "N01525", "T00176", "N01405", "N02396", "N04885", + "N01526", "N02134", + ], + vec![ + "N01205", "N01526", "T01455", "N01969", "N05250", "N01781", "N00759", "A00057", + "N03527", "N01558", + ], + ), + ( + EmailQuery { + filter: None, + sort: vec![ + email::query::Comparator::subject(), + email::query::Comparator::from(), + ], + position: 0, + anchor: get_anchor(client, "N01205").await, + anchor_offset: 10, + limit: 10, + }, + vec![ + "N01933", "N03618", "T03904", "N02398", "N02399", "N02688", "T01455", "N03051", + "N01500", "N03411", + ], + vec![ + "N01559", "N04326", "N06017", "N01553", "N01617", "N01528", "N01539", "T09439", + "N01593", "N03988", + ], + ), + ( + EmailQuery { + filter: None, + sort: vec![ + email::query::Comparator::subject(), + email::query::Comparator::from(), + ], + position: 0, + anchor: get_anchor(client, "N01205").await, + anchor_offset: -10, + limit: 10, + }, + vec![ + "N05779", "N04652", "N01534", "A00845", "N03409", "N03410", "N02061", "N02426", + "N00662", "N01205", + ], + vec![ + "N00443", "N02237", "T03025", "N01722", "N01356", "N01800", "T05475", "T01587", + "N05779", "N01205", + ], + ), + ( + EmailQuery { + filter: None, + sort: vec![ + email::query::Comparator::subject(), + email::query::Comparator::from(), + ], + position: 0, + anchor: get_anchor(client, "N01496").await, + anchor_offset: -10, + limit: 10, + }, + vec!["N01496"], + vec!["N01496"], + ), + ( + EmailQuery { + filter: None, + sort: vec![ + email::query::Comparator::subject(), + email::query::Comparator::from(), + ], + position: 0, + anchor: get_anchor(client, "AR00164").await, + anchor_offset: 10, + limit: 10, + }, + vec![], + vec![], + ), + ( + EmailQuery { + filter: None, + sort: vec![ + email::query::Comparator::subject(), + email::query::Comparator::from(), + ], + position: 0, + anchor: get_anchor(client, "AR00164").await, + anchor_offset: 0, + limit: 0, + }, + vec!["AR00164", "AR00472", "AR00178"], + vec!["AR00164"], + ), + ] { + for (test_num, expected_results) in [expected_results, expected_results_collapsed] + .into_iter() + .enumerate() + { + let mut request = client.build(); + let query_request = request + .query_email() + .sort(query.sort.clone()) + .position(query.position) + .calculate_total(true); + if query.limit > 0 { + query_request.limit(query.limit); + } + if let Some(filter) = query.filter.as_ref() { + query_request.filter(filter.clone()); + } + if let Some(anchor) = query.anchor.as_ref() { + query_request.anchor(anchor); + query_request.anchor_offset(query.anchor_offset); + } + query_request.arguments().collapse_threads(test_num == 1); + + if !expected_results.is_empty() { + let query_result_ref = query_request.result_reference(); + request + .get_email() + .ids_ref(query_result_ref) + .properties([email::Property::MessageId]); + + assert_eq!( + request + .send() + .await + .unwrap() + .unwrap_method_responses() + .pop() + .unwrap() + .unwrap_get_email() + .unwrap() + .take_list() + .into_iter() + .map(|e| e.message_id().unwrap().first().unwrap().to_string()) + .collect::>(), + expected_results, + "{:#?} ({})", + query, + test_num == 1 + ); + } else { + assert_eq!( + request.send_query_email().await.unwrap().ids(), + Vec::<&str>::new() + ); + } + } + } +} + +pub async fn create(client: &mut Client) { + let now = Instant::now(); + let mut fields = AHashMap::default(); + for (field_num, field) in FIELDS.iter().enumerate() { + fields.insert(field.to_string(), field_num); + } + + let mut total_messages = 0; + let mut total_threads = 0; + let mut thread_count = AHashMap::default(); + let mut artist_count = AHashMap::default(); + + 'outer: for record in csv::ReaderBuilder::new() + .has_headers(true) + .from_reader(&deflate_artwork_data()[..]) + .records() + { + let record = record.unwrap(); + let mut values_str = AHashMap::default(); + let mut values_int = AHashMap::default(); + + for field_name in [ + "year", + "acquisitionYear", + "accession_number", + "artist", + "artistRole", + "medium", + "title", + "creditLine", + "inscription", + ] { + let field = record.get(fields[field_name]).unwrap(); + if field.is_empty() + || (field_name == "title" && (field.contains('[') || field.contains(']'))) + { + continue 'outer; + } else if field_name == "year" || field_name == "acquisitionYear" { + let field = field.parse::().unwrap_or(0); + if field < 1000 { + continue 'outer; + } + values_int.insert(field_name.to_string(), field); + } else { + values_str.insert(field_name.to_string(), field.to_string()); + } + } + + let val = artist_count + .entry(values_str["artist"].clone()) + .or_insert(0); + if *val == 3 { + continue; + } + *val += 1; + + match thread_count.entry(values_int["year"]) { + Entry::Occupied(mut e) => { + let messages_per_thread = e.get_mut(); + if *messages_per_thread == MAX_MESSAGES_PER_THREAD { + continue; + } + *messages_per_thread += 1; + } + Entry::Vacant(e) => { + if total_threads == MAX_THREADS { + continue; + } + total_threads += 1; + e.insert(1); + } + } + + total_messages += 1; + + client + .email_import( + format!( + concat!( + "From: \"{}\" \nCc: \"{}\" \nMessage-ID: <{}>\n", + "References: <{}>\nComments: {}\nSubject: [{}]", + " Year {}\n\n{}\n{}\n" + ), + values_str["artist"], + values_str["medium"], + values_str["accession_number"], + values_int["year"], + values_str["artistRole"], + values_str["title"], + values_int["year"], + values_str["creditLine"], + values_str["inscription"] + ) + .into_bytes(), + [ + Id::new(values_int["year"] as u64).to_string(), + Id::new((values_int["acquisitionYear"] + 1000) as u64).to_string(), + ], + [ + values_str["medium"].to_string(), + values_str["artistRole"].to_string(), + values_str["accession_number"][0..1].to_string(), + format!( + "N{}", + &values_str["accession_number"][values_str["accession_number"].len() - 1..] + ), + ] + .into(), + Some(values_int["year"] as i64), + ) + .await + .unwrap(); + + if total_messages == MAX_MESSAGES { + break; + } + } + println!( + "Imported {} messages in {} ms (single thread).", + total_messages, + now.elapsed().as_millis() + ); +} + +async fn get_anchor(client: &mut Client, anchor: &str) -> Option { + client + .email_query( + email::query::Filter::header("Message-Id", anchor.into()).into(), + None::>, + ) + .await + .unwrap() + .take_ids() + .pop() + .unwrap() + .into() +} + +#[derive(Debug, Clone)] +pub struct EmailQuery { + pub filter: Option>, + pub sort: Vec>, + pub position: i32, + pub anchor: Option, + pub anchor_offset: i32, + pub limit: usize, +} diff --git a/tests/src/jmap/mod.rs b/tests/src/jmap/mod.rs index 49f7f2a4..cba56828 100644 --- a/tests/src/jmap/mod.rs +++ b/tests/src/jmap/mod.rs @@ -8,6 +8,8 @@ use tokio::sync::watch; use crate::{add_test_certs, store::TempDir}; pub mod email_get; +pub mod email_query; +pub mod thread_get; const SERVER: &str = " [server] @@ -47,6 +49,8 @@ pub async fn jmap_tests() { let delete = true; let mut params = init_jmap_tests(delete).await; email_get::test(params.server.clone(), &mut params.client).await; + //email_query::test(params.server.clone(), &mut params.client).await; + //thread_get::test(params.server.clone(), &mut params.client).await; if delete { params.temp_dir.delete(); } @@ -136,7 +140,7 @@ pub fn replace_boundaries(string: String) -> String { pub fn replace_blob_ids(string: String) -> String { let values = find_values(&string, "blobId\":"); if !values.is_empty() { - let values = BTreeSet::from_iter(values).into_iter().collect::>(); + //let values = BTreeSet::from_iter(values).into_iter().collect::>(); replace_values( string, &values, diff --git a/tests/src/jmap/thread_get.rs b/tests/src/jmap/thread_get.rs new file mode 100644 index 00000000..58c1696d --- /dev/null +++ b/tests/src/jmap/thread_get.rs @@ -0,0 +1,50 @@ +use std::sync::Arc; + +use jmap::JMAP; +use jmap_client::{client::Client, mailbox::Role}; +use jmap_proto::types::id::Id; + +pub async fn test(server: Arc, client: &mut Client) { + println!("Running Email Thread tests..."); + + let mailbox_id = "a".to_string(); + let implementer = "fd"; + /*let mailbox_id = client + .set_default_account_id(Id::new(1).to_string()) + .mailbox_create("JMAP Get", None::, Role::None) + .await + .unwrap() + .take_id();*/ + + let mut expected_result = vec!["".to_string(); 5]; + let mut thread_id = "".to_string(); + + for num in [5, 3, 1, 2, 4] { + let mut email = client + .email_import( + format!("Subject: test\nReferences: <1234>\n\n{}", num).into_bytes(), + [&mailbox_id], + None::>, + Some(10000i64 + num as i64), + ) + .await + .unwrap(); + thread_id = email.thread_id().unwrap().to_string(); + expected_result[num - 1] = email.take_id(); + } + + assert_eq!( + client + .thread_get(&thread_id) + .await + .unwrap() + .unwrap() + .email_ids(), + expected_result + ); + + let implement = "fd"; + //client.mailbox_destroy(&mailbox_id, true).await.unwrap(); + + //server.store.assert_is_empty(); +}