filip/Stalwart
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

result references

Browse Source
This commit is contained in:
Mauro D
2023-04-09 16:11:03 +00:00
parent 75bf0a240a
commit 633db406f7
34 changed files with 1441 additions and 253 deletions
Download Patch File Download Diff File Expand all files Collapse all files

240
crates/core/src/email/body.rs
View File

@@ -1,61 +1,229 @@
use mail_builder::mime::MimePart;
use mail_parser::{MessagePart, PartType};
use mail_parser::{HeaderValue, MessagePart, MimeHeaders, PartType};
use protocol::{
object::Object,
types::{
blob::{BlobId, BlobSection},
property::Property,
value::Value,
},
types::{blob::BlobId, property::Property, value::Value},
};
use store::BlobHash;
use super::headers::HeaderToValue;
pub trait ToBodyPart {
fn to_body_part(
&self,
part_id: usize,
properties: &[Property],
message_raw: &[u8],
raw_message: &[u8],
blob_id: &BlobId,
) -> Value;
}
impl ToBodyPart for MessagePart<'_> {
impl ToBodyPart for Vec<MessagePart<'_>> {
fn to_body_part(
&self,
part_id: usize,
properties: &[Property],
message_raw: &[u8],
raw_message: &[u8],
blob_id: &BlobId,
) -> Value {
let mut values = Object::with_capacity(properties.len());
let has_body = !matches!(self.body, PartType::Multipart(_));
let mut parts = vec![part_id].into_iter();
let mut parts_stack = Vec::new();
let mut subparts = Vec::with_capacity(1);
for property in properties {
let value = match property {
Property::PartId => part_id.to_string().into(),
Property::BlobId if has_body => {
let base_offset = blob_id.start_offset();
BlobId::new_section(
blob_id.hash,
self.offset_body + base_offset,
self.offset_end + base_offset,
self.encoding as u8,
)
.into()
loop {
if let Some((part_id, part)) = parts.next().map(|part_id| (part_id, &self[part_id])) {
let mut values = Object::with_capacity(properties.len());
let multipart = if let PartType::Multipart(parts) = &part.body {
parts.into()
} else {
None
};
let mut add_subparts = false;
for property in properties {
let value = match property {
Property::PartId => part_id.to_string().into(),
Property::BlobId if multipart.is_none() => {
let base_offset = blob_id.start_offset();
BlobId::new_section(
blob_id.hash,
part.offset_body + base_offset,
part.offset_end + base_offset,
part.encoding as u8,
)
.into()
}
Property::Size if multipart.is_none() => match &part.body {
PartType::Text(text) | PartType::Html(text) => text.len(),
PartType::Binary(bin) | PartType::InlineBinary(bin) => bin.len(),
PartType::Message(message) => message.root_part().raw_len(),
PartType::Multipart(_) => 0,
}
.into(),
Property::Name => part.attachment_name().into(),
Property::Type => part
.content_type()
.map(|ct| {
ct.subtype()
.map(|st| format!("{}/{}", ct.ctype(), st))
.unwrap_or_else(|| ct.ctype().to_string())
})
.or_else(|| match &part.body {
PartType::Text(_) => Some("text/plain".to_string()),
PartType::Html(_) => Some("text/html".to_string()),
PartType::Message(_) => Some("message/rfc822".to_string()),
_ => None,
})
.into(),
Property::Charset => part
.content_type()
.and_then(|ct| ct.attribute("charset"))
.or(match &part.body {
PartType::Text(_) | PartType::Html(_) => Some("utf-8"),
_ => None,
})
.into(),
Property::Disposition => {
part.content_disposition().map(|cd| cd.ctype()).into()
}
Property::Cid => part.content_id().into(),
Property::Language => match part.content_language() {
HeaderValue::Text(text) => vec![text.to_string()].into(),
HeaderValue::TextList(list) => list
.iter()
.map(|text| text.to_string().into())
.collect::<Vec<Value>>()
.into(),
_ => Value::Null,
},
Property::Location => part.content_location().into(),
Property::Header(_) => part.header_to_value(property, raw_message),
Property::Headers => part.headers_to_value(raw_message),
Property::SubParts => match multipart {
Some(multipart) if !multipart.is_empty() => {
add_subparts = true;
continue;
}
_ => Vec::<String>::new().into(),
},
_ => Value::Null,
};
values.append(property.clone(), value);
}
Property::Size if has_body => match &self.body {
PartType::Text(text) | PartType::Html(text) => text.len(),
PartType::Binary(bin) | PartType::InlineBinary(bin) => bin.len(),
PartType::Message(message) => message.root_part().raw_len(),
PartType::Multipart(_) => 0,
subparts.push(values);
if add_subparts {
let multipart = multipart.unwrap().clone();
parts_stack.push((
parts,
std::mem::replace(&mut subparts, Vec::with_capacity(multipart.len())),
));
parts = multipart.into_iter();
}
.into(),
_ => Value::Null,
};
values.append(property.clone(), value);
} else if let Some((prev_parts, mut prev_subparts)) = parts_stack.pop() {
prev_subparts
.last_mut()
.unwrap()
.append(Property::SubParts, subparts);
parts = prev_parts;
subparts = prev_subparts;
} else {
return subparts.pop().map(Into::into).unwrap_or_default();
}
}
}
}
pub(super) trait TruncateBody {
fn truncate(&self, max_len: usize) -> (bool, String);
}
impl TruncateBody for PartType<'_> {
fn truncate(&self, mut max_len: usize) -> (bool, String) {
match self {
PartType::Text(text) => {
if max_len != 0 && text.len() > max_len {
let add_dots = max_len > 6;
if add_dots {
max_len -= 3;
}
let mut result = String::with_capacity(max_len);
for ch in text.chars() {
if ch != '\r' {
if ch.len_utf8() + result.len() > max_len {
break;
}
result.push(ch);
}
}
if add_dots {
result.push_str("...");
}
(true, result)
} else {
(false, text.replace('\r', ""))
}
}
PartType::Html(html) => {
if max_len != 0 && html.len() > max_len {
let add_dots = max_len > 6;
if add_dots {
max_len -= 3;
}
let mut result = String::with_capacity(max_len);
let mut in_tag = false;
let mut in_comment = false;
let mut last_tag_end_pos = 0;
for (pos, ch) in html.char_indices() {
let mut set_last_tag = 0;
match ch {
'<' if !in_tag => {
in_tag = true;
if let Some("!--") = html.get(pos + 1..pos + 4) {
in_comment = true;
}
set_last_tag = pos;
}
'>' if in_tag => {
if in_comment {
if let Some("--") = html.get(pos - 2..pos) {
in_comment = false;
in_tag = false;
set_last_tag = pos + 1;
}
} else {
in_tag = false;
set_last_tag = pos + 1;
}
}
'\r' => continue,
_ => (),
}
if ch.len_utf8() + pos > max_len {
result.push_str(
&html[0..if (in_tag || set_last_tag > 0) && last_tag_end_pos > 0 {
last_tag_end_pos
} else {
pos
}],
);
if add_dots {
result.push_str("...");
}
break;
} else if set_last_tag > 0 {
last_tag_end_pos = set_last_tag;
}
}
(true, result)
} else {
(false, html.replace('\r', ""))
}
}
PartType::Binary(bytes) | PartType::InlineBinary(bytes) => {
PartType::Text(String::from_utf8_lossy(bytes)).truncate(max_len)
}
_ => (false, "".into()),
}
values.into()
}
}

76
crates/core/src/email/get.rs
View File

@@ -3,12 +3,14 @@ use protocol::{
error::method::MethodError,
method::get::GetResponse,
object::{email::GetArguments, Object},
types::{collection::Collection, id::Id, property::Property, value::Value},
types::{blob::BlobId, collection::Collection, id::Id, property::Property, value::Value},
};
use store::ValueKey;
use crate::{email::headers::HeaderToValue, JMAP};
use super::body::{ToBodyPart, TruncateBody};
impl JMAP {
pub async fn email_get(
&self,
@@ -59,6 +61,11 @@ impl JMAP {
Property::Location,
]
});
let fetch_text_body_values = arguments.fetch_text_body_values.unwrap_or(false);
let fetch_html_body_values = arguments.fetch_html_body_values.unwrap_or(false);
let fetch_all_body_values = arguments.fetch_all_body_values.unwrap_or(false);
let max_body_value_bytes = arguments.max_body_value_bytes.unwrap_or(0);
let mut response = GetResponse {
account_id: Some(account_id.into()),
state: self
@@ -143,7 +150,7 @@ impl JMAP {
} else {
None
};
let blob_hash = blob_id.hash;
let blob_id = BlobId::new(blob_id.hash);
// Prepare response
let mut email = Object::with_capacity(properties.len());
@@ -156,7 +163,7 @@ impl JMAP {
email.append(Property::ThreadId, id.prefix_id());
}
Property::BlobId => {
email.append(Property::BlobId, blob_hash);
email.append(Property::BlobId, blob_id.clone());
}
Property::MailboxIds | Property::Keywords => {
email.append(
@@ -205,6 +212,69 @@ impl JMAP {
);
}
}
Property::TextBody | Property::HtmlBody | Property::Attachments => {
if let Some(message) = &message {
let list = match property {
Property::TextBody => &message.text_body,
Property::HtmlBody => &message.html_body,
Property::Attachments => &message.attachments,
_ => unreachable!(),
}
.iter();
email.append(
property.clone(),
list.map(|part_id| {
message.parts.to_body_part(
*part_id,
&body_properties,
&raw_message,
&blob_id,
)
})
.collect::<Vec<_>>(),
);
}
}
Property::BodyStructure => {
if let Some(message) = &message {
email.append(
Property::BodyStructure,
message.parts.to_body_part(
0,
&body_properties,
&raw_message,
&blob_id,
),
);
}
}
Property::BodyValues => {
if let Some(message) = &message {
let mut body_values = Object::with_capacity(message.parts.len());
for (part_id, part) in message.parts.iter().enumerate() {
if (message.html_body.contains(&part_id)
&& (fetch_all_body_values || fetch_html_body_values))
|| (message.text_body.contains(&part_id)
&& (fetch_all_body_values || fetch_text_body_values))
{
let (is_truncated, value) =
part.body.truncate(max_body_value_bytes);
body_values.append(
Property::_T(part_id.to_string()),
Object::with_capacity(3)
.with_property(
Property::IsEncodingProblem,
part.is_encoding_problem,
)
.with_property(Property::IsTruncated, is_truncated)
.with_property(Property::Value, value),
);
}
}
email.append(Property::BodyValues, body_values);
}
}
_ => {
return Err(MethodError::InvalidArguments(format!(
"Invalid property {property:?}"

14
crates/core/src/email/import.rs
View File

@@ -1,3 +1,15 @@
use protocol::{
error::method::MethodError,
method::import::{ImportEmailRequest, ImportEmailResponse},
};
use crate::JMAP;
impl JMAP {}
impl JMAP {
pub async fn email_import(
&self,
request: ImportEmailRequest,
) -> Result<ImportEmailResponse, MethodError> {
todo!()
}
}

1
crates/core/src/email/mod.rs
View File

@@ -4,3 +4,4 @@ pub mod headers;
pub mod import;
pub mod index;
pub mod ingest;
pub mod query;

348
crates/core/src/email/query.rs Normal file
View File

@@ -0,0 +1,348 @@
use protocol::{
error::method::MethodError,
method::query::{Comparator, Filter, QueryRequest, QueryResponse, SortProperty},
object::email::QueryArguments,
types::{collection::Collection, keyword::Keyword, property::Property},
};
use store::{
fts::Language,
query::{self, sort::Pagination},
roaring::RoaringBitmap,
BitmapKey, ValueKey,
};
use crate::JMAP;
impl JMAP {
pub async fn email_query(
&self,
request: QueryRequest,
arguments: QueryArguments,
) -> Result<QueryResponse, MethodError> {
let account_id = request.account_id.document_id();
let mut filters = Vec::with_capacity(request.filter.len());
for cond in request.filter {
match cond {
Filter::InMailbox(mailbox) => {
filters.push(query::Filter::is_in_bitmap(Property::MailboxIds, mailbox))
}
Filter::InMailboxOtherThan(mailboxes) => {
filters.push(query::Filter::Not);
filters.push(query::Filter::Or);
for mailbox in mailboxes {
filters.push(query::Filter::is_in_bitmap(Property::MailboxIds, mailbox));
}
filters.push(query::Filter::End);
filters.push(query::Filter::End);
}
Filter::Before(date) => filters.push(query::Filter::lt(Property::ReceivedAt, date)),
Filter::After(date) => filters.push(query::Filter::gt(Property::ReceivedAt, date)),
Filter::MinSize(size) => filters.push(query::Filter::ge(Property::Size, size)),
Filter::MaxSize(size) => filters.push(query::Filter::lt(Property::Size, size)),
Filter::AllInThreadHaveKeyword(keyword) => filters.push(query::Filter::is_in_set(
self.thread_keywords(account_id, keyword, true).await?,
)),
Filter::SomeInThreadHaveKeyword(keyword) => filters.push(query::Filter::is_in_set(
self.thread_keywords(account_id, keyword, false).await?,
)),
Filter::NoneInThreadHaveKeyword(keyword) => {
filters.push(query::Filter::Not);
filters.push(query::Filter::is_in_set(
self.thread_keywords(account_id, keyword, false).await?,
));
filters.push(query::Filter::End);
}
Filter::HasKeyword(keyword) => {
filters.push(query::Filter::is_in_bitmap(Property::Keywords, keyword))
}
Filter::NotKeyword(keyword) => {
filters.push(query::Filter::Not);
filters.push(query::Filter::is_in_bitmap(Property::Keywords, keyword));
filters.push(query::Filter::End);
}
Filter::HasAttachment(has_attach) => {
if !has_attach {
filters.push(query::Filter::Not);
}
filters.push(query::Filter::is_in_bitmap(Property::HasAttachment, ()));
if !has_attach {
filters.push(query::Filter::End);
}
}
Filter::Text(text) => {
filters.push(query::Filter::Or);
filters.push(query::Filter::has_text(
Property::From,
&text,
Language::None,
));
filters.push(query::Filter::has_text(Property::To, &text, Language::None));
filters.push(query::Filter::has_text(Property::Cc, &text, Language::None));
filters.push(query::Filter::has_text(
Property::Bcc,
&text,
Language::None,
));
filters.push(query::Filter::has_text(
Property::Subject,
&text,
Language::Unknown,
));
filters.push(query::Filter::has_text(
Property::TextBody,
&text,
Language::Unknown,
));
filters.push(query::Filter::has_text(
Property::Attachments,
text,
Language::Unknown,
));
filters.push(query::Filter::End);
}
Filter::From(text) => filters.push(query::Filter::has_text(
Property::From,
text,
Language::None,
)),
Filter::To(text) => {
filters.push(query::Filter::has_text(Property::To, text, Language::None))
}
Filter::Cc(text) => {
filters.push(query::Filter::has_text(Property::Cc, text, Language::None))
}
Filter::Bcc(text) => {
filters.push(query::Filter::has_text(Property::Bcc, text, Language::None))
}
Filter::Subject(text) => filters.push(query::Filter::has_text(
Property::Subject,
text,
Language::Unknown,
)),
Filter::Body(text) => filters.push(query::Filter::has_text(
Property::TextBody,
text,
Language::Unknown,
)),
Filter::Header(header) => {
return Err(MethodError::InvalidArguments(format!(
"Querying headers '{}' is not supported.",
header.join(":")
)));
}
// Non-standard
Filter::Id(ids) => {
let mut set = RoaringBitmap::new();
for id in ids {
set.insert(id.document_id());
}
filters.push(query::Filter::is_in_set(set));
}
Filter::SentBefore(date) => filters.push(query::Filter::lt(Property::SentAt, date)),
Filter::SentAfter(date) => filters.push(query::Filter::gt(Property::SentAt, date)),
Filter::InThread(id) => {
filters.push(query::Filter::is_in_bitmap(Property::ThreadId, id))
}
other => return Err(MethodError::UnsupportedFilter(other.to_string())),
}
}
let result_set = self
.store
.filter(account_id, Collection::Email, filters)
.await?;
let total = result_set.results.len() as usize;
let (limit_total, limit) = if let Some(limit) = request.limit {
if limit > 0 {
let limit = std::cmp::min(limit, self.config.query_max_results);
(std::cmp::min(limit, total), limit)
} else {
(0, 0)
}
} else {
(
std::cmp::min(self.config.query_max_results, total),
self.config.query_max_results,
)
};
let mut response = QueryResponse {
account_id: request.account_id,
query_state: self
.store
.get_last_change_id(account_id, Collection::Email)
.await?
.into(),
can_calculate_changes: true,
position: 0,
ids: vec![],
total: if request.calculate_total.unwrap_or(false) {
Some(total)
} else {
None
},
limit: if total > limit { Some(limit) } else { None },
};
if limit_total > 0 {
// Parse sort criteria
let mut comparators = Vec::with_capacity(request.sort.as_ref().map_or(1, |s| s.len()));
for comparator in request
.sort
.and_then(|s| if !s.is_empty() { s.into() } else { None })
.unwrap_or_else(|| vec![Comparator::descending(SortProperty::ReceivedAt)])
{
comparators.push(match comparator.property {
SortProperty::ReceivedAt => {
query::Comparator::field(Property::ReceivedAt, comparator.is_ascending)
}
SortProperty::Size => {
query::Comparator::field(Property::Size, comparator.is_ascending)
}
SortProperty::From => {
query::Comparator::field(Property::From, comparator.is_ascending)
}
SortProperty::To => {
query::Comparator::field(Property::To, comparator.is_ascending)
}
SortProperty::Subject => {
query::Comparator::field(Property::Subject, comparator.is_ascending)
}
SortProperty::SentAt => {
query::Comparator::field(Property::SentAt, comparator.is_ascending)
}
SortProperty::HasKeyword => query::Comparator::set(
self.store
.get_bitmap(BitmapKey::value(
account_id,
Collection::Email,
Property::Keywords,
comparator.keyword.unwrap_or(Keyword::Seen),
))
.await?
.unwrap_or_default(),
comparator.is_ascending,
),
SortProperty::AllInThreadHaveKeyword => query::Comparator::set(
self.thread_keywords(
account_id,
comparator.keyword.unwrap_or(Keyword::Seen),
true,
)
.await?,
comparator.is_ascending,
),
SortProperty::SomeInThreadHaveKeyword => query::Comparator::set(
self.thread_keywords(
account_id,
comparator.keyword.unwrap_or(Keyword::Seen),
false,
)
.await?,
comparator.is_ascending,
),
// Non-standard
SortProperty::Cc => {
query::Comparator::field(Property::Cc, comparator.is_ascending)
}
other => return Err(MethodError::UnsupportedSort(other.to_string())),
});
}
// Sort results
let result = self
.store
.sort(
result_set,
comparators,
Pagination::new(
limit_total,
request.position.unwrap_or(0),
request.anchor.map(|a| a.document_id()),
request.anchor_offset.unwrap_or(0),
ValueKey::new(account_id, Collection::Email, 0, Property::ThreadId).into(),
arguments.collapse_threads.unwrap_or(false),
),
)
.await?;
// Prepare response
if result.found_anchor {
response.position = result.position;
response.ids = result
.ids
.into_iter()
.map(|id| id.into())
.collect::<Vec<_>>();
} else {
return Err(MethodError::AnchorNotFound);
}
}
Ok(response)
}
async fn thread_keywords(
&self,
account_id: u32,
keyword: Keyword,
match_all: bool,
) -> Result<RoaringBitmap, MethodError> {
let keyword_doc_ids = self
.store
.get_bitmap(BitmapKey::value(
account_id,
Collection::Email,
Property::Keywords,
keyword,
))
.await?
.unwrap_or_default();
let mut not_matched_ids = RoaringBitmap::new();
let mut matched_ids = RoaringBitmap::new();
for keyword_doc_id in &keyword_doc_ids {
if matched_ids.contains(keyword_doc_id) || not_matched_ids.contains(keyword_doc_id) {
continue;
}
if let Some(thread_id) = self
.store
.get_value::<u32>(ValueKey::new(
account_id,
Collection::Email,
keyword_doc_id,
Property::ThreadId,
))
.await?
{
if let Some(thread_doc_ids) = self
.store
.get_bitmap(BitmapKey::value(
account_id,
Collection::Email,
Property::ThreadId,
thread_id,
))
.await?
{
let mut thread_tag_intersection = thread_doc_ids.clone();
thread_tag_intersection &= &keyword_doc_ids;
if (match_all && thread_tag_intersection == thread_doc_ids)
|| (!match_all && !thread_tag_intersection.is_empty())
{
matched_ids |= &thread_doc_ids;
} else if !thread_tag_intersection.is_empty() {
not_matched_ids |= &thread_tag_intersection;
}
}
}
}
Ok(matched_ids)
}
}

1
crates/core/src/lib.rs
View File

@@ -9,6 +9,7 @@ pub struct JMAP {
pub struct Config {
pub default_language: Language,
pub query_max_results: usize,
}
pub enum MaybeError {