Database schema optimization - part 13

This commit is contained in:
mdecimus
2025-11-22 10:55:54 +02:00
parent dbe40829da
commit c7fc16d9a2
60 changed files with 2118 additions and 1249 deletions

View File

@@ -106,6 +106,7 @@ impl<T: SessionStream> SessionData<T> {
.email_ingest(IngestEmail {
raw_message: &message.message,
message: MessageParser::new().parse(&message.message),
blob_hash: None,
access_token: &access_token,
mailbox_ids: vec![mailbox_id],
keywords: message.flags.into_iter().map(Keyword::from).collect(),

View File

@@ -244,9 +244,7 @@ impl<T: SessionStream> SessionData<T> {
copied_ids.push((imap_id.uid, mailbox.uid.to_native()));
if is_move {
let mut new_data = data
.deserialize()
.imap_ctx(&arguments.tag, trc::location!())?;
let mut new_data = data.inner.to_builder();
new_data.remove_mailbox(src_mailbox.id.mailbox_id);
batch
.with_account_id(account_id)
@@ -255,7 +253,7 @@ impl<T: SessionStream> SessionData<T> {
.custom(
ObjectIndexBuilder::new()
.with_current(data)
.with_changes(new_data),
.with_changes(new_data.seal()),
)
.imap_ctx(&arguments.tag, trc::location!())?
.log_vanished_item(
@@ -270,9 +268,7 @@ impl<T: SessionStream> SessionData<T> {
}
// Prepare changes
let mut new_data = data
.deserialize()
.imap_ctx(&arguments.tag, trc::location!())?;
let mut new_data = data.inner.to_builder();
// Add destination folder
new_data.add_mailbox(dest_mailbox_id);
@@ -302,7 +298,7 @@ impl<T: SessionStream> SessionData<T> {
.custom(
ObjectIndexBuilder::new()
.with_current(data)
.with_changes(new_data),
.with_changes(new_data.seal()),
)
.imap_ctx(&arguments.tag, trc::location!())?;
if is_move {

View File

@@ -192,7 +192,11 @@ impl<T: SessionStream> SessionData<T> {
if metadata.inner.mailboxes.len() == 1 {
// Delete message
batch
.custom(ObjectIndexBuilder::<_, ()>::new().with_current(metadata))
.custom(
ObjectIndexBuilder::<_, ()>::new()
.with_access_token(&self.access_token)
.with_current(metadata),
)
.caused_by(trc::location!())?
.set(
ValueClass::TaskQueue(TaskQueueClass::UpdateIndex {
@@ -205,9 +209,7 @@ impl<T: SessionStream> SessionData<T> {
.commit_point();
} else {
// Untag message from this mailbox and remove Deleted flag
let mut new_metadata = metadata
.deserialize::<MessageData>()
.caused_by(trc::location!())?;
let mut new_metadata = metadata.inner.to_builder();
new_metadata.remove_mailbox(mailbox_id);
new_metadata.remove_keyword(&Keyword::Deleted);
@@ -216,7 +218,7 @@ impl<T: SessionStream> SessionData<T> {
.custom(
ObjectIndexBuilder::new()
.with_current(metadata)
.with_changes(new_metadata),
.with_changes(new_metadata.seal()),
)
.caused_by(trc::location!())?
.commit_point();

View File

@@ -15,8 +15,9 @@ use directory::Permission;
use email::{
cache::{MessageCacheFetch, email::MessageCacheAccess},
message::metadata::{
ArchivedMessageMetadata, ArchivedMessageMetadataContents, ArchivedMetadataPartType,
DecodedParts, MessageData, MessageMetadata,
ArchivedMessageMetadata, ArchivedMessageMetadataContents, ArchivedMetadataHeaderValue,
ArchivedMetadataPartType, DecodedParts, MESSAGE_RECEIVED_MASK, MessageData,
MessageMetadata, MetadataHeaderName, PART_ENCODING_PROBLEM,
},
};
use imap_proto::{
@@ -32,9 +33,6 @@ use imap_proto::{
},
receiver::Request,
};
use mail_parser::{
ArchivedAddress, ArchivedHeaderName, ArchivedHeaderValue, core::rkyv::ArchivedGetHeader,
};
use std::{borrow::Cow, sync::Arc, time::Instant};
use store::{
query::log::{Change, Query},
@@ -48,6 +46,7 @@ use types::{
id::Id,
keyword::Keyword,
};
use utils::chained_bytes::{ChainedBytes, SliceRange};
impl<T: SessionStream> Session<T> {
pub async fn handle_fetch(&mut self, requests: Vec<Request<Command>>) -> trc::Result<()> {
@@ -349,37 +348,42 @@ impl<T: SessionStream> SessionData<T> {
let metadata = metadata_
.unarchive::<MessageMetadata>()
.imap_ctx(&arguments.tag, trc::location!())?;
let raw_body;
// Fetch and parse blob
let raw_message: Cow<[u8]> = if needs_blobs {
let mut raw_message = ChainedBytes::new(metadata.raw_headers.as_ref());
if needs_blobs {
// Retrieve raw message if needed
match self
raw_body = self
.server
.blob_store()
.get_blob(metadata.blob_hash.0.as_slice(), 0..usize::MAX)
.await
.imap_ctx(&arguments.tag, trc::location!())?
{
Some(raw_message) => raw_message.into(),
None => {
trc::event!(
Store(trc::StoreEvent::NotFound),
AccountId = account_id,
DocumentId = id,
Collection = Collection::Email,
BlobId = metadata.blob_hash.0.as_slice(),
Details = "Blob not found.",
CausedBy = trc::location!(),
);
.imap_ctx(&arguments.tag, trc::location!())?;
continue;
}
if let Some(raw_body) = &raw_body {
raw_message.append(
raw_body
.get(metadata.blob_body_offset.to_native() as usize..)
.unwrap_or_default(),
);
} else {
trc::event!(
Store(trc::StoreEvent::NotFound),
AccountId = account_id,
DocumentId = id,
Collection = Collection::Email,
BlobId = metadata.blob_hash.0.as_slice(),
Details = "Blob not found.",
CausedBy = trc::location!(),
);
continue;
}
} else {
metadata.raw_headers.as_slice().into()
};
}
let message = &metadata.contents[0];
let decoded = metadata.decode_contents(raw_message.as_ref());
let decoded = metadata.decode_contents(raw_message.clone());
// Build response
let mut items = Vec::with_capacity(arguments.attributes.len());
@@ -404,7 +408,7 @@ impl<T: SessionStream> SessionData<T> {
}
Attribute::InternalDate => {
items.push(DataItem::InternalDate {
date: u64::from(metadata.received_at) as i64,
date: (metadata.rcvd_attach.to_native() & MESSAGE_RECEIVED_MASK) as i64,
});
}
Attribute::Preview { .. } => {
@@ -418,7 +422,7 @@ impl<T: SessionStream> SessionData<T> {
}
Attribute::Rfc822Size => {
items.push(DataItem::Rfc822Size {
size: u32::from(metadata.size) as usize,
size: data.size as usize,
});
}
Attribute::Uid => {
@@ -426,23 +430,21 @@ impl<T: SessionStream> SessionData<T> {
}
Attribute::Rfc822 => {
items.push(DataItem::Rfc822 {
contents: raw_message.as_ref().into(),
contents: raw_message.get_full_range(),
});
}
Attribute::Rfc822Header => {
let message = metadata.root_part();
if let Some(header) = raw_message.get(
u32::from(message.offset_header) as usize
..u32::from(message.offset_body) as usize,
) {
items.push(DataItem::Rfc822Header {
contents: header.into(),
});
let contents = raw_message.get_slice_range(
0..u32::from(metadata.root_part().offset_body) as usize,
);
if contents != SliceRange::None {
items.push(DataItem::Rfc822Header { contents });
}
}
Attribute::Rfc822Text => {
items.push(DataItem::Rfc822Text {
contents: raw_message.as_ref().into(),
contents: raw_message.get_full_range(),
});
}
Attribute::Body => {
@@ -550,9 +552,7 @@ impl<T: SessionStream> SessionData<T> {
let data = data_
.to_unarchived::<MessageData>()
.imap_ctx(&arguments.tag, trc::location!())?;
let mut new_data = data
.deserialize()
.imap_ctx(&arguments.tag, trc::location!())?;
let mut new_data = data.inner.to_builder();
new_data.keywords.push(Keyword::Seen);
batch
@@ -562,7 +562,7 @@ impl<T: SessionStream> SessionData<T> {
.custom(
ObjectIndexBuilder::new()
.with_current(data)
.with_changes(new_data),
.with_changes(new_data.seal()),
)
.imap_ctx(&arguments.tag, trc::location!())?
.commit_point();
@@ -657,15 +657,14 @@ impl AsImapDataItemPart for ArchivedMessageMetadataContents {
is_extended: bool,
) -> BodyPart<'_> {
let part = &self.parts[part_id];
let body = decoded.raw_message_section_arch(message_id, part.offset_body, part.offset_end);
let body = decoded.raw_message_section(message_id, part.body_to_end());
let (is_multipart, is_text) = match &part.body {
ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html => (false, true),
ArchivedMetadataPartType::Multipart(_) => (true, false),
_ => (false, false),
};
let content_type = part
.headers
.header_value(&ArchivedHeaderName::ContentType)
.header_value(&MetadataHeaderName::ContentType)
.and_then(|ct| ct.as_content_type());
let mut body_md5 = None;
@@ -673,13 +672,15 @@ impl AsImapDataItemPart for ArchivedMessageMetadataContents {
let mut fields = BodyPartFields::default();
if !is_multipart || is_extended {
fields.body_parameters = content_type.as_ref().and_then(|ct| {
ct.attributes.as_ref().map(|at| {
at.iter()
fields.body_parameters = content_type
.as_ref()
.map(|ct| {
ct.attributes
.iter()
.map(|k| (k.name.as_ref().into(), k.value.as_ref().into()))
.collect::<Vec<_>>()
})
})
.filter(|p| !p.is_empty())
}
if !is_multipart {
@@ -688,18 +689,15 @@ impl AsImapDataItemPart for ArchivedMessageMetadataContents {
.and_then(|ct| ct.c_subtype.as_ref().map(|cs| cs.as_ref().into()));
fields.body_id = part
.headers
.header_value(&ArchivedHeaderName::ContentId)
.header_value(&MetadataHeaderName::ContentId)
.and_then(|id| id.as_text().map(|id| format!("<{}>", id).into()));
fields.body_description = part
.headers
.header_value(&ArchivedHeaderName::ContentDescription)
.header_value(&MetadataHeaderName::ContentDescription)
.and_then(|ct| ct.as_text().map(|ct| ct.into()));
fields.body_encoding = part
.headers
.header_value(&ArchivedHeaderName::ContentTransferEncoding)
.header_value(&MetadataHeaderName::ContentTransferEncoding)
.and_then(|ct| ct.as_text().map(|ct| ct.into()));
fields.body_size_octets = body.as_ref().map(|b| b.len()).unwrap_or(0);
@@ -725,34 +723,27 @@ impl AsImapDataItemPart for ArchivedMessageMetadataContents {
}
extension.body_disposition = part
.headers
.header_value(&ArchivedHeaderName::ContentDisposition)
.header_value(&MetadataHeaderName::ContentDisposition)
.and_then(|cd| cd.as_content_type())
.map(|cd| {
(
cd.c_type.as_ref().into(),
cd.attributes
.as_ref()
.map(|at| {
at.iter()
.map(|k| (k.name.as_ref().into(), k.value.as_ref().into()))
.collect::<Vec<_>>()
})
.unwrap_or_default(),
.iter()
.map(|k| (k.name.as_ref().into(), k.value.as_ref().into()))
.collect::<Vec<_>>(),
)
});
extension.body_language = part
.headers
.header_value(&ArchivedHeaderName::ContentLanguage)
.header_value(&MetadataHeaderName::ContentLanguage)
.and_then(|hv| {
hv.as_text_list()
.map(|list| list.iter().map(|item| item.as_ref().into()).collect())
});
extension.body_location = part
.headers
.header_value(&ArchivedHeaderName::ContentLocation)
.header_value(&MetadataHeaderName::ContentLocation)
.and_then(|ct| ct.as_text().map(|ct| ct.into()));
}
@@ -805,27 +796,27 @@ impl AsImapDataItemPart for ArchivedMessageMetadataContents {
date: headers.date(),
subject: headers.subject().map(|s| s.into()),
from: headers
.header_values(ArchivedHeaderName::From)
.header_values(&MetadataHeaderName::From)
.flat_map(|a| a.as_imap_address())
.collect(),
sender: headers
.header_values(ArchivedHeaderName::Sender)
.header_values(&MetadataHeaderName::Sender)
.flat_map(|a| a.as_imap_address())
.collect(),
reply_to: headers
.header_values(ArchivedHeaderName::ReplyTo)
.header_values(&MetadataHeaderName::ReplyTo)
.flat_map(|a| a.as_imap_address())
.collect(),
to: headers
.header_values(ArchivedHeaderName::To)
.header_values(&MetadataHeaderName::To)
.flat_map(|a| a.as_imap_address())
.collect(),
cc: headers
.header_values(ArchivedHeaderName::Cc)
.header_values(&MetadataHeaderName::Cc)
.flat_map(|a| a.as_imap_address())
.collect(),
bcc: headers
.header_values(ArchivedHeaderName::Bcc)
.header_values(&MetadataHeaderName::Bcc)
.flat_map(|a| a.as_imap_address())
.collect(),
in_reply_to: headers.in_reply_to().as_text_list().map(|list| {
@@ -913,13 +904,10 @@ impl AsImapDataItem for ArchivedMessageMetadata {
) -> Option<Cow<'x, [u8]>> {
let mut part = self.root_part();
if sections.is_empty() {
return Some(
get_partial_bytes(
decoded.raw_message_section_arch(0, part.offset_header, part.offset_end)?,
partial,
)
.into(),
);
return Some(get_cow_partial_bytes(
decoded.raw_message_section(0, part.header_to_end())?,
partial,
));
}
let mut message = &self.contents[0];
@@ -931,8 +919,9 @@ impl AsImapDataItem for ArchivedMessageMetadata {
Section::Part { num } => {
part = if let Some(sub_part_ids) = part.sub_parts() {
sub_part_ids
.as_ref()
.get((*num).saturating_sub(1) as usize)
.and_then(|pos| message.parts.get(u16::from(*pos) as usize))
.and_then(|pos| message.parts.as_ref().get(u16::from(*pos) as usize))
} else if *num == 1 && (section_num == sections.len() - 1 || part.is_message())
{
Some(part)
@@ -955,17 +944,10 @@ impl AsImapDataItem for ArchivedMessageMetadata {
}
}
Section::Header => {
return Some(
get_partial_bytes(
decoded.raw_message_section_arch(
message_id,
part.offset_header,
part.offset_body,
)?,
partial,
)
.into(),
);
return Some(get_cow_partial_bytes(
decoded.raw_message_section(message_id, part.header_to_body())?,
partial,
));
}
Section::HeaderFields { not, fields } => {
let mut headers = Vec::with_capacity(
@@ -978,12 +960,8 @@ impl AsImapDataItem for ArchivedMessageMetadata {
headers.extend_from_slice(header_name.as_bytes());
headers.push(b':');
headers.extend_from_slice(
decoded
.raw_message_section_arch(
message_id,
header.offset_start,
header.offset_end,
)
&decoded
.raw_message_section(message_id, header.value_range())
.unwrap_or_default(),
);
}
@@ -998,17 +976,10 @@ impl AsImapDataItem for ArchivedMessageMetadata {
});
}
Section::Text => {
return Some(
get_partial_bytes(
decoded.raw_message_section_arch(
message_id,
part.offset_body,
part.offset_end,
)?,
partial,
)
.into(),
);
return Some(get_cow_partial_bytes(
decoded.raw_message_section(message_id, part.body_to_end())?,
partial,
));
}
Section::Mime => {
let mut headers = Vec::with_capacity(
@@ -1022,12 +993,8 @@ impl AsImapDataItem for ArchivedMessageMetadata {
headers.extend_from_slice(header.name.as_str().as_bytes());
headers.extend_from_slice(b":");
headers.extend_from_slice(
decoded
.raw_message_section_arch(
message_id,
header.offset_start,
header.offset_end,
)
&decoded
.raw_message_section(message_id, header.value_range())
.unwrap_or_default(),
);
}
@@ -1045,13 +1012,10 @@ impl AsImapDataItem for ArchivedMessageMetadata {
// BODY[x] should return both headers and body, but most clients
// expect BODY[x] to return only the body, just like BOXY[x.TEXT] does.
Some(
get_partial_bytes(
decoded.raw_message_section_arch(message_id, part.offset_body, part.offset_end)?,
partial,
)
.into(),
)
Some(get_cow_partial_bytes(
decoded.raw_message_section(message_id, part.body_to_end())?,
partial,
))
}
fn binary<'x>(
@@ -1068,8 +1032,9 @@ impl AsImapDataItem for ArchivedMessageMetadata {
while let Some((section_num, num)) = sections_iter.next() {
part = if let Some(sub_part_ids) = part.sub_parts() {
if let Some(part) = sub_part_ids
.as_ref()
.get((*num).saturating_sub(1) as usize)
.and_then(|pos| message.parts.get(u16::from(*pos) as usize))
.and_then(|pos| message.parts.as_ref().get(u16::from(*pos) as usize))
{
part
} else {
@@ -1090,7 +1055,7 @@ impl AsImapDataItem for ArchivedMessageMetadata {
}
}
if !part.is_encoding_problem {
if (part.flags & PART_ENCODING_PROBLEM) == 0 {
let part_offset = u32::from(part.offset_header) as usize;
Ok(match &part.body {
ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html => {
@@ -1117,34 +1082,24 @@ impl AsImapDataItem for ArchivedMessageMetadata {
ArchivedMetadataPartType::Message(message) => BodyContents::Bytes({
{
let part = self.message_id(*message).root_part();
get_partial_bytes(
get_cow_partial_bytes(
decoded
.raw_message_section_arch(
message_id,
part.offset_header,
part.offset_end,
)
.raw_message_section(message_id, part.header_to_end())
.unwrap_or_default(),
partial,
)
.into()
}
})
.into(),
ArchivedMetadataPartType::Multipart(_) => BodyContents::Bytes(
get_partial_bytes(
ArchivedMetadataPartType::Multipart(_) => {
BodyContents::Bytes(get_cow_partial_bytes(
decoded
.raw_message_section_arch(
message_id,
part.offset_header,
part.offset_end,
)
.raw_message_section(message_id, part.header_to_end())
.unwrap_or_default(),
partial,
)
.into(),
)
.into(),
))
.into()
}
})
} else {
Err(())
@@ -1160,8 +1115,9 @@ impl AsImapDataItem for ArchivedMessageMetadata {
while let Some((section_num, num)) = sections_iter.next() {
part = if let Some(sub_part_ids) = part.sub_parts() {
sub_part_ids
.as_ref()
.get((*num).saturating_sub(1) as usize)
.and_then(|pos| message.parts.get(u16::from(pos) as usize))
.and_then(|pos| message.parts.as_ref().get(u16::from(pos) as usize))
} else if *num == 1 && (section_num == sections.len() - 1 || part.is_message()) {
Some(part)
} else {
@@ -1205,16 +1161,29 @@ fn get_partial_bytes(bytes: &[u8], partial: Option<(u32, u32)>) -> &[u8] {
}
}
#[inline(always)]
fn get_cow_partial_bytes(bytes: Cow<'_, [u8]>, partial: Option<(u32, u32)>) -> Cow<'_, [u8]> {
if let Some((start, end)) = partial {
let range = start as usize..std::cmp::min((start + end) as usize, bytes.len());
match bytes {
Cow::Borrowed(bytes) => Cow::Borrowed(bytes.get(range).unwrap_or_default()),
Cow::Owned(bytes) => Cow::Owned(bytes.get(range).unwrap_or_default().to_vec()),
}
} else {
bytes
}
}
trait AsImapAddress {
fn as_imap_address(&'_ self) -> Vec<fetch::Address<'_>>;
}
impl AsImapAddress for ArchivedHeaderValue<'_> {
impl AsImapAddress for ArchivedMetadataHeaderValue {
fn as_imap_address(&'_ self) -> Vec<fetch::Address<'_>> {
let mut addresses = Vec::new();
match self {
ArchivedHeaderValue::Address(ArchivedAddress::List(list)) => {
ArchivedMetadataHeaderValue::AddressList(list) => {
for addr in list.iter() {
if let Some(email) = addr.address.as_ref() {
addresses.push(fetch::Address::Single(fetch::EmailAddress {
@@ -1224,7 +1193,7 @@ impl AsImapAddress for ArchivedHeaderValue<'_> {
}
}
}
ArchivedHeaderValue::Address(ArchivedAddress::Group(list)) => {
ArchivedMetadataHeaderValue::AddressGroup(list) => {
for group in list.iter() {
addresses.push(fetch::Address::Group(fetch::AddressGroup {
name: group.name.as_ref().map(|n| n.as_ref().into()),

View File

@@ -20,13 +20,8 @@ use imap_proto::{
receiver::Request,
};
use std::time::Instant;
use store::{IterateParams, roaring::RoaringBitmap, write::key::DeserializeBigEndian};
use store::{
U32_LEN, ValueKey,
write::{IndexPropertyClass, ValueClass},
};
use trc::AddContext;
use types::{collection::Collection, field::EmailField, id::Id, keyword::Keyword};
use types::{id::Id, keyword::Keyword};
impl<T: SessionStream> Session<T> {
pub async fn handle_status(&mut self, requests: Vec<Request<Command>>) -> trc::Result<()> {
@@ -219,27 +214,14 @@ impl<T: SessionStream> SessionData<T> {
for item in items_update {
let result = match item {
Status::DeletedStorage => self
.calculate_mailbox_size(
mailbox.account_id,
&RoaringBitmap::from_iter(
cache
.in_mailbox_with_keyword(mailbox.mailbox_id, &Keyword::Deleted)
.map(|x| x.document_id),
),
)
.await
.caused_by(trc::location!())?,
Status::Size => self
.calculate_mailbox_size(
mailbox.account_id,
&RoaringBitmap::from_iter(
cache.in_mailbox(mailbox.mailbox_id).map(|x| x.document_id),
),
)
.await
.caused_by(trc::location!())?,
Status::DeletedStorage => cache
.in_mailbox_with_keyword(mailbox.mailbox_id, &Keyword::Deleted)
.map(|x| x.size)
.sum::<u32>() as u64,
Status::Size => cache
.in_mailbox(mailbox.mailbox_id)
.map(|x| x.size)
.sum::<u32>() as u64,
_ => {
unreachable!()
}
@@ -287,51 +269,4 @@ impl<T: SessionStream> SessionData<T> {
items: items_response,
})
}
async fn calculate_mailbox_size(
&self,
account_id: u32,
message_ids: &RoaringBitmap,
) -> trc::Result<u64> {
let mut total_size = 0u64;
self.server
.core
.storage
.data
.iterate(
IterateParams::new(
ValueKey {
account_id,
collection: Collection::Email.into(),
document_id: 0,
class: ValueClass::IndexProperty(IndexPropertyClass::Integer {
property: EmailField::ReceivedToSize.into(),
value: 0,
}),
},
ValueKey {
account_id,
collection: Collection::Email.into(),
document_id: u32::MAX,
class: ValueClass::IndexProperty(IndexPropertyClass::Integer {
property: EmailField::ReceivedToSize.into(),
value: u64::MAX,
}),
},
)
.ascending(),
|key, value| {
let id_pos = key.len() - U32_LEN;
let document_id = key.deserialize_be_u32(id_pos)?;
if message_ids.contains(document_id) {
total_size += value.deserialize_be_u32(0)? as u64;
}
Ok(true)
},
)
.await
.caused_by(trc::location!())
.map(|_| total_size)
}
}

View File

@@ -215,9 +215,7 @@ impl<T: SessionStream> SessionData<T> {
let data = data_
.to_unarchived::<MessageData>()
.imap_ctx(response.tag.as_ref().unwrap(), trc::location!())?;
let mut new_data = data
.deserialize()
.imap_ctx(response.tag.as_ref().unwrap(), trc::location!())?;
let mut new_data = data.inner.to_builder();
// Apply changes
let mut seen_changed = false;
@@ -296,7 +294,7 @@ impl<T: SessionStream> SessionData<T> {
.custom(
ObjectIndexBuilder::new()
.with_current(data)
.with_changes(new_data),
.with_changes(new_data.seal()),
)
.imap_ctx(response.tag.as_ref().unwrap(), trc::location!())?;