Database schema optimization - part 2

crates/email/src/message/copy.rs

@@ -5,17 +5,21 @@
  */
 
 use super::{
-    index::{MAX_SORT_FIELD_LENGTH, TrimTextValue, VisitText},
     ingest::{EmailIngest, IngestedEmail},
     metadata::{MessageData, MessageMetadata},
 };
 use crate::{
     mailbox::UidMailbox,
-    message::ingest::{MergeThreadTask, ThreadInfo},
+    message::{
+        index::extractors::VisitText,
+        ingest::{MergeThreadTask, ThreadInfo},
+    },
 };
 use common::{Server, auth::ResourceToken, storage::index::ObjectIndexBuilder};
 use mail_parser::{HeaderName, HeaderValue, parsers::fields::thread::thread_name};
-use store::write::{BatchBuilder, IndexPropertyClass, TaskQueueClass, ValueClass, now};
+use store::write::{
+    BatchBuilder, IndexPropertyClass, SearchIndex, TaskQueueClass, ValueClass, now,
+};
 use trc::AddContext;
 use types::{
     blob::{BlobClass, BlobId},
@@ -123,8 +127,7 @@ impl EmailCopy for Server {
                             list.first().unwrap().as_ref()
                         }
                         _ => "",
-                    })
-                    .trim_text(MAX_SORT_FIELD_LENGTH);
+                    });
                 }
                 _ => (),
             }
@@ -196,7 +199,11 @@ impl EmailCopy for Server {
                 ThreadInfo::serialize(thread_id, &message_ids),
             )
             .set(
-                ValueClass::TaskQueue(TaskQueueClass::IndexEmail { due: now() }),
+                ValueClass::TaskQueue(TaskQueueClass::UpdateIndex {
+                    index: SearchIndex::Email,
+                    due: now(),
+                    is_insert: true,
+                }),
                 MergeThreadTask::new(thread_result).serialize(),
             );
         metadata

crates/email/src/message/delete.rs

@@ -11,15 +11,15 @@ use groupware::calendar::storage::ItipAutoExpunge;
 use std::future::Future;
 use store::rand::prelude::SliceRandom;
 use store::write::key::DeserializeBigEndian;
-use store::write::{IndexPropertyClass, TaskQueueClass, now};
-use store::{IterateParams, SerializeInfallible, U32_LEN, ValueKey};
+use store::write::{IndexPropertyClass, SearchIndex, TaskQueueClass, now};
+use store::{IterateParams, SerializeInfallible, U32_LEN, U64_LEN, ValueKey};
 use store::{
     roaring::RoaringBitmap,
     write::{BatchBuilder, ValueClass},
 };
 use trc::AddContext;
 use types::collection::{Collection, VanishedCollection};
-use types::field::{EmailField, Field};
+use types::field::{EmailField, EmailSubmissionField, Field};
 
 pub trait EmailDeletion: Sync + Send {
     fn emails_delete(
@@ -33,6 +33,12 @@ pub trait EmailDeletion: Sync + Send {
 
     fn purge_account(&self, account_id: u32) -> impl Future<Output = ()> + Send;
 
+    fn purge_email_submissions(
+        &self,
+        account_id: u32,
+        hold_period: u64,
+    ) -> impl Future<Output = trc::Result<()>> + Send;
+
     fn emails_auto_expunge(
         &self,
         account_id: u32,
@@ -72,7 +78,11 @@ impl EmailDeletion for Server {
                     .custom(ObjectIndexBuilder::<_, ()>::new().with_current(metadata))
                     .caused_by(trc::location!())?
                     .set(
-                        ValueClass::TaskQueue(TaskQueueClass::UnindexEmail { due }),
+                        ValueClass::TaskQueue(TaskQueueClass::UpdateIndex {
+                            index: SearchIndex::Email,
+                            due,
+                            is_insert: false,
+                        }),
                         0u64.serialize(),
                     )
                     .commit_point();
@@ -164,6 +174,16 @@ impl EmailDeletion for Server {
             );
         }
 
+        // Delete old e-mail submissions
+        if let Some(hold_period) = self.core.jmap.email_submission_autoexpunge_after
+            && let Err(err) = self.purge_email_submissions(account_id, hold_period).await
+        {
+            trc::error!(
+                err.details("Failed to auto-expunge e-mail submissions.")
+                    .account_id(account_id)
+            );
+        }
+
         // Purge changelogs
         if let Err(err) = self
             .delete_changes(
@@ -218,7 +238,7 @@ impl EmailDeletion for Server {
                         collection: Collection::Email.into(),
                         document_id: 0,
                         class: ValueClass::IndexProperty(IndexPropertyClass::Integer {
-                            property: EmailField::Stats.into(),
+                            property: EmailField::ReceivedToSize.into(),
                             value: 0,
                         }),
                     },
@@ -227,7 +247,7 @@ impl EmailDeletion for Server {
                         collection: Collection::Email.into(),
                         document_id: u32::MAX,
                         class: ValueClass::IndexProperty(IndexPropertyClass::Integer {
-                            property: EmailField::Stats.into(),
+                            property: EmailField::ReceivedToSize.into(),
                             value: now().saturating_sub(hold_period),
                         }),
                     },
@@ -269,6 +289,78 @@ impl EmailDeletion for Server {
         Ok(())
     }
 
+    async fn purge_email_submissions(&self, account_id: u32, hold_period: u64) -> trc::Result<()> {
+        // Filter messages by received date
+        let mut destroy_ids = Vec::new();
+        self.store()
+            .iterate(
+                IterateParams::new(
+                    ValueKey {
+                        account_id,
+                        collection: Collection::EmailSubmission.into(),
+                        document_id: 0,
+                        class: ValueClass::IndexProperty(IndexPropertyClass::Integer {
+                            property: EmailSubmissionField::Metadata.into(),
+                            value: 0,
+                        }),
+                    },
+                    ValueKey {
+                        account_id,
+                        collection: Collection::Email.into(),
+                        document_id: u32::MAX,
+                        class: ValueClass::IndexProperty(IndexPropertyClass::Integer {
+                            property: EmailSubmissionField::Metadata.into(),
+                            value: now().saturating_sub(hold_period),
+                        }),
+                    },
+                )
+                .ascending()
+                .no_values(),
+                |key, _| {
+                    destroy_ids.push((
+                        key.deserialize_be_u32(key.len() - U32_LEN)?,
+                        key.deserialize_be_u64(key.len() - U32_LEN - U64_LEN)?,
+                    ));
+
+                    Ok(true)
+                },
+            )
+            .await
+            .caused_by(trc::location!())?;
+
+        if destroy_ids.is_empty() {
+            return Ok(());
+        }
+
+        trc::event!(
+            Purge(trc::PurgeEvent::AutoExpunge),
+            Collection = Collection::EmailSubmission.as_str(),
+            AccountId = account_id,
+            Total = destroy_ids.len(),
+        );
+
+        // Delete messages
+        let mut batch = BatchBuilder::new();
+        batch
+            .with_account_id(account_id)
+            .with_collection(Collection::EmailSubmission);
+
+        for (document_id, send_at) in destroy_ids {
+            batch
+                .with_document(document_id)
+                .clear(EmailSubmissionField::Metadata)
+                .clear(ValueClass::IndexProperty(IndexPropertyClass::Integer {
+                    property: EmailSubmissionField::Metadata.into(),
+                    value: send_at,
+                }))
+                .commit_point();
+        }
+
+        self.commit_batch(batch).await?;
+
+        Ok(())
+    }
+
     /*async fn emails_purge_tombstoned(&self, account_id: u32) -> trc::Result<()> {
         // Obtain tombstoned messages
         let tombstoned_ids = self

crates/email/src/message/index/extractors.rs

@@ -0,0 +1,250 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::message::metadata::{ArchivedMessageMetadataContents, ArchivedMessageMetadataPart};
+use mail_parser::{
+    Addr, Address, ArchivedAddress, ArchivedHeaderName, ArchivedHeaderValue, Group, HeaderValue,
+    core::rkyv::ArchivedGetHeader,
+};
+use nlp::language::Language;
+use rkyv::option::ArchivedOption;
+use std::borrow::Cow;
+
+impl ArchivedMessageMetadataContents {
+    pub fn is_html_part(&self, part_id: u16) -> bool {
+        self.html_body.iter().any(|&id| id == part_id)
+    }
+
+    pub fn is_text_part(&self, part_id: u16) -> bool {
+        self.text_body.iter().any(|&id| id == part_id)
+    }
+}
+
+impl ArchivedMessageMetadataPart {
+    pub fn language(&self) -> Option<Language> {
+        self.headers
+            .header_value(&ArchivedHeaderName::ContentLanguage)
+            .and_then(|v| {
+                Language::from_iso_639(match v {
+                    ArchivedHeaderValue::Text(v) => v.as_ref(),
+                    ArchivedHeaderValue::TextList(v) => v.first()?,
+                    _ => {
+                        return None;
+                    }
+                })
+                .unwrap_or(Language::Unknown)
+                .into()
+            })
+    }
+}
+
+#[derive(Debug, PartialEq, Eq)]
+pub enum AddressElement {
+    Name,
+    Address,
+    GroupName,
+}
+
+pub trait VisitText {
+    fn visit_addresses(&self, visitor: impl FnMut(AddressElement, &str));
+    fn visit_text<'x>(&'x self, visitor: impl FnMut(&'x str));
+    fn into_visit_text(self, visitor: impl FnMut(String));
+}
+
+impl VisitText for HeaderValue<'_> {
+    fn visit_addresses(&self, mut visitor: impl FnMut(AddressElement, &str)) {
+        match self {
+            HeaderValue::Address(Address::List(addr_list)) => {
+                for addr in addr_list {
+                    if let Some(name) = &addr.name {
+                        visitor(AddressElement::Name, name);
+                    }
+                    if let Some(addr) = &addr.address {
+                        visitor(AddressElement::Address, addr);
+                    }
+                }
+            }
+            HeaderValue::Address(Address::Group(groups)) => {
+                for group in groups {
+                    if let Some(name) = &group.name {
+                        visitor(AddressElement::GroupName, name);
+                    }
+
+                    for addr in &group.addresses {
+                        if let Some(name) = &addr.name {
+                            visitor(AddressElement::Name, name);
+                        }
+                        if let Some(addr) = &addr.address {
+                            visitor(AddressElement::Address, addr);
+                        }
+                    }
+                }
+            }
+            _ => (),
+        }
+    }
+
+    fn visit_text<'x>(&'x self, mut visitor: impl FnMut(&'x str)) {
+        match &self {
+            HeaderValue::Text(text) => {
+                visitor(text.as_ref());
+            }
+            HeaderValue::TextList(texts) => {
+                for text in texts {
+                    visitor(text.as_ref());
+                }
+            }
+            _ => (),
+        }
+    }
+
+    fn into_visit_text(self, mut visitor: impl FnMut(String)) {
+        match self {
+            HeaderValue::Text(text) => {
+                visitor(text.into_owned());
+            }
+            HeaderValue::TextList(texts) => {
+                for text in texts {
+                    visitor(text.into_owned());
+                }
+            }
+            _ => (),
+        }
+    }
+}
+
+pub trait VisitTextArchived {
+    fn visit_addresses(&self, visitor: impl FnMut(AddressElement, &str));
+    fn visit_text(&self, visitor: impl FnMut(&str));
+}
+
+impl VisitTextArchived for ArchivedHeaderValue<'static> {
+    fn visit_addresses(&self, mut visitor: impl FnMut(AddressElement, &str)) {
+        match self {
+            ArchivedHeaderValue::Address(ArchivedAddress::List(addr_list)) => {
+                for addr in addr_list.iter() {
+                    if let ArchivedOption::Some(name) = &addr.name {
+                        visitor(AddressElement::Name, name);
+                    }
+                    if let ArchivedOption::Some(addr) = &addr.address {
+                        visitor(AddressElement::Address, addr);
+                    }
+                }
+            }
+            ArchivedHeaderValue::Address(ArchivedAddress::Group(groups)) => {
+                for group in groups.iter() {
+                    if let ArchivedOption::Some(name) = &group.name {
+                        visitor(AddressElement::GroupName, name);
+                    }
+
+                    for addr in group.addresses.iter() {
+                        if let ArchivedOption::Some(name) = &addr.name {
+                            visitor(AddressElement::Name, name);
+                        }
+                        if let ArchivedOption::Some(addr) = &addr.address {
+                            visitor(AddressElement::Address, addr);
+                        }
+                    }
+                }
+            }
+            _ => (),
+        }
+    }
+
+    fn visit_text(&self, mut visitor: impl FnMut(&str)) {
+        match &self {
+            ArchivedHeaderValue::Text(text) => {
+                visitor(text.as_ref());
+            }
+            ArchivedHeaderValue::TextList(texts) => {
+                for text in texts.iter() {
+                    visitor(text.as_ref());
+                }
+            }
+            _ => (),
+        }
+    }
+}
+
+pub trait TrimTextValue {
+    fn trim_text(self, length: usize) -> Self;
+}
+
+impl TrimTextValue for HeaderValue<'_> {
+    fn trim_text(self, length: usize) -> Self {
+        match self {
+            HeaderValue::Address(Address::List(v)) => {
+                HeaderValue::Address(Address::List(v.trim_text(length)))
+            }
+            HeaderValue::Address(Address::Group(v)) => {
+                HeaderValue::Address(Address::Group(v.trim_text(length)))
+            }
+            HeaderValue::Text(v) => HeaderValue::Text(v.trim_text(length)),
+            HeaderValue::TextList(v) => HeaderValue::TextList(v.trim_text(length)),
+            v => v,
+        }
+    }
+}
+
+impl TrimTextValue for Addr<'_> {
+    fn trim_text(self, length: usize) -> Self {
+        Self {
+            name: self.name.map(|v| v.trim_text(length)),
+            address: self.address.map(|v| v.trim_text(length)),
+        }
+    }
+}
+
+impl TrimTextValue for Group<'_> {
+    fn trim_text(self, length: usize) -> Self {
+        Self {
+            name: self.name.map(|v| v.trim_text(length)),
+            addresses: self.addresses.trim_text(length),
+        }
+    }
+}
+
+impl TrimTextValue for &str {
+    fn trim_text(self, length: usize) -> Self {
+        if self.len() < length {
+            self
+        } else {
+            let mut index = 0;
+
+            for (i, _) in self.char_indices() {
+                if i > length {
+                    break;
+                }
+                index = i;
+            }
+
+            &self[..index]
+        }
+    }
+}
+
+impl TrimTextValue for Cow<'_, str> {
+    fn trim_text(self, length: usize) -> Self {
+        if self.len() < length {
+            self
+        } else {
+            let mut result = String::with_capacity(length);
+            for (i, c) in self.char_indices() {
+                if i > length {
+                    break;
+                }
+                result.push(c);
+            }
+            result.into()
+        }
+    }
+}
+
+impl<T: TrimTextValue> TrimTextValue for Vec<T> {
+    fn trim_text(self, length: usize) -> Self {
+        self.into_iter().map(|v| v.trim_text(length)).collect()
+    }
+}

crates/email/src/message/index/metadata.rs

@@ -0,0 +1,232 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::message::{
+    index::{IndexMessage, MAX_MESSAGE_PARTS, PREVIEW_LENGTH},
+    metadata::{
+        ArchivedMessageMetadata, ArchivedMessageMetadataPart, MessageData, MessageMetadata,
+        MessageMetadataPart,
+    },
+};
+use common::storage::index::ObjectIndexBuilder;
+use mail_parser::{decoders::html::html_to_text, parsers::preview::preview_text};
+use store::{
+    Serialize, SerializeInfallible,
+    write::{Archiver, BatchBuilder, BlobOp, DirectoryClass, IndexPropertyClass, ValueClass},
+};
+use trc::AddContext;
+use types::{blob_hash::BlobHash, field::EmailField};
+
+impl MessageMetadata {
+    #[inline(always)]
+    pub fn root_part(&self) -> &MessageMetadataPart {
+        &self.contents[0].parts[0]
+    }
+
+    pub fn index(
+        self,
+        batch: &mut BatchBuilder,
+        account_id: u32,
+        tenant_id: Option<u32>,
+        set: bool,
+    ) -> trc::Result<()> {
+        if set {
+            // Serialize metadata
+            batch.set(
+                ValueClass::IndexProperty(IndexPropertyClass::Integer {
+                    property: EmailField::ReceivedToSize.into(),
+                    value: self.received_at,
+                }),
+                self.size.serialize(),
+            );
+        } else {
+            // Delete metadata
+            batch
+                .clear(EmailField::Metadata)
+                .clear(ValueClass::IndexProperty(IndexPropertyClass::Integer {
+                    property: EmailField::ReceivedToSize.into(),
+                    value: self.received_at,
+                }));
+        }
+
+        // Index properties
+        let quota = if set {
+            self.size as i64
+        } else {
+            -(self.size as i64)
+        };
+        batch.add(DirectoryClass::UsedQuota(account_id), quota);
+        if let Some(tenant_id) = tenant_id {
+            batch.add(DirectoryClass::UsedQuota(tenant_id), quota);
+        }
+
+        // Link blob
+        if set {
+            batch.set(
+                BlobOp::Link {
+                    hash: self.blob_hash.clone(),
+                },
+                Vec::new(),
+            );
+        } else {
+            batch.clear(BlobOp::Link {
+                hash: self.blob_hash.clone(),
+            });
+        }
+
+        if set {
+            batch.set(EmailField::Metadata, Archiver::new(self).serialize()?);
+        }
+
+        Ok(())
+    }
+}
+
+impl ArchivedMessageMetadata {
+    #[inline(always)]
+    pub fn root_part(&self) -> &ArchivedMessageMetadataPart {
+        &self.contents[0].parts[0]
+    }
+
+    pub fn unindex(&self, batch: &mut BatchBuilder, account_id: u32, tenant_id: Option<u32>) {
+        // Delete metadata
+        batch
+            .clear(EmailField::Metadata)
+            .clear(ValueClass::IndexProperty(IndexPropertyClass::Integer {
+                property: EmailField::ReceivedToSize.into(),
+                value: self.received_at.to_native(),
+            }));
+
+        // Index properties
+        let quota = -(u32::from(self.size) as i64);
+        batch.add(DirectoryClass::UsedQuota(account_id), quota);
+        if let Some(tenant_id) = tenant_id {
+            batch.add(DirectoryClass::UsedQuota(tenant_id), quota);
+        }
+
+        // Unlink blob
+        batch.clear(BlobOp::Link {
+            hash: BlobHash::from(&self.blob_hash),
+        });
+    }
+}
+
+impl IndexMessage for BatchBuilder {
+    fn index_message(
+        &mut self,
+        account_id: u32,
+        tenant_id: Option<u32>,
+        message: mail_parser::Message<'_>,
+        blob_hash: BlobHash,
+        data: MessageData,
+        received_at: u64,
+    ) -> trc::Result<&mut Self> {
+        // Index size
+        self.set(
+            ValueClass::IndexProperty(IndexPropertyClass::Integer {
+                property: EmailField::ReceivedToSize.into(),
+                value: received_at,
+            }),
+            (message.raw_message.len() as u32).serialize(),
+        )
+        .add(
+            DirectoryClass::UsedQuota(account_id),
+            message.raw_message.len() as i64,
+        );
+        if let Some(tenant_id) = tenant_id {
+            self.add(
+                DirectoryClass::UsedQuota(tenant_id),
+                message.raw_message.len() as i64,
+            );
+        }
+
+        let mut has_attachments = false;
+        let mut preview = None;
+        let preview_part_id = message
+            .text_body
+            .first()
+            .or_else(|| message.html_body.first())
+            .copied()
+            .unwrap_or(u32::MAX);
+
+        for (part_id, part) in message.parts.iter().take(MAX_MESSAGE_PARTS).enumerate() {
+            let part_id = part_id as u32;
+            match &part.body {
+                mail_parser::PartType::Text(text) => {
+                    if part_id == preview_part_id {
+                        preview =
+                            preview_text(text.replace('\r', "").into(), PREVIEW_LENGTH).into();
+                    }
+
+                    if !message.text_body.contains(&part_id)
+                        && !message.html_body.contains(&part_id)
+                    {
+                        has_attachments = true;
+                    }
+                }
+                mail_parser::PartType::Html(html) => {
+                    let text = html_to_text(html);
+                    if part_id == preview_part_id {
+                        preview =
+                            preview_text(text.replace('\r', "").into(), PREVIEW_LENGTH).into();
+                    }
+
+                    if !message.text_body.contains(&part_id)
+                        && !message.html_body.contains(&part_id)
+                    {
+                        has_attachments = true;
+                    }
+                }
+                mail_parser::PartType::Binary(_) | mail_parser::PartType::Message(_)
+                    if !has_attachments =>
+                {
+                    has_attachments = true;
+                }
+                _ => {}
+            }
+        }
+
+        // Build metadata
+        let root_part = message.root_part();
+        let metadata = MessageMetadata {
+            preview: preview.unwrap_or_default().into_owned(),
+            size: message.raw_message.len() as u32,
+            raw_headers: message
+                .raw_message
+                .as_ref()
+                .get(root_part.offset_header as usize..root_part.offset_body as usize)
+                .unwrap_or_default()
+                .to_vec(),
+            contents: vec![],
+            received_at,
+            has_attachments,
+            blob_hash,
+        }
+        .with_contents(message);
+
+        // Link blob
+        self.set(
+            BlobOp::Link {
+                hash: metadata.blob_hash.clone(),
+            },
+            Vec::new(),
+        );
+
+        // Store message data
+        self.custom(ObjectIndexBuilder::<(), _>::new().with_changes(data))
+            .caused_by(trc::location!())?;
+
+        // Store message metadata
+        self.set(
+            EmailField::Metadata,
+            Archiver::new(metadata)
+                .serialize()
+                .caused_by(trc::location!())?,
+        );
+
+        Ok(self)
+    }
+}

crates/email/src/message/index/mod.rs

@@ -0,0 +1,73 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::message::metadata::{ArchivedMessageData, MessageData};
+use common::storage::index::{IndexValue, IndexableObject};
+use types::{blob_hash::BlobHash, collection::SyncCollection};
+
+pub mod extractors;
+pub mod metadata;
+pub mod search;
+
+pub(super) const MAX_MESSAGE_PARTS: usize = 1000;
+pub const PREVIEW_LENGTH: usize = 256;
+
+impl IndexableObject for MessageData {
+    fn index_values(&self) -> impl Iterator<Item = IndexValue<'_>> {
+        [
+            IndexValue::LogItem {
+                sync_collection: SyncCollection::Email,
+                prefix: self.thread_id.into(),
+            },
+            IndexValue::LogContainerProperty {
+                sync_collection: SyncCollection::Thread,
+                ids: vec![self.thread_id],
+            },
+            IndexValue::LogContainerProperty {
+                sync_collection: SyncCollection::Email,
+                ids: self.mailboxes.iter().map(|m| m.mailbox_id).collect(),
+            },
+        ]
+        .into_iter()
+    }
+}
+
+impl IndexableObject for &ArchivedMessageData {
+    fn index_values(&self) -> impl Iterator<Item = IndexValue<'_>> {
+        [
+            IndexValue::LogItem {
+                sync_collection: SyncCollection::Email,
+                prefix: self.thread_id.to_native().into(),
+            },
+            IndexValue::LogContainerProperty {
+                sync_collection: SyncCollection::Thread,
+                ids: vec![self.thread_id.to_native()],
+            },
+            IndexValue::LogContainerProperty {
+                sync_collection: SyncCollection::Email,
+                ids: self
+                    .mailboxes
+                    .iter()
+                    .map(|m| m.mailbox_id.to_native())
+                    .collect(),
+            },
+        ]
+        .into_iter()
+    }
+}
+
+pub(super) trait IndexMessage {
+    #[allow(clippy::too_many_arguments)]
+    fn index_message(
+        &mut self,
+        account_id: u32,
+        tenant_id: Option<u32>,
+        message: mail_parser::Message<'_>,
+        blob_hash: BlobHash,
+        data: MessageData,
+        received_at: u64,
+    ) -> trc::Result<&mut Self>;
+}

crates/email/src/message/index/search.rs

@@ -0,0 +1,194 @@
+/*
+ * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC <hello@stalw.art>
+ *
+ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
+ */
+
+use crate::message::{
+    index::{MAX_MESSAGE_PARTS, extractors::VisitTextArchived},
+    metadata::{ArchivedMessageMetadata, ArchivedMetadataPartType, DecodedPartContent},
+};
+use mail_parser::{
+    ArchivedHeaderName, ArchivedHeaderValue, DateTime, HeaderName, core::rkyv::ArchivedGetHeader,
+    decoders::html::html_to_text,
+};
+use nlp::language::Language;
+use std::borrow::Cow;
+use store::{
+    ahash::AHashSet,
+    search::{EmailSearchField, IndexDocument, SearchField},
+};
+
+impl ArchivedMessageMetadata {
+    pub fn index_document(
+        &self,
+        raw_message: &[u8],
+        index_headers: &AHashSet<HeaderName<'static>>,
+    ) -> IndexDocument {
+        let mut language = Language::Unknown;
+        let message_contents = &self.contents[0];
+        let mut document = IndexDocument::with_default_language(language);
+
+        document.index_number(
+            EmailSearchField::ReceivedAt,
+            self.received_at.to_native() as i64,
+        );
+        document.index_number(EmailSearchField::Size, self.size.to_native());
+
+        for (part_id, part) in message_contents
+            .parts
+            .iter()
+            .take(MAX_MESSAGE_PARTS)
+            .enumerate()
+        {
+            let part_language = part.language().unwrap_or(language);
+            if part_id == 0 {
+                language = part_language;
+
+                for header in part.headers.iter().rev() {
+                    let header_name = HeaderName::from(&header.name);
+                    if !index_headers.is_empty() && !index_headers.contains(&header_name) {
+                        continue;
+                    }
+                    let header_name = match header_name {
+                        HeaderName::Other(name) => Cow::Owned(name.into_owned()),
+                        _ => Cow::Borrowed(header_name.as_static_str()),
+                    };
+
+                    match &header.name {
+                        ArchivedHeaderName::From => {
+                            header.value.visit_addresses(|_, value| {
+                                document.index_text(
+                                    EmailSearchField::From,
+                                    value,
+                                    Language::Unknown,
+                                );
+                            });
+                        }
+                        ArchivedHeaderName::To => {
+                            header.value.visit_addresses(|_, value| {
+                                document.index_text(EmailSearchField::To, value, Language::Unknown);
+                            });
+                        }
+                        ArchivedHeaderName::Cc => {
+                            header.value.visit_addresses(|_, value| {
+                                document.index_text(EmailSearchField::Cc, value, Language::Unknown);
+                            });
+                        }
+                        ArchivedHeaderName::Bcc => {
+                            header.value.visit_addresses(|_, value| {
+                                document.index_text(
+                                    EmailSearchField::Bcc,
+                                    value,
+                                    Language::Unknown,
+                                );
+                            });
+                        }
+                        ArchivedHeaderName::Subject => {
+                            if let Some(subject) = header.value.as_text() {
+                                document.index_text(
+                                    EmailSearchField::Subject,
+                                    subject,
+                                    part_language,
+                                );
+                            }
+                        }
+                        ArchivedHeaderName::Date => {
+                            if let Some(date) = header.value.as_datetime() {
+                                document.index_number(
+                                    EmailSearchField::SentAt,
+                                    DateTime::from(date).to_timestamp(),
+                                );
+                            }
+                        }
+                        _ => {
+                            header.value.visit_text(|text| {
+                                document.index_text(
+                                    EmailSearchField::Header(header_name.clone()),
+                                    text,
+                                    Language::Unknown,
+                                );
+                            });
+                        }
+                    }
+                }
+            }
+
+            let part_id = part_id as u16;
+            match &part.body {
+                ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html => {
+                    let text = match (part.decode_contents(raw_message), &part.body) {
+                        (DecodedPartContent::Text(text), ArchivedMetadataPartType::Text) => text,
+                        (DecodedPartContent::Text(html), ArchivedMetadataPartType::Html) => {
+                            html_to_text(html.as_ref()).into()
+                        }
+                        _ => unreachable!(),
+                    };
+
+                    if message_contents.is_html_part(part_id)
+                        || message_contents.is_text_part(part_id)
+                    {
+                        document.index_text(EmailSearchField::Body, text.as_ref(), part_language);
+                    } else {
+                        document.index_text(
+                            EmailSearchField::Attachment,
+                            text.as_ref(),
+                            part_language,
+                        );
+                    }
+                }
+                ArchivedMetadataPartType::Message(nested_message_id) => {
+                    let nested_message = self.message_id(*nested_message_id);
+                    let nested_message_language = nested_message
+                        .root_part()
+                        .language()
+                        .unwrap_or(Language::Unknown);
+                    if let Some(ArchivedHeaderValue::Text(subject)) = nested_message
+                        .root_part()
+                        .headers
+                        .header_value(&ArchivedHeaderName::Subject)
+                    {
+                        document.index_text(
+                            EmailSearchField::Attachment,
+                            subject.as_ref(),
+                            nested_message_language,
+                        );
+                    }
+
+                    for sub_part in nested_message.parts.iter().take(MAX_MESSAGE_PARTS) {
+                        let language = sub_part.language().unwrap_or(nested_message_language);
+                        match &sub_part.body {
+                            ArchivedMetadataPartType::Text | ArchivedMetadataPartType::Html => {
+                                let text =
+                                    match (sub_part.decode_contents(raw_message), &sub_part.body) {
+                                        (
+                                            DecodedPartContent::Text(text),
+                                            ArchivedMetadataPartType::Text,
+                                        ) => text,
+                                        (
+                                            DecodedPartContent::Text(html),
+                                            ArchivedMetadataPartType::Html,
+                                        ) => html_to_text(html.as_ref()).into(),
+                                        _ => unreachable!(),
+                                    };
+                                document.index_text(
+                                    EmailSearchField::Attachment,
+                                    text.as_ref(),
+                                    language,
+                                );
+                            }
+                            _ => (),
+                        }
+                    }
+                }
+                _ => {}
+            }
+        }
+
+        let has_attachment = document.has_field(&SearchField::Email(EmailSearchField::Attachment));
+
+        document.index_bool(EmailSearchField::HasAttachment, has_attachment);
+
+        document
+    }
+}

crates/email/src/message/ingest.rs

@@ -4,16 +4,13 @@
  * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
  */
 
-use super::{
-    crypto::{EncryptMessage, EncryptMessageError},
-    index::{MAX_SORT_FIELD_LENGTH, TrimTextValue},
-};
+use super::crypto::{EncryptMessage, EncryptMessageError};
 use crate::{
     cache::{MessageCacheFetch, email::MessageCacheAccess},
     mailbox::{INBOX_ID, JUNK_ID, UidMailbox},
     message::{
         crypto::EncryptionParams,
-        index::{IndexMessage, VisitText},
+        index::{IndexMessage, extractors::VisitText},
         metadata::MessageData,
     },
 };
@@ -36,8 +33,8 @@ use store::{
     IndexKeyPrefix, IterateParams, U32_LEN, ValueKey,
     ahash::AHashMap,
     write::{
-        BatchBuilder, IndexPropertyClass, TaskQueueClass, ValueClass, key::DeserializeBigEndian,
-        now,
+        BatchBuilder, IndexPropertyClass, SearchIndex, TaskQueueClass, ValueClass,
+        key::DeserializeBigEndian, now,
     },
 };
 use trc::{AddContext, MessageIngestEvent};
@@ -427,8 +424,7 @@ impl EmailIngest for Server {
                                 list.first().unwrap().as_ref()
                             }
                             _ => "",
-                        })
-                        .trim_text(MAX_SORT_FIELD_LENGTH);
+                        });
                     }
                     _ => (),
                 }
@@ -654,7 +650,11 @@ impl EmailIngest for Server {
                 ThreadInfo::serialize(thread_id, &message_ids),
             )
             .set(
-                ValueClass::TaskQueue(TaskQueueClass::IndexEmail { due }),
+                ValueClass::TaskQueue(TaskQueueClass::UpdateIndex {
+                    index: SearchIndex::Email,
+                    due,
+                    is_insert: true,
+                }),
                 MergeThreadTask::new(thread_result).serialize(),
             );
 

crates/email/src/submission/index.rs

@@ -6,30 +6,27 @@
 
 use super::{ArchivedEmailSubmission, EmailSubmission};
 use common::storage::index::{IndexValue, IndexableAndSerializableObject, IndexableObject};
+use store::{
+    U32_LEN,
+    write::{IndexPropertyClass, ValueClass, key::KeySerializer},
+};
 use types::{collection::SyncCollection, field::EmailSubmissionField};
 
 impl IndexableObject for EmailSubmission {
     fn index_values(&self) -> impl Iterator<Item = IndexValue<'_>> {
         [
-            IndexValue::Index {
-                field: EmailSubmissionField::UndoStatus.into(),
-                value: self.undo_status.as_index().into(),
-            },
-            IndexValue::Index {
-                field: EmailSubmissionField::EmailId.into(),
-                value: self.email_id.into(),
-            },
-            IndexValue::Index {
-                field: EmailSubmissionField::ThreadId.into(),
-                value: self.thread_id.into(),
-            },
-            IndexValue::Index {
-                field: EmailSubmissionField::IdentityId.into(),
-                value: self.identity_id.into(),
-            },
-            IndexValue::Index {
-                field: EmailSubmissionField::SendAt.into(),
-                value: self.send_at.into(),
+            IndexValue::Property {
+                field: ValueClass::IndexProperty(IndexPropertyClass::Integer {
+                    property: EmailSubmissionField::Metadata.into(),
+                    value: self.send_at,
+                }),
+                value: KeySerializer::new(U32_LEN * 3 + 1)
+                    .write(self.email_id)
+                    .write(self.thread_id)
+                    .write(self.identity_id)
+                    .write(self.undo_status.as_index())
+                    .finalize()
+                    .into(),
             },
             IndexValue::LogItem {
                 sync_collection: SyncCollection::EmailSubmission,
@@ -43,25 +40,18 @@ impl IndexableObject for EmailSubmission {
 impl IndexableObject for &ArchivedEmailSubmission {
     fn index_values(&self) -> impl Iterator<Item = IndexValue<'_>> {
         [
-            IndexValue::Index {
-                field: EmailSubmissionField::UndoStatus.into(),
-                value: self.undo_status.as_index().into(),
-            },
-            IndexValue::Index {
-                field: EmailSubmissionField::EmailId.into(),
-                value: self.email_id.into(),
-            },
-            IndexValue::Index {
-                field: EmailSubmissionField::ThreadId.into(),
-                value: self.thread_id.into(),
-            },
-            IndexValue::Index {
-                field: EmailSubmissionField::IdentityId.into(),
-                value: self.identity_id.into(),
-            },
-            IndexValue::Index {
-                field: EmailSubmissionField::SendAt.into(),
-                value: self.send_at.into(),
+            IndexValue::Property {
+                field: ValueClass::IndexProperty(IndexPropertyClass::Integer {
+                    property: EmailSubmissionField::Metadata.into(),
+                    value: self.send_at.to_native(),
+                }),
+                value: KeySerializer::new(U32_LEN * 3 + 1)
+                    .write(self.email_id.to_native())
+                    .write(self.thread_id.to_native())
+                    .write(self.identity_id.to_native())
+                    .write(self.undo_status.as_index())
+                    .finalize()
+                    .into(),
             },
             IndexValue::LogItem {
                 sync_collection: SyncCollection::EmailSubmission,

crates/email/src/submission/mod.rs

@@ -86,11 +86,11 @@ impl UndoStatus {
         }
     }
 
-    pub fn as_index(&self) -> &'static str {
+    pub fn as_index(&self) -> u8 {
         match self {
-            UndoStatus::Pending => "p",
-            UndoStatus::Final => "f",
-            UndoStatus::Canceled => "c",
+            UndoStatus::Pending => b'p',
+            UndoStatus::Final => b'f',
+            UndoStatus::Canceled => b'c',
         }
     }
 }
@@ -104,11 +104,11 @@ impl ArchivedUndoStatus {
         }
     }
 
-    pub fn as_index(&self) -> &'static str {
+    pub fn as_index(&self) -> u8 {
         match self {
-            ArchivedUndoStatus::Pending => "p",
-            ArchivedUndoStatus::Final => "f",
-            ArchivedUndoStatus::Canceled => "c",
+            ArchivedUndoStatus::Pending => b'p',
+            ArchivedUndoStatus::Final => b'f',
+            ArchivedUndoStatus::Canceled => b'c',
         }
     }
 }