Refactored local delivery to avoid mpsc channel
This commit is contained in:
mdecimus
78
crates/email/src/cache.rs
Normal file
78
crates/email/src/cache.rs
Normal file
@@ -0,0 +1,78 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use common::{Server, Threads};
|
||||
use jmap_proto::types::{collection::Collection, property::Property};
|
||||
use std::future::Future;
|
||||
use trc::AddContext;
|
||||
|
||||
pub trait ThreadCache: Sync + Send {
|
||||
fn get_cached_thread_ids(
|
||||
&self,
|
||||
account_id: u32,
|
||||
message_ids: impl Iterator<Item = u32> + Send,
|
||||
) -> impl Future<Output = trc::Result<Vec<(u32, u32)>>> + Send;
|
||||
}
|
||||
|
||||
impl ThreadCache for Server {
|
||||
async fn get_cached_thread_ids(
|
||||
&self,
|
||||
account_id: u32,
|
||||
message_ids: impl Iterator<Item = u32> + Send,
|
||||
) -> trc::Result<Vec<(u32, u32)>> {
|
||||
// Obtain current state
|
||||
let modseq = self
|
||||
.core
|
||||
.storage
|
||||
.data
|
||||
.get_last_change_id(account_id, Collection::Thread)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
|
||||
// Lock the cache
|
||||
let thread_cache = if let Some(thread_cache) =
|
||||
self.inner.cache.threads.get(&account_id).and_then(|t| {
|
||||
if t.modseq.unwrap_or(0) >= modseq.unwrap_or(0) {
|
||||
Some(t)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}) {
|
||||
thread_cache
|
||||
} else {
|
||||
let thread_cache = Arc::new(Threads {
|
||||
threads: self
|
||||
.get_properties::<u32, _, _>(
|
||||
account_id,
|
||||
Collection::Email,
|
||||
&(),
|
||||
Property::ThreadId,
|
||||
)
|
||||
.await?
|
||||
.into_iter()
|
||||
.collect(),
|
||||
modseq,
|
||||
});
|
||||
self.inner
|
||||
.cache
|
||||
.threads
|
||||
.insert(account_id, thread_cache.clone());
|
||||
thread_cache
|
||||
};
|
||||
|
||||
// Obtain threadIds for matching messages
|
||||
let mut thread_ids = Vec::with_capacity(message_ids.size_hint().0);
|
||||
for document_id in message_ids {
|
||||
if let Some(thread_id) = thread_cache.threads.get(&document_id) {
|
||||
thread_ids.push((document_id, *thread_id));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(thread_ids)
|
||||
}
|
||||
}
|
||||
670
crates/email/src/crypto.rs
Normal file
670
crates/email/src/crypto.rs
Normal file
@@ -0,0 +1,670 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use std::{borrow::Cow, collections::BTreeSet, fmt::Display, io::Cursor};
|
||||
|
||||
use aes::cipher::{block_padding::Pkcs7, BlockEncryptMut, KeyIvInit};
|
||||
|
||||
use mail_builder::{encoders::base64::base64_encode_mime, mime::make_boundary};
|
||||
use mail_parser::{decoders::base64::base64_decode, Message, MimeHeaders, PartType};
|
||||
use openpgp::{
|
||||
parse::Parse,
|
||||
serialize::stream,
|
||||
types::{KeyFlags, SymmetricAlgorithm},
|
||||
};
|
||||
use rasn::types::{ObjectIdentifier, OctetString};
|
||||
use rasn_cms::{
|
||||
algorithms::{AES128_CBC, AES256_CBC, RSA},
|
||||
pkcs7_compat::EncapsulatedContentInfo,
|
||||
AlgorithmIdentifier, EncryptedContent, EncryptedContentInfo, EncryptedKey, EnvelopedData,
|
||||
IssuerAndSerialNumber, KeyTransRecipientInfo, RecipientIdentifier, RecipientInfo, CONTENT_DATA,
|
||||
CONTENT_ENVELOPED_DATA,
|
||||
};
|
||||
use rsa::{pkcs1::DecodeRsaPublicKey, Pkcs1v15Encrypt, RsaPublicKey};
|
||||
use sequoia_openpgp as openpgp;
|
||||
use store::rand::{rngs::StdRng, RngCore, SeedableRng};
|
||||
use store::{
|
||||
write::{Bincode, ToBitmaps},
|
||||
Deserialize, Serialize,
|
||||
};
|
||||
|
||||
const P: openpgp::policy::StandardPolicy<'static> = openpgp::policy::StandardPolicy::new();
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum EncryptMessageError {
|
||||
AlreadyEncrypted,
|
||||
Error(String),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, serde::Serialize, serde::Deserialize)]
|
||||
pub enum Algorithm {
|
||||
Aes128,
|
||||
Aes256,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
pub enum EncryptionMethod {
|
||||
PGP,
|
||||
SMIME,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Serialize, serde::Deserialize)]
|
||||
pub struct EncryptionParams {
|
||||
pub method: EncryptionMethod,
|
||||
pub algo: Algorithm,
|
||||
pub certs: Vec<Vec<u8>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, serde::Serialize, serde::Deserialize, Default)]
|
||||
#[serde(tag = "type")]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum EncryptionType {
|
||||
PGP {
|
||||
algo: Algorithm,
|
||||
certs: String,
|
||||
},
|
||||
SMIME {
|
||||
algo: Algorithm,
|
||||
certs: String,
|
||||
},
|
||||
#[default]
|
||||
Disabled,
|
||||
}
|
||||
|
||||
#[allow(async_fn_in_trait)]
|
||||
pub trait EncryptMessage {
|
||||
async fn encrypt(&self, params: &EncryptionParams) -> Result<Vec<u8>, EncryptMessageError>;
|
||||
fn is_encrypted(&self) -> bool;
|
||||
}
|
||||
|
||||
impl EncryptMessage for Message<'_> {
|
||||
async fn encrypt(&self, params: &EncryptionParams) -> Result<Vec<u8>, EncryptMessageError> {
|
||||
let root = self.root_part();
|
||||
let raw_message = self.raw_message();
|
||||
let mut outer_message = Vec::with_capacity((raw_message.len() as f64 * 1.5) as usize);
|
||||
let mut inner_message = Vec::with_capacity(raw_message.len());
|
||||
|
||||
// Move MIME headers and body to inner message
|
||||
for header in root.headers() {
|
||||
(if header.name.is_mime_header() {
|
||||
&mut inner_message
|
||||
} else {
|
||||
&mut outer_message
|
||||
})
|
||||
.extend_from_slice(&raw_message[header.offset_field()..header.offset_end()]);
|
||||
}
|
||||
inner_message.extend_from_slice(b"\r\n");
|
||||
inner_message.extend_from_slice(&raw_message[root.raw_body_offset()..]);
|
||||
|
||||
// Encrypt inner message
|
||||
match params.method {
|
||||
EncryptionMethod::PGP => {
|
||||
// Prepare encrypted message
|
||||
let boundary = make_boundary("_");
|
||||
outer_message.extend_from_slice(
|
||||
concat!(
|
||||
"Content-Type: multipart/encrypted;\r\n\t",
|
||||
"protocol=\"application/pgp-encrypted\";\r\n\t",
|
||||
"boundary=\""
|
||||
)
|
||||
.as_bytes(),
|
||||
);
|
||||
outer_message.extend_from_slice(boundary.as_bytes());
|
||||
outer_message.extend_from_slice(
|
||||
concat!(
|
||||
"\"\r\n\r\n",
|
||||
"OpenPGP/MIME message (Automatically encrypted by Stalwart)\r\n\r\n",
|
||||
"--"
|
||||
)
|
||||
.as_bytes(),
|
||||
);
|
||||
outer_message.extend_from_slice(boundary.as_bytes());
|
||||
outer_message.extend_from_slice(
|
||||
concat!(
|
||||
"\r\nContent-Type: application/pgp-encrypted\r\n\r\n",
|
||||
"Version: 1\r\n\r\n--"
|
||||
)
|
||||
.as_bytes(),
|
||||
);
|
||||
outer_message.extend_from_slice(boundary.as_bytes());
|
||||
outer_message.extend_from_slice(
|
||||
concat!(
|
||||
"\r\nContent-Type: application/octet-stream; name=\"encrypted.asc\"\r\n",
|
||||
"Content-Disposition: inline; filename=\"encrypted.asc\"\r\n\r\n"
|
||||
)
|
||||
.as_bytes(),
|
||||
);
|
||||
|
||||
let certs = params
|
||||
.certs
|
||||
.iter()
|
||||
.map(openpgp::Cert::from_bytes)
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.map_err(|err| {
|
||||
EncryptMessageError::Error(format!(
|
||||
"Failed to parse OpenPGP public key: {}",
|
||||
err
|
||||
))
|
||||
})?;
|
||||
|
||||
// Encrypt contents (TODO: use rayon)
|
||||
let algo = params.algo;
|
||||
let encrypted_contents = tokio::task::spawn_blocking(move || {
|
||||
// Parse public key
|
||||
let mut keys = Vec::with_capacity(certs.len());
|
||||
let policy = openpgp::policy::StandardPolicy::new();
|
||||
|
||||
for cert in &certs {
|
||||
for key in cert
|
||||
.keys()
|
||||
.with_policy(&policy, None)
|
||||
.supported()
|
||||
.alive()
|
||||
.revoked(false)
|
||||
.key_flags(KeyFlags::empty().set_transport_encryption())
|
||||
{
|
||||
keys.push(key);
|
||||
}
|
||||
}
|
||||
|
||||
// Compose a writer stack corresponding to the output format and
|
||||
// packet structure we want.
|
||||
let mut sink = Vec::with_capacity(inner_message.len());
|
||||
|
||||
// Stream an OpenPGP message.
|
||||
let message = stream::Armorer::new(stream::Message::new(&mut sink))
|
||||
.build()
|
||||
.map_err(|err| {
|
||||
EncryptMessageError::Error(format!("Failed to create armorer: {}", err))
|
||||
})?;
|
||||
let message = stream::Encryptor2::for_recipients(message, keys)
|
||||
.symmetric_algo(match algo {
|
||||
Algorithm::Aes128 => SymmetricAlgorithm::AES128,
|
||||
Algorithm::Aes256 => SymmetricAlgorithm::AES256,
|
||||
})
|
||||
.build()
|
||||
.map_err(|err| {
|
||||
EncryptMessageError::Error(format!(
|
||||
"Failed to build encryptor: {}",
|
||||
err
|
||||
))
|
||||
})?;
|
||||
let mut message =
|
||||
stream::LiteralWriter::new(message).build().map_err(|err| {
|
||||
EncryptMessageError::Error(format!(
|
||||
"Failed to create literal writer: {}",
|
||||
err
|
||||
))
|
||||
})?;
|
||||
std::io::copy(&mut Cursor::new(inner_message), &mut message).map_err(
|
||||
|err| {
|
||||
EncryptMessageError::Error(format!(
|
||||
"Failed to encrypt message: {}",
|
||||
err
|
||||
))
|
||||
},
|
||||
)?;
|
||||
message.finalize().map_err(|err| {
|
||||
EncryptMessageError::Error(format!("Failed to finalize message: {}", err))
|
||||
})?;
|
||||
|
||||
String::from_utf8(sink).map_err(|err| {
|
||||
EncryptMessageError::Error(format!(
|
||||
"Failed to convert encrypted message to UTF-8: {}",
|
||||
err
|
||||
))
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(|err| {
|
||||
EncryptMessageError::Error(format!("Failed to encrypt message: {}", err))
|
||||
})??;
|
||||
outer_message.extend_from_slice(encrypted_contents.as_bytes());
|
||||
outer_message.extend_from_slice(b"\r\n--");
|
||||
outer_message.extend_from_slice(boundary.as_bytes());
|
||||
outer_message.extend_from_slice(b"--\r\n");
|
||||
}
|
||||
EncryptionMethod::SMIME => {
|
||||
// Generate random IV
|
||||
let mut rng = StdRng::from_entropy();
|
||||
let mut iv = vec![0u8; 16];
|
||||
rng.fill_bytes(&mut iv);
|
||||
|
||||
// Generate random key
|
||||
let mut key = vec![0u8; params.algo.key_size()];
|
||||
rng.fill_bytes(&mut key);
|
||||
|
||||
// Encrypt contents (TODO: use rayon)
|
||||
let algo = params.algo;
|
||||
let (encrypted_contents, key, iv) = tokio::task::spawn_blocking(move || {
|
||||
(algo.encrypt(&key, &iv, &inner_message), key, iv)
|
||||
})
|
||||
.await
|
||||
.map_err(|err| {
|
||||
EncryptMessageError::Error(format!("Failed to encrypt message: {}", err))
|
||||
})?;
|
||||
|
||||
// Encrypt key using public keys
|
||||
#[allow(clippy::mutable_key_type)]
|
||||
let mut recipient_infos = BTreeSet::new();
|
||||
for cert in ¶ms.certs {
|
||||
let cert =
|
||||
rasn::der::decode::<rasn_pkix::Certificate>(cert).map_err(|err| {
|
||||
EncryptMessageError::Error(format!(
|
||||
"Failed to parse certificate: {}",
|
||||
err
|
||||
))
|
||||
})?;
|
||||
|
||||
let public_key = RsaPublicKey::from_pkcs1_der(
|
||||
cert.tbs_certificate
|
||||
.subject_public_key_info
|
||||
.subject_public_key
|
||||
.as_raw_slice(),
|
||||
)
|
||||
.map_err(|err| {
|
||||
EncryptMessageError::Error(format!("Failed to parse public key: {}", err))
|
||||
})?;
|
||||
let encrypted_key = public_key
|
||||
.encrypt(&mut rng, Pkcs1v15Encrypt, &key[..])
|
||||
.map_err(|err| {
|
||||
EncryptMessageError::Error(format!("Failed to encrypt key: {}", err))
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
recipient_infos.insert(RecipientInfo::KeyTransRecipientInfo(
|
||||
KeyTransRecipientInfo {
|
||||
version: 0.into(),
|
||||
rid: RecipientIdentifier::IssuerAndSerialNumber(
|
||||
IssuerAndSerialNumber {
|
||||
issuer: cert.tbs_certificate.issuer,
|
||||
serial_number: cert.tbs_certificate.serial_number,
|
||||
},
|
||||
),
|
||||
key_encryption_algorithm: AlgorithmIdentifier {
|
||||
algorithm: RSA.into(),
|
||||
parameters: Some(
|
||||
rasn::der::encode(&())
|
||||
.map_err(|err| {
|
||||
EncryptMessageError::Error(format!(
|
||||
"Failed to encode RSA algorithm identifier: {}",
|
||||
err
|
||||
))
|
||||
})?
|
||||
.into(),
|
||||
),
|
||||
},
|
||||
encrypted_key: EncryptedKey::from(encrypted_key),
|
||||
},
|
||||
));
|
||||
}
|
||||
|
||||
let pkcs7 = rasn::der::encode(&EncapsulatedContentInfo {
|
||||
content_type: CONTENT_ENVELOPED_DATA.into(),
|
||||
content: Some(
|
||||
rasn::der::encode(&EnvelopedData {
|
||||
version: 0.into(),
|
||||
originator_info: None,
|
||||
recipient_infos,
|
||||
encrypted_content_info: EncryptedContentInfo {
|
||||
content_type: CONTENT_DATA.into(),
|
||||
content_encryption_algorithm: AlgorithmIdentifier {
|
||||
algorithm: params.algo.to_algorithm_identifier(),
|
||||
parameters: Some(
|
||||
rasn::der::encode(&OctetString::from(iv))
|
||||
.map_err(|err| {
|
||||
EncryptMessageError::Error(format!(
|
||||
"Failed to encode IV: {}",
|
||||
err
|
||||
))
|
||||
})?
|
||||
.into(),
|
||||
),
|
||||
},
|
||||
encrypted_content: Some(EncryptedContent::from(encrypted_contents)),
|
||||
},
|
||||
unprotected_attrs: None,
|
||||
})
|
||||
.map_err(|err| {
|
||||
EncryptMessageError::Error(format!(
|
||||
"Failed to encode EnvelopedData: {}",
|
||||
err
|
||||
))
|
||||
})?
|
||||
.into(),
|
||||
),
|
||||
})
|
||||
.map_err(|err| {
|
||||
EncryptMessageError::Error(format!("Failed to encode ContentInfo: {}", err))
|
||||
})?;
|
||||
|
||||
// Generate message
|
||||
outer_message.extend_from_slice(
|
||||
concat!(
|
||||
"Content-Type: application/pkcs7-mime;\r\n",
|
||||
"\tname=\"smime.p7m\";\r\n",
|
||||
"\tsmime-type=enveloped-data\r\n",
|
||||
"Content-Disposition: attachment;\r\n",
|
||||
"\tfilename=\"smime.p7m\"\r\n",
|
||||
"Content-Transfer-Encoding: base64\r\n\r\n"
|
||||
)
|
||||
.as_bytes(),
|
||||
);
|
||||
base64_encode_mime(&pkcs7, &mut outer_message, false).map_err(|err| {
|
||||
EncryptMessageError::Error(format!("Failed to base64 encode PKCS7: {}", err))
|
||||
})?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(outer_message)
|
||||
}
|
||||
|
||||
fn is_encrypted(&self) -> bool {
|
||||
if self.content_type().is_some_and(|ct| {
|
||||
let main_type = ct.c_type.as_ref();
|
||||
let sub_type = ct
|
||||
.c_subtype
|
||||
.as_ref()
|
||||
.map(|s| s.as_ref())
|
||||
.unwrap_or_default();
|
||||
|
||||
(main_type.eq_ignore_ascii_case("application")
|
||||
&& (sub_type.eq_ignore_ascii_case("pkcs7-mime")
|
||||
|| sub_type.eq_ignore_ascii_case("pkcs7-signature")
|
||||
|| (sub_type.eq_ignore_ascii_case("octet-stream")
|
||||
&& self.attachment_name().is_some_and(|name| {
|
||||
name.rsplit_once('.')
|
||||
.is_some_and(|(_, ext)| ["p7m", "p7s", "p7c", "p7z"].contains(&ext))
|
||||
}))))
|
||||
|| (main_type.eq_ignore_ascii_case("multipart")
|
||||
&& sub_type.eq_ignore_ascii_case("encrypted"))
|
||||
}) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if self.parts.len() <= 2 {
|
||||
let mut text_part = None;
|
||||
let mut is_multipart = false;
|
||||
|
||||
for part in &self.parts {
|
||||
match &part.body {
|
||||
PartType::Text(text) => {
|
||||
text_part = Some(text.as_ref());
|
||||
}
|
||||
PartType::Multipart(_) => {
|
||||
is_multipart = true;
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
|
||||
match text_part {
|
||||
Some(text) if self.parts.len() == 1 || is_multipart => {
|
||||
if text.trim_start().starts_with("-----BEGIN PGP MESSAGE-----") {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
impl Algorithm {
|
||||
fn key_size(&self) -> usize {
|
||||
match self {
|
||||
Algorithm::Aes128 => 16,
|
||||
Algorithm::Aes256 => 32,
|
||||
}
|
||||
}
|
||||
|
||||
fn to_algorithm_identifier(self) -> ObjectIdentifier {
|
||||
match self {
|
||||
Algorithm::Aes128 => AES128_CBC.into(),
|
||||
Algorithm::Aes256 => AES256_CBC.into(),
|
||||
}
|
||||
}
|
||||
|
||||
fn encrypt(&self, key: &[u8], iv: &[u8], contents: &[u8]) -> Vec<u8> {
|
||||
match self {
|
||||
Algorithm::Aes128 => cbc::Encryptor::<aes::Aes128>::new(key.into(), iv.into())
|
||||
.encrypt_padded_vec_mut::<Pkcs7>(contents),
|
||||
Algorithm::Aes256 => cbc::Encryptor::<aes::Aes256>::new(key.into(), iv.into())
|
||||
.encrypt_padded_vec_mut::<Pkcs7>(contents),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn try_parse_certs(
|
||||
expected_method: EncryptionMethod,
|
||||
cert: Vec<u8>,
|
||||
) -> Result<Vec<Vec<u8>>, Cow<'static, str>> {
|
||||
// Check if it's a PEM file
|
||||
let (method, certs) = if let Some(result) = try_parse_pem(&cert)? {
|
||||
result
|
||||
} else if rasn::der::decode::<rasn_pkix::Certificate>(&cert[..]).is_ok() {
|
||||
(EncryptionMethod::SMIME, vec![cert])
|
||||
} else if let Ok(cert_) = openpgp::Cert::from_bytes(&cert[..]) {
|
||||
if !has_pgp_keys(cert_) {
|
||||
(EncryptionMethod::PGP, vec![cert])
|
||||
} else {
|
||||
return Err("Could not find any suitable keys in certificate".into());
|
||||
}
|
||||
} else {
|
||||
return Err("Could not find any valid certificates".into());
|
||||
};
|
||||
|
||||
if method == expected_method {
|
||||
Ok(certs)
|
||||
} else {
|
||||
Err("No valid certificates found for the selected encryption".into())
|
||||
}
|
||||
}
|
||||
|
||||
fn has_pgp_keys(cert: openpgp::Cert) -> bool {
|
||||
cert.keys()
|
||||
.with_policy(&P, None)
|
||||
.supported()
|
||||
.alive()
|
||||
.revoked(false)
|
||||
.key_flags(KeyFlags::empty().set_transport_encryption())
|
||||
.next()
|
||||
.is_some()
|
||||
}
|
||||
|
||||
#[allow(clippy::type_complexity)]
|
||||
fn try_parse_pem(
|
||||
bytes_: &[u8],
|
||||
) -> Result<Option<(EncryptionMethod, Vec<Vec<u8>>)>, Cow<'static, str>> {
|
||||
if let Some(internal) = std::str::from_utf8(bytes_)
|
||||
.ok()
|
||||
.and_then(|cert| cert.strip_prefix("-----STALWART CERTIFICATE-----"))
|
||||
{
|
||||
return base64_decode(internal.as_bytes())
|
||||
.ok_or(Cow::from("Failed to decode base64"))
|
||||
.and_then(|bytes| {
|
||||
Bincode::<EncryptionParams>::deserialize(&bytes)
|
||||
.map_err(|_| Cow::from("Failed to deserialize internal certificate"))
|
||||
})
|
||||
.map(|params| Some((params.inner.method, params.inner.certs)));
|
||||
}
|
||||
|
||||
let mut bytes = bytes_.iter().enumerate();
|
||||
let mut buf = vec![];
|
||||
let mut method = None;
|
||||
let mut certs = vec![];
|
||||
|
||||
loop {
|
||||
// Find start of PEM block
|
||||
let mut start_pos = 0;
|
||||
for (pos, &ch) in bytes.by_ref() {
|
||||
if ch.is_ascii_whitespace() {
|
||||
continue;
|
||||
} else if ch == b'-' {
|
||||
start_pos = pos;
|
||||
break;
|
||||
} else {
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
|
||||
// Find block type
|
||||
for (_, &ch) in bytes.by_ref() {
|
||||
match ch {
|
||||
b'-' => (),
|
||||
b'\n' => break,
|
||||
_ => {
|
||||
if ch.is_ascii() {
|
||||
buf.push(ch.to_ascii_uppercase());
|
||||
} else {
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if buf.is_empty() {
|
||||
break;
|
||||
}
|
||||
|
||||
// Find type
|
||||
let tag = std::str::from_utf8(&buf).unwrap();
|
||||
if tag.contains("CERTIFICATE") {
|
||||
if method.is_some_and(|m| m == EncryptionMethod::PGP) {
|
||||
return Err("Cannot mix OpenPGP and S/MIME certificates".into());
|
||||
} else {
|
||||
method = Some(EncryptionMethod::SMIME);
|
||||
}
|
||||
} else if tag.contains("PGP") {
|
||||
if method.is_some_and(|m| m == EncryptionMethod::SMIME) {
|
||||
return Err("Cannot mix OpenPGP and S/MIME certificates".into());
|
||||
} else {
|
||||
method = Some(EncryptionMethod::PGP);
|
||||
}
|
||||
} else {
|
||||
// Ignore block
|
||||
let mut found_end = false;
|
||||
for (_, &ch) in bytes.by_ref() {
|
||||
if ch == b'-' {
|
||||
found_end = true;
|
||||
} else if ch == b'\n' && found_end {
|
||||
break;
|
||||
}
|
||||
}
|
||||
buf.clear();
|
||||
continue;
|
||||
}
|
||||
|
||||
// Collect base64
|
||||
buf.clear();
|
||||
let mut found_end = false;
|
||||
let mut end_pos = 0;
|
||||
for (pos, &ch) in bytes.by_ref() {
|
||||
match ch {
|
||||
b'-' => {
|
||||
found_end = true;
|
||||
}
|
||||
b'\n' => {
|
||||
if found_end {
|
||||
end_pos = pos;
|
||||
break;
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
if !ch.is_ascii_whitespace() {
|
||||
buf.push(ch);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Decode base64
|
||||
let cert =
|
||||
base64_decode(&buf).ok_or_else(|| Cow::from("Failed to decode base64 certificate."))?;
|
||||
match method.unwrap() {
|
||||
EncryptionMethod::PGP => match openpgp::Cert::from_bytes(bytes_) {
|
||||
Ok(cert) => {
|
||||
if !has_pgp_keys(cert) {
|
||||
return Err("Could not find any suitable keys in OpenPGP public key".into());
|
||||
}
|
||||
certs.push(
|
||||
bytes_
|
||||
.get(start_pos..end_pos + 1)
|
||||
.unwrap_or_default()
|
||||
.to_vec(),
|
||||
);
|
||||
}
|
||||
Err(err) => {
|
||||
return Err(format!("Failed to decode OpenPGP public key: {err}").into())
|
||||
}
|
||||
},
|
||||
EncryptionMethod::SMIME => {
|
||||
if let Err(err) = rasn::der::decode::<rasn_pkix::Certificate>(&cert) {
|
||||
return Err(format!("Failed to decode X509 certificate: {err}").into());
|
||||
}
|
||||
certs.push(cert);
|
||||
}
|
||||
}
|
||||
buf.clear();
|
||||
}
|
||||
|
||||
Ok(method.map(|method| (method, certs)))
|
||||
}
|
||||
|
||||
impl Serialize for &EncryptionParams {
|
||||
fn serialize(self) -> Vec<u8> {
|
||||
let len = bincode::serialized_size(&self).unwrap_or_default();
|
||||
let mut buf = Vec::with_capacity(len as usize + 1);
|
||||
buf.push(1);
|
||||
let _ = bincode::serialize_into(&mut buf, &self);
|
||||
buf
|
||||
}
|
||||
}
|
||||
|
||||
impl Deserialize for EncryptionParams {
|
||||
fn deserialize(bytes: &[u8]) -> trc::Result<Self> {
|
||||
let version = *bytes
|
||||
.first()
|
||||
.ok_or_else(|| trc::StoreEvent::DataCorruption.caused_by(trc::location!()))?;
|
||||
match version {
|
||||
1 if bytes.len() > 1 => bincode::deserialize(&bytes[1..]).map_err(|err| {
|
||||
trc::EventType::Store(trc::StoreEvent::DeserializeError)
|
||||
.from_bincode_error(err)
|
||||
.caused_by(trc::location!())
|
||||
}),
|
||||
|
||||
_ => Err(trc::StoreEvent::DeserializeError
|
||||
.into_err()
|
||||
.caused_by(trc::location!())
|
||||
.ctx(trc::Key::Value, version as u64)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ToBitmaps for &EncryptionParams {
|
||||
fn to_bitmaps(&self, _: &mut Vec<store::write::Operation>, _: u8, _: bool) {
|
||||
unreachable!()
|
||||
}
|
||||
}
|
||||
|
||||
impl Display for EncryptionMethod {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
EncryptionMethod::PGP => write!(f, "OpenPGP"),
|
||||
EncryptionMethod::SMIME => write!(f, "S/MIME"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Display for Algorithm {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Algorithm::Aes128 => write!(f, "AES-128"),
|
||||
Algorithm::Aes256 => write!(f, "AES-256"),
|
||||
}
|
||||
}
|
||||
}
|
||||
326
crates/email/src/delivery.rs
Normal file
326
crates/email/src/delivery.rs
Normal file
@@ -0,0 +1,326 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use common::Server;
|
||||
use directory::Permission;
|
||||
use jmap_proto::types::{state::StateChange, type_state::DataType};
|
||||
use mail_parser::MessageParser;
|
||||
use std::{borrow::Cow, future::Future};
|
||||
use store::ahash::AHashMap;
|
||||
use utils::BlobHash;
|
||||
|
||||
use crate::{
|
||||
ingest::{EmailIngest, IngestEmail, IngestSource},
|
||||
mailbox::INBOX_ID,
|
||||
sieve::SieveScriptIngest,
|
||||
};
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct IngestMessage {
|
||||
pub sender_address: String,
|
||||
pub recipients: Vec<String>,
|
||||
pub message_blob: BlobHash,
|
||||
pub message_size: usize,
|
||||
pub session_id: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum LocalDeliveryStatus {
|
||||
Success,
|
||||
TemporaryFailure {
|
||||
reason: Cow<'static, str>,
|
||||
},
|
||||
PermanentFailure {
|
||||
code: [u8; 3],
|
||||
reason: Cow<'static, str>,
|
||||
},
|
||||
}
|
||||
|
||||
pub struct LocalDeliveryResult {
|
||||
pub status: Vec<LocalDeliveryStatus>,
|
||||
pub autogenerated: Vec<AutogeneratedMessage>,
|
||||
}
|
||||
|
||||
pub struct AutogeneratedMessage {
|
||||
pub sender_address: String,
|
||||
pub recipients: Vec<String>,
|
||||
pub message: Vec<u8>,
|
||||
}
|
||||
|
||||
pub trait MailDelivery: Sync + Send {
|
||||
fn deliver_message(
|
||||
&self,
|
||||
message: IngestMessage,
|
||||
) -> impl Future<Output = LocalDeliveryResult> + Send;
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
let semaphore = Arc::new(Semaphore::new(
|
||||
inner
|
||||
.shared_core
|
||||
.load()
|
||||
.smtp
|
||||
.queue
|
||||
.throttle
|
||||
.local_concurrency,
|
||||
));
|
||||
|
||||
loop {
|
||||
let permit = match semaphore.clone().acquire_owned().await {
|
||||
Ok(permit) => permit,
|
||||
Err(_) => {
|
||||
trc::error!(trc::StoreEvent::UnexpectedError
|
||||
.into_err()
|
||||
.details("Semaphore error")
|
||||
.caused_by(trc::location!()));
|
||||
break;
|
||||
}
|
||||
};
|
||||
|
||||
match delivery_rx.recv().await {
|
||||
Some(event) => match event {
|
||||
DeliveryEvent::Ingest { message, result_tx } => {
|
||||
let server = inner.build_server();
|
||||
|
||||
tokio::spawn(async move {
|
||||
result_tx.send(server.deliver_message(message).await).ok();
|
||||
|
||||
drop(permit);
|
||||
});
|
||||
}
|
||||
DeliveryEvent::Stop => break,
|
||||
},
|
||||
None => {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
*/
|
||||
|
||||
impl MailDelivery for Server {
|
||||
async fn deliver_message(&self, message: IngestMessage) -> LocalDeliveryResult {
|
||||
// Obtain permit
|
||||
let _permit = match self.inner.ipc.local_delivery_sm.acquire().await {
|
||||
Ok(permit) => permit,
|
||||
Err(_) => {
|
||||
trc::error!(
|
||||
trc::Error::new(trc::EventType::Server(trc::ServerEvent::ThreadError))
|
||||
.details("Failed to obtain semaphore permit.")
|
||||
.span_id(message.session_id)
|
||||
.caused_by(trc::location!())
|
||||
);
|
||||
|
||||
return LocalDeliveryResult {
|
||||
status: (0..message.recipients.len())
|
||||
.map(|_| LocalDeliveryStatus::TemporaryFailure {
|
||||
reason: "Temporary I/O error.".into(),
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
autogenerated: vec![],
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
// Read message
|
||||
let raw_message = match self
|
||||
.core
|
||||
.storage
|
||||
.blob
|
||||
.get_blob(message.message_blob.as_slice(), 0..usize::MAX)
|
||||
.await
|
||||
{
|
||||
Ok(Some(raw_message)) => raw_message,
|
||||
Ok(None) => {
|
||||
trc::event!(
|
||||
MessageIngest(trc::MessageIngestEvent::Error),
|
||||
Reason = "Blob not found.",
|
||||
SpanId = message.session_id,
|
||||
CausedBy = trc::location!()
|
||||
);
|
||||
|
||||
return LocalDeliveryResult {
|
||||
status: (0..message.recipients.len())
|
||||
.map(|_| LocalDeliveryStatus::TemporaryFailure {
|
||||
reason: "Blob not found.".into(),
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
autogenerated: vec![],
|
||||
};
|
||||
}
|
||||
Err(err) => {
|
||||
trc::error!(err
|
||||
.details("Failed to fetch message blob.")
|
||||
.span_id(message.session_id)
|
||||
.caused_by(trc::location!()));
|
||||
|
||||
return LocalDeliveryResult {
|
||||
status: (0..message.recipients.len())
|
||||
.map(|_| LocalDeliveryStatus::TemporaryFailure {
|
||||
reason: "Temporary I/O error.".into(),
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
autogenerated: vec![],
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
// Obtain the UIDs for each recipient
|
||||
let mut uids: AHashMap<u32, usize> = AHashMap::with_capacity(message.recipients.len());
|
||||
let mut result = LocalDeliveryResult {
|
||||
status: Vec::with_capacity(message.recipients.len()),
|
||||
autogenerated: Vec::new(),
|
||||
};
|
||||
|
||||
for rcpt in message.recipients {
|
||||
let uid = match self
|
||||
.email_to_id(&self.core.storage.directory, &rcpt, message.session_id)
|
||||
.await
|
||||
{
|
||||
Ok(Some(uid)) => uid,
|
||||
Ok(None) => {
|
||||
// Something went wrong
|
||||
result.status.push(LocalDeliveryStatus::PermanentFailure {
|
||||
code: [5, 5, 0],
|
||||
reason: "Mailbox not found.".into(),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
Err(err) => {
|
||||
trc::error!(err
|
||||
.details("Failed to lookup recipient.")
|
||||
.ctx(trc::Key::To, rcpt)
|
||||
.span_id(message.session_id)
|
||||
.caused_by(trc::location!()));
|
||||
result.status.push(LocalDeliveryStatus::TemporaryFailure {
|
||||
reason: "Address lookup failed.".into(),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if let Some(status) = uids.get(&uid).and_then(|pos| result.status.get(*pos)) {
|
||||
result.status.push(status.clone());
|
||||
continue;
|
||||
}
|
||||
|
||||
// Obtain access token
|
||||
let status = match self.get_access_token(uid).await.and_then(|token| {
|
||||
token
|
||||
.assert_has_permission(Permission::EmailReceive)
|
||||
.map(|_| token)
|
||||
}) {
|
||||
Ok(access_token) => {
|
||||
// Check if there is an active sieve script
|
||||
match self.sieve_script_get_active(uid).await {
|
||||
Ok(None) => {
|
||||
// Ingest message
|
||||
self.email_ingest(IngestEmail {
|
||||
raw_message: &raw_message,
|
||||
message: MessageParser::new().parse(&raw_message),
|
||||
resource: access_token.as_resource_token(),
|
||||
mailbox_ids: vec![INBOX_ID],
|
||||
keywords: vec![],
|
||||
received_at: None,
|
||||
source: IngestSource::Smtp { deliver_to: &rcpt },
|
||||
spam_classify: access_token
|
||||
.has_permission(Permission::SpamFilterClassify),
|
||||
spam_train: self.email_bayes_can_train(&access_token),
|
||||
session_id: message.session_id,
|
||||
})
|
||||
.await
|
||||
}
|
||||
Ok(Some(active_script)) => {
|
||||
self.sieve_script_ingest(
|
||||
&access_token,
|
||||
&raw_message,
|
||||
&message.sender_address,
|
||||
&rcpt,
|
||||
message.session_id,
|
||||
active_script,
|
||||
&mut result.autogenerated,
|
||||
)
|
||||
.await
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
Err(err) => Err(err),
|
||||
};
|
||||
|
||||
let status = match status {
|
||||
Ok(ingested_message) => {
|
||||
// Notify state change
|
||||
if ingested_message.change_id != u64::MAX {
|
||||
self.broadcast_state_change(
|
||||
StateChange::new(uid)
|
||||
.with_change(DataType::EmailDelivery, ingested_message.change_id)
|
||||
.with_change(DataType::Email, ingested_message.change_id)
|
||||
.with_change(DataType::Mailbox, ingested_message.change_id)
|
||||
.with_change(DataType::Thread, ingested_message.change_id),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
LocalDeliveryStatus::Success
|
||||
}
|
||||
Err(err) => {
|
||||
let status = match err.as_ref() {
|
||||
trc::EventType::Limit(trc::LimitEvent::Quota) => {
|
||||
LocalDeliveryStatus::TemporaryFailure {
|
||||
reason: "Mailbox over quota.".into(),
|
||||
}
|
||||
}
|
||||
trc::EventType::Limit(trc::LimitEvent::TenantQuota) => {
|
||||
LocalDeliveryStatus::TemporaryFailure {
|
||||
reason: "Organization over quota.".into(),
|
||||
}
|
||||
}
|
||||
trc::EventType::Security(trc::SecurityEvent::Unauthorized) => {
|
||||
LocalDeliveryStatus::PermanentFailure {
|
||||
code: [5, 5, 0],
|
||||
reason: "This account is not authorized to receive email.".into(),
|
||||
}
|
||||
}
|
||||
trc::EventType::MessageIngest(trc::MessageIngestEvent::Error) => {
|
||||
LocalDeliveryStatus::PermanentFailure {
|
||||
code: err
|
||||
.value(trc::Key::Code)
|
||||
.and_then(|v| v.to_uint())
|
||||
.map(|n| {
|
||||
[(n / 100) as u8, ((n % 100) / 10) as u8, (n % 10) as u8]
|
||||
})
|
||||
.unwrap_or([5, 5, 0]),
|
||||
reason: err
|
||||
.value_as_str(trc::Key::Reason)
|
||||
.unwrap_or_default()
|
||||
.to_string()
|
||||
.into(),
|
||||
}
|
||||
}
|
||||
_ => LocalDeliveryStatus::TemporaryFailure {
|
||||
reason: "Transient server failure.".into(),
|
||||
},
|
||||
};
|
||||
|
||||
trc::error!(err
|
||||
.ctx(trc::Key::To, rcpt.to_string())
|
||||
.span_id(message.session_id));
|
||||
|
||||
status
|
||||
}
|
||||
};
|
||||
|
||||
// Cache response for UID to avoid duplicate deliveries
|
||||
uids.insert(uid, result.status.len());
|
||||
|
||||
result.status.push(status);
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
}
|
||||
697
crates/email/src/index.rs
Normal file
697
crates/email/src/index.rs
Normal file
@@ -0,0 +1,697 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use std::borrow::Cow;
|
||||
|
||||
use jmap_proto::types::{keyword::Keyword, property::Property};
|
||||
use mail_parser::{
|
||||
decoders::html::html_to_text,
|
||||
parsers::{fields::thread::thread_name, preview::preview_text},
|
||||
Addr, Address, GetHeader, Group, Header, HeaderName, HeaderValue, Message, MessagePart,
|
||||
PartType,
|
||||
};
|
||||
use nlp::language::Language;
|
||||
use store::{
|
||||
backend::MAX_TOKEN_LENGTH,
|
||||
fts::{index::FtsDocument, Field},
|
||||
write::{BatchBuilder, Bincode, BlobOp, DirectoryClass, F_BITMAP, F_CLEAR, F_INDEX, F_VALUE},
|
||||
};
|
||||
use utils::BlobHash;
|
||||
|
||||
use crate::mailbox::UidMailbox;
|
||||
|
||||
use super::metadata::MessageMetadata;
|
||||
|
||||
pub const MAX_MESSAGE_PARTS: usize = 1000;
|
||||
pub const MAX_ID_LENGTH: usize = 100;
|
||||
pub const MAX_SORT_FIELD_LENGTH: usize = 255;
|
||||
pub const MAX_STORED_FIELD_LENGTH: usize = 512;
|
||||
pub const PREVIEW_LENGTH: usize = 256;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct SortedAddressBuilder {
|
||||
last_is_space: bool,
|
||||
pub buf: String,
|
||||
}
|
||||
|
||||
pub(super) trait IndexMessage {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn index_message(
|
||||
&mut self,
|
||||
account_id: u32,
|
||||
tenant_id: Option<u32>,
|
||||
message: Message,
|
||||
blob_hash: BlobHash,
|
||||
keywords: Vec<Keyword>,
|
||||
mailbox_ids: Vec<UidMailbox>,
|
||||
received_at: u64,
|
||||
) -> &mut Self;
|
||||
|
||||
fn index_headers(&mut self, headers: &[Header<'_>], options: u32);
|
||||
}
|
||||
|
||||
pub trait IndexMessageText<'x>: Sized {
|
||||
fn index_message(self, message: &'x Message<'x>) -> Self;
|
||||
}
|
||||
|
||||
impl IndexMessage for BatchBuilder {
|
||||
fn index_message(
|
||||
&mut self,
|
||||
account_id: u32,
|
||||
tenant_id: Option<u32>,
|
||||
message: Message,
|
||||
blob_hash: BlobHash,
|
||||
keywords: Vec<Keyword>,
|
||||
mailbox_ids: Vec<UidMailbox>,
|
||||
received_at: u64,
|
||||
) -> &mut Self {
|
||||
// Index keywords
|
||||
self.value(Property::Keywords, keywords, F_VALUE | F_BITMAP);
|
||||
|
||||
// Index mailboxIds
|
||||
self.value(Property::MailboxIds, mailbox_ids, F_VALUE | F_BITMAP);
|
||||
|
||||
// Index size
|
||||
self.value(Property::Size, message.raw_message.len() as u32, F_INDEX)
|
||||
.add(
|
||||
DirectoryClass::UsedQuota(account_id),
|
||||
message.raw_message.len() as i64,
|
||||
);
|
||||
if let Some(tenant_id) = tenant_id {
|
||||
self.add(
|
||||
DirectoryClass::UsedQuota(tenant_id),
|
||||
message.raw_message.len() as i64,
|
||||
);
|
||||
}
|
||||
|
||||
// Index receivedAt
|
||||
self.value(Property::ReceivedAt, received_at, F_INDEX);
|
||||
|
||||
let mut has_attachments = false;
|
||||
let mut preview = None;
|
||||
let preview_part_id = message
|
||||
.text_body
|
||||
.first()
|
||||
.or_else(|| message.html_body.first())
|
||||
.copied()
|
||||
.unwrap_or(usize::MAX);
|
||||
|
||||
for (part_id, part) in message.parts.iter().take(MAX_MESSAGE_PARTS).enumerate() {
|
||||
if part_id == 0 {
|
||||
self.index_headers(&part.headers, 0);
|
||||
}
|
||||
|
||||
match &part.body {
|
||||
PartType::Text(text) => {
|
||||
if part_id == preview_part_id {
|
||||
preview =
|
||||
preview_text(text.replace('\r', "").into(), PREVIEW_LENGTH).into();
|
||||
}
|
||||
|
||||
if !message.text_body.contains(&part_id)
|
||||
&& !message.html_body.contains(&part_id)
|
||||
{
|
||||
has_attachments = true;
|
||||
}
|
||||
}
|
||||
PartType::Html(html) => {
|
||||
let text = html_to_text(html);
|
||||
if part_id == preview_part_id {
|
||||
preview =
|
||||
preview_text(text.replace('\r', "").into(), PREVIEW_LENGTH).into();
|
||||
}
|
||||
|
||||
if !message.text_body.contains(&part_id)
|
||||
&& !message.html_body.contains(&part_id)
|
||||
{
|
||||
has_attachments = true;
|
||||
}
|
||||
}
|
||||
PartType::Binary(_) | PartType::Message(_) if !has_attachments => {
|
||||
has_attachments = true;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
// Store and index hasAttachment property
|
||||
if has_attachments {
|
||||
self.tag(Property::HasAttachment, (), 0);
|
||||
}
|
||||
|
||||
// Link blob
|
||||
self.set(
|
||||
BlobOp::Link {
|
||||
hash: blob_hash.clone(),
|
||||
},
|
||||
Vec::new(),
|
||||
);
|
||||
|
||||
// Store message metadata
|
||||
let root_part = message.root_part();
|
||||
self.value(
|
||||
Property::BodyStructure,
|
||||
Bincode::new(MessageMetadata {
|
||||
preview: preview.unwrap_or_default().into_owned(),
|
||||
size: message.raw_message.len(),
|
||||
raw_headers: message
|
||||
.raw_message
|
||||
.as_ref()
|
||||
.get(root_part.offset_header..root_part.offset_body)
|
||||
.unwrap_or_default()
|
||||
.to_vec(),
|
||||
contents: message.into(),
|
||||
received_at,
|
||||
has_attachments,
|
||||
blob_hash,
|
||||
}),
|
||||
F_VALUE,
|
||||
);
|
||||
|
||||
self
|
||||
}
|
||||
|
||||
fn index_headers(&mut self, headers: &[Header<'_>], options: u32) {
|
||||
let mut seen_headers = [false; 40];
|
||||
for header in headers.iter().rev() {
|
||||
if matches!(header.name, HeaderName::Other(_)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
match header.name {
|
||||
HeaderName::MessageId => {
|
||||
header.value.visit_text(|id| {
|
||||
// Add ids to inverted index
|
||||
if id.len() < MAX_ID_LENGTH {
|
||||
self.value(Property::MessageId, id, F_INDEX | options);
|
||||
self.value(Property::References, id, F_INDEX | options);
|
||||
}
|
||||
});
|
||||
}
|
||||
HeaderName::InReplyTo | HeaderName::References | HeaderName::ResentMessageId => {
|
||||
header.value.visit_text(|id| {
|
||||
// Add ids to inverted index
|
||||
if id.len() < MAX_ID_LENGTH {
|
||||
self.value(Property::References, id, F_INDEX | options);
|
||||
}
|
||||
});
|
||||
}
|
||||
HeaderName::From | HeaderName::To | HeaderName::Cc | HeaderName::Bcc => {
|
||||
if !seen_headers[header.name.id() as usize] {
|
||||
let property = Property::from_header(&header.name);
|
||||
let mut sort_text = SortedAddressBuilder::new();
|
||||
let mut found_addr = false;
|
||||
|
||||
header.value.visit_addresses(|element, value| {
|
||||
if !found_addr {
|
||||
match element {
|
||||
AddressElement::Name => {
|
||||
found_addr = !sort_text.push(value);
|
||||
}
|
||||
AddressElement::Address => {
|
||||
sort_text.push(value);
|
||||
found_addr = true;
|
||||
}
|
||||
AddressElement::GroupName => (),
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Add address to inverted index
|
||||
self.value(u8::from(&property), sort_text.build(), F_INDEX | options);
|
||||
seen_headers[header.name.id() as usize] = true;
|
||||
}
|
||||
}
|
||||
HeaderName::Date => {
|
||||
if !seen_headers[header.name.id() as usize] {
|
||||
if let HeaderValue::DateTime(datetime) = &header.value {
|
||||
self.value(
|
||||
Property::SentAt,
|
||||
datetime.to_timestamp() as u64,
|
||||
F_INDEX | options,
|
||||
);
|
||||
}
|
||||
seen_headers[header.name.id() as usize] = true;
|
||||
}
|
||||
}
|
||||
HeaderName::Subject => {
|
||||
if !seen_headers[header.name.id() as usize] {
|
||||
// Index subject
|
||||
let subject = match &header.value {
|
||||
HeaderValue::Text(text) => text.clone(),
|
||||
HeaderValue::TextList(list) if !list.is_empty() => {
|
||||
list.first().unwrap().clone()
|
||||
}
|
||||
_ => "".into(),
|
||||
};
|
||||
|
||||
// Index thread name
|
||||
let thread_name = thread_name(&subject);
|
||||
self.value(
|
||||
Property::Subject,
|
||||
if !thread_name.is_empty() {
|
||||
thread_name.trim_text(MAX_SORT_FIELD_LENGTH)
|
||||
} else {
|
||||
"!"
|
||||
},
|
||||
F_INDEX | options,
|
||||
);
|
||||
|
||||
seen_headers[header.name.id() as usize] = true;
|
||||
}
|
||||
}
|
||||
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
|
||||
// Add subject to index if missing
|
||||
if !seen_headers[HeaderName::Subject.id() as usize] {
|
||||
self.value(Property::Subject, "!", F_INDEX | options);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'x> IndexMessageText<'x> for FtsDocument<'x, HeaderName<'x>> {
|
||||
fn index_message(mut self, message: &'x Message<'x>) -> Self {
|
||||
let mut language = Language::Unknown;
|
||||
|
||||
for (part_id, part) in message.parts.iter().take(MAX_MESSAGE_PARTS).enumerate() {
|
||||
let part_language = part.language().unwrap_or(language);
|
||||
if part_id == 0 {
|
||||
language = part_language;
|
||||
|
||||
for header in part.headers.iter().rev() {
|
||||
if matches!(header.name, HeaderName::Other(_)) {
|
||||
continue;
|
||||
}
|
||||
// Index hasHeader property
|
||||
self.index_keyword(Field::Keyword, header.name.as_str().to_ascii_lowercase());
|
||||
|
||||
match &header.name {
|
||||
HeaderName::MessageId
|
||||
| HeaderName::InReplyTo
|
||||
| HeaderName::References
|
||||
| HeaderName::ResentMessageId => {
|
||||
header.value.visit_text(|id| {
|
||||
// Index ids without stemming
|
||||
if id.len() < MAX_TOKEN_LENGTH {
|
||||
self.index_keyword(
|
||||
Field::Header(header.name.clone()),
|
||||
id.to_string(),
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
HeaderName::From | HeaderName::To | HeaderName::Cc | HeaderName::Bcc => {
|
||||
header.value.visit_addresses(|_, value| {
|
||||
// Index an address name or email without stemming
|
||||
self.index_tokenized(
|
||||
Field::Header(header.name.clone()),
|
||||
value.to_string(),
|
||||
);
|
||||
});
|
||||
}
|
||||
HeaderName::Subject => {
|
||||
// Index subject for FTS
|
||||
if let Some(subject) = header.value.as_text() {
|
||||
self.index(Field::Header(HeaderName::Subject), subject, language);
|
||||
}
|
||||
}
|
||||
HeaderName::Comments | HeaderName::Keywords | HeaderName::ListId => {
|
||||
// Index headers
|
||||
header.value.visit_text(|text| {
|
||||
self.index_tokenized(
|
||||
Field::Header(header.name.clone()),
|
||||
text.to_string(),
|
||||
);
|
||||
});
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match &part.body {
|
||||
PartType::Text(text) => {
|
||||
if message.text_body.contains(&part_id) || message.html_body.contains(&part_id)
|
||||
{
|
||||
self.index(Field::Body, text.as_ref(), part_language);
|
||||
} else {
|
||||
self.index(Field::Attachment, text.as_ref(), part_language);
|
||||
}
|
||||
}
|
||||
PartType::Html(html) => {
|
||||
let text = html_to_text(html);
|
||||
|
||||
if message.text_body.contains(&part_id) || message.html_body.contains(&part_id)
|
||||
{
|
||||
self.index(Field::Body, text, part_language);
|
||||
} else {
|
||||
self.index(Field::Attachment, text, part_language);
|
||||
}
|
||||
}
|
||||
PartType::Message(nested_message) => {
|
||||
let nested_message_language = nested_message
|
||||
.root_part()
|
||||
.language()
|
||||
.unwrap_or(Language::Unknown);
|
||||
if let Some(HeaderValue::Text(subject)) =
|
||||
nested_message.header(HeaderName::Subject)
|
||||
{
|
||||
self.index(Field::Attachment, subject.as_ref(), nested_message_language);
|
||||
}
|
||||
|
||||
for sub_part in nested_message.parts.iter().take(MAX_MESSAGE_PARTS) {
|
||||
let language = sub_part.language().unwrap_or(nested_message_language);
|
||||
match &sub_part.body {
|
||||
PartType::Text(text) => {
|
||||
self.index(Field::Attachment, text.as_ref(), language);
|
||||
}
|
||||
PartType::Html(html) => {
|
||||
self.index(Field::Attachment, html_to_text(html), language);
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
pub struct EmailIndexBuilder<'x> {
|
||||
inner: Bincode<MessageMetadata<'x>>,
|
||||
set: bool,
|
||||
}
|
||||
|
||||
impl<'x> EmailIndexBuilder<'x> {
|
||||
pub fn set(inner: MessageMetadata<'x>) -> Self {
|
||||
Self {
|
||||
inner: Bincode { inner },
|
||||
set: true,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn clear(inner: MessageMetadata<'x>) -> Self {
|
||||
Self {
|
||||
inner: Bincode { inner },
|
||||
set: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl EmailIndexBuilder<'_> {
|
||||
pub fn build(self, batch: &mut BatchBuilder, account_id: u32, tenant_id: Option<u32>) {
|
||||
let options = if self.set {
|
||||
// Serialize metadata
|
||||
batch.value(Property::BodyStructure, &self.inner, F_VALUE);
|
||||
0
|
||||
} else {
|
||||
// Delete metadata
|
||||
batch.value(Property::BodyStructure, (), F_VALUE | F_CLEAR);
|
||||
F_CLEAR
|
||||
};
|
||||
let metadata = &self.inner.inner;
|
||||
|
||||
// Index properties
|
||||
let quota = if self.set {
|
||||
metadata.size as i64
|
||||
} else {
|
||||
-(metadata.size as i64)
|
||||
};
|
||||
batch
|
||||
.value(Property::Size, metadata.size as u32, F_INDEX | options)
|
||||
.add(DirectoryClass::UsedQuota(account_id), quota);
|
||||
if let Some(tenant_id) = tenant_id {
|
||||
batch.add(DirectoryClass::UsedQuota(tenant_id), quota);
|
||||
}
|
||||
|
||||
batch.value(
|
||||
Property::ReceivedAt,
|
||||
metadata.received_at,
|
||||
F_INDEX | options,
|
||||
);
|
||||
if metadata.has_attachments {
|
||||
batch.tag(Property::HasAttachment, (), options);
|
||||
}
|
||||
|
||||
// Index headers
|
||||
batch.index_headers(&metadata.contents.parts[0].headers, options);
|
||||
|
||||
// Link blob
|
||||
if self.set {
|
||||
batch.set(
|
||||
BlobOp::Link {
|
||||
hash: metadata.blob_hash.clone(),
|
||||
},
|
||||
Vec::new(),
|
||||
);
|
||||
} else {
|
||||
batch.clear(BlobOp::Link {
|
||||
hash: metadata.blob_hash.clone(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SortedAddressBuilder {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
last_is_space: true,
|
||||
buf: String::with_capacity(32),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn push(&mut self, text: &str) -> bool {
|
||||
if !text.is_empty() {
|
||||
if !self.buf.is_empty() {
|
||||
self.buf.push(' ');
|
||||
self.last_is_space = true;
|
||||
}
|
||||
for ch in text.chars() {
|
||||
for ch in ch.to_lowercase() {
|
||||
if self.buf.len() < MAX_SORT_FIELD_LENGTH {
|
||||
let is_space = ch.is_whitespace();
|
||||
if !is_space || !self.last_is_space {
|
||||
self.buf.push(ch);
|
||||
self.last_is_space = is_space;
|
||||
}
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
pub fn build(self) -> String {
|
||||
if !self.buf.is_empty() {
|
||||
self.buf
|
||||
} else {
|
||||
"!".to_string()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for SortedAddressBuilder {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
trait GetContentLanguage {
|
||||
fn language(&self) -> Option<Language>;
|
||||
}
|
||||
|
||||
impl GetContentLanguage for MessagePart<'_> {
|
||||
fn language(&self) -> Option<Language> {
|
||||
self.headers
|
||||
.header_value(&HeaderName::ContentLanguage)
|
||||
.and_then(|v| {
|
||||
Language::from_iso_639(match v {
|
||||
HeaderValue::Text(v) => v.as_ref(),
|
||||
HeaderValue::TextList(v) => v.first()?,
|
||||
_ => {
|
||||
return None;
|
||||
}
|
||||
})
|
||||
.unwrap_or(Language::Unknown)
|
||||
.into()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub trait VisitValues<'x> {
|
||||
fn visit_addresses<'y: 'x>(&'y self, visitor: impl FnMut(AddressElement, &'x str));
|
||||
fn visit_text<'y: 'x>(&'y self, visitor: impl FnMut(&'x str));
|
||||
fn into_visit_text(self, visitor: impl FnMut(String));
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub enum AddressElement {
|
||||
Name,
|
||||
Address,
|
||||
GroupName,
|
||||
}
|
||||
|
||||
impl<'x> VisitValues<'x> for HeaderValue<'x> {
|
||||
fn visit_addresses<'y: 'x>(&'y self, mut visitor: impl FnMut(AddressElement, &'x str)) {
|
||||
match self {
|
||||
HeaderValue::Address(Address::List(addr_list)) => {
|
||||
for addr in addr_list {
|
||||
if let Some(name) = &addr.name {
|
||||
visitor(AddressElement::Name, name);
|
||||
}
|
||||
if let Some(addr) = &addr.address {
|
||||
visitor(AddressElement::Address, addr);
|
||||
}
|
||||
}
|
||||
}
|
||||
HeaderValue::Address(Address::Group(groups)) => {
|
||||
for group in groups {
|
||||
if let Some(name) = &group.name {
|
||||
visitor(AddressElement::GroupName, name);
|
||||
}
|
||||
|
||||
for addr in &group.addresses {
|
||||
if let Some(name) = &addr.name {
|
||||
visitor(AddressElement::Name, name);
|
||||
}
|
||||
if let Some(addr) = &addr.address {
|
||||
visitor(AddressElement::Address, addr);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
|
||||
fn visit_text<'y: 'x>(&'y self, mut visitor: impl FnMut(&'x str)) {
|
||||
match &self {
|
||||
HeaderValue::Text(text) => {
|
||||
visitor(text.as_ref());
|
||||
}
|
||||
HeaderValue::TextList(texts) => {
|
||||
for text in texts {
|
||||
visitor(text.as_ref());
|
||||
}
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
|
||||
fn into_visit_text(self, mut visitor: impl FnMut(String)) {
|
||||
match self {
|
||||
HeaderValue::Text(text) => {
|
||||
visitor(text.into_owned());
|
||||
}
|
||||
HeaderValue::TextList(texts) => {
|
||||
for text in texts {
|
||||
visitor(text.into_owned());
|
||||
}
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub trait TrimTextValue {
|
||||
fn trim_text(self, length: usize) -> Self;
|
||||
}
|
||||
|
||||
impl TrimTextValue for HeaderValue<'_> {
|
||||
fn trim_text(self, length: usize) -> Self {
|
||||
match self {
|
||||
HeaderValue::Address(Address::List(v)) => {
|
||||
HeaderValue::Address(Address::List(v.trim_text(length)))
|
||||
}
|
||||
HeaderValue::Address(Address::Group(v)) => {
|
||||
HeaderValue::Address(Address::Group(v.trim_text(length)))
|
||||
}
|
||||
HeaderValue::Text(v) => HeaderValue::Text(v.trim_text(length)),
|
||||
HeaderValue::TextList(v) => HeaderValue::TextList(v.trim_text(length)),
|
||||
v => v,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TrimTextValue for Addr<'_> {
|
||||
fn trim_text(self, length: usize) -> Self {
|
||||
Self {
|
||||
name: self.name.map(|v| v.trim_text(length)),
|
||||
address: self.address.map(|v| v.trim_text(length)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TrimTextValue for Group<'_> {
|
||||
fn trim_text(self, length: usize) -> Self {
|
||||
Self {
|
||||
name: self.name.map(|v| v.trim_text(length)),
|
||||
addresses: self.addresses.trim_text(length),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TrimTextValue for Cow<'_, str> {
|
||||
fn trim_text(self, length: usize) -> Self {
|
||||
if self.len() < length {
|
||||
self
|
||||
} else {
|
||||
match self {
|
||||
Cow::Borrowed(v) => v.trim_text(length).into(),
|
||||
Cow::Owned(v) => v.trim_text(length).into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TrimTextValue for &str {
|
||||
fn trim_text(self, length: usize) -> Self {
|
||||
if self.len() < length {
|
||||
self
|
||||
} else {
|
||||
let mut index = 0;
|
||||
|
||||
for (i, _) in self.char_indices() {
|
||||
if i > length {
|
||||
break;
|
||||
}
|
||||
index = i;
|
||||
}
|
||||
|
||||
&self[..index]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TrimTextValue for String {
|
||||
fn trim_text(self, length: usize) -> Self {
|
||||
if self.len() < length {
|
||||
self
|
||||
} else {
|
||||
let mut result = String::with_capacity(length);
|
||||
for (i, c) in self.char_indices() {
|
||||
if i > length {
|
||||
break;
|
||||
}
|
||||
result.push(c);
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: TrimTextValue> TrimTextValue for Vec<T> {
|
||||
fn trim_text(self, length: usize) -> Self {
|
||||
self.into_iter().map(|v| v.trim_text(length)).collect()
|
||||
}
|
||||
}
|
||||
778
crates/email/src/ingest.rs
Normal file
778
crates/email/src/ingest.rs
Normal file
@@ -0,0 +1,778 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use std::{
|
||||
borrow::Cow,
|
||||
fmt::Write,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
use common::{
|
||||
auth::{AccessToken, ResourceToken},
|
||||
Server,
|
||||
};
|
||||
use directory::Permission;
|
||||
use jmap_proto::{
|
||||
object::Object,
|
||||
types::{
|
||||
blob::BlobId, collection::Collection, id::Id, keyword::Keyword, property::Property,
|
||||
value::Value,
|
||||
},
|
||||
};
|
||||
use mail_parser::{
|
||||
parsers::fields::thread::thread_name, Header, HeaderName, HeaderValue, Message, MessageParser,
|
||||
PartType,
|
||||
};
|
||||
|
||||
use spam_filter::{
|
||||
analysis::init::SpamFilterInit, modules::bayes::BayesClassifier, SpamFilterInput,
|
||||
};
|
||||
use std::future::Future;
|
||||
use store::rand::Rng;
|
||||
use store::{
|
||||
ahash::AHashSet,
|
||||
query::Filter,
|
||||
write::{
|
||||
log::{ChangeLogBuilder, Changes, LogInsert},
|
||||
now, AssignedIds, BatchBuilder, BitmapClass, MaybeDynamicId, MaybeDynamicValue,
|
||||
SerializeWithId, TagValue, TaskQueueClass, ValueClass, F_BITMAP, F_CLEAR, F_VALUE,
|
||||
},
|
||||
BitmapKey, BlobClass, Serialize,
|
||||
};
|
||||
use trc::{AddContext, MessageIngestEvent};
|
||||
use utils::map::vec_map::VecMap;
|
||||
|
||||
use crate::{
|
||||
index::{IndexMessage, VisitValues, MAX_ID_LENGTH},
|
||||
mailbox::{UidMailbox, INBOX_ID, JUNK_ID},
|
||||
};
|
||||
|
||||
use super::{
|
||||
cache::ThreadCache,
|
||||
crypto::{EncryptMessage, EncryptMessageError, EncryptionParams},
|
||||
index::{TrimTextValue, MAX_SORT_FIELD_LENGTH},
|
||||
};
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct IngestedEmail {
|
||||
pub id: Id,
|
||||
pub change_id: u64,
|
||||
pub blob_id: BlobId,
|
||||
pub size: usize,
|
||||
pub imap_uids: Vec<u32>,
|
||||
}
|
||||
|
||||
pub struct IngestEmail<'x> {
|
||||
pub raw_message: &'x [u8],
|
||||
pub message: Option<Message<'x>>,
|
||||
pub resource: ResourceToken,
|
||||
pub mailbox_ids: Vec<u32>,
|
||||
pub keywords: Vec<Keyword>,
|
||||
pub received_at: Option<u64>,
|
||||
pub source: IngestSource<'x>,
|
||||
pub spam_classify: bool,
|
||||
pub spam_train: bool,
|
||||
pub session_id: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
|
||||
pub enum IngestSource<'x> {
|
||||
Smtp { deliver_to: &'x str },
|
||||
Jmap,
|
||||
Imap,
|
||||
Restore,
|
||||
}
|
||||
|
||||
const MAX_RETRIES: u32 = 10;
|
||||
|
||||
pub trait EmailIngest: Sync + Send {
|
||||
fn email_ingest(
|
||||
&self,
|
||||
params: IngestEmail,
|
||||
) -> impl Future<Output = trc::Result<IngestedEmail>> + Send;
|
||||
fn find_or_merge_thread(
|
||||
&self,
|
||||
account_id: u32,
|
||||
thread_name: &str,
|
||||
references: &[&str],
|
||||
) -> impl Future<Output = trc::Result<Option<u32>>> + Send;
|
||||
fn assign_imap_uid(
|
||||
&self,
|
||||
account_id: u32,
|
||||
mailbox_id: u32,
|
||||
) -> impl Future<Output = trc::Result<u32>> + Send;
|
||||
fn email_bayes_can_train(&self, access_token: &AccessToken) -> bool;
|
||||
}
|
||||
|
||||
impl EmailIngest for Server {
|
||||
#[allow(clippy::blocks_in_conditions)]
|
||||
async fn email_ingest(&self, mut params: IngestEmail<'_>) -> trc::Result<IngestedEmail> {
|
||||
// Check quota
|
||||
let start_time = Instant::now();
|
||||
let account_id = params.resource.account_id;
|
||||
let tenant_id = params.resource.tenant.map(|t| t.id);
|
||||
let mut raw_message_len = params.raw_message.len() as u64;
|
||||
self.has_available_quota(¶ms.resource, raw_message_len)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
|
||||
// Parse message
|
||||
let mut raw_message = Cow::from(params.raw_message);
|
||||
let mut message = params.message.ok_or_else(|| {
|
||||
trc::EventType::MessageIngest(trc::MessageIngestEvent::Error)
|
||||
.ctx(trc::Key::Code, 550)
|
||||
.ctx(trc::Key::Reason, "Failed to parse e-mail message.")
|
||||
})?;
|
||||
|
||||
let mut is_spam = false;
|
||||
let mut train_spam = None;
|
||||
let mut extra_headers = String::new();
|
||||
let mut extra_headers_parsed = Vec::new();
|
||||
match params.source {
|
||||
IngestSource::Smtp { deliver_to } => {
|
||||
// Add delivered to header
|
||||
if self.core.smtp.session.data.add_delivered_to {
|
||||
extra_headers = format!("Delivered-To: {deliver_to}\r\n");
|
||||
extra_headers_parsed.push(Header {
|
||||
name: HeaderName::Other("Delivered-To".into()),
|
||||
value: HeaderValue::Text(deliver_to.into()),
|
||||
offset_field: 0,
|
||||
offset_start: 13,
|
||||
offset_end: extra_headers.len(),
|
||||
});
|
||||
}
|
||||
|
||||
// Spam classification and training
|
||||
if params.spam_classify
|
||||
&& self.core.spam.enabled
|
||||
&& params.mailbox_ids == [INBOX_ID]
|
||||
{
|
||||
// Set the spam filter result
|
||||
is_spam = self
|
||||
.core
|
||||
.spam
|
||||
.headers
|
||||
.status
|
||||
.as_ref()
|
||||
.and_then(|name| message.header(name.as_str()).and_then(|v| v.as_text()))
|
||||
.is_some_and(|v| v.contains("Yes"));
|
||||
|
||||
// Classify the message with user's model
|
||||
if let Some(bayes_config) = self
|
||||
.core
|
||||
.spam
|
||||
.bayes
|
||||
.as_ref()
|
||||
.filter(|config| config.account_classify && params.spam_train)
|
||||
{
|
||||
// Initialize spam filter
|
||||
let ctx = self.spam_filter_init(SpamFilterInput::from_account_message(
|
||||
&message,
|
||||
account_id,
|
||||
params.session_id,
|
||||
));
|
||||
|
||||
// Bayes classify
|
||||
match self.bayes_classify(&ctx).await {
|
||||
Ok(Some(score)) => {
|
||||
let result = if score > bayes_config.score_spam {
|
||||
is_spam = true;
|
||||
"Yes"
|
||||
} else if score < bayes_config.score_ham {
|
||||
is_spam = false;
|
||||
"No"
|
||||
} else {
|
||||
"Unknown"
|
||||
};
|
||||
|
||||
if let Some(header) = &self.core.spam.headers.bayes_result {
|
||||
let offset_field = extra_headers.len();
|
||||
let offset_start = offset_field + header.len() + 1;
|
||||
|
||||
let _ = write!(
|
||||
&mut extra_headers,
|
||||
"{header}: {result}, {score:.2}\r\n",
|
||||
);
|
||||
|
||||
extra_headers_parsed.push(Header {
|
||||
name: HeaderName::Other(header.into()),
|
||||
value: HeaderValue::Text(
|
||||
extra_headers
|
||||
[offset_start + 1..extra_headers.len() - 2]
|
||||
.into(),
|
||||
),
|
||||
offset_field,
|
||||
offset_start,
|
||||
offset_end: extra_headers.len(),
|
||||
});
|
||||
}
|
||||
}
|
||||
Ok(None) => (),
|
||||
Err(err) => {
|
||||
trc::error!(err.caused_by(trc::location!()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if is_spam {
|
||||
params.mailbox_ids[0] = JUNK_ID;
|
||||
params.keywords.push(Keyword::Junk);
|
||||
}
|
||||
}
|
||||
}
|
||||
IngestSource::Jmap | IngestSource::Imap
|
||||
if params.spam_train && self.core.spam.enabled =>
|
||||
{
|
||||
if params.keywords.contains(&Keyword::Junk) {
|
||||
train_spam = Some(true);
|
||||
} else if params.keywords.contains(&Keyword::NotJunk) {
|
||||
train_spam = Some(false);
|
||||
} else if params.mailbox_ids[0] == JUNK_ID {
|
||||
train_spam = Some(true);
|
||||
} else if params.mailbox_ids[0] == INBOX_ID {
|
||||
train_spam = Some(false);
|
||||
}
|
||||
}
|
||||
|
||||
_ => (),
|
||||
}
|
||||
|
||||
// Obtain message references and thread name
|
||||
let mut message_id = String::new();
|
||||
let thread_id = {
|
||||
let mut references = Vec::with_capacity(5);
|
||||
let mut subject = "";
|
||||
for header in message.root_part().headers().iter().rev() {
|
||||
match &header.name {
|
||||
HeaderName::MessageId => header.value.visit_text(|id| {
|
||||
if !id.is_empty() && id.len() < MAX_ID_LENGTH {
|
||||
if message_id.is_empty() {
|
||||
message_id = id.to_string();
|
||||
}
|
||||
references.push(id);
|
||||
}
|
||||
}),
|
||||
HeaderName::InReplyTo
|
||||
| HeaderName::References
|
||||
| HeaderName::ResentMessageId => {
|
||||
header.value.visit_text(|id| {
|
||||
if !id.is_empty() && id.len() < MAX_ID_LENGTH {
|
||||
references.push(id);
|
||||
}
|
||||
});
|
||||
}
|
||||
HeaderName::Subject if subject.is_empty() => {
|
||||
subject = thread_name(match &header.value {
|
||||
HeaderValue::Text(text) => text.as_ref(),
|
||||
HeaderValue::TextList(list) if !list.is_empty() => {
|
||||
list.first().unwrap().as_ref()
|
||||
}
|
||||
_ => "",
|
||||
})
|
||||
.trim_text(MAX_SORT_FIELD_LENGTH);
|
||||
}
|
||||
_ => (),
|
||||
}
|
||||
}
|
||||
|
||||
// Check for duplicates
|
||||
if params.source.is_smtp()
|
||||
&& !message_id.is_empty()
|
||||
&& !self
|
||||
.core
|
||||
.storage
|
||||
.data
|
||||
.filter(
|
||||
account_id,
|
||||
Collection::Email,
|
||||
vec![
|
||||
Filter::eq(Property::MessageId, &message_id),
|
||||
Filter::is_in_bitmap(
|
||||
Property::MailboxIds,
|
||||
params.mailbox_ids.first().copied().unwrap_or(INBOX_ID),
|
||||
),
|
||||
],
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
.results
|
||||
.is_empty()
|
||||
{
|
||||
trc::event!(
|
||||
MessageIngest(MessageIngestEvent::Duplicate),
|
||||
SpanId = params.session_id,
|
||||
AccountId = account_id,
|
||||
MessageId = message_id,
|
||||
);
|
||||
|
||||
return Ok(IngestedEmail {
|
||||
id: Id::default(),
|
||||
change_id: u64::MAX,
|
||||
blob_id: BlobId::default(),
|
||||
imap_uids: Vec::new(),
|
||||
size: 0,
|
||||
});
|
||||
}
|
||||
|
||||
if !references.is_empty() {
|
||||
self.find_or_merge_thread(account_id, subject, &references)
|
||||
.await?
|
||||
} else {
|
||||
None
|
||||
}
|
||||
};
|
||||
|
||||
// Add additional headers to message
|
||||
if !extra_headers.is_empty() {
|
||||
let offset_start = extra_headers.len();
|
||||
raw_message_len += offset_start as u64;
|
||||
let mut new_message = Vec::with_capacity(raw_message_len as usize);
|
||||
new_message.extend_from_slice(extra_headers.as_bytes());
|
||||
new_message.extend_from_slice(raw_message.as_ref());
|
||||
raw_message = Cow::from(new_message);
|
||||
message.raw_message = raw_message.as_ref().into();
|
||||
|
||||
// Adjust offsets
|
||||
let mut part_iter_stack = Vec::new();
|
||||
let mut part_iter = message.parts.iter_mut();
|
||||
|
||||
loop {
|
||||
if let Some(part) = part_iter.next() {
|
||||
// Increment header offsets
|
||||
for header in part.headers.iter_mut() {
|
||||
header.offset_field += offset_start;
|
||||
header.offset_start += offset_start;
|
||||
header.offset_end += offset_start;
|
||||
}
|
||||
|
||||
// Adjust part offsets
|
||||
part.offset_body += offset_start;
|
||||
part.offset_end += offset_start;
|
||||
part.offset_header += offset_start;
|
||||
|
||||
if let PartType::Message(sub_message) = &mut part.body {
|
||||
if sub_message.root_part().offset_header != 0 {
|
||||
sub_message.raw_message = raw_message.as_ref().into();
|
||||
part_iter_stack.push(part_iter);
|
||||
part_iter = sub_message.parts.iter_mut();
|
||||
}
|
||||
}
|
||||
} else if let Some(iter) = part_iter_stack.pop() {
|
||||
part_iter = iter;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Add extra headers to root part
|
||||
let root_part = &mut message.parts[0];
|
||||
root_part.offset_header = 0;
|
||||
extra_headers_parsed.append(&mut root_part.headers);
|
||||
root_part.headers = extra_headers_parsed;
|
||||
}
|
||||
|
||||
// Encrypt message
|
||||
let do_encrypt = match params.source {
|
||||
IngestSource::Jmap | IngestSource::Imap => {
|
||||
self.core.jmap.encrypt && self.core.jmap.encrypt_append
|
||||
}
|
||||
IngestSource::Smtp { .. } => self.core.jmap.encrypt,
|
||||
IngestSource::Restore => false,
|
||||
};
|
||||
if do_encrypt && !message.is_encrypted() {
|
||||
if let Some(encrypt_params) = self
|
||||
.get_property::<EncryptionParams>(
|
||||
account_id,
|
||||
Collection::Principal,
|
||||
0,
|
||||
Property::Parameters,
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
{
|
||||
match message.encrypt(&encrypt_params).await {
|
||||
Ok(new_raw_message) => {
|
||||
raw_message = Cow::from(new_raw_message);
|
||||
raw_message_len = raw_message.len() as u64;
|
||||
message = MessageParser::default()
|
||||
.parse(raw_message.as_ref())
|
||||
.ok_or_else(|| {
|
||||
trc::EventType::MessageIngest(trc::MessageIngestEvent::Error)
|
||||
.ctx(trc::Key::Code, 550)
|
||||
.ctx(
|
||||
trc::Key::Reason,
|
||||
"Failed to parse encrypted e-mail message.",
|
||||
)
|
||||
})?;
|
||||
|
||||
// Remove contents from parsed message
|
||||
for part in &mut message.parts {
|
||||
match &mut part.body {
|
||||
PartType::Text(txt) | PartType::Html(txt) => {
|
||||
*txt = Cow::from("");
|
||||
}
|
||||
PartType::Binary(bin) | PartType::InlineBinary(bin) => {
|
||||
*bin = Cow::from(&[][..]);
|
||||
}
|
||||
PartType::Message(_) => {
|
||||
part.body = PartType::Binary(Cow::from(&[][..]));
|
||||
}
|
||||
PartType::Multipart(_) => (),
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(EncryptMessageError::Error(err)) => {
|
||||
trc::bail!(trc::StoreEvent::CryptoError
|
||||
.into_err()
|
||||
.caused_by(trc::location!())
|
||||
.reason(err));
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Obtain a documentId and changeId
|
||||
let change_id = self
|
||||
.assign_change_id(account_id)
|
||||
.caused_by(trc::location!())?;
|
||||
|
||||
// Store blob
|
||||
let blob_id = self
|
||||
.put_blob(account_id, raw_message.as_ref(), false)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
|
||||
// Assign IMAP UIDs
|
||||
let mut mailbox_ids = Vec::with_capacity(params.mailbox_ids.len());
|
||||
let mut imap_uids = Vec::with_capacity(params.mailbox_ids.len());
|
||||
for mailbox_id in ¶ms.mailbox_ids {
|
||||
let uid = self
|
||||
.assign_imap_uid(account_id, *mailbox_id)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
mailbox_ids.push(UidMailbox::new(*mailbox_id, uid));
|
||||
imap_uids.push(uid);
|
||||
}
|
||||
|
||||
// Prepare batch
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch
|
||||
.with_change_id(change_id)
|
||||
.with_account_id(account_id)
|
||||
.with_collection(Collection::Thread);
|
||||
if let Some(thread_id) = thread_id {
|
||||
batch.log(Changes::update([thread_id]));
|
||||
} else {
|
||||
batch.create_document().log(LogInsert());
|
||||
}
|
||||
|
||||
// Build write batch
|
||||
let mailbox_ids_event = mailbox_ids
|
||||
.iter()
|
||||
.map(|m| trc::Value::from(m.mailbox_id))
|
||||
.collect::<Vec<_>>();
|
||||
let maybe_thread_id = thread_id
|
||||
.map(MaybeDynamicId::Static)
|
||||
.unwrap_or(MaybeDynamicId::Dynamic(0));
|
||||
batch
|
||||
.with_collection(Collection::Mailbox)
|
||||
.log(Changes::child_update(params.mailbox_ids.iter().copied()))
|
||||
.with_collection(Collection::Email)
|
||||
.create_document()
|
||||
.log(LogEmailInsert(thread_id))
|
||||
.index_message(
|
||||
account_id,
|
||||
tenant_id,
|
||||
message,
|
||||
blob_id.hash.clone(),
|
||||
params.keywords,
|
||||
mailbox_ids,
|
||||
params.received_at.unwrap_or_else(now),
|
||||
)
|
||||
.value(Property::Cid, change_id, F_VALUE)
|
||||
.set(Property::ThreadId, maybe_thread_id)
|
||||
.tag(Property::ThreadId, TagValue::Id(maybe_thread_id), 0)
|
||||
.set(
|
||||
ValueClass::TaskQueue(TaskQueueClass::IndexEmail {
|
||||
seq: self.generate_snowflake_id().caused_by(trc::location!())?,
|
||||
hash: blob_id.hash.clone(),
|
||||
}),
|
||||
vec![],
|
||||
);
|
||||
|
||||
// Request spam training
|
||||
if let Some(learn_spam) = train_spam {
|
||||
batch.set(
|
||||
ValueClass::TaskQueue(TaskQueueClass::BayesTrain {
|
||||
seq: self.generate_snowflake_id()?,
|
||||
hash: blob_id.hash.clone(),
|
||||
learn_spam,
|
||||
}),
|
||||
vec![],
|
||||
);
|
||||
}
|
||||
|
||||
// Insert and obtain ids
|
||||
let ids = self
|
||||
.core
|
||||
.storage
|
||||
.data
|
||||
.write(batch.build())
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
let thread_id = match thread_id {
|
||||
Some(thread_id) => thread_id,
|
||||
None => ids.first_document_id().caused_by(trc::location!())?,
|
||||
};
|
||||
let document_id = ids.last_document_id().caused_by(trc::location!())?;
|
||||
let id = Id::from_parts(thread_id, document_id);
|
||||
|
||||
// Request FTS index
|
||||
self.notify_task_queue();
|
||||
|
||||
trc::event!(
|
||||
MessageIngest(match params.source {
|
||||
IngestSource::Smtp { .. } =>
|
||||
if !is_spam {
|
||||
MessageIngestEvent::Ham
|
||||
} else {
|
||||
MessageIngestEvent::Spam
|
||||
},
|
||||
IngestSource::Jmap | IngestSource::Restore => MessageIngestEvent::JmapAppend,
|
||||
IngestSource::Imap => MessageIngestEvent::ImapAppend,
|
||||
}),
|
||||
SpanId = params.session_id,
|
||||
AccountId = account_id,
|
||||
DocumentId = document_id,
|
||||
MailboxId = mailbox_ids_event,
|
||||
BlobId = blob_id.hash.to_hex(),
|
||||
ChangeId = change_id,
|
||||
MessageId = message_id,
|
||||
Size = raw_message_len,
|
||||
Elapsed = start_time.elapsed(),
|
||||
);
|
||||
|
||||
Ok(IngestedEmail {
|
||||
id,
|
||||
change_id,
|
||||
blob_id: BlobId {
|
||||
hash: blob_id.hash,
|
||||
class: BlobClass::Linked {
|
||||
account_id,
|
||||
collection: Collection::Email.into(),
|
||||
document_id,
|
||||
},
|
||||
section: blob_id.section,
|
||||
},
|
||||
size: raw_message_len as usize,
|
||||
imap_uids,
|
||||
})
|
||||
}
|
||||
|
||||
async fn find_or_merge_thread(
|
||||
&self,
|
||||
account_id: u32,
|
||||
thread_name: &str,
|
||||
references: &[&str],
|
||||
) -> trc::Result<Option<u32>> {
|
||||
let mut try_count = 0;
|
||||
|
||||
loop {
|
||||
// Find messages with matching references
|
||||
let mut filters = Vec::with_capacity(references.len() + 3);
|
||||
filters.push(Filter::eq(
|
||||
Property::Subject,
|
||||
if !thread_name.is_empty() {
|
||||
thread_name
|
||||
} else {
|
||||
"!"
|
||||
},
|
||||
));
|
||||
filters.push(Filter::Or);
|
||||
for reference in references {
|
||||
filters.push(Filter::eq(Property::References, *reference));
|
||||
}
|
||||
filters.push(Filter::End);
|
||||
let results = self
|
||||
.core
|
||||
.storage
|
||||
.data
|
||||
.filter(account_id, Collection::Email, filters)
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
.results;
|
||||
|
||||
if results.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
// Obtain threadIds for matching messages
|
||||
let thread_ids = self
|
||||
.get_cached_thread_ids(account_id, results.iter())
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
|
||||
if thread_ids.len() == 1 {
|
||||
return Ok(thread_ids
|
||||
.into_iter()
|
||||
.next()
|
||||
.map(|(_, thread_id)| thread_id));
|
||||
}
|
||||
|
||||
// Find the most common threadId
|
||||
let mut thread_counts = VecMap::<u32, u32>::with_capacity(thread_ids.len());
|
||||
let mut thread_id = u32::MAX;
|
||||
let mut thread_count = 0;
|
||||
for (_, thread_id_) in thread_ids.iter() {
|
||||
let tc = thread_counts.get_mut_or_insert(*thread_id_);
|
||||
*tc += 1;
|
||||
if *tc > thread_count {
|
||||
thread_count = *tc;
|
||||
thread_id = *thread_id_;
|
||||
}
|
||||
}
|
||||
|
||||
if thread_id == u32::MAX {
|
||||
return Ok(None); // This should never happen
|
||||
} else if thread_counts.len() == 1 {
|
||||
return Ok(Some(thread_id));
|
||||
}
|
||||
|
||||
// Delete all but the most common threadId
|
||||
let mut batch = BatchBuilder::new();
|
||||
let change_id = self
|
||||
.assign_change_id(account_id)
|
||||
.caused_by(trc::location!())?;
|
||||
let mut changes = ChangeLogBuilder::with_change_id(change_id);
|
||||
batch
|
||||
.with_account_id(account_id)
|
||||
.with_collection(Collection::Thread);
|
||||
for &delete_thread_id in thread_counts.keys() {
|
||||
if delete_thread_id != thread_id {
|
||||
batch.delete_document(delete_thread_id);
|
||||
changes.log_delete(Collection::Thread, delete_thread_id);
|
||||
}
|
||||
}
|
||||
|
||||
// Move messages to the new threadId
|
||||
batch.with_collection(Collection::Email);
|
||||
for old_thread_id in thread_ids
|
||||
.into_iter()
|
||||
.map(|(_, thread_id)| thread_id)
|
||||
.collect::<AHashSet<_>>()
|
||||
{
|
||||
if thread_id != old_thread_id {
|
||||
for document_id in self
|
||||
.core
|
||||
.storage
|
||||
.data
|
||||
.get_bitmap(BitmapKey {
|
||||
account_id,
|
||||
collection: Collection::Email.into(),
|
||||
class: BitmapClass::Tag {
|
||||
field: Property::ThreadId.into(),
|
||||
value: TagValue::Id(old_thread_id),
|
||||
},
|
||||
document_id: 0,
|
||||
})
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
.unwrap_or_default()
|
||||
{
|
||||
batch
|
||||
.update_document(document_id)
|
||||
.assert_value(Property::ThreadId, old_thread_id)
|
||||
.value(Property::ThreadId, old_thread_id, F_BITMAP | F_CLEAR)
|
||||
.value(Property::ThreadId, thread_id, F_VALUE | F_BITMAP);
|
||||
changes.log_move(
|
||||
Collection::Email,
|
||||
Id::from_parts(old_thread_id, document_id),
|
||||
Id::from_parts(thread_id, document_id),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
batch.custom(changes);
|
||||
|
||||
match self.core.storage.data.write(batch.build()).await {
|
||||
Ok(_) => return Ok(Some(thread_id)),
|
||||
Err(err) if err.is_assertion_failure() && try_count < MAX_RETRIES => {
|
||||
let backoff = store::rand::thread_rng().gen_range(50..=300);
|
||||
tokio::time::sleep(Duration::from_millis(backoff)).await;
|
||||
try_count += 1;
|
||||
}
|
||||
Err(err) => {
|
||||
return Err(err.caused_by(trc::location!()));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn assign_imap_uid(&self, account_id: u32, mailbox_id: u32) -> trc::Result<u32> {
|
||||
// Increment UID next
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch
|
||||
.with_account_id(account_id)
|
||||
.with_collection(Collection::Mailbox)
|
||||
.update_document(mailbox_id)
|
||||
.add_and_get(Property::EmailIds, 1);
|
||||
self.core
|
||||
.storage
|
||||
.data
|
||||
.write(batch.build())
|
||||
.await
|
||||
.and_then(|v| v.last_counter_id().map(|id| id as u32))
|
||||
}
|
||||
|
||||
fn email_bayes_can_train(&self, access_token: &AccessToken) -> bool {
|
||||
self.core.spam.bayes.as_ref().is_some_and(|bayes| {
|
||||
bayes.account_classify && access_token.has_permission(Permission::SpamFilterTrain)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub struct LogEmailInsert(Option<u32>);
|
||||
|
||||
impl LogEmailInsert {
|
||||
pub fn new(thread_id: Option<u32>) -> Self {
|
||||
Self(thread_id)
|
||||
}
|
||||
}
|
||||
|
||||
impl IngestSource<'_> {
|
||||
pub fn is_smtp(&self) -> bool {
|
||||
matches!(self, Self::Smtp { .. })
|
||||
}
|
||||
}
|
||||
|
||||
impl SerializeWithId for LogEmailInsert {
|
||||
fn serialize_with_id(&self, ids: &AssignedIds) -> trc::Result<Vec<u8>> {
|
||||
let thread_id = match self.0 {
|
||||
Some(thread_id) => thread_id,
|
||||
None => ids.first_document_id()?,
|
||||
};
|
||||
let document_id = ids.last_document_id()?;
|
||||
|
||||
Ok(Changes::insert([Id::from_parts(thread_id, document_id)]).serialize())
|
||||
}
|
||||
}
|
||||
|
||||
impl From<LogEmailInsert> for MaybeDynamicValue {
|
||||
fn from(log: LogEmailInsert) -> Self {
|
||||
MaybeDynamicValue::Dynamic(Box::new(log))
|
||||
}
|
||||
}
|
||||
|
||||
impl From<IngestedEmail> for Object<Value> {
|
||||
fn from(email: IngestedEmail) -> Self {
|
||||
Object::with_capacity(3)
|
||||
.with_property(Property::Id, email.id)
|
||||
.with_property(Property::ThreadId, Id::from(email.id.prefix_id()))
|
||||
.with_property(Property::BlobId, email.blob_id)
|
||||
.with_property(Property::Size, email.size)
|
||||
}
|
||||
}
|
||||
14
crates/email/src/lib.rs
Normal file
14
crates/email/src/lib.rs
Normal file
@@ -0,0 +1,14 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
pub mod cache;
|
||||
pub mod crypto;
|
||||
pub mod delivery;
|
||||
pub mod index;
|
||||
pub mod ingest;
|
||||
pub mod mailbox;
|
||||
pub mod metadata;
|
||||
pub mod sieve;
|
||||
495
crates/email/src/mailbox.rs
Normal file
495
crates/email/src/mailbox.rs
Normal file
@@ -0,0 +1,495 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use std::{future::Future, slice::Iter};
|
||||
|
||||
use common::{config::jmap::settings::SpecialUse, Server};
|
||||
use jmap_proto::{
|
||||
object::{
|
||||
index::{IndexAs, IndexProperty, ObjectIndexBuilder},
|
||||
Object,
|
||||
},
|
||||
types::{collection::Collection, id::Id, keyword::Keyword, property::Property, value::Value},
|
||||
};
|
||||
use store::{
|
||||
ahash::AHashSet,
|
||||
query::Filter,
|
||||
rand,
|
||||
roaring::RoaringBitmap,
|
||||
write::{
|
||||
BatchBuilder, BitmapClass, DeserializeFrom, MaybeDynamicId, Operation, SerializeInto,
|
||||
TagValue, ToBitmaps,
|
||||
},
|
||||
Serialize, U32_LEN,
|
||||
};
|
||||
use trc::AddContext;
|
||||
use utils::codec::leb128::{Leb128Iterator, Leb128Vec};
|
||||
|
||||
use crate::cache::ThreadCache;
|
||||
|
||||
pub const INBOX_ID: u32 = 0;
|
||||
pub const TRASH_ID: u32 = 1;
|
||||
pub const JUNK_ID: u32 = 2;
|
||||
pub const DRAFTS_ID: u32 = 3;
|
||||
pub const SENT_ID: u32 = 4;
|
||||
pub const ARCHIVE_ID: u32 = 5;
|
||||
pub const TOMBSTONE_ID: u32 = u32::MAX - 1;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct ExpandPath<'x> {
|
||||
pub path: Vec<&'x str>,
|
||||
pub found_names: Vec<(String, u32, u32)>,
|
||||
}
|
||||
|
||||
pub static SCHEMA: &[IndexProperty] = &[
|
||||
IndexProperty::new(Property::Name)
|
||||
.index_as(IndexAs::Text {
|
||||
tokenize: true,
|
||||
index: true,
|
||||
})
|
||||
.required(),
|
||||
IndexProperty::new(Property::Role).index_as(IndexAs::Text {
|
||||
tokenize: false,
|
||||
index: true,
|
||||
}),
|
||||
IndexProperty::new(Property::Role).index_as(IndexAs::HasProperty),
|
||||
IndexProperty::new(Property::ParentId).index_as(IndexAs::Integer),
|
||||
IndexProperty::new(Property::SortOrder).index_as(IndexAs::Integer),
|
||||
IndexProperty::new(Property::IsSubscribed).index_as(IndexAs::IntegerList),
|
||||
IndexProperty::new(Property::Acl).index_as(IndexAs::Acl),
|
||||
];
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct UidMailbox {
|
||||
pub mailbox_id: u32,
|
||||
pub uid: u32,
|
||||
}
|
||||
|
||||
pub trait MailboxFnc: Sync + Send {
|
||||
fn mailbox_get_or_create(
|
||||
&self,
|
||||
account_id: u32,
|
||||
) -> impl Future<Output = trc::Result<RoaringBitmap>> + Send;
|
||||
|
||||
fn mailbox_create_path(
|
||||
&self,
|
||||
account_id: u32,
|
||||
path: &str,
|
||||
) -> impl Future<Output = trc::Result<Option<(u32, Option<u64>)>>> + Send;
|
||||
|
||||
fn mailbox_count_threads(
|
||||
&self,
|
||||
account_id: u32,
|
||||
document_ids: Option<RoaringBitmap>,
|
||||
) -> impl Future<Output = trc::Result<usize>> + Send;
|
||||
|
||||
fn mailbox_unread_tags(
|
||||
&self,
|
||||
account_id: u32,
|
||||
document_id: u32,
|
||||
message_ids: &Option<RoaringBitmap>,
|
||||
) -> impl Future<Output = trc::Result<Option<RoaringBitmap>>> + Send;
|
||||
|
||||
fn mailbox_expand_path<'x>(
|
||||
&self,
|
||||
account_id: u32,
|
||||
path: &'x str,
|
||||
exact_match: bool,
|
||||
) -> impl Future<Output = trc::Result<Option<ExpandPath<'x>>>> + Send;
|
||||
|
||||
fn mailbox_get_by_name(
|
||||
&self,
|
||||
account_id: u32,
|
||||
path: &str,
|
||||
) -> impl Future<Output = trc::Result<Option<u32>>> + Send;
|
||||
|
||||
fn mailbox_get_by_role(
|
||||
&self,
|
||||
account_id: u32,
|
||||
role: &str,
|
||||
) -> impl Future<Output = trc::Result<Option<u32>>> + Send;
|
||||
}
|
||||
|
||||
impl MailboxFnc for Server {
|
||||
async fn mailbox_get_or_create(&self, account_id: u32) -> trc::Result<RoaringBitmap> {
|
||||
let mut mailbox_ids = self
|
||||
.get_document_ids(account_id, Collection::Mailbox)
|
||||
.await?
|
||||
.unwrap_or_default();
|
||||
if !mailbox_ids.is_empty() {
|
||||
return Ok(mailbox_ids);
|
||||
}
|
||||
|
||||
#[cfg(feature = "test_mode")]
|
||||
if mailbox_ids.is_empty() && account_id == 0 {
|
||||
return Ok(mailbox_ids);
|
||||
}
|
||||
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch
|
||||
.with_account_id(account_id)
|
||||
.with_collection(Collection::Mailbox);
|
||||
|
||||
// Create mailboxes
|
||||
let mut last_document_id = ARCHIVE_ID;
|
||||
for folder in &self.core.jmap.default_folders {
|
||||
let (role, document_id) = match folder.special_use {
|
||||
SpecialUse::Inbox => ("inbox", INBOX_ID),
|
||||
SpecialUse::Trash => ("trash", TRASH_ID),
|
||||
SpecialUse::Junk => ("junk", JUNK_ID),
|
||||
SpecialUse::Drafts => ("drafts", DRAFTS_ID),
|
||||
SpecialUse::Sent => ("sent", SENT_ID),
|
||||
SpecialUse::Archive => ("archive", ARCHIVE_ID),
|
||||
SpecialUse::None => {
|
||||
last_document_id += 1;
|
||||
("", last_document_id)
|
||||
}
|
||||
SpecialUse::Shared => unreachable!(),
|
||||
};
|
||||
|
||||
let mut object = Object::with_capacity(4)
|
||||
.with_property(Property::Name, folder.name.clone())
|
||||
.with_property(Property::ParentId, Value::Id(0u64.into()))
|
||||
.with_property(
|
||||
Property::Cid,
|
||||
Value::UnsignedInt(rand::random::<u32>() as u64),
|
||||
);
|
||||
if !role.is_empty() {
|
||||
object.set(Property::Role, role);
|
||||
}
|
||||
if folder.subscribe {
|
||||
object.set(
|
||||
Property::IsSubscribed,
|
||||
Value::List(vec![Value::Id(account_id.into())]),
|
||||
);
|
||||
}
|
||||
batch
|
||||
.create_document_with_id(document_id)
|
||||
.custom(ObjectIndexBuilder::new(SCHEMA).with_changes(object));
|
||||
mailbox_ids.insert(document_id);
|
||||
}
|
||||
|
||||
self.core
|
||||
.storage
|
||||
.data
|
||||
.write(batch.build())
|
||||
.await
|
||||
.caused_by(trc::location!())
|
||||
.map(|_| mailbox_ids)
|
||||
}
|
||||
|
||||
async fn mailbox_create_path(
|
||||
&self,
|
||||
account_id: u32,
|
||||
path: &str,
|
||||
) -> trc::Result<Option<(u32, Option<u64>)>> {
|
||||
let expanded_path =
|
||||
if let Some(expand_path) = self.mailbox_expand_path(account_id, path, false).await? {
|
||||
expand_path
|
||||
} else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let mut next_parent_id = 0;
|
||||
let mut path = expanded_path.path.into_iter().enumerate().peekable();
|
||||
'outer: while let Some((pos, name)) = path.peek() {
|
||||
let is_inbox = *pos == 0 && name.eq_ignore_ascii_case("inbox");
|
||||
|
||||
for (part, parent_id, document_id) in &expanded_path.found_names {
|
||||
if (part.eq(name) || (is_inbox && part.eq_ignore_ascii_case("inbox")))
|
||||
&& *parent_id == next_parent_id
|
||||
{
|
||||
next_parent_id = *document_id;
|
||||
path.next();
|
||||
continue 'outer;
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
// Create missing folders
|
||||
if path.peek().is_some() {
|
||||
let mut changes = self.begin_changes(account_id)?;
|
||||
|
||||
for (_, name) in path {
|
||||
if name.len() > self.core.jmap.mailbox_name_max_len {
|
||||
return Ok(None);
|
||||
}
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch
|
||||
.with_account_id(account_id)
|
||||
.with_collection(Collection::Mailbox)
|
||||
.create_document()
|
||||
.custom(
|
||||
ObjectIndexBuilder::new(SCHEMA).with_changes(
|
||||
Object::with_capacity(3)
|
||||
.with_property(Property::Name, name)
|
||||
.with_property(
|
||||
Property::ParentId,
|
||||
Value::Id(Id::from(next_parent_id)),
|
||||
)
|
||||
.with_property(
|
||||
Property::Cid,
|
||||
Value::UnsignedInt(rand::random::<u32>() as u64),
|
||||
),
|
||||
),
|
||||
);
|
||||
let document_id = self
|
||||
.store()
|
||||
.write_expect_id(batch)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
changes.log_insert(Collection::Mailbox, document_id);
|
||||
next_parent_id = document_id + 1;
|
||||
}
|
||||
let change_id = changes.change_id;
|
||||
let mut batch = BatchBuilder::new();
|
||||
|
||||
batch
|
||||
.with_account_id(account_id)
|
||||
.with_collection(Collection::Mailbox)
|
||||
.custom(changes);
|
||||
self.store()
|
||||
.write(batch.build())
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
|
||||
Ok(Some((next_parent_id - 1, Some(change_id))))
|
||||
} else {
|
||||
Ok(Some((next_parent_id - 1, None)))
|
||||
}
|
||||
}
|
||||
|
||||
async fn mailbox_count_threads(
|
||||
&self,
|
||||
account_id: u32,
|
||||
document_ids: Option<RoaringBitmap>,
|
||||
) -> trc::Result<usize> {
|
||||
if let Some(document_ids) = document_ids {
|
||||
let mut thread_ids = AHashSet::default();
|
||||
self.get_cached_thread_ids(account_id, document_ids.into_iter())
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
.into_iter()
|
||||
.for_each(|(_, thread_id)| {
|
||||
thread_ids.insert(thread_id);
|
||||
});
|
||||
Ok(thread_ids.len())
|
||||
} else {
|
||||
Ok(0)
|
||||
}
|
||||
}
|
||||
|
||||
async fn mailbox_unread_tags(
|
||||
&self,
|
||||
account_id: u32,
|
||||
document_id: u32,
|
||||
message_ids: &Option<RoaringBitmap>,
|
||||
) -> trc::Result<Option<RoaringBitmap>> {
|
||||
if let (Some(message_ids), Some(mailbox_message_ids)) = (
|
||||
message_ids,
|
||||
self.get_tag(
|
||||
account_id,
|
||||
Collection::Email,
|
||||
Property::MailboxIds,
|
||||
document_id,
|
||||
)
|
||||
.await?,
|
||||
) {
|
||||
if let Some(mut seen) = self
|
||||
.get_tag(
|
||||
account_id,
|
||||
Collection::Email,
|
||||
Property::Keywords,
|
||||
Keyword::Seen,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
seen ^= message_ids;
|
||||
seen &= &mailbox_message_ids;
|
||||
if !seen.is_empty() {
|
||||
Ok(Some(seen))
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
} else {
|
||||
Ok(mailbox_message_ids.into())
|
||||
}
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
async fn mailbox_expand_path<'x>(
|
||||
&self,
|
||||
account_id: u32,
|
||||
path: &'x str,
|
||||
exact_match: bool,
|
||||
) -> trc::Result<Option<ExpandPath<'x>>> {
|
||||
let path = path
|
||||
.split('/')
|
||||
.filter_map(|p| {
|
||||
let p = p.trim();
|
||||
if !p.is_empty() {
|
||||
p.into()
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
if path.is_empty() || path.len() > self.core.jmap.mailbox_max_depth {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let mut filter = Vec::with_capacity(path.len() + 2);
|
||||
let mut has_inbox = false;
|
||||
filter.push(Filter::Or);
|
||||
for (pos, item) in path.iter().enumerate() {
|
||||
if pos == 0 && item.eq_ignore_ascii_case("inbox") {
|
||||
has_inbox = true;
|
||||
} else {
|
||||
filter.push(Filter::eq(Property::Name, *item));
|
||||
}
|
||||
}
|
||||
filter.push(Filter::End);
|
||||
|
||||
let mut document_ids = if filter.len() > 2 {
|
||||
self.store()
|
||||
.filter(account_id, Collection::Mailbox, filter)
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
.results
|
||||
} else {
|
||||
RoaringBitmap::new()
|
||||
};
|
||||
if has_inbox {
|
||||
document_ids.insert(INBOX_ID);
|
||||
}
|
||||
if exact_match && (document_ids.len() as usize) < path.len() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let mut found_names = Vec::new();
|
||||
for document_id in document_ids {
|
||||
if let Some(mut obj) = self
|
||||
.get_property::<Object<Value>>(
|
||||
account_id,
|
||||
Collection::Mailbox,
|
||||
document_id,
|
||||
Property::Value,
|
||||
)
|
||||
.await?
|
||||
{
|
||||
if let Some(Value::Text(value)) = obj.properties.remove(&Property::Name) {
|
||||
found_names.push((
|
||||
value,
|
||||
if let Some(Value::Id(value)) = obj.properties.remove(&Property::ParentId) {
|
||||
value.document_id()
|
||||
} else {
|
||||
0
|
||||
},
|
||||
document_id + 1,
|
||||
));
|
||||
} else {
|
||||
return Ok(None);
|
||||
}
|
||||
} else {
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Some(ExpandPath { path, found_names }))
|
||||
}
|
||||
|
||||
async fn mailbox_get_by_name(&self, account_id: u32, path: &str) -> trc::Result<Option<u32>> {
|
||||
Ok(self
|
||||
.mailbox_expand_path(account_id, path, true)
|
||||
.await?
|
||||
.and_then(|ep| {
|
||||
let mut next_parent_id = 0;
|
||||
'outer: for (pos, name) in ep.path.iter().enumerate() {
|
||||
let is_inbox = pos == 0 && name.eq_ignore_ascii_case("inbox");
|
||||
|
||||
for (part, parent_id, document_id) in &ep.found_names {
|
||||
if (part.eq(name) || (is_inbox && part.eq_ignore_ascii_case("inbox")))
|
||||
&& *parent_id == next_parent_id
|
||||
{
|
||||
next_parent_id = *document_id;
|
||||
continue 'outer;
|
||||
}
|
||||
}
|
||||
return None;
|
||||
}
|
||||
Some(next_parent_id - 1)
|
||||
}))
|
||||
}
|
||||
|
||||
async fn mailbox_get_by_role(&self, account_id: u32, role: &str) -> trc::Result<Option<u32>> {
|
||||
self.store()
|
||||
.filter(
|
||||
account_id,
|
||||
Collection::Mailbox,
|
||||
vec![Filter::eq(Property::Role, role)],
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())
|
||||
.map(|r| r.results.min())
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialEq for UidMailbox {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
self.mailbox_id == other.mailbox_id
|
||||
}
|
||||
}
|
||||
|
||||
impl Eq for UidMailbox {}
|
||||
|
||||
impl ToBitmaps for UidMailbox {
|
||||
fn to_bitmaps(&self, ops: &mut Vec<Operation>, field: u8, set: bool) {
|
||||
ops.push(Operation::Bitmap {
|
||||
class: BitmapClass::Tag {
|
||||
field,
|
||||
value: TagValue::Id(MaybeDynamicId::Static(self.mailbox_id)),
|
||||
},
|
||||
set,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
impl SerializeInto for UidMailbox {
|
||||
fn serialize_into(&self, buf: &mut Vec<u8>) {
|
||||
buf.push_leb128(self.mailbox_id);
|
||||
buf.push_leb128(self.uid);
|
||||
}
|
||||
}
|
||||
|
||||
impl DeserializeFrom for UidMailbox {
|
||||
fn deserialize_from(bytes: &mut Iter<'_, u8>) -> Option<Self> {
|
||||
Some(UidMailbox {
|
||||
mailbox_id: bytes.next_leb128()?,
|
||||
uid: bytes.next_leb128()?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Serialize for UidMailbox {
|
||||
fn serialize(self) -> Vec<u8> {
|
||||
let mut buf = Vec::with_capacity(U32_LEN * 2);
|
||||
self.serialize_into(&mut buf);
|
||||
buf
|
||||
}
|
||||
}
|
||||
|
||||
impl UidMailbox {
|
||||
pub fn new(mailbox_id: u32, uid: u32) -> Self {
|
||||
UidMailbox { mailbox_id, uid }
|
||||
}
|
||||
|
||||
pub fn new_unassigned(mailbox_id: u32) -> Self {
|
||||
UidMailbox { mailbox_id, uid: 0 }
|
||||
}
|
||||
}
|
||||
299
crates/email/src/metadata.rs
Normal file
299
crates/email/src/metadata.rs
Normal file
@@ -0,0 +1,299 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use std::borrow::Cow;
|
||||
|
||||
use mail_parser::{
|
||||
decoders::{
|
||||
base64::base64_decode, charsets::map::charset_decoder,
|
||||
quoted_printable::quoted_printable_decode,
|
||||
},
|
||||
ContentType, Encoding, GetHeader, Header, HeaderName, HeaderValue, Message, MessagePart,
|
||||
MessagePartId, MimeHeaders, PartType,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use utils::BlobHash;
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct MessageMetadata<'x> {
|
||||
pub contents: MessageMetadataContents<'x>,
|
||||
pub blob_hash: BlobHash,
|
||||
pub size: usize,
|
||||
pub received_at: u64,
|
||||
pub preview: String,
|
||||
pub has_attachments: bool,
|
||||
pub raw_headers: Vec<u8>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct MessageMetadataContents<'x> {
|
||||
pub html_body: Vec<MessagePartId>,
|
||||
pub text_body: Vec<MessagePartId>,
|
||||
pub attachments: Vec<MessagePartId>,
|
||||
pub parts: Vec<MessageMetadataPart<'x>>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct MessageMetadataPart<'x> {
|
||||
pub headers: Vec<Header<'x>>,
|
||||
pub is_encoding_problem: bool,
|
||||
pub body: MetadataPartType<'x>,
|
||||
pub encoding: Encoding,
|
||||
pub size: usize,
|
||||
pub offset_header: usize,
|
||||
pub offset_body: usize,
|
||||
pub offset_end: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub enum MetadataPartType<'x> {
|
||||
Text,
|
||||
Html,
|
||||
Binary,
|
||||
InlineBinary,
|
||||
Message(MessageMetadataContents<'x>),
|
||||
Multipart(Vec<MessagePartId>),
|
||||
}
|
||||
|
||||
impl<'x> MessageMetadataContents<'x> {
|
||||
pub fn into_message(self, raw_message: &'x [u8]) -> Message<'x> {
|
||||
Message {
|
||||
html_body: self.html_body,
|
||||
text_body: self.text_body,
|
||||
attachments: self.attachments,
|
||||
parts: self
|
||||
.parts
|
||||
.into_iter()
|
||||
.map(|part| MessagePart {
|
||||
body: match part.body {
|
||||
MetadataPartType::Text
|
||||
| MetadataPartType::Html
|
||||
| MetadataPartType::Binary
|
||||
| MetadataPartType::InlineBinary
|
||||
if !raw_message.is_empty() =>
|
||||
{
|
||||
part.decode_contents(raw_message)
|
||||
}
|
||||
MetadataPartType::Message(_) if !raw_message.is_empty() => {
|
||||
match part.contents(raw_message) {
|
||||
Cow::Borrowed(_) => PartType::Message(
|
||||
part.body.unwrap_message().into_message(raw_message),
|
||||
),
|
||||
Cow::Owned(raw_message) => PartType::Message(
|
||||
part.body
|
||||
.unwrap_message()
|
||||
.into_message(&raw_message)
|
||||
.into_owned(),
|
||||
),
|
||||
}
|
||||
}
|
||||
MetadataPartType::Multipart(parts) => PartType::Multipart(parts),
|
||||
_ => PartType::Binary(Cow::Borrowed(&[])),
|
||||
},
|
||||
headers: part.headers,
|
||||
is_encoding_problem: part.is_encoding_problem,
|
||||
encoding: part.encoding,
|
||||
offset_header: part.offset_header,
|
||||
offset_body: part.offset_body,
|
||||
offset_end: part.offset_end,
|
||||
})
|
||||
.collect(),
|
||||
raw_message: raw_message.into(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn root_part(&self) -> &MessageMetadataPart<'x> {
|
||||
&self.parts[0]
|
||||
}
|
||||
}
|
||||
|
||||
impl<'x> MessageMetadataPart<'x> {
|
||||
pub fn contents<'y>(&self, raw_message: &'y [u8]) -> Cow<'y, [u8]> {
|
||||
let bytes = raw_message
|
||||
.get(self.offset_body..self.offset_end)
|
||||
.unwrap_or_default();
|
||||
match self.encoding {
|
||||
Encoding::None => bytes.into(),
|
||||
Encoding::QuotedPrintable => quoted_printable_decode(bytes).unwrap_or_default().into(),
|
||||
Encoding::Base64 => base64_decode(bytes).unwrap_or_default().into(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn decode_contents<'y>(&self, raw_message: &'y [u8]) -> PartType<'y> {
|
||||
let bytes = self.contents(raw_message);
|
||||
|
||||
match self.body {
|
||||
MetadataPartType::Text | MetadataPartType::Html => {
|
||||
let text = match (
|
||||
bytes,
|
||||
self.headers
|
||||
.header_value(&HeaderName::ContentType)
|
||||
.and_then(|c| c.as_content_type())
|
||||
.and_then(|ct| {
|
||||
ct.attribute("charset")
|
||||
.and_then(|c| charset_decoder(c.as_bytes()))
|
||||
}),
|
||||
) {
|
||||
(Cow::Owned(vec), Some(charset_decoder)) => charset_decoder(&vec).into(),
|
||||
(Cow::Owned(vec), None) => String::from_utf8(vec)
|
||||
.unwrap_or_else(|e| String::from_utf8_lossy(e.as_bytes()).into_owned())
|
||||
.into(),
|
||||
(Cow::Borrowed(bytes), Some(charset_decoder)) => charset_decoder(bytes).into(),
|
||||
(Cow::Borrowed(bytes), None) => String::from_utf8_lossy(bytes),
|
||||
};
|
||||
|
||||
if matches!(self.body, MetadataPartType::Text) {
|
||||
PartType::Text(text)
|
||||
} else {
|
||||
PartType::Html(text)
|
||||
}
|
||||
}
|
||||
MetadataPartType::Binary => PartType::Binary(bytes),
|
||||
MetadataPartType::InlineBinary => PartType::InlineBinary(bytes),
|
||||
MetadataPartType::Message(_) | MetadataPartType::Multipart(_) => unreachable!(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn remove_header(&mut self, header_name: &HeaderName) -> Option<HeaderValue<'x>> {
|
||||
for header in self.headers.iter_mut().rev() {
|
||||
if header.name == *header_name {
|
||||
return Some(std::mem::take(&mut header.value));
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
impl<'x> From<Message<'x>> for MessageMetadataContents<'x> {
|
||||
fn from(value: Message<'x>) -> Self {
|
||||
MessageMetadataContents {
|
||||
html_body: value.html_body,
|
||||
text_body: value.text_body,
|
||||
attachments: value.attachments,
|
||||
parts: value
|
||||
.parts
|
||||
.into_iter()
|
||||
.map(|part| {
|
||||
let (size, body) = match part.body {
|
||||
PartType::Text(contents) => (contents.len(), MetadataPartType::Text),
|
||||
PartType::Html(contents) => (contents.len(), MetadataPartType::Html),
|
||||
PartType::Binary(contents) => (contents.len(), MetadataPartType::Binary),
|
||||
PartType::InlineBinary(contents) => {
|
||||
(contents.len(), MetadataPartType::InlineBinary)
|
||||
}
|
||||
PartType::Message(message) => (
|
||||
message.root_part().raw_len(),
|
||||
MetadataPartType::Message(message.into()),
|
||||
),
|
||||
PartType::Multipart(parts) => (0, MetadataPartType::Multipart(parts)),
|
||||
};
|
||||
|
||||
MessageMetadataPart {
|
||||
headers: part
|
||||
.headers
|
||||
.into_iter()
|
||||
.map(|hdr| Header {
|
||||
value: if matches!(
|
||||
&hdr.name,
|
||||
HeaderName::Subject
|
||||
| HeaderName::From
|
||||
| HeaderName::To
|
||||
| HeaderName::Cc
|
||||
| HeaderName::Date
|
||||
| HeaderName::Bcc
|
||||
| HeaderName::ReplyTo
|
||||
| HeaderName::Sender
|
||||
| HeaderName::Comments
|
||||
| HeaderName::InReplyTo
|
||||
| HeaderName::Keywords
|
||||
| HeaderName::MessageId
|
||||
| HeaderName::References
|
||||
| HeaderName::ResentMessageId
|
||||
| HeaderName::ContentDescription
|
||||
| HeaderName::ContentId
|
||||
| HeaderName::ContentLanguage
|
||||
| HeaderName::ContentLocation
|
||||
| HeaderName::ContentTransferEncoding
|
||||
| HeaderName::ContentType
|
||||
| HeaderName::ContentDisposition
|
||||
| HeaderName::ListId
|
||||
) {
|
||||
hdr.value
|
||||
} else {
|
||||
HeaderValue::Empty
|
||||
},
|
||||
name: hdr.name,
|
||||
offset_field: hdr.offset_field,
|
||||
offset_start: hdr.offset_start,
|
||||
offset_end: hdr.offset_end,
|
||||
})
|
||||
.collect(),
|
||||
is_encoding_problem: part.is_encoding_problem,
|
||||
encoding: part.encoding,
|
||||
body,
|
||||
size,
|
||||
offset_header: part.offset_header,
|
||||
offset_body: part.offset_body,
|
||||
offset_end: part.offset_end,
|
||||
}
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'x> MetadataPartType<'x> {
|
||||
fn unwrap_message(self) -> MessageMetadataContents<'x> {
|
||||
match self {
|
||||
MetadataPartType::Message(message) => message,
|
||||
_ => panic!("unwrap_message called on non-message part"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'x> MimeHeaders<'x> for MessageMetadataPart<'x> {
|
||||
fn content_description(&self) -> Option<&str> {
|
||||
self.headers
|
||||
.header_value(&HeaderName::ContentDescription)
|
||||
.and_then(|header| header.as_text())
|
||||
}
|
||||
|
||||
fn content_disposition(&self) -> Option<&ContentType> {
|
||||
self.headers
|
||||
.header_value(&HeaderName::ContentDisposition)
|
||||
.and_then(|header| header.as_content_type())
|
||||
}
|
||||
|
||||
fn content_id(&self) -> Option<&str> {
|
||||
self.headers
|
||||
.header_value(&HeaderName::ContentId)
|
||||
.and_then(|header| header.as_text())
|
||||
}
|
||||
|
||||
fn content_transfer_encoding(&self) -> Option<&str> {
|
||||
self.headers
|
||||
.header_value(&HeaderName::ContentTransferEncoding)
|
||||
.and_then(|header| header.as_text())
|
||||
}
|
||||
|
||||
fn content_type(&self) -> Option<&ContentType> {
|
||||
self.headers
|
||||
.header_value(&HeaderName::ContentType)
|
||||
.and_then(|header| header.as_content_type())
|
||||
}
|
||||
|
||||
fn content_language(&self) -> &HeaderValue {
|
||||
self.headers
|
||||
.header_value(&HeaderName::ContentLanguage)
|
||||
.unwrap_or(&HeaderValue::Empty)
|
||||
}
|
||||
|
||||
fn content_location(&self) -> Option<&str> {
|
||||
self.headers
|
||||
.header_value(&HeaderName::ContentLocation)
|
||||
.and_then(|header| header.as_text())
|
||||
}
|
||||
}
|
||||
876
crates/email/src/sieve.rs
Normal file
876
crates/email/src/sieve.rs
Normal file
@@ -0,0 +1,876 @@
|
||||
/*
|
||||
* SPDX-FileCopyrightText: 2020 Stalwart Labs Ltd <hello@stalw.art>
|
||||
*
|
||||
* SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
|
||||
*/
|
||||
|
||||
use std::{borrow::Cow, sync::Arc};
|
||||
|
||||
use crate::{
|
||||
delivery::AutogeneratedMessage,
|
||||
ingest::{EmailIngest, IngestEmail, IngestSource, IngestedEmail},
|
||||
mailbox::{MailboxFnc, INBOX_ID, TRASH_ID},
|
||||
};
|
||||
use common::{auth::AccessToken, scripts::plugins::PluginContext, Server};
|
||||
use directory::{backend::internal::PrincipalField, Permission, QueryBy};
|
||||
use jmap_proto::{
|
||||
object::Object,
|
||||
types::{collection::Collection, id::Id, keyword::Keyword, property::Property, value::Value},
|
||||
};
|
||||
use mail_parser::MessageParser;
|
||||
use serde::ser::SerializeSeq;
|
||||
use sieve::{Envelope, Event, Input, Mailbox, Recipient, Sieve};
|
||||
use store::{
|
||||
ahash::AHashSet,
|
||||
blake3,
|
||||
query::Filter,
|
||||
write::{assert::HashedValue, now, BatchBuilder, Bincode, BlobOp, F_VALUE},
|
||||
Deserialize, Serialize,
|
||||
};
|
||||
use trc::{AddContext, SieveEvent};
|
||||
|
||||
use std::future::Future;
|
||||
|
||||
struct SieveMessage<'x> {
|
||||
pub raw_message: Cow<'x, [u8]>,
|
||||
pub file_into: Vec<u32>,
|
||||
pub flags: Vec<Keyword>,
|
||||
}
|
||||
|
||||
pub struct ActiveScript {
|
||||
pub document_id: u32,
|
||||
pub script_name: String,
|
||||
pub script: Arc<Sieve>,
|
||||
pub seen_ids: SeenIds,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct SeenIdHash {
|
||||
hash: [u8; 32],
|
||||
expiry: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct SeenIds {
|
||||
pub ids: AHashSet<SeenIdHash>,
|
||||
pub has_changes: bool,
|
||||
}
|
||||
|
||||
pub trait SieveScriptIngest: Sync + Send {
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn sieve_script_ingest(
|
||||
&self,
|
||||
access_token: &AccessToken,
|
||||
raw_message: &[u8],
|
||||
envelope_from: &str,
|
||||
envelope_to: &str,
|
||||
session_id: u64,
|
||||
active_script: ActiveScript,
|
||||
autogenerated: &mut Vec<AutogeneratedMessage>,
|
||||
) -> impl Future<Output = trc::Result<IngestedEmail>> + Send;
|
||||
|
||||
fn sieve_script_get_active(
|
||||
&self,
|
||||
account_id: u32,
|
||||
) -> impl Future<Output = trc::Result<Option<ActiveScript>>> + Send;
|
||||
|
||||
fn sieve_script_get_by_name(
|
||||
&self,
|
||||
account_id: u32,
|
||||
name: &str,
|
||||
) -> impl Future<Output = trc::Result<Option<Sieve>>> + Send;
|
||||
|
||||
fn sieve_script_compile(
|
||||
&self,
|
||||
account_id: u32,
|
||||
document_id: u32,
|
||||
) -> impl Future<Output = trc::Result<(Sieve, Object<Value>)>> + Send;
|
||||
}
|
||||
|
||||
impl SieveScriptIngest for Server {
|
||||
#[allow(clippy::blocks_in_conditions)]
|
||||
async fn sieve_script_ingest(
|
||||
&self,
|
||||
access_token: &AccessToken,
|
||||
raw_message: &[u8],
|
||||
envelope_from: &str,
|
||||
envelope_to: &str,
|
||||
session_id: u64,
|
||||
mut active_script: ActiveScript,
|
||||
autogenerated: &mut Vec<AutogeneratedMessage>,
|
||||
) -> trc::Result<IngestedEmail> {
|
||||
// Parse message
|
||||
let message = if let Some(message) = MessageParser::new().parse(raw_message) {
|
||||
message
|
||||
} else {
|
||||
return Err(
|
||||
trc::EventType::MessageIngest(trc::MessageIngestEvent::Error)
|
||||
.ctx(trc::Key::Code, 550)
|
||||
.ctx(trc::Key::Reason, "Failed to parse e-mail message."),
|
||||
);
|
||||
};
|
||||
|
||||
// Obtain mailboxIds
|
||||
let account_id = access_token.primary_id;
|
||||
let mailbox_ids = self
|
||||
.mailbox_get_or_create(account_id)
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
|
||||
// Create Sieve instance
|
||||
let mut instance = self.core.sieve.untrusted_runtime.filter_parsed(message);
|
||||
|
||||
// Set account name and email
|
||||
let mail_from = self
|
||||
.core
|
||||
.storage
|
||||
.directory
|
||||
.query(QueryBy::Id(account_id), false)
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
.and_then(|mut p| {
|
||||
instance.set_user_full_name(p.description().unwrap_or_else(|| p.name()));
|
||||
p.take_str_array(PrincipalField::Emails)
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.next()
|
||||
});
|
||||
|
||||
// Set account address
|
||||
let mail_from = mail_from.unwrap_or_else(|| envelope_to.to_string());
|
||||
instance.set_user_address(&mail_from);
|
||||
|
||||
// Set envelope
|
||||
instance.set_envelope(Envelope::From, envelope_from);
|
||||
instance.set_envelope(Envelope::To, envelope_to);
|
||||
|
||||
let mut input = Input::script(active_script.script_name, active_script.script.clone());
|
||||
|
||||
let mut do_discard = false;
|
||||
let mut do_deliver = false;
|
||||
|
||||
let mut new_ids = AHashSet::new();
|
||||
let mut reject_reason = None;
|
||||
let mut messages: Vec<SieveMessage> = vec![SieveMessage {
|
||||
raw_message: raw_message.into(),
|
||||
file_into: Vec::new(),
|
||||
flags: Vec::new(),
|
||||
}];
|
||||
let now = now();
|
||||
let mut ingested_message = IngestedEmail {
|
||||
id: Id::default(),
|
||||
change_id: u64::MAX,
|
||||
blob_id: Default::default(),
|
||||
size: raw_message.len(),
|
||||
imap_uids: Vec::new(),
|
||||
};
|
||||
|
||||
while let Some(event) = instance.run(input) {
|
||||
match event {
|
||||
Ok(event) => match event {
|
||||
Event::IncludeScript { name, .. } => match &name {
|
||||
sieve::Script::Personal(name_) => {
|
||||
if let Ok(Some(script)) =
|
||||
self.sieve_script_get_by_name(account_id, name_).await
|
||||
{
|
||||
input = Input::script(name, script);
|
||||
} else {
|
||||
input = false.into();
|
||||
}
|
||||
}
|
||||
sieve::Script::Global(name_) => {
|
||||
if let Some(script) = self.get_untrusted_sieve_script(name_, session_id)
|
||||
{
|
||||
input = Input::script(name, script.clone());
|
||||
} else {
|
||||
input = false.into();
|
||||
}
|
||||
}
|
||||
},
|
||||
Event::MailboxExists {
|
||||
mailboxes,
|
||||
special_use,
|
||||
} => {
|
||||
if !mailboxes.is_empty() {
|
||||
let mut special_use_ids = Vec::with_capacity(special_use.len());
|
||||
for role in special_use {
|
||||
special_use_ids.push(if role.eq_ignore_ascii_case("inbox") {
|
||||
INBOX_ID
|
||||
} else if role.eq_ignore_ascii_case("trash") {
|
||||
TRASH_ID
|
||||
} else {
|
||||
let mut mailbox_id = u32::MAX;
|
||||
let role = role.to_ascii_lowercase();
|
||||
if is_valid_role(&role) {
|
||||
if let Ok(Some(mailbox_id_)) =
|
||||
self.mailbox_get_by_role(account_id, &role).await
|
||||
{
|
||||
mailbox_id = mailbox_id_;
|
||||
}
|
||||
}
|
||||
mailbox_id
|
||||
});
|
||||
}
|
||||
|
||||
let mut result = true;
|
||||
for mailbox in mailboxes {
|
||||
match mailbox {
|
||||
Mailbox::Name(name) => {
|
||||
if !matches!(
|
||||
self.mailbox_get_by_name(account_id, &name).await,
|
||||
Ok(Some(document_id)) if special_use_ids.is_empty() ||
|
||||
special_use_ids.contains(&document_id)
|
||||
) {
|
||||
result = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
Mailbox::Id(id) => {
|
||||
if !matches!(Id::from_bytes(id.as_bytes()), Some(id) if
|
||||
mailbox_ids.contains(id.document_id()) &&
|
||||
(special_use_ids.is_empty() ||
|
||||
special_use_ids.contains(&id.document_id())))
|
||||
{
|
||||
result = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
input = result.into();
|
||||
} else if !special_use.is_empty() {
|
||||
let mut result = true;
|
||||
|
||||
for role in special_use {
|
||||
if !role.eq_ignore_ascii_case("inbox")
|
||||
&& !role.eq_ignore_ascii_case("trash")
|
||||
{
|
||||
let role = role.to_ascii_lowercase();
|
||||
if !is_valid_role(&role)
|
||||
|| !matches!(
|
||||
self.mailbox_get_by_role(account_id, &role).await,
|
||||
Ok(Some(_))
|
||||
)
|
||||
{
|
||||
result = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
input = result.into();
|
||||
} else {
|
||||
input = false.into();
|
||||
}
|
||||
}
|
||||
Event::DuplicateId { id, expiry, last } => {
|
||||
let id_hash = SeenIdHash::new(&id, expiry + now);
|
||||
let seen_id = active_script.seen_ids.ids.contains(&id_hash);
|
||||
if !seen_id || last {
|
||||
new_ids.insert(id_hash);
|
||||
}
|
||||
|
||||
input = seen_id.into();
|
||||
}
|
||||
Event::Discard => {
|
||||
do_discard = true;
|
||||
input = true.into();
|
||||
}
|
||||
Event::Reject { reason, .. } => {
|
||||
reject_reason = reason.into();
|
||||
do_discard = true;
|
||||
input = true.into();
|
||||
}
|
||||
Event::Keep { flags, message_id } => {
|
||||
if let Some(message) = messages.get_mut(message_id) {
|
||||
message.flags = flags.into_iter().map(Keyword::from).collect();
|
||||
if !message.file_into.contains(&INBOX_ID) {
|
||||
message.file_into.push(INBOX_ID);
|
||||
}
|
||||
do_deliver = true;
|
||||
} else {
|
||||
trc::event!(
|
||||
Sieve(SieveEvent::UnexpectedError),
|
||||
Details = "Unknown message id.",
|
||||
MessageId = message_id,
|
||||
SpanId = session_id
|
||||
);
|
||||
}
|
||||
input = true.into();
|
||||
}
|
||||
Event::FileInto {
|
||||
folder,
|
||||
flags,
|
||||
mailbox_id,
|
||||
special_use,
|
||||
create,
|
||||
message_id,
|
||||
} => {
|
||||
let mut target_id = u32::MAX;
|
||||
|
||||
// Find mailbox by Id
|
||||
if let Some(mailbox_id) =
|
||||
mailbox_id.and_then(|m| Id::from_bytes(m.as_bytes()))
|
||||
{
|
||||
let mailbox_id = mailbox_id.document_id();
|
||||
if mailbox_ids.contains(mailbox_id) {
|
||||
target_id = mailbox_id;
|
||||
}
|
||||
}
|
||||
|
||||
// Find mailbox by role
|
||||
if let Some(special_use) = special_use {
|
||||
if target_id == u32::MAX {
|
||||
if special_use.eq_ignore_ascii_case("inbox") {
|
||||
target_id = INBOX_ID;
|
||||
} else if special_use.eq_ignore_ascii_case("trash") {
|
||||
target_id = TRASH_ID;
|
||||
} else {
|
||||
let role = special_use.to_ascii_lowercase();
|
||||
if is_valid_role(&role) {
|
||||
if let Ok(Some(mailbox_id_)) =
|
||||
self.mailbox_get_by_role(account_id, &role).await
|
||||
{
|
||||
target_id = mailbox_id_;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Find mailbox by name
|
||||
if target_id == u32::MAX {
|
||||
if !create {
|
||||
if let Ok(Some(document_id)) =
|
||||
self.mailbox_get_by_name(account_id, &folder).await
|
||||
{
|
||||
target_id = document_id;
|
||||
}
|
||||
} else if let Ok(Some((document_id, changes))) =
|
||||
self.mailbox_create_path(account_id, &folder).await
|
||||
{
|
||||
target_id = document_id;
|
||||
if let Some(change_id) = changes {
|
||||
ingested_message.change_id = change_id;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Default to Inbox
|
||||
if target_id == u32::MAX {
|
||||
target_id = INBOX_ID;
|
||||
}
|
||||
|
||||
if let Some(message) = messages.get_mut(message_id) {
|
||||
message.flags = flags.into_iter().map(Keyword::from).collect();
|
||||
if !message.file_into.contains(&target_id) {
|
||||
message.file_into.push(target_id);
|
||||
}
|
||||
do_deliver = true;
|
||||
} else {
|
||||
trc::event!(
|
||||
Sieve(SieveEvent::UnexpectedError),
|
||||
Details = "Unknown message id.",
|
||||
MessageId = message_id,
|
||||
SpanId = session_id
|
||||
);
|
||||
}
|
||||
input = true.into();
|
||||
}
|
||||
Event::SendMessage {
|
||||
recipient,
|
||||
message_id,
|
||||
..
|
||||
} => {
|
||||
input = true.into();
|
||||
if let Some(message) = messages.get(message_id) {
|
||||
let recipients = match recipient {
|
||||
Recipient::Address(rcpt) => vec![rcpt],
|
||||
Recipient::Group(rcpts) => rcpts,
|
||||
Recipient::List(_) => {
|
||||
// Not yet implemented
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
if message.raw_message.len() <= self.core.jmap.mail_max_size {
|
||||
trc::event!(
|
||||
Sieve(SieveEvent::SendMessage),
|
||||
From = mail_from.clone(),
|
||||
To = recipients
|
||||
.iter()
|
||||
.map(|r| trc::Value::String(r.clone()))
|
||||
.collect::<Vec<_>>(),
|
||||
Size = message.raw_message.len(),
|
||||
SpanId = session_id
|
||||
);
|
||||
|
||||
autogenerated.push(AutogeneratedMessage {
|
||||
sender_address: mail_from.clone(),
|
||||
recipients,
|
||||
message: message.raw_message.to_vec(),
|
||||
});
|
||||
} else {
|
||||
trc::event!(
|
||||
Sieve(SieveEvent::MessageTooLarge),
|
||||
From = mail_from.clone(),
|
||||
To = recipients
|
||||
.iter()
|
||||
.map(|r| trc::Value::String(r.clone()))
|
||||
.collect::<Vec<_>>(),
|
||||
Size = message.raw_message.len(),
|
||||
Limit = self.core.jmap.mail_max_size,
|
||||
SpanId = session_id,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
trc::event!(
|
||||
Sieve(SieveEvent::UnexpectedError),
|
||||
Details = "Unknown message id.",
|
||||
MessageId = message_id,
|
||||
SpanId = session_id
|
||||
);
|
||||
|
||||
continue;
|
||||
}
|
||||
}
|
||||
Event::ListContains { .. }
|
||||
| Event::Notify { .. }
|
||||
| Event::SetEnvelope { .. } => {
|
||||
// Not allowed
|
||||
input = false.into();
|
||||
}
|
||||
Event::Function { id, arguments } => {
|
||||
input = self
|
||||
.core
|
||||
.run_plugin(
|
||||
id,
|
||||
PluginContext {
|
||||
session_id,
|
||||
server: self,
|
||||
message: instance.message(),
|
||||
modifications: &mut Vec::new(),
|
||||
access_token: access_token.into(),
|
||||
arguments,
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
Event::CreatedMessage { message, .. } => {
|
||||
messages.push(SieveMessage {
|
||||
raw_message: message.into(),
|
||||
file_into: Vec::new(),
|
||||
flags: Vec::new(),
|
||||
});
|
||||
input = true.into();
|
||||
}
|
||||
},
|
||||
|
||||
#[cfg(feature = "test_mode")]
|
||||
Err(sieve::runtime::RuntimeError::ScriptErrorMessage(err)) => {
|
||||
panic!("Sieve test failed: {}", err);
|
||||
}
|
||||
|
||||
Err(err) => {
|
||||
trc::event!(
|
||||
Sieve(SieveEvent::RuntimeError),
|
||||
Reason = err.to_string(),
|
||||
SpanId = session_id
|
||||
);
|
||||
|
||||
input = true.into();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fail-safe, no discard and no keep seen, assume that something went wrong and file anyway.
|
||||
if !do_deliver && !do_discard {
|
||||
messages[0].file_into.push(INBOX_ID);
|
||||
}
|
||||
|
||||
// Deliver messages
|
||||
let mut last_temp_error = None;
|
||||
let mut has_delivered = false;
|
||||
let can_spam_train = self.email_bayes_can_train(access_token);
|
||||
for (message_id, sieve_message) in messages.into_iter().enumerate() {
|
||||
if !sieve_message.file_into.is_empty() {
|
||||
// Parse message if needed
|
||||
let message = if message_id == 0 && !instance.has_message_changed() {
|
||||
instance.take_message()
|
||||
} else if let Some(message) =
|
||||
MessageParser::new().parse(sieve_message.raw_message.as_ref())
|
||||
{
|
||||
message
|
||||
} else {
|
||||
trc::event!(
|
||||
Sieve(SieveEvent::UnexpectedError),
|
||||
Details = "Failed to parse Sieve generated message.",
|
||||
SpanId = session_id
|
||||
);
|
||||
|
||||
continue;
|
||||
};
|
||||
|
||||
// Deliver message
|
||||
match self
|
||||
.email_ingest(IngestEmail {
|
||||
raw_message: &sieve_message.raw_message,
|
||||
message: message.into(),
|
||||
resource: access_token.as_resource_token(),
|
||||
mailbox_ids: sieve_message.file_into,
|
||||
keywords: sieve_message.flags,
|
||||
received_at: None,
|
||||
source: IngestSource::Smtp {
|
||||
deliver_to: envelope_to,
|
||||
},
|
||||
spam_classify: access_token.has_permission(Permission::SpamFilterClassify),
|
||||
spam_train: can_spam_train,
|
||||
session_id,
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(ingested_message_) => {
|
||||
has_delivered = true;
|
||||
ingested_message = ingested_message_;
|
||||
}
|
||||
Err(err) => {
|
||||
last_temp_error = err.into();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Save new ids script changes
|
||||
if !new_ids.is_empty() || active_script.seen_ids.has_changes {
|
||||
active_script.seen_ids.ids.extend(new_ids);
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch
|
||||
.with_account_id(account_id)
|
||||
.with_collection(Collection::SieveScript)
|
||||
.update_document(active_script.document_id)
|
||||
.value(
|
||||
Property::EmailIds,
|
||||
Bincode::new(active_script.seen_ids),
|
||||
F_VALUE,
|
||||
);
|
||||
if let Err(err) = self.store().write(batch).await.caused_by(trc::location!()) {
|
||||
trc::error!(err.details("Failed to save Sieve seen ids changes."));
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(reject_reason) = reject_reason {
|
||||
Err(
|
||||
trc::EventType::MessageIngest(trc::MessageIngestEvent::Error)
|
||||
.ctx(trc::Key::Code, 571)
|
||||
.ctx(trc::Key::Reason, reject_reason),
|
||||
)
|
||||
} else if has_delivered || last_temp_error.is_none() {
|
||||
Ok(ingested_message)
|
||||
} else {
|
||||
// There were problems during delivery
|
||||
#[allow(clippy::unnecessary_unwrap)]
|
||||
Err(last_temp_error.unwrap())
|
||||
}
|
||||
}
|
||||
|
||||
async fn sieve_script_get_active(&self, account_id: u32) -> trc::Result<Option<ActiveScript>> {
|
||||
// Find the currently active script
|
||||
if let Some(document_id) = self
|
||||
.store()
|
||||
.filter(
|
||||
account_id,
|
||||
Collection::SieveScript,
|
||||
vec![Filter::eq(Property::IsActive, 1u32)],
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
.results
|
||||
.min()
|
||||
{
|
||||
let (script, mut script_object) =
|
||||
self.sieve_script_compile(account_id, document_id).await?;
|
||||
Ok(Some(ActiveScript {
|
||||
document_id,
|
||||
script: Arc::new(script),
|
||||
script_name: script_object
|
||||
.properties
|
||||
.remove(&Property::Name)
|
||||
.and_then(|name| name.try_unwrap_string())
|
||||
.unwrap_or_else(|| account_id.to_string()),
|
||||
seen_ids: self
|
||||
.get_property::<Bincode<SeenIds>>(
|
||||
account_id,
|
||||
Collection::SieveScript,
|
||||
document_id,
|
||||
Property::EmailIds,
|
||||
)
|
||||
.await?
|
||||
.map(|seen_ids| seen_ids.inner)
|
||||
.unwrap_or_default(),
|
||||
}))
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
async fn sieve_script_get_by_name(
|
||||
&self,
|
||||
account_id: u32,
|
||||
name: &str,
|
||||
) -> trc::Result<Option<Sieve>> {
|
||||
// Find the script by name
|
||||
if let Some(document_id) = self
|
||||
.store()
|
||||
.filter(
|
||||
account_id,
|
||||
Collection::SieveScript,
|
||||
vec![Filter::eq(Property::Name, name)],
|
||||
)
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
.results
|
||||
.min()
|
||||
{
|
||||
self.sieve_script_compile(account_id, document_id)
|
||||
.await
|
||||
.map(|(sieve, _)| Some(sieve))
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::blocks_in_conditions)]
|
||||
async fn sieve_script_compile(
|
||||
&self,
|
||||
account_id: u32,
|
||||
document_id: u32,
|
||||
) -> trc::Result<(Sieve, Object<Value>)> {
|
||||
// Obtain script object
|
||||
let script_object = self
|
||||
.get_property::<HashedValue<Object<Value>>>(
|
||||
account_id,
|
||||
Collection::SieveScript,
|
||||
document_id,
|
||||
Property::Value,
|
||||
)
|
||||
.await?
|
||||
.ok_or_else(|| {
|
||||
trc::StoreEvent::NotFound
|
||||
.into_err()
|
||||
.caused_by(trc::location!())
|
||||
.document_id(document_id)
|
||||
})?;
|
||||
|
||||
// Obtain the sieve script length
|
||||
let (script_offset, blob_id) = script_object
|
||||
.inner
|
||||
.properties
|
||||
.get(&Property::BlobId)
|
||||
.and_then(|v| v.as_blob_id())
|
||||
.and_then(|v| (v.section.as_ref()?.size, v).into())
|
||||
.ok_or_else(|| {
|
||||
trc::StoreEvent::NotFound
|
||||
.into_err()
|
||||
.caused_by(trc::location!())
|
||||
.document_id(document_id)
|
||||
})?;
|
||||
|
||||
// Obtain the sieve script blob
|
||||
let script_bytes = self
|
||||
.core
|
||||
.storage
|
||||
.blob
|
||||
.get_blob(blob_id.hash.as_ref(), 0..usize::MAX)
|
||||
.await
|
||||
.caused_by(trc::location!())?
|
||||
.ok_or_else(|| {
|
||||
trc::StoreEvent::NotFound
|
||||
.into_err()
|
||||
.caused_by(trc::location!())
|
||||
.document_id(document_id)
|
||||
})?;
|
||||
|
||||
// Obtain the precompiled script
|
||||
if let Some(sieve) = script_bytes
|
||||
.get(script_offset..)
|
||||
.and_then(|bytes| Bincode::<Sieve>::deserialize(bytes).ok())
|
||||
{
|
||||
Ok((sieve.inner, script_object.inner))
|
||||
} else {
|
||||
// Deserialization failed, probably because the script compiler version changed
|
||||
match self.core.sieve.untrusted_compiler.compile(
|
||||
script_bytes.get(0..script_offset).ok_or_else(|| {
|
||||
trc::StoreEvent::NotFound
|
||||
.into_err()
|
||||
.caused_by(trc::location!())
|
||||
.document_id(document_id)
|
||||
})?,
|
||||
) {
|
||||
Ok(sieve) => {
|
||||
// Store updated compiled sieve script
|
||||
let sieve = Bincode::new(sieve);
|
||||
let compiled_bytes = (&sieve).serialize();
|
||||
let mut updated_sieve_bytes =
|
||||
Vec::with_capacity(script_offset + compiled_bytes.len());
|
||||
updated_sieve_bytes.extend_from_slice(&script_bytes[0..script_offset]);
|
||||
updated_sieve_bytes.extend_from_slice(&compiled_bytes);
|
||||
|
||||
// Store updated blob
|
||||
let mut new_blob_id = blob_id.clone();
|
||||
new_blob_id.hash = self
|
||||
.put_blob(account_id, &updated_sieve_bytes, false)
|
||||
.await?
|
||||
.hash;
|
||||
let mut new_script_object = script_object.inner.clone();
|
||||
new_script_object.set(Property::BlobId, new_blob_id.clone());
|
||||
|
||||
// Update script object
|
||||
let mut batch = BatchBuilder::new();
|
||||
batch
|
||||
.with_account_id(account_id)
|
||||
.with_collection(Collection::SieveScript)
|
||||
.update_document(document_id)
|
||||
.assert_value(Property::Value, &script_object)
|
||||
.set(Property::Value, (&new_script_object).serialize())
|
||||
.clear(BlobOp::Link {
|
||||
hash: blob_id.hash.clone(),
|
||||
})
|
||||
.set(
|
||||
BlobOp::Link {
|
||||
hash: new_blob_id.hash,
|
||||
},
|
||||
Vec::new(),
|
||||
);
|
||||
self.store()
|
||||
.write(batch.build())
|
||||
.await
|
||||
.caused_by(trc::location!())?;
|
||||
|
||||
Ok((sieve.inner, new_script_object))
|
||||
}
|
||||
Err(error) => Err(trc::StoreEvent::UnexpectedError
|
||||
.caused_by(trc::location!())
|
||||
.reason(error)
|
||||
.details("Failed to compile Sieve script")),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[inline(always)]
|
||||
pub fn is_valid_role(role: &str) -> bool {
|
||||
[
|
||||
"inbox",
|
||||
"trash",
|
||||
"spam",
|
||||
"junk",
|
||||
"drafts",
|
||||
"archive",
|
||||
"sent",
|
||||
"important",
|
||||
]
|
||||
.contains(&role)
|
||||
}
|
||||
|
||||
impl SeenIdHash {
|
||||
pub fn new(id: &str, expiry: u64) -> Self {
|
||||
let mut hasher = blake3::Hasher::new();
|
||||
hasher.update(id.as_bytes());
|
||||
SeenIdHash {
|
||||
hash: hasher.finalize().into(),
|
||||
expiry,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialOrd for SeenIdHash {
|
||||
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
|
||||
Some(self.cmp(other))
|
||||
}
|
||||
}
|
||||
|
||||
impl Ord for SeenIdHash {
|
||||
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
|
||||
self.expiry.cmp(&other.expiry)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::hash::Hash for SeenIdHash {
|
||||
fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
|
||||
self.hash.hash(state);
|
||||
}
|
||||
}
|
||||
|
||||
impl PartialEq for SeenIdHash {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
self.hash == other.hash
|
||||
}
|
||||
}
|
||||
|
||||
impl Eq for SeenIdHash {}
|
||||
|
||||
// SeenIds serializer
|
||||
impl serde::Serialize for SeenIds {
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
let mut seq = serializer.serialize_seq((self.ids.len() * 2).into())?;
|
||||
for id in &self.ids {
|
||||
seq.serialize_element(&id.expiry)?;
|
||||
seq.serialize_element(&id.hash)?;
|
||||
}
|
||||
|
||||
seq.end()
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> serde::Deserialize<'de> for SeenIds {
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
deserializer.deserialize_seq(SeenIdsVisitor)
|
||||
}
|
||||
}
|
||||
|
||||
struct SeenIdsVisitor;
|
||||
|
||||
impl<'de> serde::de::Visitor<'de> for SeenIdsVisitor {
|
||||
type Value = SeenIds;
|
||||
|
||||
fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||||
formatter.write_str("invalid SeenIds")
|
||||
}
|
||||
|
||||
fn visit_seq<A>(self, mut seq: A) -> Result<Self::Value, A::Error>
|
||||
where
|
||||
A: serde::de::SeqAccess<'de>,
|
||||
{
|
||||
let num_entries = seq.size_hint().unwrap_or(0) / 2;
|
||||
let mut seen_ids = SeenIds {
|
||||
ids: AHashSet::with_capacity(num_entries),
|
||||
has_changes: false,
|
||||
};
|
||||
let now = now();
|
||||
|
||||
for _ in 0..num_entries {
|
||||
let expiry = seq
|
||||
.next_element::<u64>()?
|
||||
.ok_or_else(|| serde::de::Error::custom("Expected expiry."))?;
|
||||
if expiry > now {
|
||||
seen_ids.ids.insert(SeenIdHash {
|
||||
hash: seq
|
||||
.next_element()?
|
||||
.ok_or_else(|| serde::de::Error::custom("Expected hash."))?,
|
||||
expiry,
|
||||
});
|
||||
} else {
|
||||
seq.next_element::<[u8; 32]>()?
|
||||
.ok_or_else(|| serde::de::Error::custom("Expected hash."))?;
|
||||
seen_ids.has_changes = true;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(seen_ids)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user