filip/Stalwart
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Zero copy everything

Browse Source
This commit is contained in:
mdecimus
2025-02-27 16:23:34 +01:00
parent 2826ff1548
commit 84be252413
86 changed files with 3749 additions and 2187 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
tests/src/imap/body_structure.rs
View File

@@ -6,12 +6,17 @@
use std::fs;
use email::message::metadata::{ArchivedMessageMetadata, MessageMetadata};
use imap::op::fetch::AsImapDataItem;
use imap_proto::{
protocol::fetch::{BodyContents, DataItem, Section},
ResponseCode, StatusResponse,
protocol::fetch::{BodyContents, DataItem, Section},
};
use mail_parser::MessageParser;
use store::{
Deserialize, Serialize,
write::{Archive, Archiver},
};
use super::resources_dir;
@@ -21,19 +26,41 @@ fn body_structure() {
for file_name in fs::read_dir(resources_dir()).unwrap() {
let mut file_name = file_name.as_ref().unwrap().path();
if file_name.extension().is_none_or( |e| e != "txt") {
if file_name.extension().is_none_or(|e| e != "txt") {
continue;
}
let raw_message = fs::read(&file_name).unwrap();
let message = MessageParser::new().parse(&raw_message).unwrap();
let mut buf = Vec::new();
let raw_message = fs::read(&file_name).unwrap();
let message_ = MessageParser::new().parse(&raw_message).unwrap();
let metadata_ = Archive::deserialize_owned(
Archiver::new(MessageMetadata {
preview: Default::default(),
size: message_.raw_message.len() as u32,
raw_headers: message_
.raw_message
.as_ref()
.get(message_.root_part().offset_header..message_.root_part().offset_body)
.unwrap_or_default()
.to_vec(),
contents: message_.into(),
received_at: 0,
has_attachments: false,
blob_hash: Default::default(),
})
.serialize()
.unwrap(),
)
.unwrap();
let metadata = metadata_.unarchive::<ArchivedMessageMetadata>().unwrap();
let message = &metadata.contents;
let decoded = message.decode_contents(&raw_message);
// Serialize body and bodystructure
for is_extended in [false, true] {
let mut buf_ = Vec::new();
message
.body_structure(is_extended)
.body_structure(&decoded, is_extended)
.serialize(&mut buf_, is_extended);
if is_extended {
buf.extend_from_slice(b"BODYSTRUCTURE ");
@@ -92,7 +119,8 @@ fn body_structure() {
true
};
if let Some(contents) = message.body_section(&body_sections, None) {
if let Some(contents) = message.body_section(&decoded, &body_sections, None)
{
DataItem::BodySection {
sections: body_sections,
origin_octet: None,
@@ -101,7 +129,7 @@ fn body_structure() {
.serialize(&mut buf);
if is_first {
match message.binary(&sections, None) {
match message.binary(&decoded, &sections, None) {
Ok(Some(contents)) => {
buf.push(b'\n');
DataItem::Binary {
@@ -135,7 +163,7 @@ fn body_structure() {
}
}
if let Some(size) = message.binary_size(&sections) {
if let Some(size) = message.binary_size(&decoded, &sections) {
buf.push(b'\n');
DataItem::BinarySize {
sections: sections.clone(),
@@ -175,14 +203,16 @@ fn body_structure() {
}],
] {
DataItem::BodySection {
contents: message.body_section(&sections, None).unwrap(),
contents: message.body_section(&decoded, &sections, None).unwrap(),
sections: sections.clone(),
origin_octet: None,
}
.serialize(&mut buf);
buf.extend_from_slice(b"\n----------------------------------\n");
DataItem::BodySection {
contents: message.body_section(&sections, (10, 25).into()).unwrap(),
contents: message
.body_section(&decoded, &sections, (10, 25).into())
.unwrap(),
sections,
origin_octet: 10.into(),
}

9
tests/src/jmap/crypto.rs
View File

@@ -11,6 +11,10 @@ use email::message::crypto::{
};
use jmap_proto::types::id::Id;
use mail_parser::{MessageParser, MimeHeaders};
use store::{
Deserialize, Serialize,
write::{Archive, Archiver},
};
use crate::{
directory::internal::TestInternalDirectory,
@@ -213,7 +217,10 @@ pub async fn import_certs_and_encrypt() {
.unwrap();
assert!(!message.is_encrypted());
params.algo = algo;
message.encrypt(&params).await.unwrap();
let arch =
Archive::deserialize_owned(Archiver::new(params.clone()).serialize().unwrap())
.unwrap();
message.encrypt(arch.unarchive().unwrap()).await.unwrap();
}
}

2
tests/src/jmap/permissions.rs
View File

@@ -597,7 +597,7 @@ pub async fn test(params: &JMAPTest) {
);
// John should not be allowed to receive email
let message_blob = BlobHash::from(TEST_MESSAGE.as_bytes());
let message_blob = BlobHash::generate(TEST_MESSAGE.as_bytes());
server
.blob_store()
.put_blob(message_blob.as_ref(), TEST_MESSAGE.as_bytes())

22
tests/src/jmap/stress_test.rs
View File

@@ -9,7 +9,7 @@ use std::{sync::Arc, time::Duration};
use crate::jmap::{mailbox::destroy_all_mailboxes_no_wait, wait_for_index};
use common::Server;
use directory::backend::internal::manage::ManageDirectory;
use email::mailbox::UidMailbox;
use email::mailbox::{ArchivedUidMailbox, UidMailbox};
use futures::future::join_all;
use jmap_client::{
client::Client,
@@ -17,7 +17,11 @@ use jmap_client::{
mailbox::{self, Mailbox, Role},
};
use jmap_proto::types::{collection::Collection, id::Id, property::Property};
use store::rand::{self, Rng};
use store::{
rand::{self, Rng},
rkyv::vec::ArchivedVec,
write::Archive,
};
use super::assert_is_empty;
@@ -70,8 +74,7 @@ async fn email_tests(server: Server, client: Arc<Client>) {
let client = client.clone();
let mailboxes = mailboxes.clone();
futures.push(tokio::spawn(async move {
let mailbox_num =
rand::rng().random_range::<usize, _>(0..mailboxes.len());
let mailbox_num = rand::rng().random_range::<usize, _>(0..mailboxes.len());
let _message_id = client
.email_import(
format!(
@@ -230,7 +233,7 @@ async fn email_tests(server: Server, client: Arc<Client>) {
for email_id in &email_ids_in_mailbox {
if let Some(mailbox_tags) = server
.get_property::<Vec<UidMailbox>>(
.get_property::<Archive>(
TEST_USER_ID,
Collection::Email,
email_id,
@@ -239,11 +242,14 @@ async fn email_tests(server: Server, client: Arc<Client>) {
.await
.unwrap()
{
let mailbox_tags = mailbox_tags
.deserialize::<ArchivedVec<ArchivedUidMailbox>, Vec<UidMailbox>>()
.unwrap();
if mailbox_tags.len() != 1 {
panic!(
"Email ORM has more than one mailbox {:?}! Id {} in mailbox {} with messages {:?}",
mailbox_tags, email_id, mailbox_id, email_ids_in_mailbox
);
"Email ORM has more than one mailbox {:?}! Id {} in mailbox {} with messages {:?}",
mailbox_tags, email_id, mailbox_id, email_ids_in_mailbox
);
}
let mailbox_tag = mailbox_tags[0];
assert!(mailbox_tag.uid != 0);

8
tests/src/smtp/inbound/mod.rs
View File

@@ -7,12 +7,12 @@
use std::time::Duration;
use common::{
ipc::{DmarcEvent, QueueEvent, QueueEventStatus, ReportingEvent, TlsEvent},
Server,
ipc::{DmarcEvent, QueueEvent, QueueEventStatus, ReportingEvent, TlsEvent},
};
use store::{
write::{key::DeserializeBigEndian, Bincode, QueueClass, ReportEvent, ValueClass},
Deserialize, IterateParams, ValueKey, U64_LEN,
Deserialize, IterateParams, U64_LEN, ValueKey,
write::{LegacyBincode, QueueClass, ReportEvent, ValueClass, key::DeserializeBigEndian},
};
use tokio::sync::mpsc::error::TryRecvError;
@@ -186,7 +186,7 @@ impl QueueReceiver {
.iterate(
IterateParams::new(from_key, to_key).descending(),
|key, value| {
let value = Bincode::<Message>::deserialize(value)?;
let value = LegacyBincode::<Message>::deserialize(value)?;
assert_eq!(key.deserialize_be_u64(0)?, value.inner.queue_id);
messages.push(value.inner);
Ok(true)

8
tests/src/smtp/queue/dsn.rs
View File

@@ -6,13 +6,13 @@
use std::{fs, path::PathBuf, time::SystemTime};
use smtp_proto::{Response, RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_SUCCESS};
use smtp_proto::{RCPT_NOTIFY_DELAY, RCPT_NOTIFY_FAILURE, RCPT_NOTIFY_SUCCESS, Response};
use store::write::now;
use utils::BlobHash;
use crate::smtp::{inbound::sign::SIGNATURES, QueueReceiver, TestSMTP};
use crate::smtp::{QueueReceiver, TestSMTP, inbound::sign::SIGNATURES};
use smtp::queue::{
dsn::SendDsn, Domain, Error, ErrorDetails, HostResponse, Message, Recipient, Schedule, Status,
Domain, Error, ErrorDetails, HostResponse, Message, Recipient, Schedule, Status, dsn::SendDsn,
};
const CONFIG: &str = r#"
@@ -87,7 +87,7 @@ async fn generate_dsn() {
flags: 0,
env_id: None,
priority: 0,
blob_hash: BlobHash::from(dsn_original.as_bytes()),
blob_hash: BlobHash::generate(dsn_original.as_bytes()),
quota_keys: vec![],
};

18
tests/src/store/blob.rs
View File

@@ -35,7 +35,7 @@ pub async fn blob_tests() {
let blob_store: BlobStore = store.clone().into();
// Blob hash exists
let hash = BlobHash::from(b"abc".as_slice());
let hash = BlobHash::generate(b"abc".as_slice());
assert!(!store.blob_exists(&hash).await.unwrap());
// Reserve blob
@@ -163,7 +163,7 @@ pub async fn blob_tests() {
.into_iter()
.enumerate()
{
let hash = BlobHash::from(blob.as_slice());
let hash = BlobHash::generate(blob.as_slice());
let blob_op = if let Some(until) = expiry_times.get(blob) {
BlobOp::Reserve {
until: *until,
@@ -256,7 +256,7 @@ pub async fn blob_tests() {
.enumerate()
{
let ct = pos == 0;
let hash = BlobHash::from(blob.as_slice());
let hash = BlobHash::generate(blob.as_slice());
assert!(store.blob_has_access(&hash, blob_class).await.unwrap() ^ ct);
assert!(store.blob_exists(&hash).await.unwrap() ^ ct);
assert!(
@@ -273,7 +273,7 @@ pub async fn blob_tests() {
assert!(
!store
.blob_has_access(
BlobHash::from(b"123".as_slice()),
BlobHash::generate(b"123".as_slice()),
BlobClass::Linked {
account_id: 0,
collection: 0,
@@ -292,7 +292,7 @@ pub async fn blob_tests() {
.with_collection(0)
.update_document(2)
.clear(BlobOp::Link {
hash: BlobHash::from(b"789".as_slice()),
hash: BlobHash::generate(b"789".as_slice()),
})
.build_batch(),
)
@@ -345,7 +345,7 @@ pub async fn blob_tests() {
.enumerate()
{
let ct = pos == 0;
let hash = BlobHash::from(blob.as_slice());
let hash = BlobHash::generate(blob.as_slice());
assert!(store.blob_has_access(&hash, blob_class).await.unwrap() ^ ct);
assert!(store.blob_exists(&hash).await.unwrap() ^ ct);
assert!(
@@ -399,7 +399,7 @@ pub async fn blob_tests() {
.enumerate()
{
let ct = pos == 0;
let hash = BlobHash::from(blob.as_slice());
let hash = BlobHash::generate(blob.as_slice());
assert!(store.blob_has_access(&hash, blob_class).await.unwrap() ^ ct);
assert!(store.blob_exists(&hash).await.unwrap() ^ ct);
assert!(
@@ -418,7 +418,7 @@ pub async fn blob_tests() {
async fn test_store(store: BlobStore) {
// Test small blob
const DATA: &[u8] = b"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Fusce erat nisl, dignissim a porttitor id, varius nec arcu. Sed mauris.";
let hash = BlobHash::from(DATA);
let hash = BlobHash::generate(DATA);
store.put_blob(hash.as_slice(), DATA).await.unwrap();
assert_eq!(
@@ -459,7 +459,7 @@ async fn test_store(store: BlobStore) {
let marker = format!(" [{}] ", data.len());
data.extend_from_slice(marker.as_bytes());
}
let hash = BlobHash::from(&data);
let hash = BlobHash::generate(&data);
store.put_blob(hash.as_slice(), &data).await.unwrap();
assert_eq!(
String::from_utf8(

4
tests/src/store/import_export.rs
View File

@@ -5,7 +5,7 @@
*/
use ahash::AHashSet;
use common::{manager::backup::BackupParams, Core};
use common::{Core, manager::backup::BackupParams};
use jmap_proto::types::{collection::Collection, property::Property};
use store::{
rand,
@@ -35,7 +35,7 @@ pub async fn test(db: Store) {
let mut blob_hashes = Vec::new();
for blob_size in [16, 128, 1024, 2056, 102400] {
let data = random_bytes(blob_size);
let hash = BlobHash::from(data.as_slice());
let hash = BlobHash::generate(data.as_slice());
blob_hashes.push(hash.clone());
core.storage
.blob