filip/Stalwart
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Email/get tests passing.

Browse Source
This commit is contained in:
Mauro D
2023-04-20 16:39:28 +00:00
parent a79dd14c5d
commit 51b14ed79e
43 changed files with 1244 additions and 257 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tests/src/jmap/email_get.rs
View File

@@ -21,7 +21,7 @@
* for more details.
*/
use std::{fs, path::PathBuf, sync::Arc};
use std::{fs, path::PathBuf, sync::Arc, time::Instant};
use jmap::JMAP;
use jmap_client::{

766
tests/src/jmap/email_query.rs Normal file
View File

@@ -0,0 +1,766 @@
use std::{collections::hash_map::Entry, sync::Arc, time::Instant};
use jmap::JMAP;
use jmap_client::{
client::Client,
core::query::{Comparator, Filter},
email,
};
use jmap_proto::types::{collection::Collection, id::Id};
use mail_parser::RfcHeader;
use store::{ahash::AHashMap, write::BatchBuilder};
use crate::store::{deflate_artwork_data, query::FIELDS};
const MAX_THREADS: usize = 100;
const MAX_MESSAGES: usize = 1000;
const MAX_MESSAGES_PER_THREAD: usize = 100;
pub async fn test(server: Arc<JMAP>, client: &mut Client) {
println!("Running Email Query tests...");
// Add some "virtual" mailbox ids so create doesn't fail
let mut batch = BatchBuilder::new();
let account_id = Id::from_bytes(client.default_account_id().as_bytes())
.unwrap()
.document_id();
batch
.with_account_id(account_id)
.with_collection(Collection::Mailbox);
for mailbox_id in 0..99999 {
batch.create_document(mailbox_id);
}
server.store.write(batch.build()).await.unwrap();
// Create test messages
println!("Inserting JMAP Mail query test messages...");
create(client).await;
// Remove mailboxes
let mut batch = BatchBuilder::new();
batch
.with_account_id(account_id)
.with_collection(Collection::Mailbox);
for mailbox_id in 0..99999 {
batch.delete_document(mailbox_id);
}
server.store.write(batch.build()).await.unwrap();
for thread_id in 0..MAX_THREADS {
assert!(
client
.thread_get(&Id::new(thread_id as u64).to_string())
.await
.unwrap()
.is_some(),
"thread {} not found",
thread_id
);
}
assert!(
client
.thread_get(&Id::new(MAX_THREADS as u64).to_string())
.await
.unwrap()
.is_none(),
"thread {} found",
MAX_THREADS
);
println!("Running JMAP Mail query tests...");
query(client).await;
println!("Running JMAP Mail query options tests...");
query_options(client).await;
println!("Deleting all messages...");
let implement = "fds";
/*let mut request = client.build();
let result_ref = request.query_email().result_reference();
request.set_email().destroy_ref(result_ref);
let response = request.send().await.unwrap();
response
.unwrap_method_responses()
.pop()
.unwrap()
.unwrap_set_email()
.unwrap();
server.store.assert_is_empty();*/
}
pub async fn query(client: &mut Client) {
for (filter, sort, expected_results) in [
(
Filter::and(vec![
(email::query::Filter::after(1850)),
(email::query::Filter::from("george")),
]),
vec![email::query::Comparator::subject()],
vec![
"N01389", "T10115", "N00618", "N03500", "T01587", "T00397", "N01561", "N05250",
"N03973", "N04973", "N04057", "N01940", "N01539", "N01612", "N04484", "N01954",
"N05998", "T02053", "AR00171", "AR00172", "AR00176",
],
),
(
Filter::and(vec![
(email::query::Filter::in_mailbox(Id::new(1768u64).to_string())),
(email::query::Filter::cc("canvas")),
]),
vec![email::query::Comparator::from()],
vec!["T01882", "N04689", "T00925", "N00121"],
),
(
Filter::and(vec![
(email::query::Filter::subject("study")),
(email::query::Filter::in_mailbox_other_than(vec![
Id::new(1991).to_string(),
Id::new(1870).to_string(),
Id::new(2011).to_string(),
Id::new(1951).to_string(),
Id::new(1902).to_string(),
Id::new(1808).to_string(),
Id::new(1963).to_string(),
])),
]),
vec![email::query::Comparator::subject()],
vec![
"T10330", "N01744", "N01743", "N04885", "N02688", "N02122", "A00059", "A00058",
"N02123", "T00651", "T09439", "N05001", "T05848", "T05508",
],
),
(
Filter::and(vec![
(email::query::Filter::has_keyword("N0")).into(),
Filter::not(vec![(email::query::Filter::from("collins"))]),
(email::query::Filter::body("bequeathed")).into(),
]),
vec![email::query::Comparator::subject()],
vec![
"N02640", "A01020", "N01250", "T03430", "N01800", "N00620", "N05250", "N04630",
"A01040",
],
),
(
email::query::Filter::not_keyword("artist").into(),
vec![email::query::Comparator::subject()],
vec!["T08626", "T09334", "T09455", "N01737", "T10965"],
),
(
Filter::and(vec![
(email::query::Filter::after(1970)),
(email::query::Filter::before(1972)),
(email::query::Filter::text("colour")),
]),
vec![email::query::Comparator::from()],
vec!["T01745", "P01436", "P01437"],
),
(
Filter::and(vec![(email::query::Filter::text("'cats and dogs'"))]),
vec![email::query::Comparator::from()],
vec!["P77623"],
),
(
Filter::and(vec![
(email::query::Filter::header(RfcHeader::Comments.to_string(), Some("attributed"))),
(email::query::Filter::from("john")),
(email::query::Filter::cc("oil")),
]),
vec![email::query::Comparator::from()],
vec!["T10965"],
),
(
Filter::and(vec![
(email::query::Filter::all_in_thread_have_keyword("N")),
(email::query::Filter::before(1800)),
]),
vec![email::query::Comparator::from()],
vec![
"N01496", "N05916", "N01046", "N00675", "N01320", "N01321", "N00273", "N01453",
"N02984",
],
),
(
Filter::and(vec![
(email::query::Filter::none_in_thread_have_keyword("N")),
(email::query::Filter::after(1995)),
]),
vec![email::query::Comparator::from()],
vec![
"AR00163", "AR00164", "AR00472", "P11481", "AR00066", "AR00178", "P77895",
"P77896", "P77897",
],
),
(
Filter::and(vec![
(email::query::Filter::some_in_thread_have_keyword("Bronze")),
(email::query::Filter::before(1878)),
]),
vec![email::query::Comparator::from()],
vec![
"N04326", "N01610", "N02920", "N01587", "T00167", "T00168", "N01554", "N01535",
"N01536", "N01622", "N01754", "N01594",
],
),
// Sorting tests
(
email::query::Filter::before(1800).into(),
vec![
email::query::Comparator::all_in_thread_have_keyword("N"),
email::query::Comparator::from(),
],
vec![
"N01496", "N05916", "N01046", "N00675", "N01320", "N01321", "N00273", "N01453",
"N02984", "T09417", "T01882", "T08820", "N04689", "T08891", "T00986", "N00316",
"N03544", "N04296", "N04297", "T08234", "N00112", "T00211", "N01497", "N02639",
"N02640", "T00925", "T11683", "T08269", "D00001", "D00002", "D00046", "N00121",
"N00126", "T08626",
],
),
(
email::query::Filter::before(1800).into(),
vec![
email::query::Comparator::all_in_thread_have_keyword("N").descending(),
email::query::Comparator::from(),
],
vec![
"T09417", "T01882", "T08820", "N04689", "T08891", "T00986", "N00316", "N03544",
"N04296", "N04297", "T08234", "N00112", "T00211", "N01497", "N02639", "N02640",
"T00925", "T11683", "T08269", "D00001", "D00002", "D00046", "N00121", "N00126",
"T08626", "N01496", "N05916", "N01046", "N00675", "N01320", "N01321", "N00273",
"N01453", "N02984",
],
),
(
Filter::and(vec![
(email::query::Filter::after(1875)),
(email::query::Filter::before(1878)),
]),
vec![
email::query::Comparator::some_in_thread_have_keyword("Bronze"),
email::query::Comparator::from(),
],
vec![
"N04326", "N01610", "N02920", "N01587", "T00167", "T00168", "N01554", "N01535",
"N01536", "N01622", "N01754", "N01594", "N01559", "N02123", "N01940", "N03594",
"N01494", "N04271",
],
),
(
Filter::and(vec![
(email::query::Filter::after(1875)),
(email::query::Filter::before(1878)),
]),
vec![
email::query::Comparator::some_in_thread_have_keyword("Bronze").descending(),
email::query::Comparator::from(),
],
vec![
"N01559", "N02123", "N01940", "N03594", "N01494", "N04271", "N04326", "N01610",
"N02920", "N01587", "T00167", "T00168", "N01554", "N01535", "N01536", "N01622",
"N01754", "N01594",
],
),
(
Filter::and(vec![
(email::query::Filter::after(1786)),
(email::query::Filter::before(1840)),
(email::query::Filter::has_keyword("T")),
]),
vec![
email::query::Comparator::has_keyword("attributed to"),
email::query::Comparator::from(),
],
vec![
"T09455", "T09334", "T10965", "T08626", "T09417", "T08951", "T01851", "T01852",
"T08761", "T08123", "T08756", "T10561", "T10562", "T10563", "T00986", "T03424",
"T03427", "T08234", "T08133", "T06866", "T08897", "T00996", "T00997", "T01095",
"T03393", "T09456", "T00188", "T02362", "T09065", "T09547", "T10330", "T09187",
"T03433", "T08635", "T02366", "T03436", "T09150", "T01861", "T09759", "T11683",
"T02368", "T02369", "T08269", "T01018", "T10066", "T01710", "T01711", "T05764",
],
),
(
Filter::and(vec![
(email::query::Filter::after(1786)),
(email::query::Filter::before(1840)),
(email::query::Filter::has_keyword("T")),
]),
vec![
email::query::Comparator::has_keyword("attributed to").descending(),
email::query::Comparator::from(),
],
vec![
"T09417", "T08951", "T01851", "T01852", "T08761", "T08123", "T08756", "T10561",
"T10562", "T10563", "T00986", "T03424", "T03427", "T08234", "T08133", "T06866",
"T08897", "T00996", "T00997", "T01095", "T03393", "T09456", "T00188", "T02362",
"T09065", "T09547", "T10330", "T09187", "T03433", "T08635", "T02366", "T03436",
"T09150", "T01861", "T09759", "T11683", "T02368", "T02369", "T08269", "T01018",
"T10066", "T01710", "T01711", "T05764", "T09455", "T09334", "T10965", "T08626",
],
),
] {
let mut request = client.build();
let query_request = request
.query_email()
.filter(filter)
.sort(sort)
.calculate_total(true);
query_request.arguments().collapse_threads(false);
let query_result_ref = query_request.result_reference();
request
.get_email()
.ids_ref(query_result_ref)
.properties([email::Property::MessageId]);
assert_eq!(
request
.send()
.await
.unwrap()
.unwrap_method_responses()
.pop()
.unwrap()
.unwrap_get_email()
.unwrap()
.take_list()
.into_iter()
.map(|e| e.message_id().unwrap().first().unwrap().to_string())
.collect::<Vec<_>>(),
expected_results
);
}
}
pub async fn query_options(client: &mut Client) {
for (query, expected_results, expected_results_collapsed) in [
(
EmailQuery {
filter: None,
sort: vec![
email::query::Comparator::subject(),
email::query::Comparator::from(),
],
position: 0,
anchor: None,
anchor_offset: 0,
limit: 10,
},
vec![
"N01496", "N01320", "N01321", "N05916", "N00273", "N01453", "N02984", "T08820",
"N00112", "T00211",
],
vec![
"N01496", "N01320", "N05916", "N01453", "T08820", "N01046", "N00675", "T08891",
"T01882", "N04296",
],
),
(
EmailQuery {
filter: None,
sort: vec![
email::query::Comparator::subject(),
email::query::Comparator::from(),
],
position: 10,
anchor: None,
anchor_offset: 0,
limit: 10,
},
vec![
"N01046", "N00675", "T08891", "N00126", "T01882", "N04689", "T00925", "N00121",
"N04296", "N04297",
],
vec![
"T08234", "T09417", "N01110", "T08123", "N01039", "T09456", "T08951", "N01273",
"N00373", "T09547",
],
),
(
EmailQuery {
filter: None,
sort: vec![
email::query::Comparator::subject(),
email::query::Comparator::from(),
],
position: -10,
anchor: None,
anchor_offset: 0,
limit: 0,
},
vec![
"T07236", "P11481", "AR00066", "P77895", "P77896", "P77897", "AR00163", "AR00164",
"AR00472", "AR00178",
],
vec![
"P07639", "P07522", "AR00089", "P02949", "T05820", "P11441", "T06971", "P11481",
"AR00163", "AR00164",
],
),
(
EmailQuery {
filter: None,
sort: vec![
email::query::Comparator::subject(),
email::query::Comparator::from(),
],
position: -20,
anchor: None,
anchor_offset: 0,
limit: 10,
},
vec![
"P20079", "AR00024", "AR00182", "P20048", "P20044", "P20045", "P20046", "T06971",
"AR00177", "P77935",
],
vec![
"T00300", "P06033", "T02310", "T02135", "P04006", "P03166", "P01358", "P07133",
"P03138", "T03562",
],
),
(
EmailQuery {
filter: None,
sort: vec![
email::query::Comparator::subject(),
email::query::Comparator::from(),
],
position: -100000,
anchor: None,
anchor_offset: 0,
limit: 1,
},
vec!["N01496"],
vec!["N01496"],
),
(
EmailQuery {
filter: None,
sort: vec![
email::query::Comparator::subject(),
email::query::Comparator::from(),
],
position: -1,
anchor: None,
anchor_offset: 0,
limit: 100000,
},
vec!["AR00178"],
vec!["AR00164"],
),
(
EmailQuery {
filter: None,
sort: vec![
email::query::Comparator::subject(),
email::query::Comparator::from(),
],
position: 0,
anchor: get_anchor(client, "N01205").await,
anchor_offset: 0,
limit: 10,
},
vec![
"N01205", "N01976", "T01139", "N01525", "T00176", "N01405", "N02396", "N04885",
"N01526", "N02134",
],
vec![
"N01205", "N01526", "T01455", "N01969", "N05250", "N01781", "N00759", "A00057",
"N03527", "N01558",
],
),
(
EmailQuery {
filter: None,
sort: vec![
email::query::Comparator::subject(),
email::query::Comparator::from(),
],
position: 0,
anchor: get_anchor(client, "N01205").await,
anchor_offset: 10,
limit: 10,
},
vec![
"N01933", "N03618", "T03904", "N02398", "N02399", "N02688", "T01455", "N03051",
"N01500", "N03411",
],
vec![
"N01559", "N04326", "N06017", "N01553", "N01617", "N01528", "N01539", "T09439",
"N01593", "N03988",
],
),
(
EmailQuery {
filter: None,
sort: vec![
email::query::Comparator::subject(),
email::query::Comparator::from(),
],
position: 0,
anchor: get_anchor(client, "N01205").await,
anchor_offset: -10,
limit: 10,
},
vec![
"N05779", "N04652", "N01534", "A00845", "N03409", "N03410", "N02061", "N02426",
"N00662", "N01205",
],
vec![
"N00443", "N02237", "T03025", "N01722", "N01356", "N01800", "T05475", "T01587",
"N05779", "N01205",
],
),
(
EmailQuery {
filter: None,
sort: vec![
email::query::Comparator::subject(),
email::query::Comparator::from(),
],
position: 0,
anchor: get_anchor(client, "N01496").await,
anchor_offset: -10,
limit: 10,
},
vec!["N01496"],
vec!["N01496"],
),
(
EmailQuery {
filter: None,
sort: vec![
email::query::Comparator::subject(),
email::query::Comparator::from(),
],
position: 0,
anchor: get_anchor(client, "AR00164").await,
anchor_offset: 10,
limit: 10,
},
vec![],
vec![],
),
(
EmailQuery {
filter: None,
sort: vec![
email::query::Comparator::subject(),
email::query::Comparator::from(),
],
position: 0,
anchor: get_anchor(client, "AR00164").await,
anchor_offset: 0,
limit: 0,
},
vec!["AR00164", "AR00472", "AR00178"],
vec!["AR00164"],
),
] {
for (test_num, expected_results) in [expected_results, expected_results_collapsed]
.into_iter()
.enumerate()
{
let mut request = client.build();
let query_request = request
.query_email()
.sort(query.sort.clone())
.position(query.position)
.calculate_total(true);
if query.limit > 0 {
query_request.limit(query.limit);
}
if let Some(filter) = query.filter.as_ref() {
query_request.filter(filter.clone());
}
if let Some(anchor) = query.anchor.as_ref() {
query_request.anchor(anchor);
query_request.anchor_offset(query.anchor_offset);
}
query_request.arguments().collapse_threads(test_num == 1);
if !expected_results.is_empty() {
let query_result_ref = query_request.result_reference();
request
.get_email()
.ids_ref(query_result_ref)
.properties([email::Property::MessageId]);
assert_eq!(
request
.send()
.await
.unwrap()
.unwrap_method_responses()
.pop()
.unwrap()
.unwrap_get_email()
.unwrap()
.take_list()
.into_iter()
.map(|e| e.message_id().unwrap().first().unwrap().to_string())
.collect::<Vec<_>>(),
expected_results,
"{:#?} ({})",
query,
test_num == 1
);
} else {
assert_eq!(
request.send_query_email().await.unwrap().ids(),
Vec::<&str>::new()
);
}
}
}
}
pub async fn create(client: &mut Client) {
let now = Instant::now();
let mut fields = AHashMap::default();
for (field_num, field) in FIELDS.iter().enumerate() {
fields.insert(field.to_string(), field_num);
}
let mut total_messages = 0;
let mut total_threads = 0;
let mut thread_count = AHashMap::default();
let mut artist_count = AHashMap::default();
'outer: for record in csv::ReaderBuilder::new()
.has_headers(true)
.from_reader(&deflate_artwork_data()[..])
.records()
{
let record = record.unwrap();
let mut values_str = AHashMap::default();
let mut values_int = AHashMap::default();
for field_name in [
"year",
"acquisitionYear",
"accession_number",
"artist",
"artistRole",
"medium",
"title",
"creditLine",
"inscription",
] {
let field = record.get(fields[field_name]).unwrap();
if field.is_empty()
|| (field_name == "title" && (field.contains('[') || field.contains(']')))
{
continue 'outer;
} else if field_name == "year" || field_name == "acquisitionYear" {
let field = field.parse::<i32>().unwrap_or(0);
if field < 1000 {
continue 'outer;
}
values_int.insert(field_name.to_string(), field);
} else {
values_str.insert(field_name.to_string(), field.to_string());
}
}
let val = artist_count
.entry(values_str["artist"].clone())
.or_insert(0);
if *val == 3 {
continue;
}
*val += 1;
match thread_count.entry(values_int["year"]) {
Entry::Occupied(mut e) => {
let messages_per_thread = e.get_mut();
if *messages_per_thread == MAX_MESSAGES_PER_THREAD {
continue;
}
*messages_per_thread += 1;
}
Entry::Vacant(e) => {
if total_threads == MAX_THREADS {
continue;
}
total_threads += 1;
e.insert(1);
}
}
total_messages += 1;
client
.email_import(
format!(
concat!(
"From: \"{}\" <artist@domain.com>\nCc: \"{}\" <cc@domain.com>\nMessage-ID: <{}>\n",
"References: <{}>\nComments: {}\nSubject: [{}]",
" Year {}\n\n{}\n{}\n"
),
values_str["artist"],
values_str["medium"],
values_str["accession_number"],
values_int["year"],
values_str["artistRole"],
values_str["title"],
values_int["year"],
values_str["creditLine"],
values_str["inscription"]
)
.into_bytes(),
[
Id::new(values_int["year"] as u64).to_string(),
Id::new((values_int["acquisitionYear"] + 1000) as u64).to_string(),
],
[
values_str["medium"].to_string(),
values_str["artistRole"].to_string(),
values_str["accession_number"][0..1].to_string(),
format!(
"N{}",
&values_str["accession_number"][values_str["accession_number"].len() - 1..]
),
]
.into(),
Some(values_int["year"] as i64),
)
.await
.unwrap();
if total_messages == MAX_MESSAGES {
break;
}
}
println!(
"Imported {} messages in {} ms (single thread).",
total_messages,
now.elapsed().as_millis()
);
}
async fn get_anchor(client: &mut Client, anchor: &str) -> Option<String> {
client
.email_query(
email::query::Filter::header("Message-Id", anchor.into()).into(),
None::<Vec<_>>,
)
.await
.unwrap()
.take_ids()
.pop()
.unwrap()
.into()
}
#[derive(Debug, Clone)]
pub struct EmailQuery {
pub filter: Option<Filter<email::query::Filter>>,
pub sort: Vec<Comparator<email::query::Comparator>>,
pub position: i32,
pub anchor: Option<String>,
pub anchor_offset: i32,
pub limit: usize,
}

6
tests/src/jmap/mod.rs
View File

@@ -8,6 +8,8 @@ use tokio::sync::watch;
use crate::{add_test_certs, store::TempDir};
pub mod email_get;
pub mod email_query;
pub mod thread_get;
const SERVER: &str = "
[server]
@@ -47,6 +49,8 @@ pub async fn jmap_tests() {
let delete = true;
let mut params = init_jmap_tests(delete).await;
email_get::test(params.server.clone(), &mut params.client).await;
//email_query::test(params.server.clone(), &mut params.client).await;
//thread_get::test(params.server.clone(), &mut params.client).await;
if delete {
params.temp_dir.delete();
}
@@ -136,7 +140,7 @@ pub fn replace_boundaries(string: String) -> String {
pub fn replace_blob_ids(string: String) -> String {
let values = find_values(&string, "blobId\":");
if !values.is_empty() {
let values = BTreeSet::from_iter(values).into_iter().collect::<Vec<_>>();
//let values = BTreeSet::from_iter(values).into_iter().collect::<Vec<_>>();
replace_values(
string,
&values,

50
tests/src/jmap/thread_get.rs Normal file
View File

@@ -0,0 +1,50 @@
use std::sync::Arc;
use jmap::JMAP;
use jmap_client::{client::Client, mailbox::Role};
use jmap_proto::types::id::Id;
pub async fn test(server: Arc<JMAP>, client: &mut Client) {
println!("Running Email Thread tests...");
let mailbox_id = "a".to_string();
let implementer = "fd";
/*let mailbox_id = client
.set_default_account_id(Id::new(1).to_string())
.mailbox_create("JMAP Get", None::<String>, Role::None)
.await
.unwrap()
.take_id();*/
let mut expected_result = vec!["".to_string(); 5];
let mut thread_id = "".to_string();
for num in [5, 3, 1, 2, 4] {
let mut email = client
.email_import(
format!("Subject: test\nReferences: <1234>\n\n{}", num).into_bytes(),
[&mailbox_id],
None::<Vec<String>>,
Some(10000i64 + num as i64),
)
.await
.unwrap();
thread_id = email.thread_id().unwrap().to_string();
expected_result[num - 1] = email.take_id();
}
assert_eq!(
client
.thread_get(&thread_id)
.await
.unwrap()
.unwrap()
.email_ids(),
expected_result
);
let implement = "fd";
//client.mailbox_destroy(&mailbox_id, true).await.unwrap();
//server.store.assert_is_empty();
}