Faster email deletion, Junk/Trash folder auto-expunge and changelog auto-expiry (closes #403)
This commit is contained in:
mdecimus
@@ -46,7 +46,11 @@ use reqwest::header;
|
||||
use serde::{de::DeserializeOwned, Deserialize, Serialize};
|
||||
use smtp::core::{SmtpSessionManager, SMTP};
|
||||
|
||||
use store::Stores;
|
||||
use store::{
|
||||
roaring::RoaringBitmap,
|
||||
write::{key::DeserializeBigEndian, AnyKey},
|
||||
IterateParams, Stores, SUBSPACE_PROPERTY,
|
||||
};
|
||||
use tokio::sync::{mpsc, watch};
|
||||
use utils::config::Config;
|
||||
|
||||
@@ -69,6 +73,7 @@ pub mod email_set;
|
||||
pub mod email_submission;
|
||||
pub mod event_source;
|
||||
pub mod mailbox;
|
||||
pub mod purge;
|
||||
pub mod push_subscription;
|
||||
pub mod quota;
|
||||
pub mod sieve_script;
|
||||
@@ -234,6 +239,12 @@ throttle = "500ms"
|
||||
throttle = "500ms"
|
||||
attempts.interval = "500ms"
|
||||
|
||||
[jmap.email]
|
||||
auto-expunge = "1s"
|
||||
|
||||
[jmap.protocol.changes]
|
||||
max-history = "1s"
|
||||
|
||||
[store."auth"]
|
||||
type = "sqlite"
|
||||
path = "{TMP}/auth.db"
|
||||
@@ -326,6 +337,7 @@ pub async fn jmap_tests() {
|
||||
quota::test(&mut params).await;
|
||||
crypto::test(&mut params).await;
|
||||
blob::test(&mut params).await;
|
||||
purge::test(&mut params).await;
|
||||
|
||||
if delete {
|
||||
params.temp_dir.delete();
|
||||
@@ -390,6 +402,9 @@ pub async fn assert_is_empty(server: Arc<JMAP>) {
|
||||
// Wait for pending FTS index tasks
|
||||
wait_for_index(&server).await;
|
||||
|
||||
// Purge accounts
|
||||
emails_purge_tombstoned(&server).await;
|
||||
|
||||
// Assert is empty
|
||||
server
|
||||
.core
|
||||
@@ -399,6 +414,38 @@ pub async fn assert_is_empty(server: Arc<JMAP>) {
|
||||
.await;
|
||||
}
|
||||
|
||||
pub async fn emails_purge_tombstoned(server: &JMAP) {
|
||||
let mut account_ids = RoaringBitmap::new();
|
||||
server
|
||||
.core
|
||||
.storage
|
||||
.data
|
||||
.iterate(
|
||||
IterateParams::new(
|
||||
AnyKey {
|
||||
subspace: SUBSPACE_PROPERTY,
|
||||
key: vec![0u8],
|
||||
},
|
||||
AnyKey {
|
||||
subspace: SUBSPACE_PROPERTY,
|
||||
key: vec![u8::MAX, u8::MAX, u8::MAX, u8::MAX],
|
||||
},
|
||||
)
|
||||
.no_values(),
|
||||
|key, _| {
|
||||
account_ids.insert(key.deserialize_be_u32(0).unwrap());
|
||||
|
||||
Ok(true)
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
for account_id in account_ids {
|
||||
server.emails_purge_tombstoned(account_id).await.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
async fn init_jmap_tests(store_id: &str, delete_if_exists: bool) -> JMAPTest {
|
||||
// Load and parse config
|
||||
let temp_dir = TempDir::new("jmap_tests", delete_if_exists);
|
||||
|
||||
200
tests/src/jmap/purge.rs
Normal file
200
tests/src/jmap/purge.rs
Normal file
@@ -0,0 +1,200 @@
|
||||
/*
|
||||
* Copyright (c) 2023 Stalwart Labs Ltd.
|
||||
*
|
||||
* This file is part of Stalwart Mail Server.
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as
|
||||
* published by the Free Software Foundation, either version 3 of
|
||||
* the License, or (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
* in the LICENSE file at the top-level directory of this distribution.
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*
|
||||
* You can be released from the requirements of the AGPLv3 license by
|
||||
* purchasing a commercial license. Please contact licensing@stalw.art
|
||||
* for more details.
|
||||
*/
|
||||
|
||||
use ahash::AHashSet;
|
||||
use jmap::{
|
||||
mailbox::{INBOX_ID, JUNK_ID, TRASH_ID},
|
||||
JMAP,
|
||||
};
|
||||
use jmap_proto::types::{collection::Collection, id::Id, property::Property};
|
||||
use store::{
|
||||
write::{key::DeserializeBigEndian, TagValue},
|
||||
IterateParams, LogKey, U32_LEN, U64_LEN,
|
||||
};
|
||||
|
||||
use super::JMAPTest;
|
||||
|
||||
pub async fn test(params: &mut JMAPTest) {
|
||||
println!("Running purge tests...");
|
||||
let server = params.server.clone();
|
||||
let client = &mut params.client;
|
||||
let inbox_id = Id::from(INBOX_ID).to_string();
|
||||
let trash_id = Id::from(TRASH_ID).to_string();
|
||||
let junk_id = Id::from(JUNK_ID).to_string();
|
||||
|
||||
// Create test messages
|
||||
client.set_default_account_id(Id::from(1u64));
|
||||
let mut message_ids = Vec::new();
|
||||
let mut pass = 0;
|
||||
let mut changes = AHashSet::new();
|
||||
|
||||
loop {
|
||||
pass += 1;
|
||||
for folder_id in [&inbox_id, &trash_id, &junk_id] {
|
||||
message_ids.push(
|
||||
client
|
||||
.email_import(
|
||||
format!(
|
||||
concat!(
|
||||
"From: bill@example.com\r\n",
|
||||
"To: jdoe@example.com\r\n",
|
||||
"Subject: TPS Report #{} {}\r\n",
|
||||
"\r\n",
|
||||
"I'm going to need those TPS reports ASAP. ",
|
||||
"So, if you could do that, that'd be great."
|
||||
),
|
||||
pass, folder_id
|
||||
)
|
||||
.into_bytes(),
|
||||
[folder_id],
|
||||
None::<Vec<&str>>,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.take_id(),
|
||||
);
|
||||
}
|
||||
|
||||
if pass == 1 {
|
||||
changes = get_changes(&server).await;
|
||||
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Make sure both messages and changes are present
|
||||
assert_eq!(
|
||||
server
|
||||
.get_document_ids(1, Collection::Email)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.len(),
|
||||
6
|
||||
);
|
||||
|
||||
// Purge junk/trash messages and old changes
|
||||
server.purge_account(1).await;
|
||||
|
||||
// Only 4 messages should remain
|
||||
assert_eq!(
|
||||
server
|
||||
.get_document_ids(1, Collection::Email)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.len(),
|
||||
4
|
||||
);
|
||||
assert_eq!(
|
||||
server
|
||||
.get_tag(
|
||||
1,
|
||||
Collection::Email,
|
||||
Property::MailboxIds,
|
||||
TagValue::Id(INBOX_ID)
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.len(),
|
||||
2
|
||||
);
|
||||
assert_eq!(
|
||||
server
|
||||
.get_tag(
|
||||
1,
|
||||
Collection::Email,
|
||||
Property::MailboxIds,
|
||||
TagValue::Id(TRASH_ID)
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
server
|
||||
.get_tag(
|
||||
1,
|
||||
Collection::Email,
|
||||
Property::MailboxIds,
|
||||
TagValue::Id(JUNK_ID)
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
|
||||
// Compare changes
|
||||
let new_changes = get_changes(&server).await;
|
||||
assert!(!changes.is_empty());
|
||||
assert!(!new_changes.is_empty());
|
||||
for change in changes {
|
||||
assert!(
|
||||
!new_changes.contains(&change),
|
||||
"Change {:?} was not purged",
|
||||
change
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_changes(server: &JMAP) -> AHashSet<(u64, u8)> {
|
||||
let mut changes = AHashSet::new();
|
||||
server
|
||||
.core
|
||||
.storage
|
||||
.data
|
||||
.iterate(
|
||||
IterateParams::new(
|
||||
LogKey {
|
||||
account_id: 0,
|
||||
collection: 0,
|
||||
change_id: 0,
|
||||
},
|
||||
LogKey {
|
||||
account_id: u32::MAX,
|
||||
collection: u8::MAX,
|
||||
change_id: u64::MAX,
|
||||
},
|
||||
)
|
||||
.ascending()
|
||||
.no_values(),
|
||||
|key, _| {
|
||||
assert_eq!(key.deserialize_be_u32(0).unwrap(), 1);
|
||||
changes.insert((
|
||||
key.deserialize_be_u64(key.len() - U64_LEN).unwrap(),
|
||||
key[U32_LEN],
|
||||
));
|
||||
Ok(true)
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
changes
|
||||
}
|
||||
@@ -22,8 +22,8 @@
|
||||
*/
|
||||
|
||||
use crate::jmap::{
|
||||
assert_is_empty, delivery::SmtpConnection, jmap_raw_request, mailbox::destroy_all_mailboxes,
|
||||
test_account_login,
|
||||
assert_is_empty, delivery::SmtpConnection, emails_purge_tombstoned, jmap_raw_request,
|
||||
mailbox::destroy_all_mailboxes, test_account_login,
|
||||
};
|
||||
use directory::backend::internal::manage::ManageDirectory;
|
||||
use jmap::{blob::upload::DISABLE_UPLOAD_QUOTA, mailbox::INBOX_ID};
|
||||
@@ -191,6 +191,7 @@ pub async fn test(params: &mut JMAPTest) {
|
||||
for message_id in message_ids {
|
||||
client.email_destroy(&message_id).await.unwrap();
|
||||
}
|
||||
emails_purge_tombstoned(&server).await;
|
||||
assert_eq!(
|
||||
server
|
||||
.get_used_quota(account_id.document_id())
|
||||
@@ -238,6 +239,7 @@ pub async fn test(params: &mut JMAPTest) {
|
||||
for message_id in message_ids {
|
||||
client.email_destroy(&message_id).await.unwrap();
|
||||
}
|
||||
emails_purge_tombstoned(&server).await;
|
||||
assert_eq!(
|
||||
server
|
||||
.get_used_quota(account_id.document_id())
|
||||
@@ -300,6 +302,7 @@ pub async fn test(params: &mut JMAPTest) {
|
||||
for message_id in message_ids {
|
||||
client.email_destroy(&message_id).await.unwrap();
|
||||
}
|
||||
emails_purge_tombstoned(&server).await;
|
||||
assert_eq!(
|
||||
server
|
||||
.get_used_quota(account_id.document_id())
|
||||
|
||||
Reference in New Issue
Block a user