diff --git a/crates/common/src/core.rs b/crates/common/src/core.rs index 7f54485d..11731ea5 100644 --- a/crates/common/src/core.rs +++ b/crates/common/src/core.rs @@ -534,16 +534,17 @@ impl Server { } pub async fn delete_changes(&self, account_id: u32, max_entries: usize) -> trc::Result<()> { - for collection in [ - SyncCollection::Email.into(), - SyncCollection::Thread.into(), - SyncCollection::Identity.into(), - SyncCollection::EmailSubmission.into(), - SyncCollection::SieveScript.into(), - SyncCollection::FileNode.into(), - SyncCollection::AddressBook.into(), - SyncCollection::Calendar.into(), + for sync_collection in [ + SyncCollection::Email, + SyncCollection::Thread, + SyncCollection::Identity, + SyncCollection::EmailSubmission, + SyncCollection::SieveScript, + SyncCollection::FileNode, + SyncCollection::AddressBook, + SyncCollection::Calendar, ] { + let collection = sync_collection.into(); let from_key = LogKey { account_id, collection, @@ -590,6 +591,27 @@ impl Server { .await .caused_by(trc::location!())?; + // Delete vanished items + if let Some(vanished_collection) = + sync_collection.vanished_collection().map(u8::from) + { + self.store() + .delete_range( + LogKey { + account_id, + collection: vanished_collection, + change_id: 0, + }, + LogKey { + account_id, + collection: vanished_collection, + change_id: first_change_id, + }, + ) + .await + .caused_by(trc::location!())?; + } + // Write truncation entry for cache let mut batch = BatchBuilder::new(); batch diff --git a/crates/dav/src/calendar/copy_move.rs b/crates/dav/src/calendar/copy_move.rs index 8c534d37..7e8c9867 100644 --- a/crates/dav/src/calendar/copy_move.rs +++ b/crates/dav/src/calendar/copy_move.rs @@ -15,7 +15,7 @@ use http_proto::HttpResponse; use hyper::StatusCode; use jmap_proto::types::{ acl::Acl, - collection::{Collection, SyncCollection}, + collection::{Collection, SyncCollection, VanishedCollection}, }; use store::write::BatchBuilder; use trc::AddContext; @@ -195,6 +195,7 @@ impl CalendarCopyMoveRequestHandler for Server { from_account_id, from_resource.document_id(), from_children_ids, + from_resources.format_collection(from_resource_name), to_account_id, to_resource.document_id().into(), to_document_ids, @@ -236,6 +237,7 @@ impl CalendarCopyMoveRequestHandler for Server { from_account_id, from_resource.document_id(), from_calendar_id, + from_resources.format_item(from_resource_name), to_account_id, to_resource.document_id().into(), to_calendar_id, @@ -302,6 +304,7 @@ impl CalendarCopyMoveRequestHandler for Server { from_account_id, from_resource.document_id(), from_calendar_id, + from_resources.format_item(from_resource_name), to_account_id, None, to_calendar_id, @@ -316,6 +319,7 @@ impl CalendarCopyMoveRequestHandler for Server { from_resource.document_id(), from_calendar_id, new_name, + from_resources.format_item(from_resource_name), ) .await } @@ -376,6 +380,7 @@ impl CalendarCopyMoveRequestHandler for Server { } else { return Err(DavError::Code(StatusCode::BAD_GATEWAY)); }, + from_resources.format_collection(from_resource_name), to_account_id, None, vec![], @@ -390,6 +395,7 @@ impl CalendarCopyMoveRequestHandler for Server { from_account_id, from_resource.document_id(), new_name, + from_resources.format_collection(from_resource_name), ) .await } @@ -404,6 +410,7 @@ impl CalendarCopyMoveRequestHandler for Server { } else { vec![] }, + from_resources.format_collection(from_resource_name), to_account_id, None, vec![], @@ -507,6 +514,7 @@ async fn copy_event( to_account_id, to_document_id, to_calendar_id, + None, &mut batch, ) .caused_by(trc::location!())?; @@ -532,6 +540,7 @@ async fn move_event( from_account_id: u32, from_document_id: u32, from_calendar_id: u32, + from_resource_path: String, to_account_id: u32, to_document_id: Option, to_calendar_id: u32, @@ -599,6 +608,7 @@ async fn move_event( &mut batch, ) .caused_by(trc::location!())?; + batch.log_vanished_item(VanishedCollection::Calendar, from_resource_path); } else { let mut new_event = event .deserialize::() @@ -614,6 +624,7 @@ async fn move_event( from_account_id, from_document_id, from_calendar_id, + from_resource_path.into(), &mut batch, ) .caused_by(trc::location!())?; @@ -645,6 +656,7 @@ async fn move_event( to_account_id, to_document_id, to_calendar_id, + None, &mut batch, ) .caused_by(trc::location!())?; @@ -671,6 +683,7 @@ async fn rename_event( document_id: u32, calendar_id: u32, new_name: &str, + from_resource_path: String, ) -> crate::Result { // Fetch event let event_ = server @@ -697,6 +710,7 @@ async fn rename_event( new_event .update(access_token, event, account_id, document_id, &mut batch) .caused_by(trc::location!())?; + batch.log_vanished_item(VanishedCollection::Calendar, from_resource_path); server .commit_batch(batch) .await @@ -712,6 +726,7 @@ async fn copy_container( from_account_id: u32, from_document_id: u32, from_children_ids: Vec, + from_resource_path: String, to_account_id: u32, to_document_id: Option, to_children_ids: Vec, @@ -736,7 +751,13 @@ async fn copy_container( if remove_source { DestroyArchive(old_calendar) - .delete(access_token, from_account_id, from_document_id, &mut batch) + .delete( + access_token, + from_account_id, + from_document_id, + from_resource_path.into(), + &mut batch, + ) .caused_by(trc::location!())?; } @@ -773,6 +794,7 @@ async fn copy_container( to_account_id, to_document_id, to_children_ids, + None, &mut batch, ) .await @@ -854,6 +876,7 @@ async fn copy_container( from_account_id, from_child_document_id, from_document_id, + None, &mut batch, ) .caused_by(trc::location!())?; @@ -903,6 +926,7 @@ async fn rename_container( account_id: u32, document_id: u32, new_name: &str, + from_resource_path: String, ) -> crate::Result { // Fetch calendar let calendar_ = server @@ -922,6 +946,7 @@ async fn rename_container( new_calendar .update(access_token, calendar, account_id, document_id, &mut batch) .caused_by(trc::location!())?; + batch.log_vanished_item(VanishedCollection::Calendar, from_resource_path); server .commit_batch(batch) .await diff --git a/crates/dav/src/calendar/delete.rs b/crates/dav/src/calendar/delete.rs index 86eefc61..27df7a47 100644 --- a/crates/dav/src/calendar/delete.rs +++ b/crates/dav/src/calendar/delete.rs @@ -116,6 +116,7 @@ impl CalendarDeleteRequestHandler for Server { .filter(|r| !r.is_container()) .map(|r| r.document_id()) .collect::>(), + resources.format_resource(delete_resource).into(), &mut batch, ) .await @@ -163,6 +164,7 @@ impl CalendarDeleteRequestHandler for Server { account_id, document_id, calendar_id, + resources.format_resource(delete_resource).into(), &mut batch, ) .caused_by(trc::location!())?; diff --git a/crates/dav/src/card/copy_move.rs b/crates/dav/src/card/copy_move.rs index 471e2536..10e63286 100644 --- a/crates/dav/src/card/copy_move.rs +++ b/crates/dav/src/card/copy_move.rs @@ -15,7 +15,7 @@ use http_proto::HttpResponse; use hyper::StatusCode; use jmap_proto::types::{ acl::Acl, - collection::{Collection, SyncCollection}, + collection::{Collection, SyncCollection, VanishedCollection}, }; use store::write::BatchBuilder; use trc::AddContext; @@ -195,6 +195,7 @@ impl CardCopyMoveRequestHandler for Server { from_account_id, from_resource.document_id(), from_children_ids, + from_resources.format_collection(from_resource_name), to_account_id, to_resource.document_id().into(), to_document_ids, @@ -236,6 +237,7 @@ impl CardCopyMoveRequestHandler for Server { from_account_id, from_resource.document_id(), from_addressbook_id, + from_resources.format_item(from_resource_name), to_account_id, to_resource.document_id().into(), to_addressbook_id, @@ -302,6 +304,7 @@ impl CardCopyMoveRequestHandler for Server { from_account_id, from_resource.document_id(), from_addressbook_id, + from_resources.format_item(from_resource_name), to_account_id, None, to_addressbook_id, @@ -316,6 +319,7 @@ impl CardCopyMoveRequestHandler for Server { from_resource.document_id(), from_addressbook_id, new_name, + from_resources.format_item(from_resource_name), ) .await } @@ -376,6 +380,7 @@ impl CardCopyMoveRequestHandler for Server { } else { return Err(DavError::Code(StatusCode::BAD_GATEWAY)); }, + from_resources.format_collection(from_resource_name), to_account_id, None, vec![], @@ -390,6 +395,7 @@ impl CardCopyMoveRequestHandler for Server { from_account_id, from_resource.document_id(), new_name, + from_resources.format_collection(from_resource_name), ) .await } @@ -404,6 +410,7 @@ impl CardCopyMoveRequestHandler for Server { } else { vec![] }, + from_resources.format_collection(from_resource_name), to_account_id, None, vec![], @@ -507,6 +514,7 @@ async fn copy_card( to_account_id, to_document_id, to_addressbook_id, + None, &mut batch, ) .caused_by(trc::location!())?; @@ -532,6 +540,7 @@ async fn move_card( from_account_id: u32, from_document_id: u32, from_addressbook_id: u32, + from_resource_path: String, to_account_id: u32, to_document_id: Option, to_addressbook_id: u32, @@ -599,6 +608,7 @@ async fn move_card( &mut batch, ) .caused_by(trc::location!())?; + batch.log_vanished_item(VanishedCollection::AddressBook, from_resource_path); } else { let mut new_card = card .deserialize::() @@ -614,6 +624,7 @@ async fn move_card( from_account_id, from_document_id, from_addressbook_id, + from_resource_path.into(), &mut batch, ) .caused_by(trc::location!())?; @@ -645,6 +656,7 @@ async fn move_card( to_account_id, to_document_id, to_addressbook_id, + None, &mut batch, ) .caused_by(trc::location!())?; @@ -671,6 +683,7 @@ async fn rename_card( document_id: u32, addressbook_id: u32, new_name: &str, + from_resource_path: String, ) -> crate::Result { // Fetch card let card_ = server @@ -697,6 +710,7 @@ async fn rename_card( new_card .update(access_token, card, account_id, document_id, &mut batch) .caused_by(trc::location!())?; + batch.log_vanished_item(VanishedCollection::AddressBook, from_resource_path); server .commit_batch(batch) .await @@ -712,6 +726,7 @@ async fn copy_container( from_account_id: u32, from_document_id: u32, from_children_ids: Vec, + from_resource_path: String, to_account_id: u32, to_document_id: Option, to_children_ids: Vec, @@ -736,7 +751,13 @@ async fn copy_container( if remove_source { DestroyArchive(old_book) - .delete(access_token, from_account_id, from_document_id, &mut batch) + .delete( + access_token, + from_account_id, + from_document_id, + from_resource_path.into(), + &mut batch, + ) .caused_by(trc::location!())?; } @@ -764,6 +785,7 @@ async fn copy_container( to_account_id, to_document_id, to_children_ids, + None, &mut batch, ) .await @@ -844,6 +866,7 @@ async fn copy_container( from_account_id, from_child_document_id, from_document_id, + None, &mut batch, ) .caused_by(trc::location!())?; @@ -893,6 +916,7 @@ async fn rename_container( account_id: u32, document_id: u32, new_name: &str, + from_resource_path: String, ) -> crate::Result { // Fetch book let book_ = server @@ -912,6 +936,7 @@ async fn rename_container( new_book .update(access_token, book, account_id, document_id, &mut batch) .caused_by(trc::location!())?; + batch.log_vanished_item(VanishedCollection::AddressBook, from_resource_path); server .commit_batch(batch) .await diff --git a/crates/dav/src/card/delete.rs b/crates/dav/src/card/delete.rs index ba2117fc..30203e4c 100644 --- a/crates/dav/src/card/delete.rs +++ b/crates/dav/src/card/delete.rs @@ -117,6 +117,7 @@ impl CardDeleteRequestHandler for Server { .filter(|r| !r.is_container()) .map(|r| r.document_id()) .collect::>(), + resources.format_resource(delete_resource).into(), &mut batch, ) .await @@ -168,6 +169,7 @@ impl CardDeleteRequestHandler for Server { account_id, document_id, addressbook_id, + resources.format_resource(delete_resource).into(), &mut batch, ) .caused_by(trc::location!())?; diff --git a/crates/dav/src/common/propfind.rs b/crates/dav/src/common/propfind.rs index 47757f6a..a25c51d5 100644 --- a/crates/dav/src/common/propfind.rs +++ b/crates/dav/src/common/propfind.rs @@ -445,22 +445,35 @@ impl PropFindRequestHandler for Server { .changes(account_id, sync_collection, Query::Since(id)) .await .caused_by(trc::location!())?; + let mut vanished: Vec = Vec::new(); // Merge changes let mut total_changes = 0; + let mut maybe_has_vanished = false; if container_has_children { let mut container_changes = RoaringBitmap::new(); let mut item_changes = RoaringBitmap::new(); for change in changes.changes { match change { - Change::InsertItem(id) | Change::UpdateItem(id) => { + Change::InsertItem(id) => { item_changes.insert(id as u32); } - Change::InsertContainer(id) | Change::UpdateContainer(id) => { + Change::UpdateItem(id) => { + maybe_has_vanished = true; + item_changes.insert(id as u32); + } + Change::InsertContainer(id) => { container_changes.insert(id as u32); } - _ => (), + Change::UpdateContainer(id) => { + maybe_has_vanished = true; + container_changes.insert(id as u32); + } + Change::DeleteContainer(_) | Change::DeleteItem(_) => { + maybe_has_vanished = true; + } + Change::UpdateContainerProperty(_) => (), } } @@ -479,10 +492,17 @@ impl PropFindRequestHandler for Server { } else { let changes = RoaringBitmap::from_iter( changes.changes.iter().filter_map(|change| match change { - Change::InsertItem(id) - | Change::UpdateItem(id) - | Change::InsertContainer(id) - | Change::UpdateContainer(id) => Some(*id as u32), + Change::InsertItem(id) | Change::InsertContainer(id) => { + Some(*id as u32) + } + Change::UpdateItem(id) | Change::UpdateContainer(id) => { + maybe_has_vanished = true; + Some(*id as u32) + } + Change::DeleteContainer(_) | Change::DeleteItem(_) => { + maybe_has_vanished = true; + None + } _ => None, }), ); @@ -495,10 +515,40 @@ impl PropFindRequestHandler for Server { } } + if maybe_has_vanished { + vanished = self + .store() + .vanished( + account_id, + sync_collection.vanished_collection().unwrap(), + Query::Since(id), + ) + .await + .caused_by(trc::location!())?; + total_changes += vanished.len(); + } + // Truncate changes if total_changes > limit { let mut offset = limit * seq as usize; let mut total_changes = 0; + + // Add vanished items to response + for item in vanished { + if offset > 0 { + offset -= 1; + } else if total_changes < limit { + response.add_response(Response::new_status( + [item], + StatusCode::NOT_FOUND, + )); + total_changes += 1; + } else { + is_sync_limited = true; + } + } + + // Add items to document set for document_ids in [&mut display_containers, &mut display_children] .into_iter() .flatten() @@ -520,6 +570,14 @@ impl PropFindRequestHandler for Server { if is_sync_limited { response.set_sync_token(Urn::Sync { id, seq: seq + 1 }.to_string()); } + } else { + // Add vanished items to response + for item in vanished { + response.add_response(Response::new_status( + [item], + StatusCode::NOT_FOUND, + )); + } } if !is_sync_limited { diff --git a/crates/dav/src/file/copy_move.rs b/crates/dav/src/file/copy_move.rs index 85eea899..bdd6b5e1 100644 --- a/crates/dav/src/file/copy_move.rs +++ b/crates/dav/src/file/copy_move.rs @@ -23,7 +23,7 @@ use http_proto::HttpResponse; use hyper::StatusCode; use jmap_proto::types::{ acl::Acl, - collection::{Collection, SyncCollection}, + collection::{Collection, SyncCollection, VanishedCollection}, }; use std::sync::Arc; use store::{ @@ -208,7 +208,19 @@ impl FileCopyMoveRequestHandler for Server { && is_move { // Rename - return rename_item(self, access_token, from_resource, destination).await; + let from_resource_path = if from_resource.resource.is_container { + from_resources.format_collection(from_resource_name) + } else { + from_resources.format_item(from_resource_name) + }; + return rename_item( + self, + access_token, + from_resource, + from_resource_path, + destination, + ) + .await; } // Validate quota @@ -239,7 +251,7 @@ impl FileCopyMoveRequestHandler for Server { let mut sorted_ids = Vec::with_capacity(ids.len()); sorted_ids.extend(ids.into_iter().map(|a| a.document_id())); DestroyArchive(sorted_ids) - .delete(self, access_token, destination.account_id) + .delete(self, access_token, destination.account_id, None) .await .caused_by(trc::location!())?; } @@ -273,10 +285,23 @@ impl FileCopyMoveRequestHandler for Server { } (false, true) => { if let Some(delete_destination) = delete_destination { - overwrite_and_delete_item(self, access_token, from_resource, delete_destination) - .await + overwrite_and_delete_item( + self, + access_token, + from_resource, + from_resources.format_item(from_resource_name), + delete_destination, + ) + .await } else { - move_item(self, access_token, from_resource, destination).await + move_item( + self, + access_token, + from_resource, + from_resources.format_item(from_resource_name), + destination, + ) + .await } } @@ -357,6 +382,10 @@ async fn move_container( ) .caused_by(trc::location!())? .etag(); + batch.with_account_id(from_account_id).log_vanished_item( + VanishedCollection::FileNode, + from_resources.format_collection(from_resource_name), + ); server .commit_batch(batch) .await @@ -490,6 +519,10 @@ async fn copy_container( .caused_by(trc::location!())? .commit_point(); } + batch.with_account_id(from_account_id).log_vanished_item( + VanishedCollection::FileNode, + from_resources.format_collection(from_resource_name), + ); } // Write changes @@ -508,6 +541,7 @@ async fn overwrite_and_delete_item( server: &Server, access_token: &AccessToken, from_resource: UriResource, + from_resource_path: String, destination: Destination, ) -> crate::Result { let from_account_id = from_resource.account_id; @@ -557,7 +591,13 @@ async fn overwrite_and_delete_item( .caused_by(trc::location!())? .etag(); DestroyArchive(source_node_) - .delete(access_token, from_account_id, from_document_id, &mut batch) + .delete( + access_token, + from_account_id, + from_document_id, + &mut batch, + from_resource_path, + ) .caused_by(trc::location!())?; server .commit_batch(batch) @@ -628,6 +668,7 @@ async fn move_item( server: &Server, access_token: &AccessToken, from_resource: UriResource, + from_resource_path: String, destination: Destination, ) -> crate::Result { let from_account_id = from_resource.account_id; @@ -652,6 +693,7 @@ async fn move_item( let mut batch = BatchBuilder::new(); let etag = if from_account_id == to_account_id { // Destination is in the same account: just update the parent id + batch.log_vanished_item(VanishedCollection::FileNode, from_resource_path); new_node .update( access_token, @@ -674,7 +716,13 @@ async fn move_item( .caused_by(trc::location!())? .etag(); DestroyArchive(node) - .delete(access_token, from_account_id, from_document_id, &mut batch) + .delete( + access_token, + from_account_id, + from_document_id, + &mut batch, + from_resource_path, + ) .caused_by(trc::location!())?; etag }; @@ -732,6 +780,7 @@ async fn rename_item( server: &Server, access_token: &AccessToken, from_resource: UriResource, + from_resource_path: String, destination: Destination, ) -> crate::Result { let from_account_id = from_resource.account_id; @@ -760,6 +809,7 @@ async fn rename_item( ) .caused_by(trc::location!())? .etag(); + batch.log_vanished_item(VanishedCollection::FileNode, from_resource_path); server .commit_batch(batch) .await diff --git a/crates/dav/src/file/delete.rs b/crates/dav/src/file/delete.rs index 5f74212e..fcb38d60 100644 --- a/crates/dav/src/file/delete.rs +++ b/crates/dav/src/file/delete.rs @@ -56,7 +56,10 @@ impl FileDeleteRequestHandler for Server { // Sort ids descending from the deepest to the root ids.sort_unstable_by_key(|b| std::cmp::Reverse(b.hierarchy_seq())); - let document_id = ids.last().map(|a| a.document_id()).unwrap(); + let (document_id, full_delete_path) = ids + .last() + .map(|a| (a.document_id(), resources.format_resource(*a))) + .unwrap(); let mut sorted_ids = Vec::with_capacity(ids.len()); sorted_ids.extend(ids.into_iter().map(|a| a.document_id())); @@ -87,7 +90,7 @@ impl FileDeleteRequestHandler for Server { .await?; DestroyArchive(sorted_ids) - .delete(self, access_token, account_id) + .delete(self, access_token, account_id, full_delete_path.into()) .await?; Ok(HttpResponse::new(StatusCode::NO_CONTENT)) diff --git a/crates/email/src/message/delete.rs b/crates/email/src/message/delete.rs index 3d892431..b92636e0 100644 --- a/crates/email/src/message/delete.rs +++ b/crates/email/src/message/delete.rs @@ -7,6 +7,7 @@ use super::metadata::MessageData; use crate::{cache::MessageCacheFetch, mailbox::*, message::metadata::MessageMetadata}; use common::{KV_LOCK_PURGE_ACCOUNT, Server, storage::index::ObjectIndexBuilder}; +use jmap_proto::types::collection::VanishedCollection; use jmap_proto::types::{collection::Collection, property::Property}; use std::future::Future; use std::time::Duration; @@ -64,15 +65,18 @@ impl EmailDeletion for Server { &document_ids, |document_id, data_| { // Add changes to batch + let metadata = data_ + .to_unarchived::() + .caused_by(trc::location!())?; + for mailbox in metadata.inner.mailboxes.iter() { + batch.log_vanished_item( + VanishedCollection::Email, + (mailbox.mailbox_id.to_native(), mailbox.uid.to_native()), + ); + } batch .update_document(document_id) - .custom( - ObjectIndexBuilder::<_, ()>::new().with_current( - data_ - .to_unarchived::() - .caused_by(trc::location!())?, - ), - ) + .custom(ObjectIndexBuilder::<_, ()>::new().with_current(metadata)) .caused_by(trc::location!())? .tag(Property::MailboxIds, TagValue::Id(TOMBSTONE_ID)) .commit_point(); diff --git a/crates/email/src/message/metadata.rs b/crates/email/src/message/metadata.rs index da3e0386..6ec29254 100644 --- a/crates/email/src/message/metadata.rs +++ b/crates/email/src/message/metadata.rs @@ -603,4 +603,11 @@ impl ArchivedMessageData { pub fn has_mailbox_id(&self, mailbox_id: u32) -> bool { self.mailboxes.iter().any(|m| m.mailbox_id == mailbox_id) } + + pub fn message_uid(&self, mailbox_id: u32) -> Option { + self.mailboxes + .iter() + .find(|m| m.mailbox_id == mailbox_id) + .map(|m| m.uid.to_native()) + } } diff --git a/crates/groupware/src/calendar/storage.rs b/crates/groupware/src/calendar/storage.rs index b03d47ee..84e12c33 100644 --- a/crates/groupware/src/calendar/storage.rs +++ b/crates/groupware/src/calendar/storage.rs @@ -6,7 +6,7 @@ use crate::DestroyArchive; use common::{Server, auth::AccessToken, storage::index::ObjectIndexBuilder}; -use jmap_proto::types::collection::Collection; +use jmap_proto::types::collection::{Collection, VanishedCollection}; use store::write::{Archive, BatchBuilder, now}; use trc::AddContext; @@ -132,6 +132,7 @@ impl Calendar { } impl DestroyArchive> { + #[allow(clippy::too_many_arguments)] pub async fn delete_with_events( self, server: &Server, @@ -139,6 +140,7 @@ impl DestroyArchive> { account_id: u32, document_id: u32, children_ids: Vec, + delete_path: Option, batch: &mut BatchBuilder, ) -> trc::Result<()> { // Process deletions @@ -158,12 +160,13 @@ impl DestroyArchive> { account_id, document_id, calendar_id, + None, batch, )?; } } - self.delete(access_token, account_id, document_id, batch) + self.delete(access_token, account_id, document_id, delete_path, batch) } pub fn delete( @@ -171,6 +174,7 @@ impl DestroyArchive> { access_token: &AccessToken, account_id: u32, document_id: u32, + delete_path: Option, batch: &mut BatchBuilder, ) -> trc::Result<()> { let calendar = self.0; @@ -184,8 +188,11 @@ impl DestroyArchive> { .with_tenant_id(access_token) .with_current(calendar), ) - .caused_by(trc::location!())? - .commit_point(); + .caused_by(trc::location!())?; + if let Some(delete_path) = delete_path { + batch.log_vanished_item(VanishedCollection::Calendar, delete_path); + } + batch.commit_point(); Ok(()) } @@ -198,6 +205,7 @@ impl DestroyArchive> { account_id: u32, document_id: u32, calendar_id: u32, + delete_path: Option, batch: &mut BatchBuilder, ) -> trc::Result<()> { let event = self.0; @@ -238,6 +246,10 @@ impl DestroyArchive> { .caused_by(trc::location!())?; } + if let Some(delete_path) = delete_path { + batch.log_vanished_item(VanishedCollection::Calendar, delete_path); + } + batch.commit_point(); } diff --git a/crates/groupware/src/contact/storage.rs b/crates/groupware/src/contact/storage.rs index 1efb638e..23a279cc 100644 --- a/crates/groupware/src/contact/storage.rs +++ b/crates/groupware/src/contact/storage.rs @@ -5,7 +5,7 @@ */ use common::{Server, auth::AccessToken, storage::index::ObjectIndexBuilder}; -use jmap_proto::types::collection::Collection; +use jmap_proto::types::collection::{Collection, VanishedCollection}; use store::write::{Archive, BatchBuilder, now}; use trc::AddContext; @@ -123,6 +123,7 @@ impl AddressBook { } impl DestroyArchive> { + #[allow(clippy::too_many_arguments)] pub async fn delete_with_cards( self, server: &Server, @@ -130,6 +131,7 @@ impl DestroyArchive> { account_id: u32, document_id: u32, children_ids: Vec, + delete_path: Option, batch: &mut BatchBuilder, ) -> trc::Result<()> { // Process deletions @@ -149,12 +151,13 @@ impl DestroyArchive> { account_id, document_id, addressbook_id, + None, batch, )?; } } - self.delete(access_token, account_id, document_id, batch) + self.delete(access_token, account_id, document_id, delete_path, batch) } pub fn delete( @@ -162,6 +165,7 @@ impl DestroyArchive> { access_token: &AccessToken, account_id: u32, document_id: u32, + delete_path: Option, batch: &mut BatchBuilder, ) -> trc::Result<()> { let book = self.0; @@ -175,8 +179,13 @@ impl DestroyArchive> { .with_tenant_id(access_token) .with_current(book), ) - .caused_by(trc::location!())? - .commit_point(); + .caused_by(trc::location!())?; + + if let Some(delete_path) = delete_path { + batch.log_vanished_item(VanishedCollection::AddressBook, delete_path); + } + + batch.commit_point(); Ok(()) } @@ -189,6 +198,7 @@ impl DestroyArchive> { account_id: u32, document_id: u32, addressbook_id: u32, + delete_path: Option, batch: &mut BatchBuilder, ) -> trc::Result<()> { let card = self.0; @@ -229,6 +239,10 @@ impl DestroyArchive> { .caused_by(trc::location!())?; } + if let Some(delete_path) = delete_path { + batch.log_vanished_item(VanishedCollection::AddressBook, delete_path); + } + batch.commit_point(); } diff --git a/crates/groupware/src/file/storage.rs b/crates/groupware/src/file/storage.rs index 1f4e5fbb..3f7cc007 100644 --- a/crates/groupware/src/file/storage.rs +++ b/crates/groupware/src/file/storage.rs @@ -5,7 +5,7 @@ */ use common::{Server, auth::AccessToken, storage::index::ObjectIndexBuilder}; -use jmap_proto::types::collection::Collection; +use jmap_proto::types::collection::{Collection, VanishedCollection}; use store::write::{Archive, BatchBuilder, now}; use trc::AddContext; @@ -71,6 +71,7 @@ impl DestroyArchive> { account_id: u32, document_id: u32, batch: &mut BatchBuilder, + path: String, ) -> trc::Result<()> { // Prepare write batch batch @@ -82,6 +83,7 @@ impl DestroyArchive> { .with_current(self.0) .with_tenant_id(access_token), )? + .log_vanished_item(VanishedCollection::FileNode, path) .commit_point(); Ok(()) } @@ -93,6 +95,7 @@ impl DestroyArchive> { server: &Server, access_token: &AccessToken, account_id: u32, + delete_path: Option, ) -> trc::Result<()> { // Process deletions let mut batch = BatchBuilder::new(); @@ -122,6 +125,9 @@ impl DestroyArchive> { // Write changes if !batch.is_empty() { + if let Some(delete_path) = delete_path { + batch.log_vanished_item(VanishedCollection::FileNode, delete_path); + } server .commit_batch(batch) .await diff --git a/crates/imap/src/op/copy_move.rs b/crates/imap/src/op/copy_move.rs index c137e5b5..13da7977 100644 --- a/crates/imap/src/op/copy_move.rs +++ b/crates/imap/src/op/copy_move.rs @@ -22,7 +22,12 @@ use imap_proto::{ }; use jmap_proto::{ error::set::SetErrorType, - types::{acl::Acl, collection::Collection, state::StateChange, type_state::DataType}, + types::{ + acl::Acl, + collection::{Collection, VanishedCollection}, + state::StateChange, + type_state::DataType, + }, }; use std::{sync::Arc, time::Instant}; use store::{ @@ -250,6 +255,12 @@ impl SessionData { .with_changes(new_data), ) .imap_ctx(&arguments.tag, trc::location!())?; + if is_move { + batch.log_vanished_item( + VanishedCollection::Email, + (src_mailbox.id.mailbox_id, imap_id.uid), + ); + } // Add bayes train task if can_spam_train { diff --git a/crates/imap/src/op/expunge.rs b/crates/imap/src/op/expunge.rs index f1e23be0..093032f5 100644 --- a/crates/imap/src/op/expunge.rs +++ b/crates/imap/src/op/expunge.rs @@ -11,16 +11,25 @@ use common::{listener::SessionStream, storage::index::ObjectIndexBuilder}; use directory::Permission; use email::{ cache::{MessageCacheFetch, email::MessageCacheAccess}, - message::{delete::EmailDeletion, metadata::MessageData}, + mailbox::TOMBSTONE_ID, + message::metadata::MessageData, }; use imap_proto::{ Command, ResponseCode, ResponseType, StatusResponse, parser::parse_sequence_set, receiver::{Request, Token}, }; -use jmap_proto::types::{acl::Acl, collection::Collection, keyword::Keyword}; +use jmap_proto::types::{ + acl::Acl, + collection::{Collection, VanishedCollection}, + keyword::Keyword, + property::Property, +}; use std::{sync::Arc, time::Instant}; -use store::{roaring::RoaringBitmap, write::BatchBuilder}; +use store::{ + roaring::RoaringBitmap, + write::{BatchBuilder, TagValue}, +}; use trc::AddContext; impl Session { @@ -158,53 +167,61 @@ impl SessionData { deleted_ids: &RoaringBitmap, batch: &mut BatchBuilder, ) -> trc::Result<()> { - let mut destroy_ids = RoaringBitmap::new(); batch .with_account_id(account_id) .with_collection(Collection::Email); self.server - .get_archives(account_id, Collection::Email, deleted_ids, |id, data_| { - let data = data_ - .to_unarchived::() - .caused_by(trc::location!())?; + .get_archives( + account_id, + Collection::Email, + deleted_ids, + |document_id, data_| { + let metadata = data_ + .to_unarchived::() + .caused_by(trc::location!())?; - if !data.inner.has_mailbox_id(mailbox_id) { - return Ok(true); - } else if data.inner.mailboxes.len() == 1 { - destroy_ids.insert(id); - return Ok(true); - } + if let Some(message_uid) = metadata.inner.message_uid(mailbox_id) { + // Add vanished items + batch.update_document(document_id); + batch.log_vanished_item( + VanishedCollection::Email, + (mailbox_id, message_uid), + ); - // Untag message from this mailbox and remove Deleted flag - let mut new_data = data.deserialize().caused_by(trc::location!())?; - new_data.remove_mailbox(mailbox_id); - new_data.remove_keyword(&Keyword::Deleted); + if metadata.inner.mailboxes.len() == 1 { + // Tombstone message + batch + .custom(ObjectIndexBuilder::<_, ()>::new().with_current(metadata)) + .caused_by(trc::location!())? + .tag(Property::MailboxIds, TagValue::Id(TOMBSTONE_ID)) + .commit_point(); + } else { + // Untag message from this mailbox and remove Deleted flag + let mut new_metadata = metadata + .deserialize::() + .caused_by(trc::location!())?; + new_metadata.remove_mailbox(mailbox_id); + new_metadata.remove_keyword(&Keyword::Deleted); - // Write changes - batch - .update_document(id) - .custom( - ObjectIndexBuilder::new() - .with_current(data) - .with_changes(new_data), - ) - .caused_by(trc::location!())? - .commit_point(); + // Write changes + batch + .custom( + ObjectIndexBuilder::new() + .with_current(metadata) + .with_changes(new_metadata), + ) + .caused_by(trc::location!())? + .commit_point(); + } + } - Ok(true) - }) + Ok(true) + }, + ) .await .caused_by(trc::location!())?; - if !destroy_ids.is_empty() { - // Delete message from all mailboxes - self.server - .emails_tombstone(account_id, batch, destroy_ids) - .await - .caused_by(trc::location!())?; - } - Ok(()) } } diff --git a/crates/imap/src/op/fetch.rs b/crates/imap/src/op/fetch.rs index 6cf94137..c77dd918 100644 --- a/crates/imap/src/op/fetch.rs +++ b/crates/imap/src/op/fetch.rs @@ -34,7 +34,7 @@ use imap_proto::{ }; use jmap_proto::types::{ acl::Acl, - collection::{Collection, SyncCollection}, + collection::{Collection, SyncCollection, VanishedCollection}, id::Id, keyword::Keyword, property::Property, @@ -190,9 +190,25 @@ impl SessionData { // Send vanished UIDs if arguments.include_vanished && has_vanished { // Add to vanished all known destroyed Ids - let vanished = mailbox - .sequence_expand_missing(&arguments.sequence_set, true) - .await; + let vanished = self + .server + .store() + .vanished::<(u32, u32)>( + account_id, + VanishedCollection::Email, + Query::from_modseq(changed_since), + ) + .await + .imap_ctx(&arguments.tag, trc::location!())? + .into_iter() + .filter_map(|(mailbox_id, uid)| { + if mailbox.id.mailbox_id == mailbox_id { + Some(uid) + } else { + None + } + }) + .collect::>(); if !vanished.is_empty() { let mut buf = Vec::with_capacity(vanished.len() * 3); diff --git a/crates/jmap-proto/src/types/collection.rs b/crates/jmap-proto/src/types/collection.rs index 4afe2232..14bc48bc 100644 --- a/crates/jmap-proto/src/types/collection.rs +++ b/crates/jmap-proto/src/types/collection.rs @@ -49,6 +49,15 @@ pub enum SyncCollection { None = 8, } +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)] +#[repr(u8)] +pub enum VanishedCollection { + Email = 251, + Calendar = 252, + AddressBook = 253, + FileNode = 254, +} + impl Collection { pub fn main_collection(&self) -> Collection { match self { @@ -122,6 +131,16 @@ impl SyncCollection { SyncCollection::None => Collection::None, } } + + pub fn vanished_collection(&self) -> Option { + match self { + SyncCollection::Email => Some(VanishedCollection::Email), + SyncCollection::Calendar => Some(VanishedCollection::Calendar), + SyncCollection::AddressBook => Some(VanishedCollection::AddressBook), + SyncCollection::FileNode => Some(VanishedCollection::FileNode), + _ => None, + } + } } impl From for SyncCollection { @@ -215,6 +234,12 @@ impl From for u8 { } } +impl From for u8 { + fn from(v: VanishedCollection) -> Self { + v as u8 + } +} + impl From for u64 { fn from(collection: Collection) -> u64 { collection as u64 diff --git a/crates/jmap/src/email/set.rs b/crates/jmap/src/email/set.rs index a1dcea83..44f252a8 100644 --- a/crates/jmap/src/email/set.rs +++ b/crates/jmap/src/email/set.rs @@ -25,7 +25,7 @@ use jmap_proto::{ response::references::EvalObjectReferences, types::{ acl::Acl, - collection::{Collection, SyncCollection}, + collection::{Collection, SyncCollection, VanishedCollection}, keyword::Keyword, property::Property, state::{State, StateChange}, @@ -43,7 +43,7 @@ use mail_builder::{ }; use mail_parser::MessageParser; use std::future::Future; -use store::{ahash::AHashSet, roaring::RoaringBitmap, write::BatchBuilder}; +use store::{ahash::AHashMap, roaring::RoaringBitmap, write::BatchBuilder}; use trc::AddContext; pub trait EmailSet: Sync + Send { @@ -734,7 +734,7 @@ impl EmailSet for Server { // Process updates let mut batch = BatchBuilder::new(); - let mut changed_mailboxes = AHashSet::new(); + let mut changed_mailboxes: AHashMap> = AHashMap::new(); let mut will_update = Vec::with_capacity(request.update.as_ref().map_or(0, |u| u.len())); 'update: for (id, object) in request.unwrap_update() { // Make sure id won't be destroyed @@ -848,7 +848,7 @@ impl EmailSet for Server { .any(|keyword| keyword == &Keyword::Seen) { for mailbox_id in new_data.mailboxes.iter() { - changed_mailboxes.insert(mailbox_id.mailbox_id); + changed_mailboxes.insert(mailbox_id.mailbox_id, Vec::new()); } } } @@ -872,7 +872,7 @@ impl EmailSet for Server { // Verify permissions on shared accounts if !matches!(&can_add_mailbox_ids, Some(ids) if !ids.contains(mailbox_id.mailbox_id)) { - changed_mailboxes.insert(mailbox_id.mailbox_id); + changed_mailboxes.insert(mailbox_id.mailbox_id, Vec::new()); } else { response.not_updated.append( id, @@ -902,7 +902,10 @@ impl EmailSet for Server { // Verify permissions on shared accounts if !matches!(&can_delete_mailbox_ids, Some(ids) if !ids.contains(u32::from(mailbox_id.mailbox_id))) { - changed_mailboxes.insert(u32::from(mailbox_id.mailbox_id)); + changed_mailboxes + .entry(mailbox_id.mailbox_id.to_native()) + .or_default() + .push(mailbox_id.uid.to_native()); } else { response.not_updated.append( id, @@ -943,8 +946,11 @@ impl EmailSet for Server { if !batch.is_empty() { // Log mailbox changes - for parent_id in changed_mailboxes { + for (parent_id, deleted_uids) in changed_mailboxes { batch.log_container_property_change(SyncCollection::Email, parent_id); + for deleted_uid in deleted_uids { + batch.log_vanished_item(VanishedCollection::Email, (parent_id, deleted_uid)); + } } match self diff --git a/crates/store/src/query/log.rs b/crates/store/src/query/log.rs index a391e2d8..e0e64398 100644 --- a/crates/store/src/query/log.rs +++ b/crates/store/src/query/log.rs @@ -7,7 +7,7 @@ use trc::AddContext; use utils::codec::leb128::Leb128Iterator; -use crate::{IterateParams, LogKey, Store, U64_LEN, write::key::DeserializeBigEndian}; +use crate::{IterateParams, LogKey, Store, U32_LEN, U64_LEN, write::key::DeserializeBigEndian}; #[derive(Debug, PartialEq, Eq, Clone, Copy)] pub enum Change { @@ -38,6 +38,10 @@ pub enum Query { RangeInclusive(u64, u64), } +pub trait DeserializeVanished: Sized + Sync + Send { + fn deserialize_vanished<'x>(bytes: &mut impl Iterator) -> Option; +} + impl Default for Changes { fn default() -> Self { Self { @@ -122,6 +126,62 @@ impl Store { Ok(changelog) } + pub async fn vanished( + &self, + account_id: u32, + collection: impl Into + Sync + Send, + query: Query, + ) -> trc::Result> { + let collection = collection.into(); + let (is_inclusive, from_change_id, to_change_id) = match query { + Query::All => (true, 0, u64::MAX), + Query::Since(change_id) => (false, change_id, u64::MAX), + Query::SinceInclusive(change_id) => (true, change_id, u64::MAX), + Query::RangeInclusive(from_change_id, to_change_id) => { + (true, from_change_id, to_change_id) + } + }; + let from_key = LogKey { + account_id, + collection, + change_id: from_change_id, + }; + let to_key = LogKey { + account_id, + collection, + change_id: to_change_id, + }; + + let mut vanished = Vec::default(); + + self.iterate( + IterateParams::new(from_key, to_key).ascending(), + |key, value| { + let change_id = key.deserialize_be_u64(key.len() - U64_LEN)?; + if is_inclusive || change_id != from_change_id { + let mut iter = value.iter().peekable(); + + while iter.peek().is_some() { + if let Some(item) = T::deserialize_vanished(&mut iter) { + vanished.push(item); + } else { + return Err(trc::Error::corrupted_key( + key, + value.into(), + trc::location!(), + )); + } + } + } + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; + + Ok(vanished) + } + pub async fn get_last_change_id( &self, account_id: u32, @@ -369,3 +429,41 @@ impl Change { ) } } + +impl DeserializeVanished for u64 { + fn deserialize_vanished<'x>(bytes: &mut impl Iterator) -> Option { + let mut num = [0u8; U64_LEN]; + for i in num.iter_mut() { + *i = *bytes.next()?; + } + Some(u64::from_be_bytes(num)) + } +} + +impl DeserializeVanished for (u32, u32) { + fn deserialize_vanished<'x>(bytes: &mut impl Iterator) -> Option { + let mut num1 = [0u8; U32_LEN]; + let mut num2 = [0u8; U32_LEN]; + for i in num1.iter_mut().chain(num2.iter_mut()) { + *i = *bytes.next()?; + } + Some((u32::from_be_bytes(num1), u32::from_be_bytes(num2))) + } +} + +impl DeserializeVanished for String { + fn deserialize_vanished<'x>(bytes: &mut impl Iterator) -> Option { + let mut name = Vec::with_capacity(16); + + loop { + let byte = bytes.next()?; + if *byte != 0 { + name.push(*byte); + } else { + break; + } + } + + String::from_utf8(name).ok() + } +} diff --git a/crates/store/src/write/batch.rs b/crates/store/src/write/batch.rs index 71348088..5e6fb3e2 100644 --- a/crates/store/src/write/batch.rs +++ b/crates/store/src/write/batch.rs @@ -6,7 +6,7 @@ use super::{ Batch, BatchBuilder, BitmapClass, ChangedCollection, IntoOperations, Operation, TagValue, - ValueClass, ValueOp, assert::ToAssertValue, + ValueClass, ValueOp, assert::ToAssertValue, log::VanishedItem, }; use crate::{SerializeInfallible, U32_LEN}; use utils::map::{bitmap::ShortId, vec_map::VecMap}; @@ -331,12 +331,30 @@ impl BatchBuilder { self } + pub fn log_vanished_item( + &mut self, + collection: impl Into, + item: impl Into, + ) -> &mut Self { + if let Some(account_id) = self.current_account_id { + let item = item.into(); + self.batch_size += item.serialized_size(); + let collection = collection.into(); + debug_assert!(collection > 200); + self.changes + .get_mut_or_insert(account_id) + .log_vanished_item(collection, item); + } + self + } + fn serialize_changes(&mut self) { if !self.changes.is_empty() { for (account_id, changelog) in std::mem::take(&mut self.changes) { self.with_account_id(account_id); - for (collection, changes) in changelog.into_iterator() { + // Serialize changes + for (collection, changes) in changelog.changes.into_iter() { let cc = self.changed_collections.get_mut_or_insert(account_id); if changes.has_container_changes() { cc.changed_containers.insert(ShortId(collection)); @@ -350,6 +368,14 @@ impl BatchBuilder { set: changes.serialize(), }); } + + // Serialize vanished items + for (collection, vanished) in changelog.vanished.into_iter() { + self.ops.push(Operation::Log { + collection, + set: vanished.serialize(), + }); + } } } } diff --git a/crates/store/src/write/log.rs b/crates/store/src/write/log.rs index 7e686e62..8ef00c09 100644 --- a/crates/store/src/write/log.rs +++ b/crates/store/src/write/log.rs @@ -4,16 +4,28 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use crate::{SerializeInfallible, U64_LEN}; use ahash::AHashSet; use utils::{codec::leb128::Leb128Vec, map::vec_map::VecMap}; -use crate::SerializeInfallible; +use super::key::KeySerializer; #[derive(Default, Debug)] pub(crate) struct ChangeLogBuilder { pub changes: VecMap, + pub vanished: VecMap, } +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub enum VanishedItem { + Name(String), + Id(u64), + IdPair(u32, u32), +} + +#[derive(Default, Debug)] +pub(crate) struct VanishedItems(Vec); + #[derive(Default, Debug)] pub struct Changes { pub item_inserts: AHashSet, @@ -27,15 +39,13 @@ pub struct Changes { } impl ChangeLogBuilder { - pub fn into_iterator(self) -> impl Iterator { - self.changes.into_iter() - } - pub fn log_container_insert(&mut self, collection: impl Into, document_id: u32) { - self.changes - .get_mut_or_insert(collection.into()) - .container_inserts - .insert(document_id); + let changes = self.changes.get_mut_or_insert(collection.into()); + if changes.container_deletes.remove(&document_id) { + changes.container_updates.insert(document_id); + } else { + changes.container_inserts.insert(document_id); + } } pub fn log_item_insert( @@ -44,10 +54,13 @@ impl ChangeLogBuilder { prefix: Option, document_id: u32, ) { - self.changes - .get_mut_or_insert(collection.into()) - .item_inserts - .insert(build_id(prefix, document_id)); + let id = build_id(prefix, document_id); + let changes = self.changes.get_mut_or_insert(collection.into()); + if changes.item_deletes.remove(&id) { + changes.item_updates.insert(id); + } else { + changes.item_inserts.insert(id); + } } pub fn log_container_update(&mut self, collection: impl Into, document_id: u32) { @@ -95,6 +108,13 @@ impl ChangeLogBuilder { changes.item_updates.remove(&id); changes.item_deletes.insert(id); } + + pub fn log_vanished_item(&mut self, collection: impl Into, item: impl Into) { + self.vanished + .get_mut_or_insert(collection.into()) + .0 + .push(item.into()); + } } #[inline(always)] @@ -162,3 +182,48 @@ impl SerializeInfallible for Changes { buf } } + +impl From for VanishedItem { + fn from(value: String) -> Self { + VanishedItem::Name(value) + } +} + +impl From for VanishedItem { + fn from(value: u64) -> Self { + VanishedItem::Id(value) + } +} + +impl From<(u32, u32)> for VanishedItem { + fn from(value: (u32, u32)) -> Self { + VanishedItem::Id((value.0 as u64) << 32 | value.1 as u64) + } +} + +impl VanishedItem { + pub fn serialized_size(&self) -> usize { + match self { + VanishedItem::Name(name) => name.len() + 1, + VanishedItem::Id(_) | VanishedItem::IdPair(..) => U64_LEN, + } + } +} + +impl SerializeInfallible for VanishedItems { + fn serialize(&self) -> Vec { + let mut buf = KeySerializer::new(64); + + for item in &self.0 { + buf = match item { + VanishedItem::Name(name) => buf.write(name.as_bytes()).write(0u8), + VanishedItem::Id(id) => buf.write(id.to_be_bytes().as_slice()), + VanishedItem::IdPair(a, b) => buf + .write(a.to_be_bytes().as_slice()) + .write(b.to_be_bytes().as_slice()), + }; + } + + buf.finalize() + } +} diff --git a/tests/src/imap/condstore.rs b/tests/src/imap/condstore.rs index 1c1186ae..143b7199 100644 --- a/tests/src/imap/condstore.rs +++ b/tests/src/imap/condstore.rs @@ -166,7 +166,7 @@ pub async fn test(imap: &mut ImapConnection, imap_check: &mut ImapConnection) { imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_count("VANISHED", 1) - .assert_contains("VANISHED (EARLIER) 1:2") // .assert_contains("VANISHED (EARLIER) 2") + .assert_contains("VANISHED (EARLIER) 2") .assert_count("FETCH (", 3); // Fetch changes since SEQ 4 @@ -178,7 +178,7 @@ pub async fn test(imap: &mut ImapConnection, imap_check: &mut ImapConnection) { imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_count("VANISHED", 1) - .assert_contains("VANISHED (EARLIER) 1:2") // .assert_contains("VANISHED (EARLIER) 2") + .assert_contains("VANISHED (EARLIER) 2") .assert_count("FETCH (", 2); // Fetch changes since SEQ 6 @@ -190,7 +190,7 @@ pub async fn test(imap: &mut ImapConnection, imap_check: &mut ImapConnection) { imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_count("VANISHED", 1) - .assert_contains("VANISHED (EARLIER) 1:2") // .assert_contains("VANISHED (EARLIER) 2") + .assert_contains("VANISHED (EARLIER) 2") .assert_count("FETCH (", 1); // Fetch changes since SEQ 7 @@ -202,7 +202,7 @@ pub async fn test(imap: &mut ImapConnection, imap_check: &mut ImapConnection) { imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_count("VANISHED", 1) - .assert_contains("VANISHED (EARLIER) 1:2") // .assert_contains("VANISHED (EARLIER) 2") + .assert_contains("VANISHED (EARLIER) 2") .assert_count("FETCH (", 0); // Fetch changes since SEQ 8 @@ -276,5 +276,5 @@ pub async fn test(imap: &mut ImapConnection, imap_check: &mut ImapConnection) { imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_count("FETCH (", 3) - .assert_contains("VANISHED (EARLIER) 1:2"); // .assert_contains("VANISHED (EARLIER) 2"); + .assert_contains("VANISHED (EARLIER) 2"); } diff --git a/tests/src/webdav/copy_move.rs b/tests/src/webdav/copy_move.rs index d79fa60f..544abbbf 100644 --- a/tests/src/webdav/copy_move.rs +++ b/tests/src/webdav/copy_move.rs @@ -131,6 +131,24 @@ pub async fn test(test: &WebDavTest) { replace_prefix(&mut hierarchy, &hierarchy_root, &new_hierarchy_root); assert_result(&response, &hierarchy); client.validate_values(&hierarchy).await; + // Validate changes + let changes = client + .sync_collection( + &user_base_path, + sync_token, + Depth::Infinity, + None, + ["D:getetag"], + ) + .await + .with_href_count(2) + .into_propfind_response(None); + changes + .properties(&hierarchy_root) + .with_status(StatusCode::NOT_FOUND); + changes + .properties(&new_hierarchy_root) + .with_status(StatusCode::OK); let hierarchy_root = new_hierarchy_root; // Test 2: Copy container @@ -167,6 +185,17 @@ pub async fn test(test: &WebDavTest) { // Test 4: Create a shallow container and overwrite the previous one using MOVE let (new_hierarchy_root, mut hierarchy) = client.create_hierarchy(&user_base_path, 0, 0, 3).await; + let sync_token = client + .sync_collection( + &user_base_path, + sync_token, + Depth::Infinity, + None, + ["D:getetag"], + ) + .await + .sync_token() + .to_string(); client .request_with_headers( "MOVE", @@ -182,6 +211,23 @@ pub async fn test(test: &WebDavTest) { replace_prefix(&mut hierarchy, &new_hierarchy_root, &hierarchy_root); assert_result(&response, &hierarchy); client.validate_values(&hierarchy).await; + // Validate changes + let changes = client + .sync_collection( + &user_base_path, + &sync_token, + Depth::Infinity, + None, + ["D:getetag"], + ) + .await + .into_propfind_response(None); + changes + .properties(&new_hierarchy_root) + .with_status(StatusCode::NOT_FOUND); + changes + .properties(&hierarchy_root) + .with_status(StatusCode::OK); // Test 5: Create a deep container and overwrite the previous one using COPY let (new_hierarchy_root, new_hierarchy) = client diff --git a/tests/src/webdav/prop.rs b/tests/src/webdav/prop.rs index ffedb231..35cccc57 100644 --- a/tests/src/webdav/prop.rs +++ b/tests/src/webdav/prop.rs @@ -710,7 +710,11 @@ pub struct DavItem { } #[derive(Debug, serde::Serialize)] -pub struct DavProperties(Vec); +pub struct DavProperties { + #[serde(skip)] + status: StatusCode, + props: Vec, +} impl DavMultiStatus { pub fn properties(&self, href: &str) -> DavPropertyResult<'_> { @@ -752,7 +756,7 @@ impl DavPropertyResult<'_> { pub fn get(&self, name: impl AsRef) -> DavQueryResult<'_> { let name = name.as_ref(); self.properties - .0 + .props .iter() .find_map(|prop| { prop.values.get(name).map(|values| DavQueryResult { @@ -765,15 +769,26 @@ impl DavPropertyResult<'_> { self.response.dump_response(); panic!( "No property found for name: {name} in {}", - serde_json::to_string_pretty(&self.properties.0).unwrap() + serde_json::to_string_pretty(&self.properties.props).unwrap() ) }) } + pub fn with_status(&self, status: StatusCode) -> &Self { + if self.properties.status != status { + self.response.dump_response(); + panic!( + "Expected status {status}, but got {}", + self.properties.status + ); + } + self + } + pub fn is_defined(&self, name: impl AsRef) -> &Self { if self .properties - .0 + .props .iter() .any(|prop| prop.values.contains_key(name.as_ref())) { @@ -787,7 +802,7 @@ impl DavPropertyResult<'_> { pub fn is_undefined(&self, name: impl AsRef) -> &Self { if self .properties - .0 + .props .iter() .any(|prop| prop.values.contains_key(name.as_ref())) { @@ -931,6 +946,7 @@ impl DavResponse { hrefs: AHashMap::new(), }; let mut href = None; + let mut href_status = StatusCode::OK; let mut props = Vec::new(); let mut prop = DavItem::default(); @@ -941,12 +957,25 @@ impl DavResponse { if !prop.is_empty() { props.push(std::mem::take(&mut prop)); } - result - .hrefs - .insert(href, DavProperties(std::mem::take(&mut props))); + result.hrefs.insert( + href, + DavProperties { + status: href_status, + props: std::mem::take(&mut props), + }, + ); + href_status = StatusCode::OK; } href = Some(value.to_string()); } + "D:multistatus.D:response.D:status" => { + href_status = value + .split_ascii_whitespace() + .nth(1) + .unwrap_or_default() + .parse() + .unwrap(); + } "D:multistatus.D:response.D:propstat.D:status" => { prop.status = value .split_ascii_whitespace() @@ -958,7 +987,6 @@ impl DavResponse { "D:multistatus.D:response.D:propstat.D:responsedescription" => { prop.description = Some(value.to_string()); } - _ => { if let Some(prop_name) = key.strip_prefix("D:multistatus.D:response.D:propstat.D:prop.") @@ -990,7 +1018,13 @@ impl DavResponse { if !prop.is_empty() { props.push(prop); } - result.hrefs.insert(href, DavProperties(props)); + result.hrefs.insert( + href, + DavProperties { + status: href_status, + props, + }, + ); } result diff --git a/tests/src/webdav/sync.rs b/tests/src/webdav/sync.rs index 3206cc3e..7f6162bd 100644 --- a/tests/src/webdav/sync.rs +++ b/tests/src/webdav/sync.rs @@ -223,15 +223,68 @@ pub async fn test(test: &WebDavTest) { } assert!(expected_changes.is_empty(), "{:?}", expected_changes); + // Test 10: Expect changes after deletion client - .request("DELETE", &folder_name, "") + .request("DELETE", &new_file, "") .await .with_status(StatusCode::NO_CONTENT); - + let response = client + .sync_collection( + &user_base_path, + &sync_token, + Depth::Infinity, + None, + ["D:getetag"], + ) + .await; + sync_token = response.sync_token().to_string(); + response + .with_href_count(1) + .with_value("D:multistatus.D:response.D:href", &new_file) + .with_value( + "D:multistatus.D:response.D:status", + "HTTP/1.1 404 Not Found", + ); client .request("DELETE", &new_collection, "") .await .with_status(StatusCode::NO_CONTENT); + let response = client + .sync_collection( + &user_base_path, + &sync_token, + Depth::Infinity, + None, + ["D:getetag"], + ) + .await; + sync_token = response.sync_token().to_string(); + response + .with_href_count(1) + .with_value("D:multistatus.D:response.D:href", &new_collection) + .with_value( + "D:multistatus.D:response.D:status", + "HTTP/1.1 404 Not Found", + ); + client + .request("DELETE", &folder_name, "") + .await + .with_status(StatusCode::NO_CONTENT); + client + .sync_collection( + &user_base_path, + &sync_token, + Depth::Infinity, + None, + ["D:getetag"], + ) + .await + .with_href_count(1) + .with_value("D:multistatus.D:response.D:href", &folder_name) + .with_value( + "D:multistatus.D:response.D:status", + "HTTP/1.1 404 Not Found", + ); } client.delete_default_containers().await;