Database schema optimization - part 13 (fixes #1882 fixes #2415)

tests/src/jmap/auth/permissions.rs

@@ -610,11 +610,10 @@ pub async fn test(params: &JMAPTest) {
         );
 
     // John should not be allowed to receive email
-    let message_blob = server
-        .put_blob(tenant_user_id, TEST_MESSAGE.as_bytes(), false)
+    let (message_blob, _) = server
+        .put_temporary_blob(tenant_user_id, TEST_MESSAGE.as_bytes(), 60)
         .await
-        .unwrap()
-        .hash;
+        .unwrap();
     assert_eq!(
         server
             .deliver_message(IngestMessage {

tests/src/jmap/mail/delivery.rs

@@ -15,7 +15,7 @@ use email::{
 };
 use groupware::DavResourceName;
 use jmap::blob::download::BlobDownload;
-use std::time::Duration;
+use std::{sync::Arc, time::Duration};
 use tokio::{
     io::{AsyncBufReadExt, AsyncWriteExt, BufReader, Lines, ReadHalf, WriteHalf},
     net::TcpStream,
@@ -31,7 +31,11 @@ use utils::chained_bytes::ChainedBytes;
 pub async fn test(params: &mut JMAPTest) {
     println!("Running message delivery tests...");
 
-    let todo = "enable delivered to for test";
+    // Enable delivered to
+    let old_core = params.server.core.clone();
+    let mut new_core = old_core.as_ref().clone();
+    new_core.smtp.session.data.add_delivered_to = true;
+    params.server.inner.shared_core.store(Arc::new(new_core));
 
     // Create a domain name and a test account
     let server = params.server.clone();
@@ -277,37 +281,48 @@ END:VCARD
                 .unwrap()
                 .unwrap();
             let metadata = archive.to_unarchived::<MessageMetadata>().unwrap();
-            let body = server
-                .blob_download(
-                    &BlobId {
-                        hash: BlobHash::from(&metadata.inner.blob_hash),
-                        class: BlobClass::Linked {
-                            account_id,
-                            collection: Collection::Email.into(),
-                            document_id,
-                        },
-                        section: None,
-                    },
-                    &access_token,
-                )
+            let partial_message = server
+                .store()
+                .get_blob(metadata.inner.blob_hash.0.as_ref(), 0..usize::MAX)
                 .await
                 .unwrap()
                 .unwrap();
             assert_ne!(metadata.inner.blob_body_offset.to_native(), 0);
-            let raw_message = ChainedBytes::new(metadata.inner.raw_headers.as_ref()).with_last(
-                body.get(metadata.inner.blob_body_offset.to_native() as usize..)
-                    .unwrap_or_default(),
-            );
-            let full_message = String::from_utf8(raw_message.to_bytes()).unwrap();
+            let expected_full_message = String::from_utf8(
+                ChainedBytes::new(metadata.inner.raw_headers.as_ref())
+                    .with_last(
+                        partial_message
+                            .get(metadata.inner.blob_body_offset.to_native() as usize..)
+                            .unwrap_or_default(),
+                    )
+                    .to_bytes(),
+            )
+            .unwrap();
             assert!(
-                full_message.contains("Delivered-To:") && full_message.contains("Subject:"),
-                "for {account_id}: {full_message}"
-            );
-            println!(
-                "full message for {}:\n{}",
-                account.id_string(),
-                full_message
+                expected_full_message.contains("Delivered-To:")
+                    && expected_full_message.contains("Subject:"),
+                "for {account_id}: {expected_full_message}"
             );
+            let full_message = String::from_utf8(
+                server
+                    .blob_download(
+                        &BlobId {
+                            hash: BlobHash::from(&metadata.inner.blob_hash),
+                            class: BlobClass::Linked {
+                                account_id,
+                                collection: Collection::Email.into(),
+                                document_id,
+                            },
+                            section: None,
+                        },
+                        &access_token,
+                    )
+                    .await
+                    .unwrap()
+                    .unwrap(),
+            )
+            .unwrap();
+            assert_eq!(full_message, expected_full_message, "for {account_id}");
         }
     }
 
@@ -317,6 +332,9 @@ END:VCARD
     }
     params.assert_is_empty().await;
 
+    // Restore core
+    params.server.inner.shared_core.store(old_core);
+
     // Check webhook events
     params.webhook.assert_contains(&[
         "message-ingest.",

tests/src/jmap/mod.rs

@@ -127,6 +127,8 @@ async fn jmap_tests() {
     server::purge::test(&mut params).await;
     server::enterprise::test(&mut params).await;
 
+    assert_is_empty(&params.server).await;
+
     if delete {
         params.temp_dir.delete();
     }
@@ -266,7 +268,7 @@ pub async fn assert_is_empty(server: &Server) {
         .unwrap();
 
     // Assert is empty
-    store_assert_is_empty(server.store(), server.core.storage.blob.clone()).await;
+    store_assert_is_empty(server.store(), server.core.storage.blob.clone(), false).await;
     search_store_destroy(server.search_store()).await;
 
     // Clean caches

tests/src/jmap/server/enterprise.rs

@@ -25,14 +25,16 @@ use common::{
     core::BuildServer,
     enterprise::{
         Enterprise, MetricStore, TraceStore, Undelete, config::parse_metric_alerts,
-        license::LicenseKey, undelete::DeletedBlob,
+        license::LicenseKey,
     },
     telemetry::{
         metrics::store::{Metric, MetricsStore, SharedMetricHistory},
         tracers::store::TracingStore,
     },
 };
-use http::management::enterprise::undelete::{UndeleteRequest, UndeleteResponse};
+use http::management::enterprise::undelete::{
+    DeletedBlobResponse, DeletedItemResponse, UndeleteRequest, UndeleteResponse,
+};
 use imap_proto::ResponseType;
 use nlp::language::Language;
 use std::{sync::Arc, time::Duration};
@@ -435,13 +437,22 @@ async fn undelete(params: &mut JMAPTest) {
     wait_for_index(&params.server).await;
     tokio::time::sleep(Duration::from_millis(200)).await;
     let deleted = api
-        .get::<List<DeletedBlob<String, String, String>>>("/api/store/undelete/jdoe@example.com")
+        .get::<List<DeletedBlobResponse>>("/api/store/undelete/jdoe@example.com")
         .await
         .unwrap()
         .unwrap_data()
         .items;
     assert_eq!(deleted.len(), 1);
     let deleted = deleted.into_iter().next().unwrap();
+    match deleted.item {
+        DeletedItemResponse::Email { from, subject, .. } => {
+            assert_eq!(subject.as_ref(), "undelete test");
+            assert_eq!(from.as_ref(), "john@example.com");
+        }
+        other => {
+            panic!("Unexpected deleted item response: {:?}", other);
+        }
+    }
 
     // Undelete
     let result = api
@@ -449,7 +460,7 @@ async fn undelete(params: &mut JMAPTest) {
             "/api/store/undelete/jdoe@example.com",
             &vec![UndeleteRequest {
                 hash: deleted.hash,
-                collection: deleted.collection,
+                collection: "email".to_string(),
                 time: deleted.deleted_at,
                 cancel_deletion: deleted.expires_at.into(),
             }],

tests/src/jmap/server/purge.rs

@@ -16,12 +16,9 @@ use email::{
     mailbox::{INBOX_ID, JUNK_ID, TRASH_ID},
     message::delete::EmailDeletion,
 };
+use http::management::stores::destroy_account_data;
 use imap_proto::ResponseType;
-use store::{
-    IterateParams, LogKey, U32_LEN, U64_LEN,
-    search::SearchQuery,
-    write::{SearchIndex, key::DeserializeBigEndian},
-};
+use store::{IterateParams, LogKey, U32_LEN, U64_LEN, write::key::DeserializeBigEndian};
 use types::id::Id;
 
 pub async fn test(params: &mut JMAPTest) {
@@ -153,25 +150,13 @@ pub async fn test(params: &mut JMAPTest) {
 
     // Delete account
     server
-        .core
-        .storage
-        .data
+        .store()
         .delete_principal(QueryBy::Id(account.id().document_id()))
         .await
         .unwrap();
-    for index in [
-        SearchIndex::Email,
-        SearchIndex::Contacts,
-        SearchIndex::Calendar,
-    ] {
-        server
-            .core
-            .storage
-            .fts
-            .unindex(SearchQuery::new(index).with_account_id(account.id().document_id()))
-            .await
-            .unwrap();
-    }
+    destroy_account_data(&server, account.id().document_id(), true)
+        .await
+        .unwrap();
     params.assert_is_empty().await;
 }