Database schema optimization - part 12

tests/src/jmap/auth/quota.rs

@@ -8,6 +8,7 @@ use crate::{
     directory::internal::TestInternalDirectory,
     jmap::{JMAPTest, mail::delivery::SmtpConnection, wait_for_index},
     smtp::queue::QueuedEvents,
+    store::cleanup::store_blob_expire_all,
 };
 use common::config::smtp::queue::QueueName;
 use email::{cache::MessageCacheFetch, mailbox::INBOX_ID};
@@ -42,7 +43,7 @@ pub async fn test(params: &mut JMAPTest) {
     server.inner.cache.access_tokens.clear();
 
     // Delete temporary blobs from previous tests
-    server.core.storage.data.blob_expire_all().await;
+    store_blob_expire_all(&server.core.storage.data).await;
 
     // Test temporary blob quota (3 files)
     DISABLE_UPLOAD_QUOTA.store(false, std::sync::atomic::Ordering::Relaxed);
@@ -65,7 +66,7 @@ pub async fn test(params: &mut JMAPTest) {
         jmap_client::Error::Problem(err) if err.detail().unwrap().contains("quota") => (),
         other => panic!("Unexpected error: {:?}", other),
     }
-    server.core.storage.data.blob_expire_all().await;
+    store_blob_expire_all(&server.core.storage.data).await;
 
     // Test temporary blob quota (50000 bytes)
     for i in 0..2 {
@@ -86,7 +87,7 @@ pub async fn test(params: &mut JMAPTest) {
         jmap_client::Error::Problem(err) if err.detail().unwrap().contains("quota") => (),
         other => panic!("Unexpected error: {:?}", other),
     }
-    server.core.storage.data.blob_expire_all().await;
+    store_blob_expire_all(&server.core.storage.data).await;
 
     // Test JMAP Quotas extension
     let response = account

tests/src/jmap/calendar/alarm.rs

@@ -55,7 +55,9 @@ pub async fn test(params: &mut JMAPTest) {
     let (stream_tx, mut stream_rx) = mpsc::channel::<WebSocketMessage>(100);
     tokio::spawn(async move {
         while let Some(change) = ws_stream.next().await {
-            stream_tx.send(change.unwrap()).await.unwrap();
+            if stream_tx.send(change.unwrap()).await.is_err() {
+                break;
+            }
         }
     });
     client_ws

tests/src/jmap/calendar/event.rs

@@ -5,7 +5,7 @@
  */
 
 use crate::{
-    jmap::{ChangeType, IntoJmapSet, JMAPTest, JmapUtils},
+    jmap::{ChangeType, IntoJmapSet, JMAPTest, JmapUtils, wait_for_index},
     webdav::DummyWebDavClient,
 };
 use ahash::AHashSet;
@@ -452,6 +452,7 @@ pub async fn test(params: &mut JMAPTest) {
     }));
 
     // Query tests
+    wait_for_index(&params.server).await;
     assert_eq!(
         account
             .jmap_query(

tests/src/jmap/calendar/notification.rs

@@ -4,7 +4,7 @@
  * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
  */
 
-use crate::jmap::{IntoJmapSet, JMAPTest, JmapUtils};
+use crate::jmap::{IntoJmapSet, JMAPTest, JmapUtils, wait_for_index};
 use calcard::jscalendar::JSCalendarProperty;
 use jmap_proto::{
     object::calendar_event_notification::CalendarEventNotificationProperty,
@@ -73,6 +73,7 @@ pub async fn test(params: &mut JMAPTest) {
     let john_event_id = response.created(0).id().to_string();
 
     tokio::time::sleep(std::time::Duration::from_millis(600)).await;
+    wait_for_index(&params.server).await;
 
     // Verify Jane and Bill received the share notification
     let mut jane_event_id = String::new();

tests/src/jmap/core/blob.rs

@@ -4,7 +4,7 @@
  * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
  */
 
-use crate::jmap::JMAPTest;
+use crate::{jmap::JMAPTest, store::cleanup::store_blob_expire_all};
 use email::mailbox::INBOX_ID;
 use serde_json::{Value, json};
 use types::id::Id;
@@ -13,7 +13,7 @@ pub async fn test(params: &mut JMAPTest) {
     println!("Running blob tests...");
     let server = params.server.clone();
     let account = params.account("jdoe@example.com");
-    server.core.storage.data.blob_expire_all().await;
+    store_blob_expire_all(&server.core.storage.data).await;
 
     // Blob/set simple test
     let response = account.jmap_method_call("Blob/upload", json!({
@@ -139,7 +139,7 @@ pub async fn test(params: &mut JMAPTest) {
         );
     }
 
-    server.core.storage.data.blob_expire_all().await;
+    store_blob_expire_all(&server.core.storage.data).await;
 
     // Blob/upload Complex Example
     let response = account
@@ -226,7 +226,7 @@ pub async fn test(params: &mut JMAPTest) {
             "Pointer {pointer:?} Response: {response:?}",
         );
     }
-    server.core.storage.data.blob_expire_all().await;
+    store_blob_expire_all(&server.core.storage.data).await;
 
     // Blob/get Example with Range and Encoding Errors
     let response = account.jmap_method_calls(json!([
@@ -353,7 +353,7 @@ pub async fn test(params: &mut JMAPTest) {
             "Pointer {pointer:?} Response: {response:?}",
         );
     }
-    server.core.storage.data.blob_expire_all().await;
+    store_blob_expire_all(&server.core.storage.data).await;
 
     // Blob/lookup
     let client = account.client();

tests/src/jmap/mail/acl.rs

@@ -4,10 +4,7 @@
  * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL
  */
 
-use crate::{
-    directory::internal::TestInternalDirectory,
-    jmap::{JMAPTest},
-};
+use crate::{directory::internal::TestInternalDirectory, jmap::JMAPTest};
 use ::email::mailbox::{INBOX_ID, TRASH_ID};
 use jmap_client::{
     core::{

tests/src/jmap/mail/query.rs

@@ -92,8 +92,10 @@ pub async fn test(params: &mut JMAPTest, insert: bool) {
         wait_for_index(&server).await;
     }
 
+    let can_stem = !params.server.search_store().is_mysql();
+
     println!("Running JMAP Mail query tests...");
-    query(client).await;
+    query(client, can_stem).await;
 
     println!("Running JMAP Mail query options tests...");
     query_options(client).await;
@@ -114,7 +116,7 @@ pub async fn test(params: &mut JMAPTest, insert: bool) {
     params.assert_is_empty().await;
 }
 
-pub async fn query(client: &Client) {
+pub async fn query(client: &Client, can_stem: bool) {
     for (filter, sort, expected_results) in [
         (
             Filter::and(vec![
@@ -144,7 +146,7 @@ pub async fn query(client: &Client) {
         ),
         (
             Filter::and(vec![
-                (email::query::Filter::text("study")),
+                (email::query::Filter::text(if can_stem { "study" } else { "studies" })),
                 (email::query::Filter::in_mailbox_other_than(vec![
                     Id::new(1991).to_string(),
                     Id::new(1870).to_string(),

tests/src/jmap/mail/search_snippet.rs

@@ -49,6 +49,8 @@ pub async fn test(params: &mut JMAPTest) {
     }
     wait_for_index(&server).await;
 
+    let can_stem = params.server.search_store().internal_fts().is_some();
+
     // Run tests
     for (filter, email_name, snippet_subject, snippet_preview) in [
         (
@@ -121,7 +123,12 @@ pub async fn test(params: &mut JMAPTest) {
             )),
         ),
         (
-            Filter::text("es:galería vasto biblioteca").into(),
+            Filter::text(if can_stem {
+                "es:galería vasto biblioteca"
+            } else {
+                "es:galería vastos biblioteca"
+            })
+            .into(),
             "mixed",
             Some("<mark>Biblioteca</mark> de Babel"),
             Some(concat!(

tests/src/jmap/mod.rs

@@ -11,7 +11,10 @@ use crate::{
         enterprise::{EnterpriseCore, insert_test_metrics},
         webhooks::{MockWebhookEndpoint, spawn_mock_webhook_endpoint},
     },
-    store::{TempDir, build_store_config},
+    store::{
+        TempDir, build_store_config,
+        cleanup::{search_store_destroy, store_assert_is_empty, store_destroy},
+    },
 };
 use ahash::AHashMap;
 use base64::{
@@ -75,9 +78,9 @@ async fn jmap_tests() {
 
     server::webhooks::test(&mut params).await;
 
-    /*mail::get::test(&mut params).await;
+    mail::get::test(&mut params).await;
     mail::set::test(&mut params).await;
-    mail::parse::test(&mut params).await;*/
+    mail::parse::test(&mut params).await;
     mail::query::test(&mut params, delete).await;
     mail::search_snippet::test(&mut params).await;
     mail::changes::test(&mut params).await;
@@ -263,10 +266,8 @@ pub async fn assert_is_empty(server: &Server) {
         .unwrap();
 
     // Assert is empty
-    server
-        .store()
-        .assert_is_empty(server.core.storage.blob.clone())
-        .await;
+    store_assert_is_empty(server.store(), server.core.storage.blob.clone()).await;
+    search_store_destroy(server.search_store()).await;
 
     // Clean caches
     for cache in [
@@ -321,6 +322,7 @@ async fn init_jmap_tests(delete_if_exists: bool) -> JMAPTest {
     let data = Data::parse(&mut config);
     let cache = Caches::parse(&mut config);
     let store = core.storage.data.clone();
+    let search_store = core.storage.fts.clone();
     let (ipc, mut ipc_rxs) = build_ipc(false);
     let inner = Arc::new(Inner {
         shared_core: core.into_shared(),
@@ -330,7 +332,8 @@ async fn init_jmap_tests(delete_if_exists: bool) -> JMAPTest {
     });
 
     if delete_if_exists {
-        store.destroy().await;
+        store_destroy(&store).await;
+        search_store_destroy(&search_store).await;
     }
 
     // Parse acceptors

tests/src/jmap/server/purge.rs

@@ -17,7 +17,11 @@ use email::{
     message::delete::EmailDeletion,
 };
 use imap_proto::ResponseType;
-use store::{IterateParams, LogKey, U32_LEN, U64_LEN, write::key::DeserializeBigEndian};
+use store::{
+    IterateParams, LogKey, U32_LEN, U64_LEN,
+    search::SearchQuery,
+    write::{SearchIndex, key::DeserializeBigEndian},
+};
 use types::id::Id;
 
 pub async fn test(params: &mut JMAPTest) {
@@ -155,6 +159,19 @@ pub async fn test(params: &mut JMAPTest) {
         .delete_principal(QueryBy::Id(account.id().document_id()))
         .await
         .unwrap();
+    for index in [
+        SearchIndex::Email,
+        SearchIndex::Contacts,
+        SearchIndex::Calendar,
+    ] {
+        server
+            .core
+            .storage
+            .fts
+            .unindex(SearchQuery::new(index).with_account_id(account.id().document_id()))
+            .await
+            .unwrap();
+    }
     params.assert_is_empty().await;
 }