From dbe40829daf661986d165b60cad6cfe7d8e24315 Mon Sep 17 00:00:00 2001 From: mdecimus <11444311+mdecimus@users.noreply.github.com> Date: Sat, 15 Nov 2025 12:44:04 +0100 Subject: [PATCH] Database schema optimization - part 12 --- crates/imap/src/op/search.rs | 27 +- crates/jmap/src/email/query.rs | 15 +- crates/store/src/backend/elastic/main.rs | 45 +- crates/store/src/backend/mysql/main.rs | 3 +- crates/store/src/backend/mysql/mod.rs | 2 +- crates/store/src/backend/mysql/search.rs | 30 +- crates/store/src/backend/mysql/write.rs | 4 +- crates/store/src/backend/postgres/mod.rs | 2 +- crates/store/src/backend/postgres/search.rs | 2 - crates/store/src/dispatch/search.rs | 182 ++--- crates/store/src/dispatch/store.rs | 311 --------- crates/store/src/search/bm_u32.rs | 22 +- crates/store/src/search/bm_u64.rs | 20 +- crates/store/src/search/index.rs | 6 +- crates/store/src/search/mod.rs | 6 +- crates/store/src/search/query.rs | 7 +- crates/store/src/search/split.rs | 708 ++++++++++++++++++++ crates/store/src/search/term.rs | 52 +- crates/store/src/write/key.rs | 6 +- crates/store/src/write/mod.rs | 5 +- tests/Cargo.toml | 4 +- tests/src/cluster/mod.rs | 3 +- tests/src/directory/internal.rs | 7 +- tests/src/directory/sql.rs | 7 +- tests/src/imap/mod.rs | 13 +- tests/src/imap/search.rs | 12 +- tests/src/imap/thread.rs | 19 +- tests/src/jmap/auth/quota.rs | 7 +- tests/src/jmap/calendar/alarm.rs | 4 +- tests/src/jmap/calendar/event.rs | 3 +- tests/src/jmap/calendar/notification.rs | 3 +- tests/src/jmap/core/blob.rs | 10 +- tests/src/jmap/mail/acl.rs | 5 +- tests/src/jmap/mail/query.rs | 8 +- tests/src/jmap/mail/search_snippet.rs | 9 +- tests/src/jmap/mod.rs | 19 +- tests/src/jmap/server/purge.rs | 19 +- tests/src/smtp/inbound/data.rs | 6 +- tests/src/smtp/mod.rs | 4 +- tests/src/smtp/queue/concurrent.rs | 11 +- tests/src/smtp/queue/virtualq.rs | 11 +- tests/src/store/blob.rs | 4 +- tests/src/store/cleanup.rs | 342 ++++++++++ tests/src/store/lookup.rs | 15 +- tests/src/store/mod.rs | 23 +- tests/src/store/ops.rs | 4 +- tests/src/store/query.rs | 12 +- tests/src/webdav/cal_scheduling.rs | 4 + tests/src/webdav/mod.rs | 15 +- 49 files changed, 1364 insertions(+), 694 deletions(-) create mode 100644 crates/store/src/search/split.rs create mode 100644 tests/src/store/cleanup.rs diff --git a/crates/imap/src/op/search.rs b/crates/imap/src/op/search.rs index 19d1aabf..a480c128 100644 --- a/crates/imap/src/op/search.rs +++ b/crates/imap/src/op/search.rs @@ -498,41 +498,44 @@ impl SessionData { )); } Filter::Text(text) => { + let (text, language) = + Language::detect(text, self.server.core.jmap.default_language); + filters.push(SearchFilter::Or); filters.push(SearchFilter::has_text( EmailSearchField::From, - text.as_str(), + &text, Language::None, )); filters.push(SearchFilter::has_text( EmailSearchField::To, - text.as_str(), + &text, Language::None, )); filters.push(SearchFilter::has_text( EmailSearchField::Cc, - text.as_str(), + &text, Language::None, )); filters.push(SearchFilter::has_text( EmailSearchField::Bcc, - text.as_str(), + &text, Language::None, )); - filters.push(SearchFilter::has_text_detect( + filters.push(SearchFilter::has_text( EmailSearchField::Subject, - text.as_str(), - self.server.core.jmap.default_language, + &text, + language, )); - filters.push(SearchFilter::has_text_detect( + filters.push(SearchFilter::has_text( EmailSearchField::Body, - text.as_str(), - self.server.core.jmap.default_language, + &text, + language, )); - filters.push(SearchFilter::has_text_detect( + filters.push(SearchFilter::has_text( EmailSearchField::Attachment, text, - self.server.core.jmap.default_language, + language, )); filters.push(SearchFilter::End); } diff --git a/crates/jmap/src/email/query.rs b/crates/jmap/src/email/query.rs index 832a179a..b1d6e208 100644 --- a/crates/jmap/src/email/query.rs +++ b/crates/jmap/src/email/query.rs @@ -51,6 +51,9 @@ impl EmailQuery for Server { match filter { Filter::Property(cond) => match cond { EmailFilter::Text(text) => { + let (text, language) = + Language::detect(text, self.core.jmap.default_language); + filters.push(SearchFilter::Or); filters.push(SearchFilter::has_text( EmailSearchField::From, @@ -72,20 +75,20 @@ impl EmailQuery for Server { &text, Language::None, )); - filters.push(SearchFilter::has_text_detect( + filters.push(SearchFilter::has_text( EmailSearchField::Subject, &text, - self.core.jmap.default_language, + language, )); - filters.push(SearchFilter::has_text_detect( + filters.push(SearchFilter::has_text( EmailSearchField::Body, &text, - self.core.jmap.default_language, + language, )); - filters.push(SearchFilter::has_text_detect( + filters.push(SearchFilter::has_text( EmailSearchField::Attachment, text, - self.core.jmap.default_language, + language, )); filters.push(SearchFilter::End); } diff --git a/crates/store/src/backend/elastic/main.rs b/crates/store/src/backend/elastic/main.rs index 1c9b2a0a..1ef872ba 100644 --- a/crates/store/src/backend/elastic/main.rs +++ b/crates/store/src/backend/elastic/main.rs @@ -41,39 +41,32 @@ impl ElasticSearchStore { .unwrap_or(false); #[cfg(feature = "test_mode")] - es.drop_indexes().await.unwrap(); + let _ = es.drop_indexes().await; - if let Err(err) = es - .create_index::(shards, replicas, with_source) - .await - { - config.new_build_error(prefix.as_str(), err.to_string()); - } - - if let Err(err) = es - .create_index::(shards, replicas, with_source) - .await - { - config.new_build_error(prefix.as_str(), err.to_string()); - } - - if let Err(err) = es - .create_index::(shards, replicas, with_source) - .await - { - config.new_build_error(prefix.as_str(), err.to_string()); - } - - if let Err(err) = es - .create_index::(shards, replicas, with_source) - .await - { + if let Err(err) = es.create_indexes(shards, replicas, with_source).await { config.new_build_error(prefix.as_str(), err.to_string()); } Some(es) } + pub async fn create_indexes( + &self, + shards: usize, + replicas: usize, + with_source: bool, + ) -> trc::Result<()> { + self.create_index::(shards, replicas, with_source) + .await?; + self.create_index::(shards, replicas, with_source) + .await?; + self.create_index::(shards, replicas, with_source) + .await?; + self.create_index::(shards, replicas, with_source) + .await?; + Ok(()) + } + async fn create_index( &self, shards: usize, diff --git a/crates/store/src/backend/mysql/main.rs b/crates/store/src/backend/mysql/main.rs index 0f31287f..5e233f8a 100644 --- a/crates/store/src/backend/mysql/main.rs +++ b/crates/store/src/backend/mysql/main.rs @@ -46,7 +46,8 @@ impl MysqlStore { .property::>((&prefix, "timeout")) .unwrap_or_default() .map(|t| t.as_secs() as usize), - ); + ) + .client_found_rows(true); if let Some(port) = config.property((&prefix, "port")) { opts = opts.tcp_port(port); } diff --git a/crates/store/src/backend/mysql/mod.rs b/crates/store/src/backend/mysql/mod.rs index 8a702886..7f41d35e 100644 --- a/crates/store/src/backend/mysql/mod.rs +++ b/crates/store/src/backend/mysql/mod.rs @@ -31,7 +31,7 @@ fn into_error(err: impl Display) -> trc::Error { } impl SearchIndex { - pub(crate) fn mysql_table(&self) -> &'static str { + pub fn mysql_table(&self) -> &'static str { match self { SearchIndex::Email => "s_email", SearchIndex::Calendar => "s_cal", diff --git a/crates/store/src/backend/mysql/search.rs b/crates/store/src/backend/mysql/search.rs index 795ffe42..265018c1 100644 --- a/crates/store/src/backend/mysql/search.rs +++ b/crates/store/src/backend/mysql/search.rs @@ -5,7 +5,10 @@ */ use crate::{ - backend::mysql::{MysqlSearchField, MysqlStore, into_error}, + backend::{ + MAX_TOKEN_LENGTH, + mysql::{MysqlSearchField, MysqlStore, into_error}, + }, search::{ IndexDocument, SearchComparator, SearchDocumentId, SearchFilter, SearchOperator, SearchQuery, SearchValue, @@ -13,6 +16,7 @@ use crate::{ write::SearchIndex, }; use mysql_async::{IsolationLevel, TxOpts, Value, prelude::Queryable}; +use nlp::tokenizers::word::WordTokenizer; use std::fmt::Write; impl MysqlStore { @@ -135,32 +139,30 @@ fn build_filter(query: &mut String, filters: &[SearchFilter]) -> Vec { if field.is_text() && matches!(op, SearchOperator::Equal | SearchOperator::Contains) { - let value = match (value, op) { - (SearchValue::Text { value, .. }, SearchOperator::Equal) => { + let (value, mode) = match (value, op) { + (SearchValue::Text { value, .. }, SearchOperator::Equal) => ( + Value::Bytes(format!("{value:?}").into_bytes()), + "NATURAL LANGUAGE", + ), + (SearchValue::Text { value, .. }, ..) => { let mut text_query = String::with_capacity(value.len() + 1); - for item in value.split_whitespace() { + for item in WordTokenizer::new(value, MAX_TOKEN_LENGTH) { if !text_query.is_empty() { text_query.push(' '); } - let _ = write!(text_query, "+{item}"); + text_query.push('+'); + text_query.push_str(&item.word); } - Value::Bytes(text_query.into_bytes()) - } - (SearchValue::Text { value, .. }, ..) => { - Value::Bytes(format!("{value:?}").into_bytes()) + (Value::Bytes(text_query.into_bytes()), "BOOLEAN") } _ => { debug_assert!(false, "Invalid search value for text field"); continue; } }; - let _ = write!( - query, - "MATCH({}) AGAINST(? IN BOOLEAN MODE)", - field.column() - ); + let _ = write!(query, "MATCH({}) AGAINST(? IN {mode} MODE)", field.column()); values.push(value); } else if let SearchValue::KeyValues(kv) = value { let (key, value) = kv.iter().next().unwrap(); diff --git a/crates/store/src/backend/mysql/write.rs b/crates/store/src/backend/mysql/write.rs index 066611df..a03c318f 100644 --- a/crates/store/src/backend/mysql/write.rs +++ b/crates/store/src/backend/mysql/write.rs @@ -155,7 +155,7 @@ impl MysqlStore { .await { Ok(_) => { - if exists.is_some() && trx.affected_rows() == 0 { + if trx.affected_rows() == 0 { trx.rollback().await?; return Err(trc::StoreEvent::AssertValueFailed .into_err() @@ -193,7 +193,7 @@ impl MysqlStore { match trx.exec_drop(&s, params! {"k" => key, "v" => &value}).await { Ok(_) => { - if exists.is_some() && trx.affected_rows() == 0 { + if trx.affected_rows() == 0 { trx.rollback().await?; return Err(trc::StoreEvent::AssertValueFailed .into_err() diff --git a/crates/store/src/backend/postgres/mod.rs b/crates/store/src/backend/postgres/mod.rs index 121a0ad9..6f08df3b 100644 --- a/crates/store/src/backend/postgres/mod.rs +++ b/crates/store/src/backend/postgres/mod.rs @@ -50,7 +50,7 @@ fn into_pool_error(err: deadpool::managed::PoolError) -> } impl SearchIndex { - pub(crate) fn psql_table(&self) -> &'static str { + pub fn psql_table(&self) -> &'static str { match self { SearchIndex::Email => "s_email", SearchIndex::Calendar => "s_cal", diff --git a/crates/store/src/backend/postgres/search.rs b/crates/store/src/backend/postgres/search.rs index af272564..b81d3754 100644 --- a/crates/store/src/backend/postgres/search.rs +++ b/crates/store/src/backend/postgres/search.rs @@ -124,8 +124,6 @@ impl PostgresStore { let conn = self.conn_pool.get().await.map_err(into_pool_error)?; let s = conn.prepare_cached(&query).await.map_err(into_error)?; - let c = println!("Executing search query: {} and values {:?}", query, params); - conn.query(&s, params.as_slice()) .await .and_then(|rows| { diff --git a/crates/store/src/dispatch/search.rs b/crates/store/src/dispatch/search.rs index 40740859..fa17c5aa 100644 --- a/crates/store/src/dispatch/search.rs +++ b/crates/store/src/dispatch/search.rs @@ -11,6 +11,7 @@ use crate::{ search::{ IndexDocument, SearchComparator, SearchField, SearchFilter, SearchOperator, SearchQuery, SearchValue, + split::{SplitFilter, split_filters}, }, write::SearchIndex, }; @@ -37,6 +38,7 @@ impl SearchStore { // If all filters and comparators are external, delegate to the underlying store let mut account_id = u32::MAX; let mut has_local_filters = false; + let mut has_external_filters = false; for filter in &query.filters { match filter { SearchFilter::Operator { @@ -49,6 +51,9 @@ impl SearchStore { SearchFilter::DocumentSet(_) => { has_local_filters = true; } + SearchFilter::Operator { .. } => { + has_external_filters = true; + } _ => (), } } @@ -71,143 +76,36 @@ impl SearchStore { .caused_by(trc::location!()); } - // Decompose filters into external and local filters - let mut filters = Vec::with_capacity(query.filters.len()); - let mut iter = query.filters.into_iter(); - let mut logical_op = None; + let filters = if has_external_filters { + // Split filters + let split_filters = split_filters(query.filters).ok_or_else(|| { + trc::StoreEvent::UnexpectedError + .reason("Invalid filter query") + .caused_by(trc::location!()) + })?; - while let Some(item) = iter.next() { - match &item { - SearchFilter::Operator { - field: SearchField::AccountId, - .. - } => {} - SearchFilter::Operator { .. } => { - let mut depth = 0; - let mut external = Vec::with_capacity(5); - - // Add the logical operator if there is one - let in_logical_op = if let Some(op) = logical_op.take() { - external.push(op); - true - } else { - false - }; - external.push(item); - - for item in iter.by_ref() { - match item { - SearchFilter::And | SearchFilter::Or | SearchFilter::Not => { - depth += 1; - external.push(item); - } - SearchFilter::End if depth > 0 => { - depth -= 1; - external.push(item); - } - SearchFilter::Operator { - field: SearchField::AccountId, - .. - } => {} - SearchFilter::Operator { .. } => { - external.push(item); - } - _ => { - let mut new_filters = Vec::new(); - let mut pop_count = depth; - while pop_count > 0 { - let prev_item = external.pop().unwrap(); - if matches!( - prev_item, - SearchFilter::And | SearchFilter::Or | SearchFilter::Not - ) { - pop_count -= 1; - } - new_filters.push(prev_item); - } - let is_end = matches!(item, SearchFilter::End); - new_filters.push(item); - - if !is_end { - if logical_op.is_some() { - depth += 1; - } - for item in iter { - match item { - SearchFilter::And - | SearchFilter::Or - | SearchFilter::Not => { - depth += 1; - new_filters.push(item); - } - SearchFilter::End => { - depth -= 1; - new_filters.push(item); - } - SearchFilter::Operator { - field: SearchField::AccountId, - .. - } => {} - SearchFilter::Operator { .. } if depth == 0 => { - external.push(item); - } - _ => { - new_filters.push(item); - } - } - } - } else { - new_filters.extend(iter); - } - iter = new_filters.into_iter(); - break; - } - } + let mut filters = Vec::with_capacity(split_filters.len()); + for split_filter in split_filters { + match split_filter { + SplitFilter::External(external) => { + // Execute sub-query + filters.push(SearchFilter::DocumentSet( + self.sub_query(query.index, &external, &[]) + .await? + .into_iter() + .collect(), + )); } - - if in_logical_op { - external.push(SearchFilter::End); + SplitFilter::Internal(filter) => { + filters.push(filter); } - - // Add account id - if external.len() == 1 { - external.push(SearchFilter::Operator { - field: SearchField::AccountId, - op: SearchOperator::Equal, - value: SearchValue::Uint(account_id as u64), - }); - } else { - external.insert(0, SearchFilter::And); - external.push(SearchFilter::Operator { - field: SearchField::AccountId, - op: SearchOperator::Equal, - value: SearchValue::Uint(account_id as u64), - }); - external.push(SearchFilter::End); - } - - // Execute sub-query - filters.push(SearchFilter::DocumentSet( - self.sub_query(query.index, &external, &[]) - .await? - .into_iter() - .collect(), - )); - } - _ => { - match &item { - SearchFilter::Or => { - logical_op = Some(SearchFilter::Or); - } - SearchFilter::And | SearchFilter::Not => { - logical_op = Some(SearchFilter::And); - } - _ => {} - } - filters.push(item); } } - } + + filters + } else { + query.filters + }; // Merge results locally let results = SearchQuery::new(query.index) @@ -403,6 +301,26 @@ impl SearchStore { _ => None, } } + + pub fn is_mysql(&self) -> bool { + match self { + #[cfg(feature = "mysql")] + SearchStore::Store(Store::MySQL(_)) => true, + _ => false, + } + } + + pub fn is_postgres(&self) -> bool { + match self { + #[cfg(feature = "postgres")] + SearchStore::Store(Store::PostgreSQL(_)) => true, + _ => false, + } + } + + pub fn is_elasticsearch(&self) -> bool { + matches!(self, SearchStore::ElasticSearch(_)) + } } impl SearchFilter { diff --git a/crates/store/src/dispatch/store.rs b/crates/store/src/dispatch/store.rs index e609e8b7..42d08efd 100644 --- a/crates/store/src/dispatch/store.rs +++ b/crates/store/src/dispatch/store.rs @@ -452,315 +452,4 @@ impl Store { } .caused_by(trc::location!()) } - - #[cfg(feature = "test_mode")] - pub async fn destroy(&self) { - use crate::*; - - #[cfg(any(feature = "postgres", feature = "mysql"))] - { - use crate::write::SearchIndex; - - for index in [ - SearchIndex::Email, - SearchIndex::Calendar, - SearchIndex::Contacts, - SearchIndex::Tracing, - ] { - self.sql_query::(&format!("TRUNCATE TABLE {}", index.psql_table()), vec![]) - .await - .unwrap(); - } - } - - for subspace in [ - SUBSPACE_ACL, - SUBSPACE_DIRECTORY, - SUBSPACE_TASK_QUEUE, - SUBSPACE_INDEXES, - SUBSPACE_BLOB_RESERVE, - SUBSPACE_BLOB_LINK, - SUBSPACE_LOGS, - SUBSPACE_IN_MEMORY_COUNTER, - SUBSPACE_IN_MEMORY_VALUE, - SUBSPACE_COUNTER, - SUBSPACE_PROPERTY, - SUBSPACE_SETTINGS, - SUBSPACE_BLOBS, - SUBSPACE_QUEUE_MESSAGE, - SUBSPACE_QUEUE_EVENT, - SUBSPACE_QUOTA, - SUBSPACE_REPORT_OUT, - SUBSPACE_REPORT_IN, - SUBSPACE_TELEMETRY_SPAN, - SUBSPACE_TELEMETRY_METRIC, - SUBSPACE_SEARCH_INDEX, - ] { - if subspace == SUBSPACE_SEARCH_INDEX && self.is_pg_or_mysql() { - continue; - } - - self.delete_range( - AnyKey { - subspace, - key: vec![0u8], - }, - AnyKey { - subspace, - key: vec![u8::MAX; 16], - }, - ) - .await - .unwrap(); - } - } - - #[cfg(feature = "test_mode")] - pub async fn blob_expire_all(&self) { - use crate::{U64_LEN, write::BlobOp}; - - // Delete all temporary hashes - let from_key = ValueKey { - account_id: 0, - collection: 0, - document_id: 0, - class: ValueClass::Blob(BlobOp::Reserve { - hash: types::blob_hash::BlobHash::default(), - until: 0, - }), - }; - let to_key = ValueKey { - account_id: u32::MAX, - collection: 0, - document_id: 0, - class: ValueClass::Blob(BlobOp::Reserve { - hash: types::blob_hash::BlobHash::default(), - until: 0, - }), - }; - let mut batch = BatchBuilder::new(); - let mut last_account_id = u32::MAX; - self.iterate( - IterateParams::new(from_key, to_key).ascending().no_values(), - |key, _| { - let account_id = key.deserialize_be_u32(0).caused_by(trc::location!())?; - if account_id != last_account_id { - last_account_id = account_id; - batch.with_account_id(account_id); - } - - batch.any_op(Operation::Value { - class: ValueClass::Blob(BlobOp::Reserve { - hash: types::blob_hash::BlobHash::try_from_hash_slice( - key.get(U32_LEN..U32_LEN + types::blob_hash::BLOB_HASH_LEN) - .unwrap(), - ) - .unwrap(), - until: key - .deserialize_be_u64(key.len() - U64_LEN) - .caused_by(trc::location!())?, - }), - op: ValueOp::Clear, - }); - - Ok(true) - }, - ) - .await - .unwrap(); - self.write(batch.build_all()).await.unwrap(); - } - - #[cfg(feature = "test_mode")] - pub async fn lookup_expire_all(&self) { - use crate::write::InMemoryClass; - - // Delete all temporary counters - let from_key = ValueKey::from(ValueClass::InMemory(InMemoryClass::Key(vec![0u8]))); - let to_key = ValueKey::from(ValueClass::InMemory(InMemoryClass::Key(vec![u8::MAX; 10]))); - - let mut expired_keys = Vec::new(); - let mut expired_counters = Vec::new(); - - self.iterate(IterateParams::new(from_key, to_key), |key, value| { - let expiry = value.deserialize_be_u64(0).caused_by(trc::location!())?; - if expiry == 0 { - expired_counters.push(key.to_vec()); - } else if expiry != u64::MAX { - expired_keys.push(key.to_vec()); - } - Ok(true) - }) - .await - .unwrap(); - - if !expired_keys.is_empty() { - let mut batch = BatchBuilder::new(); - for key in expired_keys { - batch.any_op(Operation::Value { - class: ValueClass::InMemory(InMemoryClass::Key(key)), - op: ValueOp::Clear, - }); - if batch.is_large_batch() { - self.write(batch.build_all()).await.unwrap(); - batch = BatchBuilder::new(); - } - } - if !batch.is_empty() { - self.write(batch.build_all()).await.unwrap(); - } - } - - if !expired_counters.is_empty() { - let mut batch = BatchBuilder::new(); - for key in expired_counters { - batch.any_op(Operation::Value { - class: ValueClass::InMemory(InMemoryClass::Counter(key.clone())), - op: ValueOp::Clear, - }); - batch.any_op(Operation::Value { - class: ValueClass::InMemory(InMemoryClass::Key(key)), - op: ValueOp::Clear, - }); - if batch.is_large_batch() { - self.write(batch.build_all()).await.unwrap(); - batch = BatchBuilder::new(); - } - } - if !batch.is_empty() { - self.write(batch.build_all()).await.unwrap(); - } - } - } - - #[cfg(feature = "test_mode")] - #[allow(unused_variables)] - pub async fn assert_is_empty(&self, blob_store: crate::BlobStore) { - use crate::*; - - self.blob_expire_all().await; - self.lookup_expire_all().await; - self.purge_blobs(blob_store).await.unwrap(); - self.purge_store().await.unwrap(); - - let store = self.clone(); - let mut failed = false; - - for (subspace, with_values) in [ - (SUBSPACE_ACL, true), - //(SUBSPACE_DIRECTORY, true), - (SUBSPACE_TASK_QUEUE, true), - (SUBSPACE_IN_MEMORY_VALUE, true), - (SUBSPACE_IN_MEMORY_COUNTER, false), - (SUBSPACE_PROPERTY, true), - (SUBSPACE_SETTINGS, true), - (SUBSPACE_QUEUE_MESSAGE, true), - (SUBSPACE_QUEUE_EVENT, true), - (SUBSPACE_REPORT_OUT, true), - (SUBSPACE_REPORT_IN, true), - (SUBSPACE_BLOB_RESERVE, true), - (SUBSPACE_BLOB_LINK, true), - (SUBSPACE_BLOBS, true), - (SUBSPACE_COUNTER, false), - (SUBSPACE_QUOTA, false), - (SUBSPACE_BLOBS, true), - (SUBSPACE_INDEXES, false), - (SUBSPACE_TELEMETRY_SPAN, true), - (SUBSPACE_TELEMETRY_METRIC, true), - (SUBSPACE_SEARCH_INDEX, true), - ] { - if subspace == SUBSPACE_SEARCH_INDEX && store.is_pg_or_mysql() { - continue; - } - - let from_key = crate::write::AnyKey { - subspace, - key: vec![0u8], - }; - let to_key = crate::write::AnyKey { - subspace, - key: vec![u8::MAX; 10], - }; - - self.iterate( - IterateParams::new(from_key, to_key).set_values(with_values), - |key, value| { - match subspace { - SUBSPACE_COUNTER if key.len() == U32_LEN + 1 || key.len() == U32_LEN => { - // Message ID and change ID counters - return Ok(true); - } - SUBSPACE_INDEXES => { - println!( - concat!( - "Found index key, account {}, collection {}, ", - "document {}, property {}, value {:?}: {:?}" - ), - u32::from_be_bytes(key[0..4].try_into().unwrap()), - key[4], - u32::from_be_bytes(key[key.len() - 4..].try_into().unwrap()), - key[5], - String::from_utf8_lossy(&key[6..key.len() - 4]), - key - ); - } - _ => { - println!( - "Found key in {:?}: {:?} ({:?}) = {:?} ({:?})", - char::from(subspace), - key, - String::from_utf8_lossy(key), - value, - String::from_utf8_lossy(value) - ); - } - } - failed = true; - - Ok(true) - }, - ) - .await - .unwrap(); - } - - // Delete logs and counters - self.delete_range( - AnyKey { - subspace: SUBSPACE_LOGS, - key: &[0u8], - }, - AnyKey { - subspace: SUBSPACE_LOGS, - key: &[ - u8::MAX, - u8::MAX, - u8::MAX, - u8::MAX, - u8::MAX, - u8::MAX, - u8::MAX, - ], - }, - ) - .await - .unwrap(); - - self.delete_range( - AnyKey { - subspace: SUBSPACE_COUNTER, - key: &[0u8], - }, - AnyKey { - subspace: SUBSPACE_COUNTER, - key: (u32::MAX / 2).to_be_bytes().as_slice(), - }, - ) - .await - .unwrap(); - - if failed { - panic!("Store is not empty."); - } - } } diff --git a/crates/store/src/search/bm_u32.rs b/crates/store/src/search/bm_u32.rs index 8d386f9c..32040fe9 100644 --- a/crates/store/src/search/bm_u32.rs +++ b/crates/store/src/search/bm_u32.rs @@ -145,11 +145,6 @@ pub(crate) async fn range_to_bitmap( ), }; - let len = from_value.len().min(SEARCH_INDEX_MAX_FIELD_LEN); - let mut data = [0u8; SEARCH_INDEX_MAX_FIELD_LEN]; - if len > 0 { - data[..len].copy_from_slice(&from_value[..len]); - } let begin = ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { index, id: SearchIndexId::Account { @@ -159,17 +154,11 @@ pub(crate) async fn range_to_bitmap( typ: SearchIndexType::Index { field: SearchIndexField { field_id: from_field, - len: len as u8, - data, + data: from_value.to_vec(), }, }, })); - let len = end_value.len().min(SEARCH_INDEX_MAX_FIELD_LEN); - let mut data = [0u8; SEARCH_INDEX_MAX_FIELD_LEN]; - if len > 0 { - data[..len].copy_from_slice(&end_value[..len]); - } let end = ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { index, id: SearchIndexId::Account { @@ -179,8 +168,7 @@ pub(crate) async fn range_to_bitmap( typ: SearchIndexType::Index { field: SearchIndexField { field_id: end_field, - len: len as u8, - data, + data: end_value.to_vec(), }, }, })); @@ -246,8 +234,7 @@ pub(crate) async fn sort_order( typ: SearchIndexType::Index { field: SearchIndexField { field_id, - len: SEARCH_INDEX_MAX_FIELD_LEN as u8, - data: [0u8; SEARCH_INDEX_MAX_FIELD_LEN], + data: vec![0u8], }, }, })); @@ -260,8 +247,7 @@ pub(crate) async fn sort_order( typ: SearchIndexType::Index { field: SearchIndexField { field_id, - len: SEARCH_INDEX_MAX_FIELD_LEN as u8, - data: [u8::MAX; SEARCH_INDEX_MAX_FIELD_LEN], + data: vec![u8::MAX; SEARCH_INDEX_MAX_FIELD_LEN], }, }, })); diff --git a/crates/store/src/search/bm_u64.rs b/crates/store/src/search/bm_u64.rs index f347e17c..3fda59a7 100644 --- a/crates/store/src/search/bm_u64.rs +++ b/crates/store/src/search/bm_u64.rs @@ -8,8 +8,8 @@ use crate::{ IterateParams, Store, U64_LEN, ValueKey, search::*, write::{ - SEARCH_INDEX_MAX_FIELD_LEN, SearchIndex, SearchIndexClass, SearchIndexField, SearchIndexId, - SearchIndexType, ValueClass, + SearchIndex, SearchIndexClass, SearchIndexField, SearchIndexId, SearchIndexType, + ValueClass, key::{DeserializeBigEndian, KeySerializer}, }, }; @@ -137,36 +137,24 @@ pub(crate) async fn range_to_treemap( ), }; - let len = from_value.len().min(SEARCH_INDEX_MAX_FIELD_LEN); - let mut data = [0u8; SEARCH_INDEX_MAX_FIELD_LEN]; - if len > 0 { - data[..len].copy_from_slice(&from_value[..len]); - } let begin = ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { index, id: SearchIndexId::Global { id: from_id }, typ: SearchIndexType::Index { field: SearchIndexField { field_id: from_field, - len: len as u8, - data, + data: from_value.to_vec(), }, }, })); - let len = end_value.len().min(SEARCH_INDEX_MAX_FIELD_LEN); - let mut data = [0u8; SEARCH_INDEX_MAX_FIELD_LEN]; - if len > 0 { - data[..len].copy_from_slice(&end_value[..len]); - } let end = ValueKey::from(ValueClass::SearchIndex(SearchIndexClass { index, id: SearchIndexId::Global { id: end_id }, typ: SearchIndexType::Index { field: SearchIndexField { field_id: end_field, - len: len as u8, - data, + data: end_value.to_vec(), }, }, })); diff --git a/crates/store/src/search/index.rs b/crates/store/src/search/index.rs index e90b5bdd..ad47da53 100644 --- a/crates/store/src/search/index.rs +++ b/crates/store/src/search/index.rs @@ -210,8 +210,7 @@ impl Store { typ: SearchIndexType::Index { field: SearchIndexField { field_id: 0, - len: 1, - data: [0; SEARCH_INDEX_MAX_FIELD_LEN], + data: vec![0u8], }, }, })), @@ -224,8 +223,7 @@ impl Store { typ: SearchIndexType::Index { field: SearchIndexField { field_id: u8::MAX, - len: 1, - data: [u8::MAX; SEARCH_INDEX_MAX_FIELD_LEN], + data: vec![u8::MAX; SEARCH_INDEX_MAX_FIELD_LEN], }, }, })), diff --git a/crates/store/src/search/mod.rs b/crates/store/src/search/mod.rs index 3588ba9f..19d33cce 100644 --- a/crates/store/src/search/mod.rs +++ b/crates/store/src/search/mod.rs @@ -11,6 +11,7 @@ pub mod fields; pub mod index; pub mod local; pub mod query; +pub mod split; pub mod term; use crate::write::SearchIndex; @@ -122,7 +123,7 @@ pub struct SearchQuery { pub(crate) mask: RoaringBitmap, } -#[derive(Debug)] +#[derive(Debug, PartialEq, Clone, Default)] pub enum SearchFilter { Operator { field: SearchField, @@ -133,6 +134,7 @@ pub enum SearchFilter { And, Or, Not, + #[default] End, } @@ -319,3 +321,5 @@ impl ParseValue for SearchField { }) } } + +impl Eq for SearchFilter {} diff --git a/crates/store/src/search/query.rs b/crates/store/src/search/query.rs index 382301a8..4b155cc4 100644 --- a/crates/store/src/search/query.rs +++ b/crates/store/src/search/query.rs @@ -13,6 +13,7 @@ use crate::{ bm_u32::{BitmapCache, range_to_bitmap, sort_order}, bm_u64::{TreemapCache, range_to_treemap}, }, + write::SEARCH_INDEX_MAX_FIELD_LEN, }; use nlp::{language::stemmer::Stemmer, tokenizers::space::SpaceTokenizer}; use roaring::{RoaringBitmap, RoaringTreemap}; @@ -172,7 +173,11 @@ impl Store { } } else if field.is_indexed() { let value = match value { - SearchValue::Text { value, .. } => value.into_bytes(), + SearchValue::Text { value, .. } => { + let mut value = value.into_bytes(); + value.truncate(SEARCH_INDEX_MAX_FIELD_LEN); + value + } SearchValue::Int(v) => (v as u64).to_be_bytes().to_vec(), SearchValue::Uint(v) => v.to_be_bytes().to_vec(), SearchValue::Boolean(v) => vec![v as u8], diff --git a/crates/store/src/search/split.rs b/crates/store/src/search/split.rs new file mode 100644 index 00000000..4a5a767c --- /dev/null +++ b/crates/store/src/search/split.rs @@ -0,0 +1,708 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::search::*; + +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum SplitFilter { + Internal(SearchFilter), + External(Vec), +} + +pub(crate) fn split_filters(filters_in: Vec) -> Option> { + let mut account_id = u64::MAX; + let mut filters: Vec = Vec::with_capacity(filters_in.len()); + let mut op_stack = Vec::new(); + let mut document_sets: AHashMap = AHashMap::new(); + let mut operators: AHashMap> = AHashMap::new(); + + for filter in filters_in { + match filter { + op @ (SearchFilter::And | SearchFilter::Or | SearchFilter::Not) => { + op_stack.push(op.clone()); + filters.push(op); + } + SearchFilter::End => { + if let Some(ops) = operators.remove(&op_stack.len()) { + filters.extend(ops); + } + if let Some(docs) = document_sets.remove(&op_stack.len()) { + filters.push(SearchFilter::DocumentSet(docs)); + } + filters.push(SearchFilter::End); + op_stack.pop()?; + } + SearchFilter::Operator { + field: SearchField::AccountId, + value: SearchValue::Uint(id), + .. + } => { + account_id = id; + } + SearchFilter::Operator { .. } => { + operators.entry(op_stack.len()).or_default().push(filter); + } + SearchFilter::DocumentSet(docs) => match document_sets.entry(op_stack.len()) { + Entry::Occupied(mut entry) => { + if matches!(op_stack.last(), Some(SearchFilter::Or)) { + entry.get_mut().bitor_assign(&docs); + } else { + entry.get_mut().bitand_assign(&docs); + } + } + Entry::Vacant(entry) => { + entry.insert(docs); + } + }, + } + } + + if let Some(ops) = operators.remove(&0) { + filters.extend(ops); + } + if let Some(docs) = document_sets.remove(&0) { + filters.push(SearchFilter::DocumentSet(docs)); + } + + if account_id == u64::MAX { + return None; + } + + let mut split: Vec = Vec::new(); + let mut i = 0; + + 'outer: while i < filters.len() { + let mut j = i; + let mut depth = 0; + + while j < filters.len() { + match &filters[j] { + SearchFilter::And | SearchFilter::Or | SearchFilter::Not => { + depth += 1; + } + SearchFilter::End => { + depth -= 1; + if depth < 0 { + if j > i { + break; + } else { + split.push(SplitFilter::Internal(SearchFilter::End)); + i += 1; + continue 'outer; + } + } + } + SearchFilter::Operator { .. } => {} + SearchFilter::DocumentSet(_) => { + if depth == 0 && j > i { + break; + } else { + split.push(SplitFilter::Internal(std::mem::take(&mut filters[i]))); + i += 1; + continue 'outer; + } + } + } + j += 1; + } + + let mut external_filters = vec![SearchFilter::Operator { + field: SearchField::AccountId, + op: SearchOperator::Equal, + value: SearchValue::Uint(account_id), + }]; + let add_or = + matches!(split.last(), Some(SplitFilter::Internal(SearchFilter::Or))) && j > i + 1; + if add_or { + external_filters.push(SearchFilter::Or); + } + external_filters.extend(&mut filters[i..j].iter_mut().map(std::mem::take)); + if add_or { + external_filters.push(SearchFilter::End); + } + split.push(SplitFilter::External(external_filters)); + + i = j; + } + + Some(split) +} + +// Test cases +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_split_filters_exhaustive() { + let test_cases: Vec<(&str, Vec, Vec)> = vec![ + // Test 1: Operator followed by document set at depth 0 + ( + "Operator then document set at depth 0", + vec![account_id(42), other_op("test"), doc_set(&[1, 2, 3])], + vec![ + SplitFilter::External(vec![account_id(42), other_op("test")]), + SplitFilter::Internal(doc_set(&[1, 2, 3])), + ], + ), + // Test 2: Document set followed by operator at depth 0 + ( + "Document set then operator at depth 0", + vec![account_id(42), doc_set(&[1, 2, 3]), other_op("test")], + vec![ + SplitFilter::External(vec![account_id(42), other_op("test")]), + SplitFilter::Internal(doc_set(&[1, 2, 3])), + ], + ), + // Test 3: Multiple document sets with operator in between + ( + "Multiple document sets at depth 0 with operator", + vec![ + account_id(42), + doc_set(&[1, 2]), + other_op("middle"), + doc_set(&[2, 4]), + ], + vec![ + SplitFilter::External(vec![account_id(42), other_op("middle")]), + SplitFilter::Internal(doc_set(&[2])), + ], + ), + // Test 4: Document set at depth 0, then AND group + ( + "Document set then AND group", + vec![ + account_id(42), + doc_set(&[1, 2]), + SearchFilter::And, + other_op("a"), + other_op("b"), + SearchFilter::End, + ], + vec![ + SplitFilter::External(vec![ + account_id(42), + SearchFilter::And, + other_op("a"), + other_op("b"), + SearchFilter::End, + ]), + SplitFilter::Internal(doc_set(&[1, 2])), + ], + ), + // Test 5: AND group followed by document set at depth 0 + ( + "AND group then document set", + vec![ + account_id(42), + SearchFilter::And, + other_op("a"), + other_op("b"), + SearchFilter::End, + doc_set(&[1, 2]), + ], + vec![ + SplitFilter::External(vec![ + account_id(42), + SearchFilter::And, + other_op("a"), + other_op("b"), + SearchFilter::End, + ]), + SplitFilter::Internal(doc_set(&[1, 2])), + ], + ), + // Test 6: Operator at depth 0, then OR group, then document set + ( + "Operator, OR group, then document set", + vec![ + account_id(42), + other_op("pre"), + SearchFilter::Or, + other_op("a"), + other_op("b"), + SearchFilter::End, + doc_set(&[1, 2, 3]), + ], + vec![ + SplitFilter::External(vec![ + account_id(42), + SearchFilter::Or, + other_op("a"), + other_op("b"), + SearchFilter::End, + other_op("pre"), + ]), + SplitFilter::Internal(doc_set(&[1, 2, 3])), + ], + ), + // Test 7: Document set, OR group, operator + ( + "Document set, OR group, operator", + vec![ + account_id(42), + doc_set(&[1, 2]), + SearchFilter::Or, + other_op("a"), + other_op("b"), + SearchFilter::End, + other_op("post"), + ], + vec![ + SplitFilter::External(vec![ + account_id(42), + SearchFilter::Or, + other_op("a"), + other_op("b"), + SearchFilter::End, + other_op("post"), + ]), + SplitFilter::Internal(doc_set(&[1, 2])), + ], + ), + // Test 8: Multiple OR branches with document sets between + ( + "Multiple OR branches with document sets between", + vec![ + account_id(42), + SearchFilter::Or, + other_op("a"), + SearchFilter::End, + doc_set(&[1, 2]), + SearchFilter::Or, + other_op("b"), + SearchFilter::End, + doc_set(&[1, 2, 5, 6]), + ], + vec![ + SplitFilter::External(vec![ + account_id(42), + SearchFilter::Or, + other_op("a"), + SearchFilter::End, + SearchFilter::Or, + other_op("b"), + SearchFilter::End, + ]), + SplitFilter::Internal(doc_set(&[1, 2])), + ], + ), + // Test 9: Document sets at different depths - depth 0 and inside AND + ( + "Document sets at different depths in AND", + vec![ + account_id(42), + doc_set(&[1, 2]), + SearchFilter::And, + other_op("a"), + doc_set(&[2, 3]), + SearchFilter::End, + ], + vec![ + SplitFilter::Internal(SearchFilter::And), + SplitFilter::External(vec![account_id(42), other_op("a")]), + SplitFilter::Internal(doc_set(&[2, 3])), + SplitFilter::Internal(SearchFilter::End), + SplitFilter::Internal(doc_set(&[1, 2])), + ], + ), + // Test 10: Operator, AND group with doc set inside, operator + ( + "Operator, AND(operator, doc_set), operator", + vec![ + account_id(42), + other_op("pre"), + SearchFilter::And, + other_op("a"), + doc_set(&[1, 2, 3]), + SearchFilter::End, + other_op("post"), + ], + vec![ + SplitFilter::Internal(SearchFilter::And), + SplitFilter::External(vec![account_id(42), other_op("a")]), + SplitFilter::Internal(doc_set(&[1, 2, 3])), + SplitFilter::Internal(SearchFilter::End), + SplitFilter::External(vec![account_id(42), other_op("pre"), other_op("post")]), + ], + ), + // Test 11: Document set, nested groups, document set + ( + "Doc set, AND(OR(a,b)), doc set", + vec![ + account_id(42), + SearchFilter::Or, + doc_set(&[1, 2]), + other_op("c"), + SearchFilter::And, + other_op("a"), + other_op("b"), + SearchFilter::End, + doc_set(&[3, 4]), + SearchFilter::End, + ], + vec![ + SplitFilter::Internal(SearchFilter::Or), + SplitFilter::External(vec![ + account_id(42), + SearchFilter::Or, + SearchFilter::And, + other_op("a"), + other_op("b"), + SearchFilter::End, + other_op("c"), + SearchFilter::End, + ]), + SplitFilter::Internal(doc_set(&[1, 2, 3, 4])), + SplitFilter::Internal(SearchFilter::End), + ], + ), + // Test 12: OR with nested AND containing document sets, followed by operator + ( + "OR(AND(doc_set, doc_set), operator) followed by operator", + vec![ + account_id(42), + SearchFilter::Or, + SearchFilter::And, + doc_set(&[1, 2]), + doc_set(&[2, 3]), + SearchFilter::End, + other_op("b"), + SearchFilter::End, + other_op("post"), + ], + vec![ + SplitFilter::Internal(SearchFilter::Or), + SplitFilter::Internal(SearchFilter::And), + SplitFilter::Internal(doc_set(&[2])), + SplitFilter::Internal(SearchFilter::End), + SplitFilter::External(vec![account_id(42), other_op("b")]), + SplitFilter::Internal(SearchFilter::End), + SplitFilter::External(vec![account_id(42), other_op("post")]), + ], + ), + // Test 13: Complex: doc set, AND group, doc set, OR group, doc set + ( + "Complex: doc, AND, doc, OR, doc", + vec![ + account_id(42), + doc_set(&[1, 2, 3]), + SearchFilter::And, + other_op("a"), + SearchFilter::End, + doc_set(&[1, 2, 3, 5]), + SearchFilter::Or, + other_op("b"), + SearchFilter::End, + doc_set(&[1, 2, 3, 6]), + ], + vec![ + SplitFilter::External(vec![ + account_id(42), + SearchFilter::And, + other_op("a"), + SearchFilter::End, + SearchFilter::Or, + other_op("b"), + SearchFilter::End, + ]), + SplitFilter::Internal(doc_set(&[1, 2, 3])), + ], + ), + // Test 14: Operator, NOT group, document set + ( + "Operator, NOT(operator), document set", + vec![ + account_id(42), + other_op("pre"), + SearchFilter::Not, + other_op("a"), + SearchFilter::End, + doc_set(&[1, 2]), + ], + vec![ + SplitFilter::External(vec![ + account_id(42), + SearchFilter::Not, + other_op("a"), + SearchFilter::End, + other_op("pre"), + ]), + SplitFilter::Internal(doc_set(&[1, 2])), + ], + ), + // Test 15: Document set, NOT group, operator + ( + "Document set, NOT(operator), operator", + vec![ + account_id(42), + doc_set(&[1, 2]), + SearchFilter::Not, + other_op("a"), + doc_set(&[3, 4]), + SearchFilter::End, + other_op("post"), + ], + vec![ + SplitFilter::Internal(SearchFilter::Not), + SplitFilter::External(vec![account_id(42), other_op("a")]), + SplitFilter::Internal(doc_set(&[3, 4])), + SplitFilter::Internal(SearchFilter::End), + SplitFilter::External(vec![account_id(42), other_op("post")]), + SplitFilter::Internal(doc_set(&[1, 2])), + ], + ), + // Test 16: Alternating doc sets and operators + ( + "Alternating: doc, op, doc, op, doc", + vec![ + account_id(42), + doc_set(&[1]), + other_op("a"), + doc_set(&[1, 2]), + other_op("b"), + doc_set(&[1, 3]), + ], + vec![ + SplitFilter::External(vec![account_id(42), other_op("a"), other_op("b")]), + SplitFilter::Internal(doc_set(&[1])), + ], + ), + // Test 17: Multiple operators, then OR group with doc set inside, then doc set + ( + "Multiple ops, OR(op, doc_set), doc", + vec![ + account_id(42), + other_op("a"), + SearchFilter::Or, + other_op("c"), + doc_set(&[1, 2]), + SearchFilter::End, + other_op("b"), + doc_set(&[3, 4]), + ], + vec![ + SplitFilter::Internal(SearchFilter::Or), + SplitFilter::External(vec![account_id(42), other_op("c")]), + SplitFilter::Internal(doc_set(&[1, 2])), + SplitFilter::Internal(SearchFilter::End), + SplitFilter::External(vec![account_id(42), other_op("a"), other_op("b")]), + SplitFilter::Internal(doc_set(&[3, 4])), + ], + ), + // Test 18: Doc set before and after nested OR(AND(op)) + ( + "Doc, OR(AND(op)), doc", + vec![ + account_id(42), + doc_set(&[1]), + SearchFilter::Or, + SearchFilter::And, + other_op("a"), + other_op("c"), + SearchFilter::End, + other_op("b"), + SearchFilter::End, + doc_set(&[2]), + ], + vec![ + SplitFilter::External(vec![ + account_id(42), + SearchFilter::Or, + SearchFilter::And, + other_op("a"), + other_op("c"), + SearchFilter::End, + other_op("b"), + SearchFilter::End, + ]), + SplitFilter::Internal(doc_set(&[])), + ], + ), + // Test 19: AND group with doc set, operator between, OR group with doc set + ( + "AND(op, doc), op, OR(op, doc)", + vec![ + account_id(42), + SearchFilter::And, + other_op("a"), + doc_set(&[1, 2]), + SearchFilter::End, + other_op("middle"), + SearchFilter::Or, + other_op("b"), + other_op("c"), + doc_set(&[3, 4]), + SearchFilter::End, + ], + vec![ + SplitFilter::Internal(SearchFilter::And), + SplitFilter::External(vec![account_id(42), other_op("a")]), + SplitFilter::Internal(doc_set(&[1, 2])), + SplitFilter::Internal(SearchFilter::End), + SplitFilter::Internal(SearchFilter::Or), + SplitFilter::External(vec![ + account_id(42), + SearchFilter::Or, + other_op("b"), + other_op("c"), + SearchFilter::End, + ]), + SplitFilter::Internal(doc_set(&[3, 4])), + SplitFilter::Internal(SearchFilter::End), + SplitFilter::External(vec![account_id(42), other_op("middle")]), + ], + ), + // Test 20: Deep nesting with document sets at multiple levels + ( + "Deep nesting: doc, AND(doc, OR(doc, AND(op, doc)))", + vec![ + account_id(42), + doc_set(&[1]), + SearchFilter::And, + doc_set(&[2]), + SearchFilter::Or, + doc_set(&[3]), + SearchFilter::And, + other_op("a"), + doc_set(&[4]), + SearchFilter::End, + SearchFilter::End, + SearchFilter::End, + ], + vec![ + SplitFilter::Internal(SearchFilter::And), + SplitFilter::Internal(SearchFilter::Or), + SplitFilter::Internal(SearchFilter::And), + SplitFilter::External(vec![account_id(42), other_op("a")]), + SplitFilter::Internal(doc_set(&[4])), + SplitFilter::Internal(SearchFilter::End), + SplitFilter::Internal(doc_set(&[3])), + SplitFilter::Internal(SearchFilter::End), + SplitFilter::Internal(doc_set(&[2])), + SplitFilter::Internal(SearchFilter::End), + SplitFilter::Internal(doc_set(&[1])), + ], + ), + ]; + + for (description, input, expected) in test_cases { + println!("------ Running test: {} ------", description); + let result = split_filters(input.clone()); + assert!(result.is_some(), "Test '{}' returned None", description); + + let result = result.unwrap(); + if result != expected { + print_split_filter_code(&result); + } + assert_eq!(result, expected, "Test '{description}' failed",); + } + } + + fn account_id(id: u64) -> SearchFilter { + SearchFilter::Operator { + field: SearchField::AccountId, + op: SearchOperator::Equal, + value: SearchValue::Uint(id), + } + } + + fn other_op(value: &str) -> SearchFilter { + SearchFilter::Operator { + field: SearchField::DocumentId, + op: SearchOperator::Equal, + value: SearchValue::Text { + value: value.to_string(), + language: Language::None, + }, + } + } + + fn doc_set(ids: &[u32]) -> SearchFilter { + let mut bitmap = RoaringBitmap::new(); + for id in ids { + bitmap.insert(*id); + } + SearchFilter::DocumentSet(bitmap) + } + + fn print_split_filter_code(splits: &[SplitFilter]) { + println!("vec!["); + for split in splits { + match split { + SplitFilter::Internal(filter) => { + print!(" SplitFilter::Internal("); + print_search_filter_code(filter, 0); + println!("),"); + } + SplitFilter::External(filters) => { + println!(" SplitFilter::External(vec!["); + for filter in filters { + print!(" "); + print_search_filter_code(filter, 2); + println!(","); + } + println!(" ]),"); + } + } + } + println!("]"); + } + + fn print_search_filter_code(filter: &SearchFilter, indent_level: usize) { + let indent = " ".repeat(indent_level); + match filter { + SearchFilter::Operator { field, op, value } => match (field, op, value) { + (SearchField::AccountId, SearchOperator::Equal, SearchValue::Uint(id)) => { + print!("account_id({})", id); + } + ( + SearchField::DocumentId, + SearchOperator::Equal, + SearchValue::Text { value, .. }, + ) => { + print!("other_op(\"{}\")", value); + } + _ => { + println!("SearchFilter::Operator {{"); + println!("{} field: {:?},", indent, field); + println!("{} op: {:?},", indent, op); + println!("{} value: {:?},", indent, value); + print!("{}}}", indent); + } + }, + SearchFilter::DocumentSet(bitmap) => { + let ids: Vec = bitmap.iter().collect(); + if ids.is_empty() { + print!("doc_set(&[])"); + } else if ids.len() <= 5 { + print!("doc_set(&["); + for (i, id) in ids.iter().enumerate() { + if i > 0 { + print!(", "); + } + print!("{}", id); + } + print!("])"); + } else { + // For large bitmaps, create inline + println!("{{"); + println!("{} let mut bitmap = RoaringBitmap::new();", indent); + for id in ids { + println!("{} bitmap.insert({});", indent, id); + } + print!("{} doc_set_bitmap(bitmap)", indent); + println!(); + print!("{}}}", indent); + } + } + SearchFilter::And => print!("SearchFilter::And"), + SearchFilter::Or => print!("SearchFilter::Or"), + SearchFilter::Not => print!("SearchFilter::Not"), + SearchFilter::End => print!("SearchFilter::End"), + } + } +} diff --git a/crates/store/src/search/term.rs b/crates/store/src/search/term.rs index 2e657af1..d7d79fea 100644 --- a/crates/store/src/search/term.rs +++ b/crates/store/src/search/term.rs @@ -5,7 +5,7 @@ */ use crate::{ - Serialize, U64_LEN, + Serialize, backend::MAX_TOKEN_LENGTH, search::*, write::{ @@ -52,12 +52,9 @@ impl TermIndexBuilder { match field { SearchField::Id => { if let SearchValue::Uint(v) = value { - let mut data = [0u8; SEARCH_INDEX_MAX_FIELD_LEN]; - data[..U64_LEN].copy_from_slice(&v.to_be_bytes()); fields.push(SearchIndexField { field_id: field.u8_id(), - len: U64_LEN as u8, - data, + data: v.to_be_bytes().to_vec(), }); id = Some(v); } @@ -120,15 +117,11 @@ impl TermIndexBuilder { } if field.is_indexed() { - let bytes = value.as_bytes(); - let len = bytes.len().min(SEARCH_INDEX_MAX_FIELD_LEN); - let mut data = [0u8; SEARCH_INDEX_MAX_FIELD_LEN]; - - data[..len].copy_from_slice(&bytes[..len]); + let mut data = value.into_bytes(); + data.truncate(SEARCH_INDEX_MAX_FIELD_LEN); SearchIndexField { field_id: field.u8_id(), - len: len as u8, data, } } else { @@ -151,30 +144,17 @@ impl TermIndexBuilder { continue; } - SearchValue::Int(v) => { - let mut data = [0u8; SEARCH_INDEX_MAX_FIELD_LEN]; - data[..U64_LEN].copy_from_slice(&(v as u64).to_be_bytes()); - - SearchIndexField { - field_id: field.u8_id(), - len: U64_LEN as u8, - data, - } - } - SearchValue::Uint(v) => { - let mut data = [0u8; SEARCH_INDEX_MAX_FIELD_LEN]; - data[..U64_LEN].copy_from_slice(&v.to_be_bytes()); - - SearchIndexField { - field_id: field.u8_id(), - len: U64_LEN as u8, - data, - } - } + SearchValue::Int(v) => SearchIndexField { + field_id: field.u8_id(), + data: (v as u64).to_be_bytes().to_vec(), + }, + SearchValue::Uint(v) => SearchIndexField { + field_id: field.u8_id(), + data: v.to_be_bytes().to_vec(), + }, SearchValue::Boolean(v) => SearchIndexField { field_id: field.u8_id(), - len: 1, - data: [v as u8; SEARCH_INDEX_MAX_FIELD_LEN], + data: vec![v as u8], }, }; @@ -286,8 +266,7 @@ impl TermIndex { for field in old_term.fields.iter() { old_fields.insert(SearchIndexField { field_id: field.field_id, - len: field.len, - data: field.data, + data: field.data.to_vec(), }); } @@ -366,8 +345,7 @@ impl ArchivedTermIndex { typ: SearchIndexType::Index { field: SearchIndexField { field_id: field.field_id, - len: field.len, - data: field.data, + data: field.data.to_vec(), }, }, })); diff --git a/crates/store/src/write/key.rs b/crates/store/src/write/key.rs index 6ca6c623..1e2657ca 100644 --- a/crates/store/src/write/key.rs +++ b/crates/store/src/write/key.rs @@ -473,12 +473,12 @@ impl ValueClass { .write(class) .write(*account_id) .write(field.field_id) - .write(&field.data[..field.len as usize]) + .write(field.data.as_slice()) .write(*document_id), SearchIndexId::Global { id } => serializer .write(class) .write(field.field_id) - .write(&field.data[..field.len as usize]) + .write(field.data.as_slice()) .write(*id), } } @@ -601,7 +601,7 @@ impl ValueClass { ValueClass::ShareNotification { .. } => U32_LEN + U64_LEN + 1, ValueClass::SearchIndex(v) => match &v.typ { SearchIndexType::Term { hash, .. } => U64_LEN + hash.len() + 2, - SearchIndexType::Index { field, .. } => 1 + field.len as usize + U64_LEN, + SearchIndexType::Index { field, .. } => 1 + field.data.len() + U64_LEN, SearchIndexType::Document => match &v.id { SearchIndexId::Account { .. } => 1 + U32_LEN * 2, SearchIndexId::Global { .. } => 1 + U64_LEN, diff --git a/crates/store/src/write/mod.rs b/crates/store/src/write/mod.rs index b326f01c..ba55930c 100644 --- a/crates/store/src/write/mod.rs +++ b/crates/store/src/write/mod.rs @@ -205,13 +205,12 @@ pub enum SearchIndexType { Document, } -pub(crate) const SEARCH_INDEX_MAX_FIELD_LEN: usize = 16; +pub(crate) const SEARCH_INDEX_MAX_FIELD_LEN: usize = 128; #[derive(Debug, PartialEq, Eq, Clone, Hash, rkyv::Serialize, rkyv::Deserialize, rkyv::Archive)] pub struct SearchIndexField { pub(crate) field_id: u8, - pub(crate) len: u8, - pub(crate) data: [u8; SEARCH_INDEX_MAX_FIELD_LEN], + pub(crate) data: Vec, } #[derive(Debug, PartialEq, Clone, Copy, Eq, Hash)] diff --git a/tests/Cargo.toml b/tests/Cargo.toml index e53f6b95..4d0b57ab 100644 --- a/tests/Cargo.toml +++ b/tests/Cargo.toml @@ -4,8 +4,8 @@ version = "0.14.1" edition = "2024" [features] -#default = ["sqlite", "postgres", "mysql", "rocks", "s3", "redis", "nats", "azure", "foundationdb", "enterprise"] -default = ["postgres"] +#default = ["sqlite", "postgres", "mysql", "rocks", "s3", "redis", "nats", "azure", "foundationdb"] +default = ["rocks", "foundationdb"] sqlite = ["store/sqlite"] foundationdb = ["store/foundation", "common/foundation"] postgres = ["store/postgres"] diff --git a/tests/src/cluster/mod.rs b/tests/src/cluster/mod.rs index 9fb329bc..2b588652 100644 --- a/tests/src/cluster/mod.rs +++ b/tests/src/cluster/mod.rs @@ -9,6 +9,7 @@ use crate::{ directory::internal::TestInternalDirectory, imap::{ImapConnection, Type}, jmap::server::enterprise::EnterpriseCore, + store::cleanup::store_destroy, }; use ahash::AHashMap; use common::{ @@ -103,7 +104,7 @@ async fn init_cluster_tests(delete_if_exists: bool) -> ClusterTest { let store = servers.first().unwrap().store().clone(); if delete_if_exists { - store.destroy().await; + store_destroy(&store).await; } // Create test users diff --git a/tests/src/directory/internal.rs b/tests/src/directory/internal.rs index b82a1ab6..7395562b 100644 --- a/tests/src/directory/internal.rs +++ b/tests/src/directory/internal.rs @@ -4,7 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::directory::{DirectoryTest, IntoTestPrincipal, TestPrincipal}; +use crate::{ + directory::{DirectoryTest, IntoTestPrincipal, TestPrincipal}, + store::cleanup::store_destroy, +}; use ahash::AHashSet; use directory::{ Permission, QueryBy, QueryParams, Type, @@ -30,7 +33,7 @@ async fn internal_directory() { for (store_id, store) in config.stores.stores { println!("Testing internal directory with store {:?}", store_id); - store.destroy().await; + store_destroy(&store).await; // A principal without name should fail assert_eq!( diff --git a/tests/src/directory/sql.rs b/tests/src/directory/sql.rs index aee49a5a..c71265cc 100644 --- a/tests/src/directory/sql.rs +++ b/tests/src/directory/sql.rs @@ -13,8 +13,9 @@ use mail_send::Credentials; #[allow(unused_imports)] use store::{InMemoryStore, Store}; -use crate::directory::{ - DirectoryTest, IntoTestPrincipal, TestPrincipal, map_account_id, map_account_ids, +use crate::{ + directory::{DirectoryTest, IntoTestPrincipal, TestPrincipal, map_account_id, map_account_ids}, + store::cleanup::store_destroy, }; use super::DirectoryStore; @@ -43,7 +44,7 @@ async fn sql_directory() { let core = config.server; // Create tables - base_store.destroy().await; + store_destroy(base_store).await; store.create_test_directory().await; // Create test users diff --git a/tests/src/imap/mod.rs b/tests/src/imap/mod.rs index c95828e6..0fb0a801 100644 --- a/tests/src/imap/mod.rs +++ b/tests/src/imap/mod.rs @@ -23,7 +23,10 @@ pub mod thread; use crate::{ AssertConfig, add_test_certs, directory::internal::TestInternalDirectory, - store::{TempDir, build_store_config}, + store::{ + TempDir, build_store_config, + cleanup::{search_store_destroy, store_destroy}, + }, }; use ::managesieve::core::ManageSieveSessionManager; use ::store::Stores; @@ -88,11 +91,11 @@ pub async fn imap_tests() { mailbox::test(&mut imap, &mut imap_check).await; append::test(&mut imap, &mut imap_check, &handle).await; - search::test(&mut imap, &mut imap_check).await; + search::test(&mut imap, &mut imap_check, &handle).await; fetch::test(&mut imap, &mut imap_check).await; store::test(&mut imap, &mut imap_check, &handle).await; copy_move::test(&mut imap, &mut imap_check).await; - thread::test(&mut imap, &mut imap_check).await; + thread::test(&mut imap, &mut imap_check, &handle).await; idle::test(&mut imap, &mut imap_check, false).await; condstore::test(&mut imap, &mut imap_check).await; acl::test(&mut imap, &mut imap_check).await; @@ -166,6 +169,7 @@ async fn init_imap_tests(delete_if_exists: bool) -> IMAPTest { let cache = Caches::parse(&mut config); let store = core.storage.data.clone(); + let search_store = core.storage.fts.clone(); let (ipc, mut ipc_rxs) = build_ipc(false); let inner = Arc::new(Inner { shared_core: core.into_shared(), @@ -222,7 +226,8 @@ async fn init_imap_tests(delete_if_exists: bool) -> IMAPTest { }); if delete_if_exists { - store.destroy().await; + store_destroy(&store).await; + search_store_destroy(&search_store).await; } // Create tables and test accounts diff --git a/tests/src/imap/search.rs b/tests/src/imap/search.rs index 8a593f3d..d87bc2d0 100644 --- a/tests/src/imap/search.rs +++ b/tests/src/imap/search.rs @@ -4,11 +4,11 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use super::{AssertResult, ImapConnection, Type}; +use crate::imap::IMAPTest; use imap_proto::ResponseType; -use super::{AssertResult, ImapConnection, Type}; - -pub async fn test(imap: &mut ImapConnection, imap_check: &mut ImapConnection) { +pub async fn test(imap: &mut ImapConnection, imap_check: &mut ImapConnection, handle: &IMAPTest) { println!("Running SEARCH tests..."); // Searches without selecting a mailbox should fail. @@ -119,5 +119,9 @@ pub async fn test(imap: &mut ImapConnection, imap_check: &mut ImapConnection) { .await; imap.assert_read(Type::Tagged, ResponseType::Ok) .await - .assert_contains("COUNT 10 ALL 6,4:5,1,10,3,7:8,2,9"); //6,4:5,1,10,9,3,7:8,2"); + .assert_contains(if !handle.server.search_store().is_mysql() { + "COUNT 10 ALL 6,4:5,1,10,3,7:8,2,9" + } else { + "COUNT 10 ALL 9,3,7:8,2,6,4:5,1,10" + }); //6,4:5,1,10,9,3,7:8,2"); } diff --git a/tests/src/imap/thread.rs b/tests/src/imap/thread.rs index 04227835..2439ba33 100644 --- a/tests/src/imap/thread.rs +++ b/tests/src/imap/thread.rs @@ -6,11 +6,11 @@ use imap_proto::ResponseType; -use crate::imap::{AssertResult, expand_uid_list}; +use crate::imap::{AssertResult, IMAPTest, expand_uid_list}; use super::{ImapConnection, Type, append::build_messages}; -pub async fn test(imap: &mut ImapConnection, _imap_check: &mut ImapConnection) { +pub async fn test(imap: &mut ImapConnection, _imap_check: &mut ImapConnection, handle: &IMAPTest) { println!("Running THREAD tests..."); // Create test messages @@ -80,12 +80,15 @@ pub async fn test(imap: &mut ImapConnection, _imap_check: &mut ImapConnection) { .assert_contains("(5 6 7 8)") .assert_contains("(9 10 11 12)"); - imap.send("THREAD REFERENCES UTF-8 SUBJECT T1").await; - imap.assert_read(Type::Tagged, ResponseType::Ok) - .await - .assert_contains("(5 6 7 8)") - .assert_count("(1 2 3 4)", 0) - .assert_count("(9 10 11 12)", 0); + // Filter by subject (mySQL does not support searching for short keywords) + if !handle.server.search_store().is_mysql() { + imap.send("THREAD REFERENCES UTF-8 SUBJECT T1").await; + imap.assert_read(Type::Tagged, ResponseType::Ok) + .await + .assert_contains("(5 6 7 8)") + .assert_count("(1 2 3 4)", 0) + .assert_count("(9 10 11 12)", 0); + } // Filter by threadId and messageId imap.send(&format!( diff --git a/tests/src/jmap/auth/quota.rs b/tests/src/jmap/auth/quota.rs index 35a732a3..a5e243db 100644 --- a/tests/src/jmap/auth/quota.rs +++ b/tests/src/jmap/auth/quota.rs @@ -8,6 +8,7 @@ use crate::{ directory::internal::TestInternalDirectory, jmap::{JMAPTest, mail::delivery::SmtpConnection, wait_for_index}, smtp::queue::QueuedEvents, + store::cleanup::store_blob_expire_all, }; use common::config::smtp::queue::QueueName; use email::{cache::MessageCacheFetch, mailbox::INBOX_ID}; @@ -42,7 +43,7 @@ pub async fn test(params: &mut JMAPTest) { server.inner.cache.access_tokens.clear(); // Delete temporary blobs from previous tests - server.core.storage.data.blob_expire_all().await; + store_blob_expire_all(&server.core.storage.data).await; // Test temporary blob quota (3 files) DISABLE_UPLOAD_QUOTA.store(false, std::sync::atomic::Ordering::Relaxed); @@ -65,7 +66,7 @@ pub async fn test(params: &mut JMAPTest) { jmap_client::Error::Problem(err) if err.detail().unwrap().contains("quota") => (), other => panic!("Unexpected error: {:?}", other), } - server.core.storage.data.blob_expire_all().await; + store_blob_expire_all(&server.core.storage.data).await; // Test temporary blob quota (50000 bytes) for i in 0..2 { @@ -86,7 +87,7 @@ pub async fn test(params: &mut JMAPTest) { jmap_client::Error::Problem(err) if err.detail().unwrap().contains("quota") => (), other => panic!("Unexpected error: {:?}", other), } - server.core.storage.data.blob_expire_all().await; + store_blob_expire_all(&server.core.storage.data).await; // Test JMAP Quotas extension let response = account diff --git a/tests/src/jmap/calendar/alarm.rs b/tests/src/jmap/calendar/alarm.rs index c5323a66..f8acb485 100644 --- a/tests/src/jmap/calendar/alarm.rs +++ b/tests/src/jmap/calendar/alarm.rs @@ -55,7 +55,9 @@ pub async fn test(params: &mut JMAPTest) { let (stream_tx, mut stream_rx) = mpsc::channel::(100); tokio::spawn(async move { while let Some(change) = ws_stream.next().await { - stream_tx.send(change.unwrap()).await.unwrap(); + if stream_tx.send(change.unwrap()).await.is_err() { + break; + } } }); client_ws diff --git a/tests/src/jmap/calendar/event.rs b/tests/src/jmap/calendar/event.rs index ae3f5416..18070825 100644 --- a/tests/src/jmap/calendar/event.rs +++ b/tests/src/jmap/calendar/event.rs @@ -5,7 +5,7 @@ */ use crate::{ - jmap::{ChangeType, IntoJmapSet, JMAPTest, JmapUtils}, + jmap::{ChangeType, IntoJmapSet, JMAPTest, JmapUtils, wait_for_index}, webdav::DummyWebDavClient, }; use ahash::AHashSet; @@ -452,6 +452,7 @@ pub async fn test(params: &mut JMAPTest) { })); // Query tests + wait_for_index(¶ms.server).await; assert_eq!( account .jmap_query( diff --git a/tests/src/jmap/calendar/notification.rs b/tests/src/jmap/calendar/notification.rs index d459508d..f844bb95 100644 --- a/tests/src/jmap/calendar/notification.rs +++ b/tests/src/jmap/calendar/notification.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::jmap::{IntoJmapSet, JMAPTest, JmapUtils}; +use crate::jmap::{IntoJmapSet, JMAPTest, JmapUtils, wait_for_index}; use calcard::jscalendar::JSCalendarProperty; use jmap_proto::{ object::calendar_event_notification::CalendarEventNotificationProperty, @@ -73,6 +73,7 @@ pub async fn test(params: &mut JMAPTest) { let john_event_id = response.created(0).id().to_string(); tokio::time::sleep(std::time::Duration::from_millis(600)).await; + wait_for_index(¶ms.server).await; // Verify Jane and Bill received the share notification let mut jane_event_id = String::new(); diff --git a/tests/src/jmap/core/blob.rs b/tests/src/jmap/core/blob.rs index 0af56c53..ab043d68 100644 --- a/tests/src/jmap/core/blob.rs +++ b/tests/src/jmap/core/blob.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::jmap::JMAPTest; +use crate::{jmap::JMAPTest, store::cleanup::store_blob_expire_all}; use email::mailbox::INBOX_ID; use serde_json::{Value, json}; use types::id::Id; @@ -13,7 +13,7 @@ pub async fn test(params: &mut JMAPTest) { println!("Running blob tests..."); let server = params.server.clone(); let account = params.account("jdoe@example.com"); - server.core.storage.data.blob_expire_all().await; + store_blob_expire_all(&server.core.storage.data).await; // Blob/set simple test let response = account.jmap_method_call("Blob/upload", json!({ @@ -139,7 +139,7 @@ pub async fn test(params: &mut JMAPTest) { ); } - server.core.storage.data.blob_expire_all().await; + store_blob_expire_all(&server.core.storage.data).await; // Blob/upload Complex Example let response = account @@ -226,7 +226,7 @@ pub async fn test(params: &mut JMAPTest) { "Pointer {pointer:?} Response: {response:?}", ); } - server.core.storage.data.blob_expire_all().await; + store_blob_expire_all(&server.core.storage.data).await; // Blob/get Example with Range and Encoding Errors let response = account.jmap_method_calls(json!([ @@ -353,7 +353,7 @@ pub async fn test(params: &mut JMAPTest) { "Pointer {pointer:?} Response: {response:?}", ); } - server.core.storage.data.blob_expire_all().await; + store_blob_expire_all(&server.core.storage.data).await; // Blob/lookup let client = account.client(); diff --git a/tests/src/jmap/mail/acl.rs b/tests/src/jmap/mail/acl.rs index 28dd44af..43c2c5b3 100644 --- a/tests/src/jmap/mail/acl.rs +++ b/tests/src/jmap/mail/acl.rs @@ -4,10 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{ - directory::internal::TestInternalDirectory, - jmap::{JMAPTest}, -}; +use crate::{directory::internal::TestInternalDirectory, jmap::JMAPTest}; use ::email::mailbox::{INBOX_ID, TRASH_ID}; use jmap_client::{ core::{ diff --git a/tests/src/jmap/mail/query.rs b/tests/src/jmap/mail/query.rs index 549df3a2..02750192 100644 --- a/tests/src/jmap/mail/query.rs +++ b/tests/src/jmap/mail/query.rs @@ -92,8 +92,10 @@ pub async fn test(params: &mut JMAPTest, insert: bool) { wait_for_index(&server).await; } + let can_stem = !params.server.search_store().is_mysql(); + println!("Running JMAP Mail query tests..."); - query(client).await; + query(client, can_stem).await; println!("Running JMAP Mail query options tests..."); query_options(client).await; @@ -114,7 +116,7 @@ pub async fn test(params: &mut JMAPTest, insert: bool) { params.assert_is_empty().await; } -pub async fn query(client: &Client) { +pub async fn query(client: &Client, can_stem: bool) { for (filter, sort, expected_results) in [ ( Filter::and(vec![ @@ -144,7 +146,7 @@ pub async fn query(client: &Client) { ), ( Filter::and(vec![ - (email::query::Filter::text("study")), + (email::query::Filter::text(if can_stem { "study" } else { "studies" })), (email::query::Filter::in_mailbox_other_than(vec![ Id::new(1991).to_string(), Id::new(1870).to_string(), diff --git a/tests/src/jmap/mail/search_snippet.rs b/tests/src/jmap/mail/search_snippet.rs index da913f58..1a2454bc 100644 --- a/tests/src/jmap/mail/search_snippet.rs +++ b/tests/src/jmap/mail/search_snippet.rs @@ -49,6 +49,8 @@ pub async fn test(params: &mut JMAPTest) { } wait_for_index(&server).await; + let can_stem = params.server.search_store().internal_fts().is_some(); + // Run tests for (filter, email_name, snippet_subject, snippet_preview) in [ ( @@ -121,7 +123,12 @@ pub async fn test(params: &mut JMAPTest) { )), ), ( - Filter::text("es:galería vasto biblioteca").into(), + Filter::text(if can_stem { + "es:galería vasto biblioteca" + } else { + "es:galería vastos biblioteca" + }) + .into(), "mixed", Some("Biblioteca de Babel"), Some(concat!( diff --git a/tests/src/jmap/mod.rs b/tests/src/jmap/mod.rs index 4c62ab67..f8c84403 100644 --- a/tests/src/jmap/mod.rs +++ b/tests/src/jmap/mod.rs @@ -11,7 +11,10 @@ use crate::{ enterprise::{EnterpriseCore, insert_test_metrics}, webhooks::{MockWebhookEndpoint, spawn_mock_webhook_endpoint}, }, - store::{TempDir, build_store_config}, + store::{ + TempDir, build_store_config, + cleanup::{search_store_destroy, store_assert_is_empty, store_destroy}, + }, }; use ahash::AHashMap; use base64::{ @@ -75,9 +78,9 @@ async fn jmap_tests() { server::webhooks::test(&mut params).await; - /*mail::get::test(&mut params).await; + mail::get::test(&mut params).await; mail::set::test(&mut params).await; - mail::parse::test(&mut params).await;*/ + mail::parse::test(&mut params).await; mail::query::test(&mut params, delete).await; mail::search_snippet::test(&mut params).await; mail::changes::test(&mut params).await; @@ -263,10 +266,8 @@ pub async fn assert_is_empty(server: &Server) { .unwrap(); // Assert is empty - server - .store() - .assert_is_empty(server.core.storage.blob.clone()) - .await; + store_assert_is_empty(server.store(), server.core.storage.blob.clone()).await; + search_store_destroy(server.search_store()).await; // Clean caches for cache in [ @@ -321,6 +322,7 @@ async fn init_jmap_tests(delete_if_exists: bool) -> JMAPTest { let data = Data::parse(&mut config); let cache = Caches::parse(&mut config); let store = core.storage.data.clone(); + let search_store = core.storage.fts.clone(); let (ipc, mut ipc_rxs) = build_ipc(false); let inner = Arc::new(Inner { shared_core: core.into_shared(), @@ -330,7 +332,8 @@ async fn init_jmap_tests(delete_if_exists: bool) -> JMAPTest { }); if delete_if_exists { - store.destroy().await; + store_destroy(&store).await; + search_store_destroy(&search_store).await; } // Parse acceptors diff --git a/tests/src/jmap/server/purge.rs b/tests/src/jmap/server/purge.rs index 2223cfdb..65f2e7e6 100644 --- a/tests/src/jmap/server/purge.rs +++ b/tests/src/jmap/server/purge.rs @@ -17,7 +17,11 @@ use email::{ message::delete::EmailDeletion, }; use imap_proto::ResponseType; -use store::{IterateParams, LogKey, U32_LEN, U64_LEN, write::key::DeserializeBigEndian}; +use store::{ + IterateParams, LogKey, U32_LEN, U64_LEN, + search::SearchQuery, + write::{SearchIndex, key::DeserializeBigEndian}, +}; use types::id::Id; pub async fn test(params: &mut JMAPTest) { @@ -155,6 +159,19 @@ pub async fn test(params: &mut JMAPTest) { .delete_principal(QueryBy::Id(account.id().document_id())) .await .unwrap(); + for index in [ + SearchIndex::Email, + SearchIndex::Contacts, + SearchIndex::Calendar, + ] { + server + .core + .storage + .fts + .unindex(SearchQuery::new(index).with_account_id(account.id().document_id())) + .await + .unwrap(); + } params.assert_is_empty().await; } diff --git a/tests/src/smtp/inbound/data.rs b/tests/src/smtp/inbound/data.rs index f0de915a..b42fb970 100644 --- a/tests/src/smtp/inbound/data.rs +++ b/tests/src/smtp/inbound/data.rs @@ -15,6 +15,7 @@ use crate::{ inbound::TestMessage, session::{TestSession, VerifyResponse, load_test_message}, }, + store::cleanup::store_assert_is_empty, }; use smtp::core::Session; @@ -233,8 +234,5 @@ async fn data() { // Make sure store is empty qr.clear_queue(&test.server).await; - test.server - .store() - .assert_is_empty(test.server.blob_store().clone()) - .await; + store_assert_is_empty(test.server.store(), test.server.blob_store().clone()).await; } diff --git a/tests/src/smtp/mod.rs b/tests/src/smtp/mod.rs index a4e9441d..c447b77d 100644 --- a/tests/src/smtp/mod.rs +++ b/tests/src/smtp/mod.rs @@ -29,7 +29,7 @@ use store::{BlobStore, Store, Stores}; use tokio::sync::{mpsc, watch}; use utils::config::Config; -use crate::AssertConfig; +use crate::{AssertConfig, store::cleanup::store_destroy}; pub mod config; pub mod inbound; @@ -236,7 +236,7 @@ impl TestSMTP { let stores = Stores::parse_all(&mut config, false).await; let core = Core::parse(&mut config, stores, Default::default()).await; let data = Data::parse(&mut config); - core.storage.data.destroy().await; + store_destroy(&core.storage.data).await; Self::from_core_and_tempdir(core, data, Some(temp_dir)) } diff --git a/tests/src/smtp/queue/concurrent.rs b/tests/src/smtp/queue/concurrent.rs index 4827c3b9..e22ec4e2 100644 --- a/tests/src/smtp/queue/concurrent.rs +++ b/tests/src/smtp/queue/concurrent.rs @@ -9,7 +9,10 @@ use std::time::{Duration, Instant}; use common::{config::server::ServerProtocol, core::BuildServer, ipc::QueueEvent}; use mail_auth::MX; -use crate::smtp::{DnsCache, TestSMTP, session::TestSession}; +use crate::{ + smtp::{DnsCache, TestSMTP, session::TestSession}, + store::cleanup::store_assert_is_empty, +}; use smtp::queue::manager::Queue; const LOCAL: &str = r#" @@ -149,9 +152,5 @@ async fn concurrent_queue() { assert_eq!(remote_messages.len(), NUM_MESSAGES); // Make sure local store is queue - core.core - .storage - .data - .assert_is_empty(core.core.storage.blob.clone()) - .await; + store_assert_is_empty(&core.core.storage.data, core.core.storage.blob.clone()).await; } diff --git a/tests/src/smtp/queue/virtualq.rs b/tests/src/smtp/queue/virtualq.rs index 1847257d..5c1be93a 100644 --- a/tests/src/smtp/queue/virtualq.rs +++ b/tests/src/smtp/queue/virtualq.rs @@ -13,7 +13,10 @@ use common::{ }; use mail_auth::MX; -use crate::smtp::{DnsCache, TestSMTP, session::TestSession}; +use crate::{ + smtp::{DnsCache, TestSMTP, session::TestSession}, + store::cleanup::store_assert_is_empty, +}; use smtp::queue::manager::Queue; const LOCAL: &str = r#" @@ -205,9 +208,5 @@ async fn virtual_queue() { assert_eq!(remote_messages.len(), NUM_MESSAGES * 2); // Make sure local store is queue - core.core - .storage - .data - .assert_is_empty(core.core.storage.blob.clone()) - .await; + store_assert_is_empty(&core.core.storage.data, core.core.storage.blob.clone()).await; } diff --git a/tests/src/store/blob.rs b/tests/src/store/blob.rs index 1a7820b7..5c123669 100644 --- a/tests/src/store/blob.rs +++ b/tests/src/store/blob.rs @@ -12,7 +12,7 @@ use store::{ use types::{blob::BlobClass, blob_hash::BlobHash, collection::Collection}; use utils::config::Config; -use crate::store::{CONFIG, TempDir}; +use crate::store::{CONFIG, TempDir, cleanup::store_destroy}; #[tokio::test] pub async fn blob_tests() { @@ -30,7 +30,7 @@ pub async fn blob_tests() { println!("Testing blob management on store {}...", store_id); // Init store - store.destroy().await; + store_destroy(&store).await; // Test internal blob store let blob_store: BlobStore = store.clone().into(); diff --git a/tests/src/store/cleanup.rs b/tests/src/store/cleanup.rs new file mode 100644 index 00000000..f38c0e42 --- /dev/null +++ b/tests/src/store/cleanup.rs @@ -0,0 +1,342 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use store::{ + ValueKey, + write::{key::DeserializeBigEndian, *}, + *, +}; +use trc::AddContext; + +pub async fn store_destroy(store: &Store) { + store_destroy_sql_indexes(store).await; + + for subspace in [ + SUBSPACE_ACL, + SUBSPACE_DIRECTORY, + SUBSPACE_TASK_QUEUE, + SUBSPACE_INDEXES, + SUBSPACE_BLOB_RESERVE, + SUBSPACE_BLOB_LINK, + SUBSPACE_LOGS, + SUBSPACE_IN_MEMORY_COUNTER, + SUBSPACE_IN_MEMORY_VALUE, + SUBSPACE_COUNTER, + SUBSPACE_PROPERTY, + SUBSPACE_SETTINGS, + SUBSPACE_BLOBS, + SUBSPACE_QUEUE_MESSAGE, + SUBSPACE_QUEUE_EVENT, + SUBSPACE_QUOTA, + SUBSPACE_REPORT_OUT, + SUBSPACE_REPORT_IN, + SUBSPACE_TELEMETRY_SPAN, + SUBSPACE_TELEMETRY_METRIC, + SUBSPACE_SEARCH_INDEX, + ] { + if subspace == SUBSPACE_SEARCH_INDEX && store.is_pg_or_mysql() { + continue; + } + + store + .delete_range( + AnyKey { + subspace, + key: vec![0u8], + }, + AnyKey { + subspace, + key: vec![u8::MAX; 16], + }, + ) + .await + .unwrap(); + } +} + +pub async fn search_store_destroy(store: &SearchStore) { + match &store { + SearchStore::Store(store) => { + store_destroy_sql_indexes(store).await; + } + SearchStore::ElasticSearch(store) => { + if let Err(err) = store.drop_indexes().await { + eprintln!("Failed to drop elasticsearch indexes: {}", err); + } + store.create_indexes(3, 0, false).await.unwrap(); + } + } +} + +#[allow(unused_variables)] +async fn store_destroy_sql_indexes(store: &Store) { + #[cfg(any(feature = "postgres", feature = "mysql"))] + { + if store.is_pg_or_mysql() { + for index in [ + SearchIndex::Email, + SearchIndex::Calendar, + SearchIndex::Contacts, + SearchIndex::Tracing, + ] { + #[cfg(feature = "postgres")] + let table = index.psql_table(); + #[cfg(feature = "mysql")] + let table = index.mysql_table(); + + store + .sql_query::(&format!("TRUNCATE TABLE {table}"), vec![]) + .await + .unwrap(); + } + } + } +} + +pub async fn store_blob_expire_all(store: &Store) { + // Delete all temporary hashes + let from_key = ValueKey { + account_id: 0, + collection: 0, + document_id: 0, + class: ValueClass::Blob(BlobOp::Reserve { + hash: types::blob_hash::BlobHash::default(), + until: 0, + }), + }; + let to_key = ValueKey { + account_id: u32::MAX, + collection: 0, + document_id: 0, + class: ValueClass::Blob(BlobOp::Reserve { + hash: types::blob_hash::BlobHash::default(), + until: 0, + }), + }; + let mut batch = BatchBuilder::new(); + let mut last_account_id = u32::MAX; + store + .iterate( + IterateParams::new(from_key, to_key).ascending().no_values(), + |key, _| { + let account_id = key.deserialize_be_u32(0).caused_by(trc::location!())?; + if account_id != last_account_id { + last_account_id = account_id; + batch.with_account_id(account_id); + } + + batch.any_op(Operation::Value { + class: ValueClass::Blob(BlobOp::Reserve { + hash: types::blob_hash::BlobHash::try_from_hash_slice( + key.get(U32_LEN..U32_LEN + types::blob_hash::BLOB_HASH_LEN) + .unwrap(), + ) + .unwrap(), + until: key + .deserialize_be_u64(key.len() - U64_LEN) + .caused_by(trc::location!())?, + }), + op: ValueOp::Clear, + }); + + Ok(true) + }, + ) + .await + .unwrap(); + store.write(batch.build_all()).await.unwrap(); +} + +pub async fn store_lookup_expire_all(store: &Store) { + // Delete all temporary counters + let from_key = ValueKey::from(ValueClass::InMemory(InMemoryClass::Key(vec![0u8]))); + let to_key = ValueKey::from(ValueClass::InMemory(InMemoryClass::Key(vec![u8::MAX; 10]))); + + let mut expired_keys = Vec::new(); + let mut expired_counters = Vec::new(); + + store + .iterate(IterateParams::new(from_key, to_key), |key, value| { + let expiry = value.deserialize_be_u64(0).caused_by(trc::location!())?; + if expiry == 0 { + expired_counters.push(key.to_vec()); + } else if expiry != u64::MAX { + expired_keys.push(key.to_vec()); + } + Ok(true) + }) + .await + .unwrap(); + + if !expired_keys.is_empty() { + let mut batch = BatchBuilder::new(); + for key in expired_keys { + batch.any_op(Operation::Value { + class: ValueClass::InMemory(InMemoryClass::Key(key)), + op: ValueOp::Clear, + }); + if batch.is_large_batch() { + store.write(batch.build_all()).await.unwrap(); + batch = BatchBuilder::new(); + } + } + if !batch.is_empty() { + store.write(batch.build_all()).await.unwrap(); + } + } + + if !expired_counters.is_empty() { + let mut batch = BatchBuilder::new(); + for key in expired_counters { + batch.any_op(Operation::Value { + class: ValueClass::InMemory(InMemoryClass::Counter(key.clone())), + op: ValueOp::Clear, + }); + batch.any_op(Operation::Value { + class: ValueClass::InMemory(InMemoryClass::Key(key)), + op: ValueOp::Clear, + }); + if batch.is_large_batch() { + store.write(batch.build_all()).await.unwrap(); + batch = BatchBuilder::new(); + } + } + if !batch.is_empty() { + store.write(batch.build_all()).await.unwrap(); + } + } +} + +#[allow(unused_variables)] +pub async fn store_assert_is_empty(store: &Store, blob_store: BlobStore) { + store_blob_expire_all(store).await; + store_lookup_expire_all(store).await; + store.purge_blobs(blob_store).await.unwrap(); + store.purge_store().await.unwrap(); + + let store = store.clone(); + let mut failed = false; + + for (subspace, with_values) in [ + (SUBSPACE_ACL, true), + //(SUBSPACE_DIRECTORY, true), + (SUBSPACE_TASK_QUEUE, true), + (SUBSPACE_IN_MEMORY_VALUE, true), + (SUBSPACE_IN_MEMORY_COUNTER, false), + (SUBSPACE_PROPERTY, true), + (SUBSPACE_SETTINGS, true), + (SUBSPACE_QUEUE_MESSAGE, true), + (SUBSPACE_QUEUE_EVENT, true), + (SUBSPACE_REPORT_OUT, true), + (SUBSPACE_REPORT_IN, true), + (SUBSPACE_BLOB_RESERVE, true), + (SUBSPACE_BLOB_LINK, true), + (SUBSPACE_BLOBS, true), + (SUBSPACE_COUNTER, false), + (SUBSPACE_QUOTA, false), + (SUBSPACE_BLOBS, true), + (SUBSPACE_INDEXES, false), + (SUBSPACE_TELEMETRY_SPAN, true), + (SUBSPACE_TELEMETRY_METRIC, true), + (SUBSPACE_SEARCH_INDEX, true), + ] { + if subspace == SUBSPACE_SEARCH_INDEX && store.is_pg_or_mysql() { + continue; + } + + let from_key = AnyKey { + subspace, + key: vec![0u8], + }; + let to_key = AnyKey { + subspace, + key: vec![u8::MAX; 10], + }; + + store + .iterate( + IterateParams::new(from_key, to_key).set_values(with_values), + |key, value| { + match subspace { + SUBSPACE_COUNTER if key.len() == U32_LEN + 1 || key.len() == U32_LEN => { + // Message ID and change ID counters + return Ok(true); + } + SUBSPACE_INDEXES => { + println!( + concat!( + "Found index key, account {}, collection {}, ", + "document {}, property {}, value {:?}: {:?}" + ), + u32::from_be_bytes(key[0..4].try_into().unwrap()), + key[4], + u32::from_be_bytes(key[key.len() - 4..].try_into().unwrap()), + key[5], + String::from_utf8_lossy(&key[6..key.len() - 4]), + key + ); + } + _ => { + println!( + "Found key in {:?}: {:?} ({:?}) = {:?} ({:?})", + char::from(subspace), + key, + String::from_utf8_lossy(key), + value, + String::from_utf8_lossy(value) + ); + } + } + failed = true; + + Ok(true) + }, + ) + .await + .unwrap(); + } + + // Delete logs and counters + store + .delete_range( + AnyKey { + subspace: SUBSPACE_LOGS, + key: &[0u8], + }, + AnyKey { + subspace: SUBSPACE_LOGS, + key: &[ + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + u8::MAX, + ], + }, + ) + .await + .unwrap(); + + store + .delete_range( + AnyKey { + subspace: SUBSPACE_COUNTER, + key: &[0u8], + }, + AnyKey { + subspace: SUBSPACE_COUNTER, + key: (u32::MAX / 2).to_be_bytes().as_slice(), + }, + ) + .await + .unwrap(); + + if failed { + panic!("Store is not empty."); + } +} diff --git a/tests/src/store/lookup.rs b/tests/src/store/lookup.rs index 5745ebca..c2d6598b 100644 --- a/tests/src/store/lookup.rs +++ b/tests/src/store/lookup.rs @@ -11,7 +11,10 @@ use utils::config::{Config, Rate}; use crate::{ AssertConfig, - store::{CONFIG, TempDir}, + store::{ + CONFIG, TempDir, + cleanup::{store_assert_is_empty, store_destroy}, + }, }; #[tokio::test] @@ -30,7 +33,7 @@ pub async fn lookup_tests() { for (store_id, store) in stores.in_memory_stores { println!("Testing in-memory store {}...", store_id); if let InMemoryStore::Store(store) = &store { - store.destroy().await; + store_destroy(store).await; } else { // Reset redis counter store @@ -65,7 +68,7 @@ pub async fn lookup_tests() { store.purge_in_memory_store().await.unwrap(); if let InMemoryStore::Store(store) = &store { - store.assert_is_empty(store.clone().into()).await; + store_assert_is_empty(store, store.clone().into()).await; } // Test counter @@ -123,7 +126,7 @@ pub async fn lookup_tests() { tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; store.purge_in_memory_store().await.unwrap(); if let InMemoryStore::Store(store) = &store { - store.assert_is_empty(store.clone().into()).await; + store_assert_is_empty(store, store.clone().into()).await; } // Test locking @@ -149,7 +152,7 @@ pub async fn lookup_tests() { } store.purge_in_memory_store().await.unwrap(); if let InMemoryStore::Store(store) = &store { - store.assert_is_empty(store.clone().into()).await; + store_assert_is_empty(store, store.clone().into()).await; } // Test prefix delete @@ -281,7 +284,7 @@ pub async fn lookup_tests() { ); if let InMemoryStore::Store(store) = &store { - store.assert_is_empty(store.clone().into()).await; + store_assert_is_empty(store, store.clone().into()).await; } } } diff --git a/tests/src/store/mod.rs b/tests/src/store/mod.rs index 05c0fd37..992981e8 100644 --- a/tests/src/store/mod.rs +++ b/tests/src/store/mod.rs @@ -6,13 +6,17 @@ pub mod blob; //pub mod import_export; +pub mod cleanup; pub mod lookup; pub mod ops; pub mod query; -use crate::AssertConfig; +use crate::{ + AssertConfig, + store::cleanup::{search_store_destroy, store_destroy}, +}; use std::io::Read; -use store::{SearchStore, Stores}; +use store::Stores; use utils::config::Config; pub struct TempDir { @@ -38,7 +42,7 @@ pub async fn store_tests() { println!("Testing store {}...", store_id); if insert { - store.destroy().await; + store_destroy(&store).await; } //import_export::test(store.clone()).await; @@ -68,10 +72,7 @@ pub async fn search_tests() { println!("Testing store {}...", store_id); if insert { - match &store { - SearchStore::Store(store) => store.destroy().await, - SearchStore::ElasticSearch(_) => (), - } + search_store_destroy(&store).await; } query::test(store, insert).await; @@ -179,10 +180,10 @@ type = "redis" urls = "redis://127.0.0.1" redis-type = "single" -[store."psql-replica"] -type = "sql-read-replica" -primary = "postgresql" -replicas = "postgresql" +#[store."psql-replica"] +#type = "sql-read-replica" +#primary = "postgresql" +#replicas = "postgresql" [storage] data = "{STORE}" diff --git a/tests/src/store/ops.rs b/tests/src/store/ops.rs index aa4d6239..e1c67a3a 100644 --- a/tests/src/store/ops.rs +++ b/tests/src/store/ops.rs @@ -16,6 +16,8 @@ use store::{ }; use types::collection::{Collection, SyncCollection}; +use crate::store::cleanup::store_assert_is_empty; + // FDB max value const MAX_VALUE_SIZE: usize = 100000; @@ -471,6 +473,6 @@ pub async fn test(db: Store) { db.write(batch.build_all()).await.unwrap(); // Make sure everything is deleted - db.assert_is_empty(db.clone().into()).await; + store_assert_is_empty(&db, db.clone().into()).await; } } diff --git a/tests/src/store/query.rs b/tests/src/store/query.rs index 5c575d78..b7d3393a 100644 --- a/tests/src/store/query.rs +++ b/tests/src/store/query.rs @@ -300,11 +300,7 @@ pub async fn test(store: SearchStore, do_insert: bool) { } async fn test_filter(store: SearchStore, fields: &AHashMap, mask: &RoaringBitmap) { - #[cfg(feature = "mysql")] - let can_stem = !matches!(store, SearchStore::Store(store::Store::MySQL(_))); - - #[cfg(not(feature = "mysql"))] - let can_stem = true; + let can_stem = !store.is_mysql(); let tests = [ ( @@ -482,11 +478,7 @@ async fn test_filter(store: SearchStore, fields: &AHashMap, mask: & } async fn test_sort(store: SearchStore, fields: &AHashMap, mask: &RoaringBitmap) { - #[cfg(feature = "postgres")] - let is_reversed = matches!(store, SearchStore::Store(store::Store::PostgreSQL(_))); - - #[cfg(not(feature = "postgres"))] - let is_reversed = false; + let is_reversed = store.is_postgres(); let tests = [ ( diff --git a/tests/src/webdav/cal_scheduling.rs b/tests/src/webdav/cal_scheduling.rs index 142f5715..b3901a75 100644 --- a/tests/src/webdav/cal_scheduling.rs +++ b/tests/src/webdav/cal_scheduling.rs @@ -259,6 +259,8 @@ pub async fn test(test: &WebDavTest) { ); // Check that John received the RSVP + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + test.wait_for_index().await; let itips = fetch_and_remove_itips(john_client).await; assert_eq!(itips.len(), 1); assert!( @@ -448,6 +450,8 @@ pub async fn test(test: &WebDavTest) { let main_event_href = cal.href; // Check that Bill received the update + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + test.wait_for_index().await; let mut itips = fetch_and_remove_itips(bill_client).await; itips.sort_unstable_by(|a, _| { if a.contains("Lunch") { diff --git a/tests/src/webdav/mod.rs b/tests/src/webdav/mod.rs index e94a1f30..d191ea9c 100644 --- a/tests/src/webdav/mod.rs +++ b/tests/src/webdav/mod.rs @@ -7,8 +7,11 @@ use crate::{ AssertConfig, TEST_USERS, add_test_certs, directory::internal::TestInternalDirectory, - jmap::assert_is_empty, - store::{TempDir, build_store_config}, + jmap::{assert_is_empty, wait_for_index}, + store::{ + TempDir, build_store_config, + cleanup::{search_store_destroy, store_destroy}, + }, }; use ::managesieve::core::ManageSieveSessionManager; use ::store::Stores; @@ -150,6 +153,7 @@ async fn init_webdav_tests(assisted_discovery: bool, delete_if_exists: bool) -> let cache = Caches::parse(&mut config); let store = core.storage.data.clone(); + let search_store = core.storage.fts.clone(); let (ipc, mut ipc_rxs) = build_ipc(false); let inner = Arc::new(Inner { shared_core: core.into_shared(), @@ -206,7 +210,8 @@ async fn init_webdav_tests(assisted_discovery: bool, delete_if_exists: bool) -> }); if delete_if_exists { - store.destroy().await; + store_destroy(&store).await; + search_store_destroy(&search_store).await; } // Create test accounts @@ -270,6 +275,10 @@ impl WebDavTest { assert_is_empty(&self.server).await; self.clear_cache(); } + + pub async fn wait_for_index(&self) { + wait_for_index(&self.server).await; + } } #[allow(dead_code)]