diff --git a/CHANGELOG.md b/CHANGELOG.md index d5dcd2e1..bca5497a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,8 @@ All notable changes to this project will be documented in this file. This projec ## [0.15.2] - 2025-12-22 -This version includes **multiple breaking changes**. If you are upgrading from v0.14.x and below, please read the [upgrading documentation](https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING/v0_15.md) for more information on how to upgrade from previous versions. If you are upgrading from v0.15.x, replace the binary and update the webadmin. +If you are upgrading from v0.14.x and below, this version includes **multiple breaking changes**. Please read the [upgrading documentation](https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING/v0_15.md) for more information on how to upgrade from previous versions. +If you are upgrading from v0.15.x, replace the binary and update the webadmin. ## Added - OAuth: Add device authorization endpoint (#2225). @@ -15,6 +16,7 @@ This version includes **multiple breaking changes**. If you are upgrading from v ## Fixed - mySQL search: Use `MEDIUMTEXT` field type for email body and attachments (#2544). - PostgreSQL search: Truncate large text fields. +- ElasticSearch: Implement pagination (#2552). - Antispam: Fix `NO_SPACE_IN_FROM` spam tag detection logic (#2372). - IMAP: Fix shared folder double nesting (test suite credits to @ochnygosch) (#2358). - JMAP: Use latest `Received` header in JMAP `Email/import` (credits to @apexskier) (#2374). diff --git a/crates/store/src/backend/elastic/mod.rs b/crates/store/src/backend/elastic/mod.rs index a242cc52..bb9b0fa4 100644 --- a/crates/store/src/backend/elastic/mod.rs +++ b/crates/store/src/backend/elastic/mod.rs @@ -37,6 +37,7 @@ pub struct Total { pub struct Hit { #[serde(rename = "_id", deserialize_with = "deserialize_string_to_u64")] pub id: u64, + pub sort: Option, } #[derive(Debug, Deserialize)] diff --git a/crates/store/src/backend/elastic/search.rs b/crates/store/src/backend/elastic/search.rs index 85b21320..b741ec2c 100644 --- a/crates/store/src/backend/elastic/search.rs +++ b/crates/store/src/backend/elastic/search.rs @@ -60,45 +60,58 @@ impl ElasticSearchStore { filters: &[SearchFilter], sort: &[SearchComparator], ) -> trc::Result> { - let query = Map::from_iter( - [ - Some(("query".to_string(), build_query(filters))), - Some(("size".to_string(), Value::from(10_000))), - Some(("_source".to_string(), Value::from(false))), - (!sort.is_empty()).then(|| ("sort".to_string(), build_sort(sort))), - ] - .into_iter() - .flatten(), - ); + let mut search_after: Option = None; + let mut results = Vec::new(); + let mut has_more = true; - let response = assert_success( - self.client - .post(format!("{}/{}/_search", self.url, index.index_name())) - .body(serde_json::to_string(&query).unwrap_or_default()) - .send() - .await, - ) - .await?; + while has_more { + let query = Map::from_iter( + [ + Some(("query".to_string(), build_query(filters))), + Some(("size".to_string(), Value::from(10_000))), + Some(("_source".to_string(), Value::from(false))), + Some(( + "sort".to_string(), + build_sort(sort, R::field().field_name()), + )), + search_after + .take() + .map(|sa| ("search_after".to_string(), sa)), + ] + .into_iter() + .flatten(), + ); - let text = response - .text() - .await - .map_err(|err| trc::StoreEvent::ElasticsearchError.reason(err))?; + let response = assert_success( + self.client + .post(format!("{}/{}/_search", self.url, index.index_name())) + .body(serde_json::to_string(&query).unwrap_or_default()) + .send() + .await, + ) + .await?; - serde_json::from_str::(&text) - .map(|results| { - results - .hits - .hits - .into_iter() - .map(|hit| R::from_u64(hit.id)) - .collect() - }) - .map_err(|err| { + let text = response + .text() + .await + .map_err(|err| trc::StoreEvent::ElasticsearchError.reason(err))?; + + let response = serde_json::from_str::(&text).map_err(|err| { trc::StoreEvent::ElasticsearchError .reason(err) .details(text) - }) + })?; + + has_more = response.hits.hits.len() == 10_000 + && response.hits.hits.last().unwrap().sort.is_some(); + + for hit in response.hits.hits { + search_after = hit.sort; + results.push(R::from_u64(hit.id)); + } + } + + Ok(results) } pub async fn unindex(&self, filter: SearchQuery) -> trc::Result { @@ -278,7 +291,7 @@ fn build_query(filters: &[SearchFilter]) -> Value { } } -fn build_sort(sort: &[SearchComparator]) -> Value { +fn build_sort(sort: &[SearchComparator], tie_breaker: &str) -> Value { Value::Array( sort.iter() .filter_map(|comp| match comp { @@ -295,6 +308,9 @@ fn build_sort(sort: &[SearchComparator]) -> Value { } _ => None, }) + .chain([json!({ + tie_breaker: "asc" + })]) .collect(), ) } diff --git a/crates/store/src/search/mod.rs b/crates/store/src/search/mod.rs index 9a2b969f..ef6cc461 100644 --- a/crates/store/src/search/mod.rs +++ b/crates/store/src/search/mod.rs @@ -20,6 +20,7 @@ use nlp::language::Language; use roaring::RoaringBitmap; use std::cmp::Ordering; use std::collections::hash_map::Entry; +use std::fmt::Display; use std::ops::{BitAndAssign, BitOrAssign, BitXorAssign}; use utils::config::utils::ParseValue; use utils::map::vec_map::VecMap; @@ -110,7 +111,7 @@ pub enum SearchValue { Boolean(bool), } -pub trait SearchDocumentId: Sized { +pub trait SearchDocumentId: Sized + Copy + Display { fn from_u64(id: u64) -> Self; fn field() -> SearchField; } diff --git a/tests/src/store/query.rs b/tests/src/store/query.rs index a4514b7f..f0ffd54a 100644 --- a/tests/src/store/query.rs +++ b/tests/src/store/query.rs @@ -285,6 +285,33 @@ pub async fn test(store: SearchStore, do_insert: bool) { println!("\nInsert took {} ms.", now.elapsed().as_millis()); } + let ids = store + .query_account( + SearchQuery::new(SearchIndex::Email) + .with_filters(vec![SearchFilter::eq(SearchField::AccountId, 0u32)]) + .with_comparator(SearchComparator::ascending(EmailSearchField::ReceivedAt)) + .with_mask(mask.clone()), + ) + .await + .unwrap() + .into_iter() + .collect::(); + assert_eq!(ids, mask); + let ids = store + .query_account( + SearchQuery::new(SearchIndex::Email) + .with_filters(vec![ + SearchFilter::eq(SearchField::AccountId, 0u32), + SearchFilter::ge(SearchField::DocumentId, 0u32), + ]) + .with_mask(mask.clone()), + ) + .await + .unwrap() + .into_iter() + .collect::(); + assert_eq!(ids, mask); + println!("Running account filter tests..."); let now = Instant::now(); test_filter(store.clone(), &fields, &mask).await;