ElasticSearch: Implement pagination (#2551)

CHANGELOG.md

@@ -4,7 +4,8 @@ All notable changes to this project will be documented in this file. This projec
 
 ## [0.15.2] - 2025-12-22
 
-This version includes **multiple breaking changes**. If you are upgrading from v0.14.x and below, please read the [upgrading documentation](https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING/v0_15.md) for more information on how to upgrade from previous versions. If you are upgrading from v0.15.x, replace the binary and update the webadmin.
+If you are upgrading from v0.14.x and below, this version includes **multiple breaking changes**. Please read the [upgrading documentation](https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING/v0_15.md) for more information on how to upgrade from previous versions.
+If you are upgrading from v0.15.x, replace the binary and update the webadmin.
 
 ## Added
 - OAuth: Add device authorization endpoint (#2225).
@@ -15,6 +16,7 @@ This version includes **multiple breaking changes**. If you are upgrading from v
 ## Fixed
 - mySQL search: Use `MEDIUMTEXT` field type for email body and attachments (#2544).
 - PostgreSQL search: Truncate large text fields.
+- ElasticSearch: Implement pagination (#2552).
 - Antispam: Fix `NO_SPACE_IN_FROM` spam tag detection logic (#2372).
 - IMAP: Fix shared folder double nesting (test suite credits to @ochnygosch) (#2358).
 - JMAP: Use latest `Received` header in JMAP `Email/import` (credits to @apexskier) (#2374).

crates/store/src/backend/elastic/mod.rs

@@ -37,6 +37,7 @@ pub struct Total {
 pub struct Hit {
     #[serde(rename = "_id", deserialize_with = "deserialize_string_to_u64")]
     pub id: u64,
+    pub sort: Option<Value>,
 }
 
 #[derive(Debug, Deserialize)]

crates/store/src/backend/elastic/search.rs

@@ -60,12 +60,23 @@ impl ElasticSearchStore {
         filters: &[SearchFilter],
         sort: &[SearchComparator],
     ) -> trc::Result<Vec<R>> {
+        let mut search_after: Option<Value> = None;
+        let mut results = Vec::new();
+        let mut has_more = true;
+
+        while has_more {
             let query = Map::from_iter(
                 [
                     Some(("query".to_string(), build_query(filters))),
                     Some(("size".to_string(), Value::from(10_000))),
                     Some(("_source".to_string(), Value::from(false))),
-                (!sort.is_empty()).then(|| ("sort".to_string(), build_sort(sort))),
+                    Some((
+                        "sort".to_string(),
+                        build_sort(sort, R::field().field_name()),
+                    )),
+                    search_after
+                        .take()
+                        .map(|sa| ("search_after".to_string(), sa)),
                 ]
                 .into_iter()
                 .flatten(),
@@ -85,20 +96,22 @@ impl ElasticSearchStore {
                 .await
                 .map_err(|err| trc::StoreEvent::ElasticsearchError.reason(err))?;
 
-        serde_json::from_str::<SearchResponse>(&text)
-            .map(|results| {
-                results
-                    .hits
-                    .hits
-                    .into_iter()
-                    .map(|hit| R::from_u64(hit.id))
-                    .collect()
-            })
-            .map_err(|err| {
+            let response = serde_json::from_str::<SearchResponse>(&text).map_err(|err| {
                 trc::StoreEvent::ElasticsearchError
                     .reason(err)
                     .details(text)
-            })
+            })?;
+
+            has_more = response.hits.hits.len() == 10_000
+                && response.hits.hits.last().unwrap().sort.is_some();
+
+            for hit in response.hits.hits {
+                search_after = hit.sort;
+                results.push(R::from_u64(hit.id));
+            }
+        }
+
+        Ok(results)
     }
 
     pub async fn unindex(&self, filter: SearchQuery) -> trc::Result<u64> {
@@ -278,7 +291,7 @@ fn build_query(filters: &[SearchFilter]) -> Value {
     }
 }
 
-fn build_sort(sort: &[SearchComparator]) -> Value {
+fn build_sort(sort: &[SearchComparator], tie_breaker: &str) -> Value {
     Value::Array(
         sort.iter()
             .filter_map(|comp| match comp {
@@ -295,6 +308,9 @@ fn build_sort(sort: &[SearchComparator]) -> Value {
                 }
                 _ => None,
             })
+            .chain([json!({
+                tie_breaker: "asc"
+            })])
             .collect(),
     )
 }

crates/store/src/search/mod.rs

@@ -20,6 +20,7 @@ use nlp::language::Language;
 use roaring::RoaringBitmap;
 use std::cmp::Ordering;
 use std::collections::hash_map::Entry;
+use std::fmt::Display;
 use std::ops::{BitAndAssign, BitOrAssign, BitXorAssign};
 use utils::config::utils::ParseValue;
 use utils::map::vec_map::VecMap;
@@ -110,7 +111,7 @@ pub enum SearchValue {
     Boolean(bool),
 }
 
-pub trait SearchDocumentId: Sized {
+pub trait SearchDocumentId: Sized + Copy + Display {
     fn from_u64(id: u64) -> Self;
     fn field() -> SearchField;
 }

tests/src/store/query.rs

@@ -285,6 +285,33 @@ pub async fn test(store: SearchStore, do_insert: bool) {
         println!("\nInsert took {} ms.", now.elapsed().as_millis());
     }
 
+    let ids = store
+        .query_account(
+            SearchQuery::new(SearchIndex::Email)
+                .with_filters(vec![SearchFilter::eq(SearchField::AccountId, 0u32)])
+                .with_comparator(SearchComparator::ascending(EmailSearchField::ReceivedAt))
+                .with_mask(mask.clone()),
+        )
+        .await
+        .unwrap()
+        .into_iter()
+        .collect::<RoaringBitmap>();
+    assert_eq!(ids, mask);
+    let ids = store
+        .query_account(
+            SearchQuery::new(SearchIndex::Email)
+                .with_filters(vec![
+                    SearchFilter::eq(SearchField::AccountId, 0u32),
+                    SearchFilter::ge(SearchField::DocumentId, 0u32),
+                ])
+                .with_mask(mask.clone()),
+        )
+        .await
+        .unwrap()
+        .into_iter()
+        .collect::<RoaringBitmap>();
+    assert_eq!(ids, mask);
+
     println!("Running account filter tests...");
     let now = Instant::now();
     test_filter(store.clone(), &fields, &mask).await;