diff --git a/crates/store/src/backend/postgres/search.rs b/crates/store/src/backend/postgres/search.rs index a1bde853..c06c7e96 100644 --- a/crates/store/src/backend/postgres/search.rs +++ b/crates/store/src/backend/postgres/search.rs @@ -318,9 +318,8 @@ impl ToSql for SearchValue { SearchValue::Text { value, .. } => { // Truncate large text fields to avoid Postgres errors (see https://www.postgresql.org/docs/current/textsearch-limitations.html) - if value.len() > 1_048_574 { - let pos = value.floor_char_boundary(1_048_574); - (&value[..pos]).to_sql(ty, out) + if value.len() > 650_000 { + (&value[..value.floor_char_boundary(650_000)]).to_sql(ty, out) } else { value.to_sql(ty, out) } @@ -356,9 +355,8 @@ impl ToSql for SearchValue { SearchValue::Text { value, .. } => { // Truncate large text fields to avoid Postgres errors (see https://www.postgresql.org/docs/current/textsearch-limitations.html) - if value.len() > 1_048_574 { - let pos = value.floor_char_boundary(1_048_574); - (&value[..pos]).to_sql_checked(ty, out) + if value.len() > 650_000 { + (&value[..value.floor_char_boundary(650_000)]).to_sql_checked(ty, out) } else { value.to_sql_checked(ty, out) } diff --git a/tests/src/store/query.rs b/tests/src/store/query.rs index cd7c1bf5..a51553f0 100644 --- a/tests/src/store/query.rs +++ b/tests/src/store/query.rs @@ -15,6 +15,7 @@ use std::{ use store::{ SearchStore, ahash::AHashMap, + rand::{self, Rng, distr::Alphanumeric}, roaring::RoaringBitmap, search::{ EmailSearchField, IndexDocument, SearchComparator, SearchField, SearchFilter, @@ -120,6 +121,47 @@ pub async fn test(store: SearchStore, do_insert: bool) { println!("Running global id filtering tests..."); test_global(store.clone()).await; + // Large document insert test + println!("Running large document insert tests..."); + let mut large_text = String::with_capacity(20 * 1024 * 1024); + while large_text.len() < 20 * 1024 * 1024 { + let word = rand::rng() + .sample_iter(&Alphanumeric) + .take(rand::rng().random_range(3..10)) + .map(char::from) + .collect::(); + large_text.push_str(&word); + large_text.push(' '); + } + let mut document = IndexDocument::new(SearchIndex::Email) + .with_account_id(1) + .with_document_id(1); + for field in [ + EmailSearchField::From, + EmailSearchField::To, + EmailSearchField::Cc, + EmailSearchField::Bcc, + EmailSearchField::Subject, + ] { + document.index_text(field, &large_text[..10 * 1024], Language::English); + } + for field in [EmailSearchField::Body, EmailSearchField::Attachment] { + document.index_text(field, &large_text, Language::English); + } + for field in [ + EmailSearchField::ReceivedAt, + EmailSearchField::SentAt, + EmailSearchField::Size, + ] { + document.index_unsigned(field, rand::rng().random_range(100u64..1_000_000u64)); + } + store.index(vec![document]).await.unwrap(); + // Refresh + if let SearchStore::ElasticSearch(store) = &store { + store.refresh_index(SearchIndex::Email).await.unwrap(); + } + + println!("Running account filtering tests..."); let filter_ids = std::env::var("QUICK_TEST").is_ok().then(|| { let mut ids = AHashSet::new(); for &id in ALL_IDS {