From 62941c968cec7501836d332e7f9ca78811aa9dcc Mon Sep 17 00:00:00 2001 From: mdecimus <11444311+mdecimus@users.noreply.github.com> Date: Sat, 20 Dec 2025 10:21:40 +0100 Subject: [PATCH] PostgreSQL: Truncate ts_vector to 1MB --- CHANGELOG.md | 22 ++++++++- README.md | 4 +- crates/store/src/backend/mysql/search.rs | 10 ++++- crates/store/src/backend/postgres/search.rs | 50 ++++++++++++++------- 4 files changed, 67 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2af9ac20..d5dcd2e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,9 +2,29 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). +## [0.15.2] - 2025-12-22 + +This version includes **multiple breaking changes**. If you are upgrading from v0.14.x and below, please read the [upgrading documentation](https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING/v0_15.md) for more information on how to upgrade from previous versions. If you are upgrading from v0.15.x, replace the binary and update the webadmin. + +## Added +- OAuth: Add device authorization endpoint (#2225). + +## Changed +- Antispam: Only auto-learn spam from traps or multiple RBL hits. + +## Fixed +- mySQL search: Use `MEDIUMTEXT` field type for email body and attachments (#2544). +- PostgreSQL search: Truncate large text fields. +- Antispam: Fix `NO_SPACE_IN_FROM` spam tag detection logic (#2372). +- IMAP: Fix shared folder double nesting (test suite credits to @ochnygosch) (#2358). +- JMAP: Use latest `Received` header in JMAP `Email/import` (credits to @apexskier) (#2374). +- JMAP: Return unsorted search results when the index is not ready (#2544). +- LDAP: Lowercase attribute comparison (credits to @pdf) (#2363). +- CLI: Fix same-host JMAP redirection on non-standard ports (#2271). + ## [0.15.1] - 2025-12-17 -This version includes **multiple breaking changes**. Please read the [upgrading documentation](https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING/v0_15.md) for more information on how to upgrade from previous versions. +This version includes **multiple breaking changes**. If you are upgrading from v0.14.x and below, please read the [upgrading documentation](https://github.com/stalwartlabs/stalwart/blob/main/UPGRADING/v0_15.md) for more information on how to upgrade from previous versions. ## Added diff --git a/README.md b/README.md index ac5cbb60..e139cf73 100644 --- a/README.md +++ b/README.md @@ -77,8 +77,8 @@ Key features: - **Greylisting** to temporarily defer unknown senders. - **Spam traps** to set up decoy email addresses that catch and analyze spam. - **Flexible**: - - Pluggable storage backends with **RocksDB**, **FoundationDB**, **PostgreSQL**, **mySQL**, **SQLite**, **S3-Compatible**, **Azure**, **Redis** and **ElasticSearch** support. - - Full-text search available in 17 languages. + - Pluggable storage backends with **RocksDB**, **FoundationDB**, **PostgreSQL**, **mySQL**, **SQLite**, **S3-Compatible**, **Azure** and **Redis** support. + - Full-text search available in 17 languages using the built-in search engine or via **Meilisearch**, **ElasticSearch**, **OpenSearch**, **PostgreSQL** or **mySQL** backends. - Sieve scripting language with support for all [registered extensions](https://www.iana.org/assignments/sieve-extensions/sieve-extensions.xhtml). - Email aliases, mailing lists, subaddressing and catch-all addresses support. - Automatic account configuration and discovery with [autoconfig](https://www.ietf.org/id/draft-bucksch-autoconfig-02.html) and [autodiscover](https://learn.microsoft.com/en-us/exchange/architecture/client-access/autodiscover?view=exchserver-2019). diff --git a/crates/store/src/backend/mysql/search.rs b/crates/store/src/backend/mysql/search.rs index 236e984c..be3804a2 100644 --- a/crates/store/src/backend/mysql/search.rs +++ b/crates/store/src/backend/mysql/search.rs @@ -289,7 +289,15 @@ impl SearchOperator { impl From for Value { fn from(value: SearchValue) -> Self { match value { - SearchValue::Text { value, .. } => Value::Bytes(value.into_bytes()), + SearchValue::Text { mut value, .. } => { + // Truncate values larger than 16MB to avoid MySQL errors + if value.len() > 16_777_214 { + let pos = value.floor_char_boundary(16_777_214); + value.truncate(pos); + } + + Value::Bytes(value.into_bytes()) + } SearchValue::KeyValues(vec_map) => serde_json::to_string(&vec_map) .map(|v| Value::Bytes(v.into_bytes())) .unwrap_or(Value::NULL), diff --git a/crates/store/src/backend/postgres/search.rs b/crates/store/src/backend/postgres/search.rs index 39fafd8c..a1bde853 100644 --- a/crates/store/src/backend/postgres/search.rs +++ b/crates/store/src/backend/postgres/search.rs @@ -58,21 +58,23 @@ impl PostgresStore { if let Some(value) = fields.get(field) { let value_ref = format!("${}", values.len() + 1); - let truncate_text = matches!(value, SearchValue::Text { value, .. } - if value.len() > 255); + let (text_len, language) = if let SearchValue::Text { value, language } = value + { + ( + value.len(), + if self.languages.contains(language) { + pg_lang(language).unwrap_or("simple") + } else { + "simple" + }, + ) + } else { + (0, "simple") + }; if field.is_text() { - let language = match &value { - SearchValue::Text { language, .. } - if self.languages.contains(language) => - { - pg_lang(language).unwrap_or("simple") - } - _ => "simple", - }; - let _ = write!(&mut query, "to_tsvector('{language}',{value_ref})"); - } else if truncate_text { + } else if text_len > 512 { query.push_str("left("); query.push_str(&value_ref); query.push_str(",512)"); @@ -81,7 +83,7 @@ impl PostgresStore { } if field.sort_column().is_some() { - if truncate_text { + if text_len > 255 { query.push_str(",left("); query.push_str(&value_ref); query.push_str(",255)"); @@ -313,7 +315,16 @@ impl ToSql for SearchValue { Self: Sized, { match self { - SearchValue::Text { value, .. } => value.to_sql(ty, out), + SearchValue::Text { value, .. } => { + // Truncate large text fields to avoid Postgres errors (see https://www.postgresql.org/docs/current/textsearch-limitations.html) + + if value.len() > 1_048_574 { + let pos = value.floor_char_boundary(1_048_574); + (&value[..pos]).to_sql(ty, out) + } else { + value.to_sql(ty, out) + } + } SearchValue::Int(v) => match *ty { Type::INT4 => (*v as i32).to_sql(ty, out), _ => v.to_sql(ty, out), @@ -342,7 +353,16 @@ impl ToSql for SearchValue { out: &mut bytes::BytesMut, ) -> Result> { match self { - SearchValue::Text { value, .. } => value.to_sql_checked(ty, out), + SearchValue::Text { value, .. } => { + // Truncate large text fields to avoid Postgres errors (see https://www.postgresql.org/docs/current/textsearch-limitations.html) + + if value.len() > 1_048_574 { + let pos = value.floor_char_boundary(1_048_574); + (&value[..pos]).to_sql_checked(ty, out) + } else { + value.to_sql_checked(ty, out) + } + } SearchValue::Int(v) => match *ty { Type::INT4 => (*v as i32).to_sql_checked(ty, out), _ => v.to_sql_checked(ty, out),