From f33b5f5d6650aecb87cfa9d408686bd8ddecd15e Mon Sep 17 00:00:00 2001 From: mdecimus <11444311+mdecimus@users.noreply.github.com> Date: Wed, 5 Nov 2025 16:42:11 +0100 Subject: [PATCH] Database schema optimization - part 6 --- crates/common/src/config/mod.rs | 51 +- crates/groupware/src/calendar/index.rs | 5 +- crates/groupware/src/contact/index.rs | 7 +- crates/main/Cargo.toml | 3 +- crates/services/src/task_manager/index.rs | 66 +- crates/store/Cargo.lock | 1540 ------------------- crates/store/Cargo.toml | 6 +- crates/store/src/backend/elastic/index.rs | 94 -- crates/store/src/backend/elastic/main.rs | 134 ++ crates/store/src/backend/elastic/mod.rs | 188 +-- crates/store/src/backend/elastic/query.rs | 124 -- crates/store/src/backend/elastic/search.rs | 5 + crates/store/src/backend/mod.rs | 1 - crates/store/src/backend/mysql/mod.rs | 4 +- crates/store/src/backend/mysql/search.rs | 294 +++- crates/store/src/backend/postgres/main.rs | 4 +- crates/store/src/backend/postgres/mod.rs | 4 +- crates/store/src/backend/postgres/search.rs | 130 +- crates/store/src/config.rs | 10 +- crates/store/src/dispatch/search.rs | 13 +- crates/store/src/lib.rs | 2 - crates/store/src/search/mod.rs | 17 +- crates/utils/src/config/http.rs | 89 ++ crates/utils/src/config/mod.rs | 1 + 24 files changed, 679 insertions(+), 2113 deletions(-) delete mode 100644 crates/store/Cargo.lock delete mode 100644 crates/store/src/backend/elastic/index.rs create mode 100644 crates/store/src/backend/elastic/main.rs delete mode 100644 crates/store/src/backend/elastic/query.rs create mode 100644 crates/store/src/backend/elastic/search.rs create mode 100644 crates/utils/src/config/http.rs diff --git a/crates/common/src/config/mod.rs b/crates/common/src/config/mod.rs index c922682e..fc364eb6 100644 --- a/crates/common/src/config/mod.rs +++ b/crates/common/src/config/mod.rs @@ -23,7 +23,7 @@ use hyper::{ use ring::signature::{EcdsaKeyPair, RsaKeyPair}; use spamfilter::SpamFilterConfig; use std::{str::FromStr, sync::Arc}; -use store::{BlobBackend, BlobStore, SearchStore, InMemoryStore, Store, Stores}; +use store::{BlobBackend, BlobStore, InMemoryStore, SearchStore, Store, Stores}; use telemetry::Metrics; use utils::config::{Config, utils::AsKey}; @@ -261,52 +261,3 @@ pub fn build_ecdsa_pem( Ok(None) => Err("No ECDSA key found in PEM".to_string()), } } - -pub(crate) fn parse_http_headers(config: &mut Config, prefix: impl AsKey) -> HeaderMap { - let prefix = prefix.as_key(); - let mut headers = HeaderMap::new(); - - for (header, value) in config - .values((&prefix, "headers")) - .map(|(_, v)| { - if let Some((k, v)) = v.split_once(':') { - Ok(( - HeaderName::from_str(k.trim()).map_err(|err| { - format!("Invalid header found in property \"{prefix}.headers\": {err}",) - })?, - HeaderValue::from_str(v.trim()).map_err(|err| { - format!("Invalid header found in property \"{prefix}.headers\": {err}",) - })?, - )) - } else { - Err(format!( - "Invalid header found in property \"{prefix}.headers\": {v}", - )) - } - }) - .collect::, String>>() - .map_err(|e| config.new_parse_error((&prefix, "headers"), e)) - .unwrap_or_default() - { - headers.insert(header, value); - } - - if let (Some(name), Some(secret)) = ( - config.value((&prefix, "auth.username")), - config.value((&prefix, "auth.secret")), - ) { - headers.insert( - AUTHORIZATION, - format!( - "Basic {}", - general_purpose::STANDARD.encode(format!("{}:{}", name, secret)) - ) - .parse() - .unwrap(), - ); - } else if let Some(token) = config.value((&prefix, "auth.token")) { - headers.insert(AUTHORIZATION, format!("Bearer {}", token).parse().unwrap()); - } - - headers -} diff --git a/crates/groupware/src/calendar/index.rs b/crates/groupware/src/calendar/index.rs index 188143a1..30bf2896 100644 --- a/crates/groupware/src/calendar/index.rs +++ b/crates/groupware/src/calendar/index.rs @@ -358,7 +358,7 @@ impl ArchivedCalendarEvent { .filter(|e| e.component_type.is_scheduling_object()) { for entry in component.entries.iter() { - let (is_lang, field) = SearchField::Calendar(match entry.name { + let (is_lang, field) = match entry.name { ArchivedICalendarProperty::Summary => (true, CalendarSearchField::Title), ArchivedICalendarProperty::Description => { (true, CalendarSearchField::Description) @@ -368,7 +368,8 @@ impl ArchivedCalendarEvent { ArchivedICalendarProperty::Attendee => (false, CalendarSearchField::Attendee), ArchivedICalendarProperty::Uid => (false, CalendarSearchField::Uid), _ => continue, - }); + }; + let field = SearchField::Calendar(field); if index_fields.is_empty() || index_fields.contains(&field) { for value in entry diff --git a/crates/groupware/src/contact/index.rs b/crates/groupware/src/contact/index.rs index 97a6741a..03169f00 100644 --- a/crates/groupware/src/contact/index.rs +++ b/crates/groupware/src/contact/index.rs @@ -265,7 +265,7 @@ impl ArchivedContactCard { let mut detector = LanguageDetector::new(); for entry in self.card.entries.iter() { - let (is_text, field) = SearchField::Contact(match entry.name { + let (is_text, field) = match entry.name { ArchivedVCardProperty::N => (false, ContactSearchField::Name), ArchivedVCardProperty::Nickname => (false, ContactSearchField::Nickname), ArchivedVCardProperty::Org => (false, ContactSearchField::Organization), @@ -280,7 +280,8 @@ impl ArchivedContactCard { ArchivedVCardProperty::Uid => (false, ContactSearchField::Uid), ArchivedVCardProperty::Member => (false, ContactSearchField::Member), _ => continue, - }); + }; + let field = SearchField::Contact(field); if index_fields.is_empty() || index_fields.contains(&field) { for value in entry.values.iter() { @@ -310,7 +311,7 @@ impl ArchivedContactCard { for param in entry.params.iter() { if let ArchivedVCardParameterValue::Text(value) = ¶m.value { let lang = if is_text { - detector.detect(v.as_str(), MIN_LANGUAGE_SCORE); + detector.detect(value.as_str(), MIN_LANGUAGE_SCORE); Language::Unknown } else { Language::None diff --git a/crates/main/Cargo.toml b/crates/main/Cargo.toml index 0190a904..9c61dede 100644 --- a/crates/main/Cargo.toml +++ b/crates/main/Cargo.toml @@ -40,14 +40,13 @@ tokio = { version = "1.47", features = ["full"] } jemallocator = "0.5.0" [features] -#default = ["sqlite", "postgres", "mysql", "rocks", "elastic", "s3", "redis", "azure", "nats", "enterprise"] +#default = ["sqlite", "postgres", "mysql", "rocks", "s3", "redis", "azure", "nats", "enterprise"] default = ["rocks", "enterprise"] sqlite = ["store/sqlite"] foundationdb = ["store/foundation", "common/foundation"] postgres = ["store/postgres"] mysql = ["store/mysql"] rocks = ["store/rocks"] -elastic = ["store/elastic"] s3 = ["store/s3"] redis = ["store/redis"] nats = ["store/nats"] diff --git a/crates/services/src/task_manager/index.rs b/crates/services/src/task_manager/index.rs index 2d5ebee0..e05bab7b 100644 --- a/crates/services/src/task_manager/index.rs +++ b/crates/services/src/task_manager/index.rs @@ -72,40 +72,35 @@ impl SearchIndexTask for Server { async fn index(&self, tasks: &[Task]) -> Vec { let mut results: Vec = Vec::with_capacity(tasks.len()); let mut batch = BatchBuilder::new(); - let mut document_insertions: [Vec; NUM_INDEXES] = - std::array::from_fn(|_| Vec::new()); + let mut document_insertions = Vec::new(); let mut document_deletions: [AHashMap>; NUM_INDEXES] = std::array::from_fn(|_| AHashMap::new()); for task in tasks { if task.action.is_insert { - let (idx, document) = match task.action.index { - SearchIndex::Email => ( - 0, - build_email_document(self, task.account_id, task.document_id).await, - ), - SearchIndex::Calendar => ( - 1, - build_calendar_document(self, task.account_id, task.document_id).await, - ), - SearchIndex::Contacts => ( - 2, - build_contact_document(self, task.account_id, task.document_id).await, - ), + let document = match task.action.index { + SearchIndex::Email => { + build_email_document(self, task.account_id, task.document_id).await + } + SearchIndex::Calendar => { + build_calendar_document(self, task.account_id, task.document_id).await + } + SearchIndex::Contacts => { + build_contact_document(self, task.account_id, task.document_id).await + } SearchIndex::File => { // File indexing not implemented yet continue; } - SearchIndex::Tracing => ( - 4, - build_tracing_span_document(self, task.account_id, task.document_id).await, - ), + SearchIndex::Tracing => { + build_tracing_span_document(self, task.account_id, task.document_id).await + } SearchIndex::InMemory => unreachable!(), }; let result = match document { Ok(Some(doc)) if !doc.is_empty() => { - document_insertions[idx].push(doc); + document_insertions.push(doc); TaskStatus::Success } Err(err) => { @@ -200,28 +195,19 @@ impl SearchIndexTask for Server { } // Index documents - for (documents, index) in document_insertions.into_iter().zip([ - SearchIndex::Email, - SearchIndex::Calendar, - SearchIndex::Contacts, - SearchIndex::File, - SearchIndex::Tracing, - ]) { - if !documents.is_empty() - && let Err(err) = self.search_store().index(index, documents).await - { - trc::error!( - err.caused_by(trc::location!()) - .details("Failed to index documents") - .ctx(trc::Key::Collection, index.name()) - ); - for r in results.iter_mut() { - if r.task_type == TaskType::Delete && r.status == TaskStatus::Success { - r.status = TaskStatus::Failed; - } + if !document_insertions.is_empty() + && let Err(err) = self.search_store().index(document_insertions).await + { + trc::error!( + err.caused_by(trc::location!()) + .details("Failed to index documents") + ); + for r in results.iter_mut() { + if r.task_type == TaskType::Delete && r.status == TaskStatus::Success { + r.status = TaskStatus::Failed; } - return results; } + return results; } // Delete documents diff --git a/crates/store/Cargo.lock b/crates/store/Cargo.lock deleted file mode 100644 index 421639c3..00000000 --- a/crates/store/Cargo.lock +++ /dev/null @@ -1,1540 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 3 - -[[package]] -name = "adler" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" - -[[package]] -name = "ahash" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" -dependencies = [ - "getrandom", - "once_cell", - "version_check", -] - -[[package]] -name = "ahash" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" -dependencies = [ - "cfg-if", - "getrandom", - "once_cell", - "serde", - "version_check", -] - -[[package]] -name = "aho-corasick" -version = "0.7.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac" -dependencies = [ - "memchr", -] - -[[package]] -name = "arrayref" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" - -[[package]] -name = "arrayvec" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" - -[[package]] -name = "async-recursion" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e97ce7de6cf12de5d7226c73f5ba9811622f4db3a5b91b55c53e987e5f91cba" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.3", -] - -[[package]] -name = "async-trait" -version = "0.1.67" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86ea188f25f0255d8f92797797c97ebf5631fa88178beb1a46fdf5622c9a00e4" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.3", -] - -[[package]] -name = "atty" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" -dependencies = [ - "hermit-abi 0.1.19", - "libc", - "winapi", -] - -[[package]] -name = "autocfg" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" - -[[package]] -name = "bindgen" -version = "0.60.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "062dddbc1ba4aca46de6338e2bf87771414c335f7b2f2036e8f3e9befebf88e6" -dependencies = [ - "bitflags 1.3.2", - "cexpr", - "clang-sys", - "clap", - "env_logger", - "lazy_static", - "lazycell", - "log", - "peeking_take_while", - "proc-macro2", - "quote", - "regex", - "rustc-hash", - "shlex", - "which", -] - -[[package]] -name = "bindgen" -version = "0.64.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4243e6031260db77ede97ad86c27e501d646a27ab57b59a574f725d98ab1fb4" -dependencies = [ - "bitflags 1.3.2", - "cexpr", - "clang-sys", - "lazy_static", - "lazycell", - "peeking_take_while", - "proc-macro2", - "quote", - "regex", - "rustc-hash", - "shlex", - "syn 1.0.109", -] - -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "bitflags" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "487f1e0fcbe47deb8b0574e646def1c903389d95241dd1bbcc6ce4a715dfc0c1" - -[[package]] -name = "bitpacking" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8c7d2ac73c167c06af4a5f37e6e59d84148d57ccbe4480b76f0273eefea82d7" -dependencies = [ - "crunchy", -] - -[[package]] -name = "blake3" -version = "1.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42ae2468a89544a466886840aa467a25b766499f4f04bf7d9fcd10ecee9fccef" -dependencies = [ - "arrayref", - "arrayvec", - "cc", - "cfg-if", - "constant_time_eq", - "digest", -] - -[[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" -dependencies = [ - "generic-array", -] - -[[package]] -name = "bytemuck" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17febce684fd15d89027105661fec94afb475cb995fbc59d2865198446ba2eea" - -[[package]] -name = "byteorder" -version = "1.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" - -[[package]] -name = "bytes" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" - -[[package]] -name = "bzip2-sys" -version = "0.1.11+1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" -dependencies = [ - "cc", - "libc", - "pkg-config", -] - -[[package]] -name = "cc" -version = "1.0.79" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" -dependencies = [ - "jobserver", -] - -[[package]] -name = "cedarwood" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d910bedd62c24733263d0bed247460853c9d22e8956bd4cd964302095e04e90" -dependencies = [ - "smallvec", -] - -[[package]] -name = "cexpr" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" -dependencies = [ - "nom", -] - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "clang-sys" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ed9a53e5d4d9c573ae844bfac6872b159cb1d1585a83b29e7a64b7eef7332a" -dependencies = [ - "glob", - "libc", - "libloading", -] - -[[package]] -name = "clap" -version = "3.2.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5" -dependencies = [ - "atty", - "bitflags 1.3.2", - "clap_lex", - "indexmap", - "strsim", - "termcolor", - "textwrap", -] - -[[package]] -name = "clap_lex" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5" -dependencies = [ - "os_str_bytes", -] - -[[package]] -name = "constant_time_eq" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13418e745008f7349ec7e449155f419a61b92b58a99cc3616942b926825ec76b" - -[[package]] -name = "crc32fast" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "crossbeam-channel" -version = "0.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf2b3e8478797446514c91ef04bafcb59faba183e621ad488df88983cc14128c" -dependencies = [ - "cfg-if", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-deque" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" -dependencies = [ - "cfg-if", - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695" -dependencies = [ - "autocfg", - "cfg-if", - "crossbeam-utils", - "memoffset", - "scopeguard", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "crunchy" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" - -[[package]] -name = "crypto-common" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" -dependencies = [ - "generic-array", - "typenum", -] - -[[package]] -name = "csv" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b015497079b9a9d69c02ad25de6c0a6edef051ea6360a327d0bd05802ef64ad" -dependencies = [ - "csv-core", - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "csv-core" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b2466559f260f48ad25fe6317b3c8dac77b5bdb5763ac7d9d6103530663bc90" -dependencies = [ - "memchr", -] - -[[package]] -name = "digest" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" -dependencies = [ - "block-buffer", - "crypto-common", - "subtle", -] - -[[package]] -name = "either" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" - -[[package]] -name = "env_logger" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7" -dependencies = [ - "atty", - "humantime", - "log", - "regex", - "termcolor", -] - -[[package]] -name = "fallible-iterator" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" - -[[package]] -name = "fallible-streaming-iterator" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" - -[[package]] -name = "farmhash" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f35ce9c8fb9891c75ceadbc330752951a4e369b50af10775955aeb9af3eee34b" - -[[package]] -name = "flate2" -version = "1.0.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a2db397cb1c8772f31494cb8917e48cd1e64f0fa7efac59fbd741a0a8ce841" -dependencies = [ - "crc32fast", - "libz-sys", - "miniz_oxide", -] - -[[package]] -name = "foundationdb" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69adb701525370e5f8958454b46e8459b276d81ce6391edbf84eae32eeddff75" -dependencies = [ - "async-recursion", - "async-trait", - "foundationdb-gen", - "foundationdb-macros", - "foundationdb-sys", - "futures", - "memchr", - "rand", - "static_assertions", - "uuid", -] - -[[package]] -name = "foundationdb-gen" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "134e1c986a2bb78904f426d4924a55e8c14162ba764e229501eb6f95c8c37489" -dependencies = [ - "xml-rs", -] - -[[package]] -name = "foundationdb-macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2163c7326208be8edc605e10303ec6ae45cf106c12540754a9970bcce0f80cae" -dependencies = [ - "quote", - "syn 1.0.109", -] - -[[package]] -name = "foundationdb-sys" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eb26eee771096794dbee1a2a9defa455443a2c150a810386331aa0d6603d356" -dependencies = [ - "bindgen 0.60.1", -] - -[[package]] -name = "futures" -version = "0.3.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "531ac96c6ff5fd7c62263c5e3c67a603af4fcaee2e1a0ae5565ba3a11e69e549" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-channel" -version = "0.3.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "164713a5a0dcc3e7b4b1ed7d3b433cabc18025386f9339346e8daf15963cf7ac" -dependencies = [ - "futures-core", - "futures-sink", -] - -[[package]] -name = "futures-core" -version = "0.3.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86d7a0c1aa76363dac491de0ee99faf6941128376f1cf96f07db7603b7de69dd" - -[[package]] -name = "futures-executor" -version = "0.3.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1997dd9df74cdac935c76252744c1ed5794fac083242ea4fe77ef3ed60ba0f83" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-io" -version = "0.3.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89d422fa3cbe3b40dca574ab087abb5bc98258ea57eea3fd6f1fa7162c778b91" - -[[package]] -name = "futures-macro" -version = "0.3.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eb14ed937631bd8b8b8977f2c198443447a8355b6e3ca599f38c975e5a963b6" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "futures-sink" -version = "0.3.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec93083a4aecafb2a80a885c9de1f0ccae9dbd32c2bb54b0c3a65690e0b8d2f2" - -[[package]] -name = "futures-task" -version = "0.3.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd65540d33b37b16542a0438c12e6aeead10d4ac5d05bd3f805b8f35ab592879" - -[[package]] -name = "futures-util" -version = "0.3.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ef6b17e481503ec85211fed8f39d1970f128935ca1f814cd32ac4a6842e84ab" -dependencies = [ - "futures-channel", - "futures-core", - "futures-io", - "futures-macro", - "futures-sink", - "futures-task", - "memchr", - "pin-project-lite", - "pin-utils", - "slab", -] - -[[package]] -name = "fxhash" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" -dependencies = [ - "byteorder", -] - -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - -[[package]] -name = "getrandom" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" -dependencies = [ - "cfg-if", - "libc", - "wasi", -] - -[[package]] -name = "glob" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" - -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" -dependencies = [ - "ahash 0.7.6", -] - -[[package]] -name = "hashlink" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69fe1fcf8b4278d860ad0548329f892a3631fb63f82574df68275f34cdbe0ffa" -dependencies = [ - "hashbrown", -] - -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - -[[package]] -name = "hermit-abi" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" -dependencies = [ - "libc", -] - -[[package]] -name = "humantime" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" - -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown", -] - -[[package]] -name = "itoa" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" - -[[package]] -name = "jieba-rs" -version = "0.6.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37228e06c75842d1097432d94d02f37fe3ebfca9791c2e8fef6e9db17ed128c1" -dependencies = [ - "cedarwood", - "fxhash", - "hashbrown", - "lazy_static", - "phf", - "phf_codegen", - "regex", -] - -[[package]] -name = "jobserver" -version = "0.1.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" -dependencies = [ - "libc", -] - -[[package]] -name = "lazy_static" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" - -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - -[[package]] -name = "libc" -version = "0.2.140" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99227334921fae1a979cf0bfdfcc6b3e5ce376ef57e16fb6fb3ea2ed6095f80c" - -[[package]] -name = "libloading" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" -dependencies = [ - "cfg-if", - "winapi", -] - -[[package]] -name = "librocksdb-sys" -version = "0.10.0+7.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fe4d5874f5ff2bc616e55e8c6086d478fcda13faf9495768a4aa1c22042d30b" -dependencies = [ - "bindgen 0.64.0", - "bzip2-sys", - "cc", - "glob", - "libc", - "libz-sys", - "lz4-sys", - "zstd-sys", -] - -[[package]] -name = "libsqlite3-sys" -version = "0.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afc22eff61b133b115c6e8c74e818c628d6d5e7a502afea6f64dee076dd94326" -dependencies = [ - "cc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "libz-sys" -version = "1.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9702761c3935f8cc2f101793272e202c72b99da8f4224a19ddcf1279a6450bbf" -dependencies = [ - "cc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "linked-hash-map" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" - -[[package]] -name = "lock_api" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" -dependencies = [ - "autocfg", - "scopeguard", -] - -[[package]] -name = "log" -version = "0.4.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "lru-cache" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" -dependencies = [ - "linked-hash-map", -] - -[[package]] -name = "lz4-sys" -version = "1.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57d27b317e207b10f69f5e75494119e391a96f48861ae870d1da6edac98ca900" -dependencies = [ - "cc", - "libc", -] - -[[package]] -name = "maplit" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" - -[[package]] -name = "maybe-async" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f1b8c13cb1f814b634a96b2c725449fe7ed464a7b8781de8688be5ffbd3f305" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "memchr" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" - -[[package]] -name = "memoffset" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" -dependencies = [ - "autocfg", -] - -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - -[[package]] -name = "miniz_oxide" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b275950c28b37e794e8c55d88aeb5e139d0ce23fdbbeda68f8d7174abdf9e8fa" -dependencies = [ - "adler", -] - -[[package]] -name = "mio" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" -dependencies = [ - "libc", - "log", - "wasi", - "windows-sys", -] - -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - -[[package]] -name = "num_cpus" -version = "1.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" -dependencies = [ - "hermit-abi 0.2.6", - "libc", -] - -[[package]] -name = "once_cell" -version = "1.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" - -[[package]] -name = "os_str_bytes" -version = "6.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ceedf44fb00f2d1984b0bc98102627ce622e083e49a5bacdb3e514fa4238e267" - -[[package]] -name = "parking_lot" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-sys", -] - -[[package]] -name = "peeking_take_while" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" - -[[package]] -name = "phf" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "928c6535de93548188ef63bb7c4036bd415cd8f36ad25af44b9789b2ee72a48c" -dependencies = [ - "phf_shared", -] - -[[package]] -name = "phf_codegen" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56ac890c5e3ca598bbdeaa99964edb5b0258a583a9eb6ef4e89fc85d9224770" -dependencies = [ - "phf_generator", - "phf_shared", -] - -[[package]] -name = "phf_generator" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1181c94580fa345f50f19d738aaa39c0ed30a600d95cb2d3e23f94266f14fbf" -dependencies = [ - "phf_shared", - "rand", -] - -[[package]] -name = "phf_shared" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1fb5f6f826b772a8d4c0394209441e7d37cbbb967ae9c7e0e8134365c9ee676" -dependencies = [ - "siphasher", -] - -[[package]] -name = "pin-project-lite" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" - -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - -[[package]] -name = "pkg-config" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" - -[[package]] -name = "ppv-lite86" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" - -[[package]] -name = "proc-macro2" -version = "1.0.52" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d0e1ae9e836cc3beddd63db0df682593d7e2d3d891ae8c9083d2113e1744224" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "quote" -version = "1.0.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "r2d2" -version = "0.8.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93" -dependencies = [ - "log", - "parking_lot", - "scheduled-thread-pool", -] - -[[package]] -name = "rand" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" -dependencies = [ - "libc", - "rand_chacha", - "rand_core", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom", -] - -[[package]] -name = "rayon" -version = "1.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" -dependencies = [ - "crossbeam-channel", - "crossbeam-deque", - "crossbeam-utils", - "num_cpus", -] - -[[package]] -name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags 1.3.2", -] - -[[package]] -name = "regex" -version = "1.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.6.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848" - -[[package]] -name = "retain_mut" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c31b5c4033f8fdde8700e4657be2c497e7288f01515be52168c631e2e4d4086" - -[[package]] -name = "roaring" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef0fb5e826a8bde011ecae6a8539dd333884335c57ff0f003fbe27c25bbe8f71" -dependencies = [ - "bytemuck", - "byteorder", - "retain_mut", -] - -[[package]] -name = "rocksdb" -version = "0.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "015439787fce1e75d55f279078d33ff14b4af5d93d995e8838ee4631301c8a99" -dependencies = [ - "libc", - "librocksdb-sys", -] - -[[package]] -name = "rusqlite" -version = "0.29.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "549b9d036d571d42e6e85d1c1425e2ac83491075078ca9a15be021c56b1641f2" -dependencies = [ - "bitflags 2.0.2", - "fallible-iterator", - "fallible-streaming-iterator", - "hashlink", - "libsqlite3-sys", - "smallvec", -] - -[[package]] -name = "rust-stemmers" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" -dependencies = [ - "serde", - "serde_derive", -] - -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - -[[package]] -name = "ryu" -version = "1.0.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" - -[[package]] -name = "scheduled-thread-pool" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19" -dependencies = [ - "parking_lot", -] - -[[package]] -name = "scopeguard" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" - -[[package]] -name = "serde" -version = "1.0.158" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "771d4d9c4163ee138805e12c710dd365e4f44be8be0503cb1bb9eb989425d9c9" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.158" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e801c1712f48475582b7696ac71e0ca34ebb30e09338425384269d9717c62cad" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.3", -] - -[[package]] -name = "shlex" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" - -[[package]] -name = "signal-hook-registry" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" -dependencies = [ - "libc", -] - -[[package]] -name = "siphasher" -version = "0.3.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" - -[[package]] -name = "slab" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" -dependencies = [ - "autocfg", -] - -[[package]] -name = "smallvec" -version = "1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" - -[[package]] -name = "socket2" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" -dependencies = [ - "libc", - "winapi", -] - -[[package]] -name = "static_assertions" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - -[[package]] -name = "store" -version = "0.1.0" -dependencies = [ - "ahash 0.8.3", - "bitpacking", - "blake3", - "csv", - "farmhash", - "flate2", - "foundationdb", - "futures", - "jieba-rs", - "lazy_static", - "lru-cache", - "maybe-async", - "parking_lot", - "r2d2", - "rand", - "rayon", - "roaring", - "rocksdb", - "rusqlite", - "rust-stemmers", - "serde", - "siphasher", - "tinysegmenter", - "tokio", - "utils", - "whatlang", - "xxhash-rust", -] - -[[package]] -name = "strsim" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" - -[[package]] -name = "subtle" -version = "2.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" - -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "2.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8234ae35e70582bfa0f1fedffa6daa248e41dd045310b19800c4a36382c8f60" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "termcolor" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be55cf8942feac5c765c2c993422806843c9a9a45d4d5c407ad6dd2ea95eb9b6" -dependencies = [ - "winapi-util", -] - -[[package]] -name = "textwrap" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" - -[[package]] -name = "tinysegmenter" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1755695d17d470baf2d937a59ab4e86de3034b056fc8700e21411b0efca36497" -dependencies = [ - "lazy_static", - "maplit", -] - -[[package]] -name = "tokio" -version = "1.26.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03201d01c3c27a29c8a5cee5b55a93ddae1ccf6f08f65365c2c918f8c1b76f64" -dependencies = [ - "autocfg", - "bytes", - "libc", - "memchr", - "mio", - "num_cpus", - "parking_lot", - "pin-project-lite", - "signal-hook-registry", - "socket2", - "tokio-macros", - "windows-sys", -] - -[[package]] -name = "tokio-macros" -version = "1.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" -dependencies = [ - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "typenum" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" - -[[package]] -name = "unicode-ident" -version = "1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" - -[[package]] -name = "utils" -version = "0.1.0" -dependencies = [ - "serde", -] - -[[package]] -name = "uuid" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1674845326ee10d37ca60470760d4288a6f80f304007d92e5c53bab78c9cfd79" - -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - -[[package]] -name = "version_check" -version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" - -[[package]] -name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" - -[[package]] -name = "whatlang" -version = "0.16.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c531a2dc4c462b833788be2c07eef4e621d0e9edbd55bf280cc164c1c1aa043" -dependencies = [ - "hashbrown", - "once_cell", -] - -[[package]] -name = "which" -version = "4.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" -dependencies = [ - "either", - "libc", - "once_cell", -] - -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-util" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" -dependencies = [ - "winapi", -] - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows-sys" -version = "0.45.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" -dependencies = [ - "windows-targets", -] - -[[package]] -name = "windows-targets" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" - -[[package]] -name = "windows_i686_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" - -[[package]] -name = "windows_i686_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.42.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" - -[[package]] -name = "xml-rs" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2d7d3948613f75c98fd9328cfdcc45acc4d360655289d0a7d4ec931392200a3" - -[[package]] -name = "xxhash-rust" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "735a71d46c4d68d71d4b24d03fdc2b98e38cea81730595801db779c04fe80d70" - -[[package]] -name = "zstd-sys" -version = "2.0.7+zstd.1.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94509c3ba2fe55294d752b79842c530ccfab760192521df74a081a78d2b3c7f5" -dependencies = [ - "cc", - "libc", - "pkg-config", -] diff --git a/crates/store/Cargo.toml b/crates/store/Cargo.toml index 213bcc1d..7fe4a7bd 100644 --- a/crates/store/Cargo.toml +++ b/crates/store/Cargo.toml @@ -41,8 +41,7 @@ rustls-pki-types = { version = "1", optional = true } ring = { version = "0.17", optional = true } bytes = { version = "1.10", optional = true } mysql_async = { version = "0.36", default-features = false, features = ["default-rustls-ring", "minimal"], optional = true } -elasticsearch = { version = "8.5.0-alpha.1", default-features = false, features = ["rustls-tls"], optional = true } -serde_json = {version = "1.0.64", optional = true } +serde_json = { version = "1.0.64" } regex = "1.12" flate2 = "1.1" redis = { version = "0.32", features = [ "tokio-comp", "tokio-rustls-comp", "tls-rustls-insecure", "tls-rustls-webpki-roots", "cluster-async"], optional = true } @@ -72,9 +71,6 @@ fdb-chunked-bm = [] s3 = ["rust-s3"] azure = ["azure_core", "azure_storage", "azure_storage_blobs"] -# Full-text stores -elastic = ["elasticsearch", "serde_json"] - # In-memory stores redis = ["dep:redis", "deadpool", "futures"] diff --git a/crates/store/src/backend/elastic/index.rs b/crates/store/src/backend/elastic/index.rs deleted file mode 100644 index 3915b513..00000000 --- a/crates/store/src/backend/elastic/index.rs +++ /dev/null @@ -1,94 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use super::{ElasticSearchStore, assert_success}; -use crate::{backend::elastic::INDEX_NAMES, dispatch::DocumentSet, search::IndexDocument}; -use elasticsearch::{DeleteByQueryParts, IndexParts}; -use serde::{Deserialize, Serialize}; -use serde_json::json; -use std::{borrow::Cow, fmt::Display}; -use types::collection::Collection; - -#[derive(Serialize, Deserialize, Default)] -struct Document<'x> { - document_id: u32, - account_id: u32, - body: Vec>, - attachments: Vec>, - keywords: Vec>, - header: Vec>, -} - -#[derive(Serialize, Deserialize)] -struct Header<'x> { - name: Cow<'x, str>, - value: Cow<'x, str>, -} - -impl ElasticSearchStore { - pub async fn index_insert(&self, document: IndexDocument) -> trc::Result<()> { - todo!() - /*assert_success( - self.index - .index(IndexParts::Index(INDEX_NAMES[document.collection as usize])) - .body(Document::from(document)) - .send() - .await, - ) - .await - .map(|_| ())*/ - } - - pub async fn index_remove( - &self, - account_id: u32, - collection: Collection, - document_ids: &impl DocumentSet, - ) -> trc::Result<()> { - let document_ids = document_ids.iterate().collect::>(); - - assert_success( - self.index - .delete_by_query(DeleteByQueryParts::Index(&[ - INDEX_NAMES[collection as usize] - ])) - .body(json!({ - "query": { - "bool": { - "must": [ - { "match": { "account_id": account_id } }, - { "terms": { "document_id": document_ids } } - ] - } - } - })) - .send() - .await, - ) - .await - .map(|_| ()) - } - - pub async fn index_remove_all(&self, account_id: u32) -> trc::Result<()> { - assert_success( - self.index - .delete_by_query(DeleteByQueryParts::Index(INDEX_NAMES)) - .body(json!({ - "query": { - "bool": { - "must": [ - { "match": { "account_id": account_id } }, - ] - } - } - })) - .send() - .await, - ) - .await - .map(|_| ()) - } -} diff --git a/crates/store/src/backend/elastic/main.rs b/crates/store/src/backend/elastic/main.rs new file mode 100644 index 00000000..dfe13280 --- /dev/null +++ b/crates/store/src/backend/elastic/main.rs @@ -0,0 +1,134 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use std::time::Duration; + +use crate::backend::elastic::ElasticSearchStore; +use reqwest::Client; +use serde_json::json; +use utils::config::{Config, http::build_http_client, utils::AsKey}; + +pub(crate) static INDEX_NAMES: &[&str] = &["stalwart_email"]; + +impl ElasticSearchStore { + pub async fn open(config: &mut Config, prefix: impl AsKey) -> Option { + let client = build_http_client(config, prefix.clone())?; + let url = config + .value_require((&prefix, "url"))? + .trim_end_matches("/"); + Url::parse(url) + .map_err(|e| config.new_parse_error((&prefix, "url"), format!("Invalid URL: {e}",))) + .ok()?; + let es = Self { + client, + url: url.to_string(), + }; + + if let Err(err) = es + .create_index( + config + .property_or_default((&prefix, "index.shards"), "3") + .unwrap_or(3), + config + .property_or_default((&prefix, "index.replicas"), "0") + .unwrap_or(0), + ) + .await + { + config.new_build_error(prefix.as_str(), err.to_string()); + } + + Some(es) + } + + async fn create_index(&self, shards: usize, replicas: usize) -> trc::Result<()> { + let exists = self + .index + .indices() + .exists(IndicesExistsParts::Index(&[INDEX_NAMES[0]])) + .send() + .await + .map_err(|err| trc::StoreEvent::ElasticsearchError.reason(err))?; + + if exists.status_code() == StatusCode::NOT_FOUND { + let response = self + .index + .indices() + .create(IndicesCreateParts::Index(INDEX_NAMES[0])) + .body(json!({ + "mappings": { + "properties": { + "document_id": { + "type": "integer" + }, + "account_id": { + "type": "integer" + }, + "header": { + "type": "object", + "properties": { + "name": { + "type": "keyword" + }, + "value": { + "type": "text", + "analyzer": "default_analyzer", + } + } + }, + "body": { + "analyzer": "default_analyzer", + "type": "text" + }, + "attachment": { + "analyzer": "default_analyzer", + "type": "text" + }, + "keyword": { + "type": "keyword" + } + } + }, + "settings": { + "index.number_of_shards": shards, + "index.number_of_replicas": replicas, + "analysis": { + "analyzer": { + "default_analyzer": { + "type": "custom", + "tokenizer": "standard", + "filter": ["lowercase"] + } + } + } + } + })) + .send() + .await; + + assert_success(response).await?; + } + + Ok(()) + } +} + +/*pub(crate) async fn assert_success(response: Result) -> trc::Result { + match response { + Ok(response) => { + let status = response.status_code(); + if status.is_success() { + Ok(response) + } else { + Err(trc::StoreEvent::ElasticsearchError + .reason(response.text().await.unwrap_or_default()) + .ctx(trc::Key::Code, status.as_u16())) + } + } + Err(err) => Err(trc::StoreEvent::ElasticsearchError.reason(err)), + } +} +*/ diff --git a/crates/store/src/backend/elastic/mod.rs b/crates/store/src/backend/elastic/mod.rs index f5449346..4ec30ed5 100644 --- a/crates/store/src/backend/elastic/mod.rs +++ b/crates/store/src/backend/elastic/mod.rs @@ -4,190 +4,12 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use elasticsearch::{ - Elasticsearch, Error, - auth::Credentials, - cert::CertificateValidation, - http::{ - StatusCode, Url, - response::Response, - transport::{SingleNodeConnectionPool, Transport, TransportBuilder}, - }, - indices::{IndicesCreateParts, IndicesExistsParts}, -}; -use serde_json::json; -use utils::config::{Config, utils::AsKey}; +use reqwest::Client; -pub mod index; -pub mod query; +pub mod main; +pub mod search; pub struct ElasticSearchStore { - index: Elasticsearch, -} - -pub(crate) static INDEX_NAMES: &[&str] = &["stalwart_email"]; - -impl ElasticSearchStore { - pub async fn open(config: &mut Config, prefix: impl AsKey) -> Option { - let prefix = prefix.as_key(); - let credentials = if let Some(user) = config.value((&prefix, "user")) { - let user = user.to_string(); - let password = config - .value_require((&prefix, "password")) - .unwrap_or_default(); - Some(Credentials::Basic(user, password.to_string())) - } else { - None - }; - - let es = if let Some(url) = config.value((&prefix, "url")) { - let url = Url::parse(url) - .map_err(|e| config.new_parse_error((&prefix, "url"), format!("Invalid URL: {e}",))) - .ok()?; - let conn_pool = SingleNodeConnectionPool::new(url); - let mut builder = TransportBuilder::new(conn_pool); - if let Some(credentials) = credentials { - builder = builder.auth(credentials); - } - if config - .property_or_default::((&prefix, "tls.allow-invalid-certs"), "false") - .unwrap_or(false) - { - builder = builder.cert_validation(CertificateValidation::None); - } - - Self { - index: Elasticsearch::new( - builder - .build() - .map_err(|err| config.new_build_error(prefix.as_str(), err.to_string())) - .ok()?, - ), - } - } else { - let credentials = credentials.unwrap_or_else(|| { - config.new_build_error((&prefix, "user"), "Missing property"); - Credentials::Basic("".to_string(), "".to_string()) - }); - - if let Some(cloud_id) = config.value((&prefix, "cloud-id")) { - Self { - index: Elasticsearch::new( - Transport::cloud(cloud_id, credentials) - .map_err(|err| config.new_build_error(prefix.as_str(), err.to_string())) - .ok()?, - ), - } - } else { - config.new_parse_error( - prefix.as_str(), - "Missing url or cloud_id for ElasticSearch store", - ); - return None; - } - }; - - if let Err(err) = es - .create_index( - config - .property_or_default((&prefix, "index.shards"), "3") - .unwrap_or(3), - config - .property_or_default((&prefix, "index.replicas"), "0") - .unwrap_or(0), - ) - .await - { - config.new_build_error(prefix.as_str(), err.to_string()); - } - - Some(es) - } - - async fn create_index(&self, shards: usize, replicas: usize) -> trc::Result<()> { - let exists = self - .index - .indices() - .exists(IndicesExistsParts::Index(&[INDEX_NAMES[0]])) - .send() - .await - .map_err(|err| trc::StoreEvent::ElasticsearchError.reason(err))?; - - if exists.status_code() == StatusCode::NOT_FOUND { - let response = self - .index - .indices() - .create(IndicesCreateParts::Index(INDEX_NAMES[0])) - .body(json!({ - "mappings": { - "properties": { - "document_id": { - "type": "integer" - }, - "account_id": { - "type": "integer" - }, - "header": { - "type": "object", - "properties": { - "name": { - "type": "keyword" - }, - "value": { - "type": "text", - "analyzer": "default_analyzer", - } - } - }, - "body": { - "analyzer": "default_analyzer", - "type": "text" - }, - "attachment": { - "analyzer": "default_analyzer", - "type": "text" - }, - "keyword": { - "type": "keyword" - } - } - }, - "settings": { - "index.number_of_shards": shards, - "index.number_of_replicas": replicas, - "analysis": { - "analyzer": { - "default_analyzer": { - "type": "custom", - "tokenizer": "standard", - "filter": ["lowercase"] - } - } - } - } - })) - .send() - .await; - - assert_success(response).await?; - } - - Ok(()) - } -} - -pub(crate) async fn assert_success(response: Result) -> trc::Result { - match response { - Ok(response) => { - let status = response.status_code(); - if status.is_success() { - Ok(response) - } else { - Err(trc::StoreEvent::ElasticsearchError - .reason(response.text().await.unwrap_or_default()) - .ctx(trc::Key::Code, status.as_u16())) - } - } - Err(err) => Err(trc::StoreEvent::ElasticsearchError.reason(err)), - } + client: Client, + url: String, } diff --git a/crates/store/src/backend/elastic/query.rs b/crates/store/src/backend/elastic/query.rs deleted file mode 100644 index a1e7100f..00000000 --- a/crates/store/src/backend/elastic/query.rs +++ /dev/null @@ -1,124 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use crate::search::{SearchComparator, SearchFilter}; - -use super::{ElasticSearchStore, INDEX_NAMES, assert_success}; -use elasticsearch::SearchParts; -use roaring::RoaringBitmap; -use serde_json::{Value, json}; -use std::{borrow::Cow, fmt::Display}; - -impl ElasticSearchStore { - pub async fn index_query( - &self, - account_id: u32, - collection: impl Into, - filters: Vec, - comparators: Vec, - ) -> trc::Result> { - todo!() - - /*let mut stack: Vec<(FtsFilter, Vec)> = vec![]; - let mut conditions = vec![json!({ "match": { "account_id": account_id } })]; - let mut logical_op = FtsFilter::And; - - for filter in filters { - let is_exact = matches!(filter, FtsFilter::Exact { .. }); - match filter { - FtsFilter::Exact { field, text, .. } - | FtsFilter::Contains { field, text, .. } - | FtsFilter::Keyword { field, text, .. } => { - let match_type = if is_exact { "term" } else { "match" }; - - if let Field::Header(name) = field { - conditions.push(json!({"bool": { - "must": [ - { - "term": { - "header.name": name.to_string() - } - }, - { - match_type: { - "header.value": text - } - } - ] - }})); - } else { - conditions.push(json!({ - match_type: { field.name(): text } - })); - } - } - FtsFilter::And | FtsFilter::Or | FtsFilter::Not => { - stack.push((logical_op, conditions)); - logical_op = filter; - conditions = Vec::new(); - } - FtsFilter::End => { - if let Some((prev_logical_op, mut prev_conditions)) = stack.pop() { - if !conditions.is_empty() { - match logical_op { - FtsFilter::And => { - prev_conditions.push(json!({ "bool": { "must": conditions } })); - } - FtsFilter::Or => { - prev_conditions - .push(json!({ "bool": { "should": conditions } })); - } - FtsFilter::Not => { - prev_conditions - .push(json!({ "bool": { "must_not": conditions } })); - } - _ => unreachable!(), - } - } - logical_op = prev_logical_op; - conditions = prev_conditions; - } - } - } - } - - // TODO implement pagination - let response = assert_success( - self.index - .search(SearchParts::Index(&[ - INDEX_NAMES[collection.into() as usize] - ])) - .body(json!({ - "query": { - "bool": { - "must": conditions, - } - }, - "size": 10000, - "_source": ["document_id"] - })) - .send() - .await, - ) - .await?; - - let json: Value = response - .json() - .await - .map_err(|err| trc::StoreEvent::ElasticsearchError.reason(err))?; - let mut results = RoaringBitmap::new(); - - for hit in json["hits"]["hits"].as_array().ok_or_else(|| { - trc::StoreEvent::ElasticsearchError.reason("Invalid response from ElasticSearch") - })? { - results.insert(hit["_source"]["document_id"].as_u64().ok_or_else(|| { - trc::StoreEvent::ElasticsearchError.reason("Invalid response from ElasticSearch") - })? as u32); - } - - Ok(results)*/ - } -} diff --git a/crates/store/src/backend/elastic/search.rs b/crates/store/src/backend/elastic/search.rs new file mode 100644 index 00000000..69b01f91 --- /dev/null +++ b/crates/store/src/backend/elastic/search.rs @@ -0,0 +1,5 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ diff --git a/crates/store/src/backend/mod.rs b/crates/store/src/backend/mod.rs index bac97ce4..dea9e5aa 100644 --- a/crates/store/src/backend/mod.rs +++ b/crates/store/src/backend/mod.rs @@ -6,7 +6,6 @@ #[cfg(feature = "azure")] pub mod azure; -#[cfg(feature = "elastic")] pub mod elastic; #[cfg(feature = "foundation")] pub mod foundationdb; diff --git a/crates/store/src/backend/mysql/mod.rs b/crates/store/src/backend/mysql/mod.rs index ebe36cab..2d1e0efc 100644 --- a/crates/store/src/backend/mysql/mod.rs +++ b/crates/store/src/backend/mysql/mod.rs @@ -31,7 +31,7 @@ fn into_error(err: impl Display) -> trc::Error { } impl SearchIndex { - pub(super) fn mysql_table(&self) -> &'static str { + fn mysql_table(&self) -> &'static str { match self { SearchIndex::Email => "s_email", SearchIndex::Calendar => "s_cal", @@ -43,7 +43,7 @@ impl SearchIndex { } } -pub(super) trait MysqlSearchField { +trait MysqlSearchField { fn column(&self) -> &'static str; fn column_type(&self) -> &'static str; } diff --git a/crates/store/src/backend/mysql/search.rs b/crates/store/src/backend/mysql/search.rs index 61ff4d55..f2046b3a 100644 --- a/crates/store/src/backend/mysql/search.rs +++ b/crates/store/src/backend/mysql/search.rs @@ -5,25 +5,299 @@ */ use crate::{ - backend::mysql::MysqlStore, - search::{IndexDocument, SearchDocumentId, SearchQuery}, + backend::mysql::{MysqlSearchField, MysqlStore, into_error}, + search::{ + IndexDocument, SearchComparator, SearchDocumentId, SearchFilter, SearchOperator, + SearchQuery, SearchValue, + }, write::SearchIndex, }; +use mysql_async::{IsolationLevel, TxOpts, Value, prelude::Queryable}; +use std::fmt::Write; impl MysqlStore { - pub async fn query(&self, query: SearchQuery) -> trc::Result> { - todo!() + pub async fn index(&self, documents: Vec) -> trc::Result<()> { + let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?; + let mut tx_opts = TxOpts::default(); + tx_opts + .with_consistent_snapshot(false) + .with_isolation_level(IsolationLevel::ReadCommitted); + let mut trx = conn.start_transaction(tx_opts).await.map_err(into_error)?; + + for document in documents { + let index = document.index; + let primary_keys = index.primary_keys(); + let all_fields = index.all_fields(); + let mut fields = document.fields; + let mut values = Vec::with_capacity(fields.len() + 2); + let mut query = format!("INSERT INTO {} (", index.mysql_table()); + + for (i, field) in primary_keys.iter().chain(all_fields).enumerate() { + if i > 0 { + query.push(','); + } + query.push_str(field.column()); + } + + query.push_str(") VALUES ("); + + for (i, field) in primary_keys.iter().chain(all_fields).enumerate() { + if i > 0 { + query.push(','); + } + + if let Some(value) = fields.remove(field) { + let _ = write!(&mut query, "${}", values.len() + 1); + values.push(value); + } else { + query.push_str("NULL"); + } + } + + query.push_str(") ON DUPLICATE KEY UPDATE "); + for (i, field) in all_fields.iter().enumerate() { + if i > 0 { + query.push(','); + } + let column = field.column(); + let _ = write!(&mut query, "{column} = VALUES({column})"); + } + + let s = trx.prep(&query).await.map_err(into_error)?; + + trx.exec_drop(&s, values).await.map_err(into_error)?; + } + + trx.commit().await.map_err(into_error) } - pub async fn index( + pub async fn query( &self, index: SearchIndex, - documents: Vec, - ) -> trc::Result<()> { - todo!() + filters: &[SearchFilter], + sort: &[SearchComparator], + ) -> trc::Result> { + let mut query = format!( + "SELECT {} FROM {} ", + R::field().column(), + index.mysql_table() + ); + let params = build_filter(&mut query, filters); + if !sort.is_empty() { + build_sort(&mut query, sort); + } + let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?; + let s = conn.prep(query).await.map_err(into_error)?; + + conn.exec::(s, params) + .await + .map(|r| r.into_iter().map(|r| R::from_u64(r as u64)).collect()) + .map_err(into_error) } - pub async fn unindex(&self, query: SearchQuery) -> trc::Result<()> { - todo!() + pub async fn unindex(&self, filter: SearchQuery) -> trc::Result { + let mut query = format!("DELETE FROM {} ", filter.index.mysql_table()); + let params = build_filter(&mut query, &filter.filters); + + let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?; + let s = conn.prep(query).await.map_err(into_error)?; + + conn.exec_drop(s, params) + .await + .map(|_| conn.affected_rows() as u64) + .map_err(into_error) + } +} + +fn build_filter(query: &mut String, filters: &[SearchFilter]) -> Vec { + query.push_str("WHERE "); + let mut operator_stack = Vec::new(); + let mut operator = &SearchFilter::And; + let mut is_first = true; + let mut values: Vec = Vec::new(); + + for filter in filters { + match filter { + SearchFilter::Operator { field, op, value } => { + if !is_first { + match operator { + SearchFilter::And => query.push_str(" AND "), + SearchFilter::Or => query.push_str(" OR "), + _ => (), + } + } else { + is_first = false; + } + + let value_pos = values.len() + 1; + if field.is_text() { + let value = match (value, op) { + (SearchValue::Text { value, .. }, SearchOperator::Equal) => { + Value::Bytes(format!("{value:?}").into_bytes()) + } + (SearchValue::Text { value, .. }, ..) => { + let mut text_query = String::with_capacity(value.len() + 1); + + for item in value.split_whitespace() { + if !text_query.is_empty() { + text_query.push(' '); + } + let _ = write!(text_query, "+{item}"); + } + + Value::Bytes(text_query.into_bytes()) + } + _ => { + debug_assert!(false, "Invalid search value for text field"); + continue; + } + }; + let _ = write!( + query, + "MATCH({}) AGAINST(${value_pos} IN BOOLEAN MODE)", + field.column() + ); + values.push(value); + } else if let SearchValue::KeyValues(kv) = value { + let (key, value) = kv.iter().next().unwrap(); + + values.push(Value::Bytes(format!("$.{key}").into_bytes())); + + if !value.is_empty() { + if op == &SearchOperator::Equal { + let _ = write!( + query, + "JSON_EXTRACT({}, ${}) = ${}", + field.column(), + value_pos, + values.len() + 1 + ); + values.push(Value::Bytes(format!("{value:?}").into_bytes())); + } else { + let _ = write!( + query, + "JSON_EXTRACT({}, ${}) LIKE ${}", + field.column(), + value_pos, + values.len() + 1 + ); + values.push(Value::Bytes(format!("%{value}%").into_bytes())); + } + } else { + let _ = write!( + query, + "JSON_CONTAINS_PATH({}, 'one', ${})", + field.column(), + value_pos + ); + } + } else { + query.push_str(field.column()); + query.push(' '); + op.write_mysql(query, value_pos); + values.push(to_mysql(value)); + } + } + SearchFilter::And | SearchFilter::Or => { + operator_stack.push((operator, is_first)); + operator = filter; + query.push('('); + } + SearchFilter::Not => { + operator_stack.push((operator, is_first)); + operator = &SearchFilter::And; + query.push_str("NOT ("); + } + SearchFilter::End => { + let p = operator_stack.pop().unwrap_or((&SearchFilter::And, true)); + operator = p.0; + is_first = p.1; + query.push(')'); + } + SearchFilter::DocumentSet(_) => { + debug_assert!( + false, + "DocumentSet filters are not supported in Postgres backend" + ) + } + } + } + + values +} + +fn build_sort(query: &mut String, sort: &[SearchComparator]) { + query.push_str(" ORDER BY "); + for (i, comparator) in sort.iter().enumerate() { + if i > 0 { + query.push_str(", "); + } + match comparator { + SearchComparator::Field { field, ascending } => { + query.push_str(field.column()); + if *ascending { + query.push_str(" ASC"); + } else { + query.push_str(" DESC"); + } + } + SearchComparator::DocumentSet { .. } | SearchComparator::SortedSet { .. } => { + debug_assert!( + false, + "DocumentSet and SortedSet comparators are not supported " + ); + } + } + } +} + +impl SearchOperator { + fn write_mysql(&self, query: &mut String, value_pos: usize) { + match self { + SearchOperator::LowerThan => { + let _ = write!(query, " < ${value_pos}"); + } + SearchOperator::LowerEqualThan => { + let _ = write!(query, " <= ${value_pos}"); + } + SearchOperator::GreaterThan => { + let _ = write!(query, " > ${value_pos}"); + } + SearchOperator::GreaterEqualThan => { + let _ = write!(query, " >= ${value_pos}"); + } + SearchOperator::Equal => { + let _ = write!(query, " = ${value_pos}"); + } + SearchOperator::Contains => { + let _ = write!(query, " LIKE '%' CONCAT('%', ${value_pos}, '%')"); + } + } + } +} + +impl From for Value { + fn from(value: SearchValue) -> Self { + match value { + SearchValue::Text { value, .. } => Value::Bytes(value.into_bytes()), + SearchValue::KeyValues(vec_map) => serde_json::to_string(&vec_map) + .map(|v| Value::Bytes(v.into_bytes())) + .unwrap_or(Value::NULL), + SearchValue::Int(i) => Value::Int(i), + SearchValue::Uint(i) => Value::Int(i as i64), + SearchValue::Boolean(b) => Value::Int(b as i64), + } + } +} + +fn to_mysql(value: &SearchValue) -> Value { + match value { + SearchValue::Text { value, .. } => Value::Bytes(value.as_bytes().to_vec()), + SearchValue::KeyValues(vec_map) => serde_json::to_string(&vec_map) + .map(|v| Value::Bytes(v.into_bytes())) + .unwrap_or(Value::NULL), + SearchValue::Int(i) => Value::Int(*i), + SearchValue::Uint(i) => Value::Int(*i as i64), + SearchValue::Boolean(b) => Value::Int(*b as i64), } } diff --git a/crates/store/src/backend/postgres/main.rs b/crates/store/src/backend/postgres/main.rs index 79ec9845..f2382c39 100644 --- a/crates/store/src/backend/postgres/main.rs +++ b/crates/store/src/backend/postgres/main.rs @@ -216,10 +216,10 @@ async fn create_search_tables( // Create indexes for field in T::all_fields() { - if field.is_text() { + if field.is_text() || field.is_json() { let column_name = field.column(); let create_index_query = format!( - "CREATE INDEX IF NOT EXISTS fts_{table_name}_{column_name} ON {table_name} USING GIN({column_name})", + "CREATE INDEX IF NOT EXISTS gin_{table_name}_{column_name} ON {table_name} USING GIN({column_name})", ); conn.execute(&create_index_query, &[]) .await diff --git a/crates/store/src/backend/postgres/mod.rs b/crates/store/src/backend/postgres/mod.rs index f68b0ea0..c11edc7f 100644 --- a/crates/store/src/backend/postgres/mod.rs +++ b/crates/store/src/backend/postgres/mod.rs @@ -35,7 +35,7 @@ fn into_error(err: impl Display) -> trc::Error { } impl SearchIndex { - pub(super) fn psql_table(&self) -> &'static str { + fn psql_table(&self) -> &'static str { match self { SearchIndex::Email => "s_email", SearchIndex::Calendar => "s_cal", @@ -47,7 +47,7 @@ impl SearchIndex { } } -pub(super) trait PsqlSearchField { +trait PsqlSearchField { fn column(&self) -> &'static str; fn column_type(&self) -> &'static str; fn sort_column_type(&self) -> Option<&'static str>; diff --git a/crates/store/src/backend/postgres/search.rs b/crates/store/src/backend/postgres/search.rs index c0aac554..050d44cf 100644 --- a/crates/store/src/backend/postgres/search.rs +++ b/crates/store/src/backend/postgres/search.rs @@ -13,6 +13,7 @@ use crate::{ write::SearchIndex, }; use nlp::language::Language; +use std::fmt::Write; use tokio_postgres::{ IsolationLevel, types::{ToSql, Type}, @@ -68,7 +69,7 @@ impl PostgresStore { _ => "simple", }; - query.push_str(&format!("to_tsvector('{language}',{value_ref})")); + let _ = write!(&mut query, "to_tsvector('{language}',{value_ref})"); } else { query.push_str(&value_ref); } @@ -100,7 +101,7 @@ impl PostgresStore { query.push(','); } let column = field.column(); - query.push_str(&format!("{column} = EXCLUDED.{column}")); + let _ = write!(&mut query, "{column} = EXCLUDED.{column}"); } trx.execute(&query, &values).await.map_err(into_error)?; @@ -120,12 +121,32 @@ impl PostgresStore { R::field().column(), index.psql_table() ); + let params = self.build_filter(&mut query, filters); + if !sort.is_empty() { + build_sort(&mut query, sort); + } + let conn = self.conn_pool.get().await.map_err(into_error)?; + let s = conn.prepare_cached(&query).await.map_err(into_error)?; - todo!() + conn.query(&s, params.as_slice()) + .await + .and_then(|rows| { + rows.into_iter() + .map(|row| row.try_get::<_, i64>(0).map(|v| R::from_u64(v as u64))) + .collect::, _>>() + }) + .map_err(into_error) } - pub async fn unindex(&self, query: SearchQuery) -> trc::Result<()> { - todo!() + pub async fn unindex(&self, filter: SearchQuery) -> trc::Result { + let mut query = format!("DELETE FROM {} ", filter.index.psql_table()); + let params = self.build_filter(&mut query, &filter.filters); + let conn = self.conn_pool.get().await.map_err(into_error)?; + let s = conn.prepare_cached(&query).await.map_err(into_error)?; + + conn.execute(&s, params.as_slice()) + .await + .map_err(into_error) } fn build_filter<'x>( @@ -137,7 +158,7 @@ impl PostgresStore { let mut operator_stack = Vec::new(); let mut operator = &SearchFilter::And; let mut is_first = true; - let values = Vec::new(); + let mut values = Vec::new(); for filter in filters { match filter { @@ -154,7 +175,7 @@ impl PostgresStore { query.push_str(field.column()); query.push(' '); - let value_ref = format!("${}", values.len() + 1); + let value_pos = values.len() + 1; if field.is_text() { let language = match &value { SearchValue::Text { language, .. } @@ -168,30 +189,22 @@ impl PostgresStore { SearchOperator::Equal => "phraseto_tsquery", _ => "plainto_tsquery", }; - query.push_str(&format!("@@ {method}('{language}', {value_ref})")); - } else { - let todo = "jsonb query"; - match op { - SearchOperator::LowerThan => { - query.push_str(" < "); - } - SearchOperator::LowerEqualThan => { - query.push_str(" <= "); - } - SearchOperator::GreaterThan => { - query.push_str(" > "); - } - SearchOperator::GreaterEqualThan => { - query.push_str(" >= "); - } - SearchOperator::Equal => { - query.push_str(" = "); - } - SearchOperator::Contains => { - query.push_str(" LIKE "); - } + let _ = write!(query, "@@ {method}('{language}', ${value_pos})"); + values.push(value as &(dyn ToSql + Sync)); + } else if let SearchValue::KeyValues(kv) = value { + let (key, value) = kv.iter().next().unwrap(); + values.push(key as &(dyn ToSql + Sync)); + + if !value.is_empty() { + query.push_str("->>?"); + op.write_pqsql(query, values.len() + 1); + values.push(value as &(dyn ToSql + Sync)); + } else { + let _ = write!(query, " ? ${value_pos}"); } - query.push_str(&value_ref); + } else { + op.write_pqsql(query, value_pos); + values.push(value as &(dyn ToSql + Sync)); } } SearchFilter::And | SearchFilter::Or => { @@ -210,7 +223,12 @@ impl PostgresStore { is_first = p.1; query.push(')'); } - SearchFilter::DocumentSet(_) => (), + SearchFilter::DocumentSet(_) => { + debug_assert!( + false, + "DocumentSet filters are not supported in Postgres backend" + ) + } } } @@ -218,6 +236,31 @@ impl PostgresStore { } } +fn build_sort(query: &mut String, sort: &[SearchComparator]) { + query.push_str(" ORDER BY "); + for (i, comparator) in sort.iter().enumerate() { + if i > 0 { + query.push_str(", "); + } + match comparator { + SearchComparator::Field { field, ascending } => { + query.push_str(field.sort_column().unwrap_or(field.column())); + if *ascending { + query.push_str(" ASC"); + } else { + query.push_str(" DESC"); + } + } + SearchComparator::DocumentSet { .. } | SearchComparator::SortedSet { .. } => { + debug_assert!( + false, + "DocumentSet and SortedSet comparators are not supported " + ); + } + } + } +} + impl ToSql for SearchValue { fn to_sql( &self, @@ -274,6 +317,31 @@ impl ToSql for SearchValue { } } +impl SearchOperator { + fn write_pqsql(&self, query: &mut String, value_pos: usize) { + match self { + SearchOperator::LowerThan => { + let _ = write!(query, " < ${value_pos}"); + } + SearchOperator::LowerEqualThan => { + let _ = write!(query, " <= ${value_pos}"); + } + SearchOperator::GreaterThan => { + let _ = write!(query, " > ${value_pos}"); + } + SearchOperator::GreaterEqualThan => { + let _ = write!(query, " >= ${value_pos}"); + } + SearchOperator::Equal => { + let _ = write!(query, " = ${value_pos}"); + } + SearchOperator::Contains => { + let _ = write!(query, " LIKE '%' || ${value_pos} || '%'"); + } + } + } +} + #[inline(always)] fn pg_lang(lang: &Language) -> Option<&'static str> { match lang { diff --git a/crates/store/src/config.rs b/crates/store/src/config.rs index 63e72fcc..3055a076 100644 --- a/crates/store/src/config.rs +++ b/crates/store/src/config.rs @@ -6,7 +6,7 @@ use crate::{ BlobStore, CompressionAlgo, InMemoryStore, PurgeSchedule, PurgeStore, Store, Stores, - backend::fs::FsStore, + backend::{elastic::ElasticSearchStore, fs::FsStore}, }; use utils::config::{Config, cron::SimpleCron, utils::ParseValue}; @@ -203,12 +203,10 @@ impl Stores { .insert(store_id, db.with_compression(compression_algo)); } } - #[cfg(feature = "elastic")] "elasticsearch" => { - if let Some(db) = - crate::backend::elastic::ElasticSearchStore::open(config, prefix) - .await - .map(crate::SearchStore::from) + if let Some(db) = ElasticSearchStore::open(config, prefix) + .await + .map(crate::SearchStore::from) { self.search_stores.insert(store_id, db); } diff --git a/crates/store/src/dispatch/search.rs b/crates/store/src/dispatch/search.rs index eef7d0c2..82e82b1d 100644 --- a/crates/store/src/dispatch/search.rs +++ b/crates/store/src/dispatch/search.rs @@ -7,7 +7,6 @@ use super::DocumentSet; use crate::{ SearchStore, - backend::elastic::query, search::{IndexDocument, SearchComparator, SearchDocumentId, SearchFilter, SearchQuery}, write::SearchIndex, }; @@ -23,7 +22,7 @@ impl SearchStore { .index_query(account_id, collection, filters, comparators) .await } - #[cfg(feature = "elastic")] + SearchStore::ElasticSearch(store) => { store .index_query(account_id, collection, filters, comparators) @@ -33,15 +32,11 @@ impl SearchStore { .caused_by(trc::location!())*/ } - pub async fn index( - &self, - index: SearchIndex, - documents: Vec, - ) -> trc::Result<()> { + pub async fn index(&self, documents: Vec) -> trc::Result<()> { todo!() /*match self { SearchStore::Store(store) => store.index_insert(document).await, - #[cfg(feature = "elastic")] + SearchStore::ElasticSearch(store) => store.index_insert(document).await, } .caused_by(trc::location!())*/ @@ -55,7 +50,7 @@ impl SearchStore { .index_remove(account_id, collection, document_ids) .await } - #[cfg(feature = "elastic")] + SearchStore::ElasticSearch(store) => { store .index_remove(account_id, collection, document_ids) diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index e7c8aa62..3955e345 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -185,7 +185,6 @@ pub enum BlobBackend { #[derive(Clone)] pub enum SearchStore { Store(Store), - #[cfg(feature = "elastic")] ElasticSearch(Arc), } @@ -282,7 +281,6 @@ impl From for BlobStore { } } -#[cfg(feature = "elastic")] impl From for SearchStore { fn from(store: backend::elastic::ElasticSearchStore) -> Self { Self::ElasticSearch(Arc::new(store)) diff --git a/crates/store/src/search/mod.rs b/crates/store/src/search/mod.rs index 4a29c16c..768ab321 100644 --- a/crates/store/src/search/mod.rs +++ b/crates/store/src/search/mod.rs @@ -478,7 +478,7 @@ impl SearchQuery { }; let mut stack = Vec::new(); let mut filters = self.filters.into_iter().peekable(); - let not_mask = self.mask; + let mask = self.mask; while let Some(filter) = filters.next() { let mut result = match filter { @@ -519,7 +519,7 @@ impl SearchQuery { } SearchFilter::Not => { if let Some(mut result) = result { - result.bitxor_assign(¬_mask); + result.bitxor_assign(&mask); dest.bitand_assign(result); } } @@ -527,11 +527,11 @@ impl SearchQuery { } } else if let Some(ref mut result_) = result { if let SearchFilter::Not = state.op { - result_.bitxor_assign(¬_mask); + result_.bitxor_assign(&mask); } state.bm = result; } else if let SearchFilter::Not = state.op { - state.bm = Some(not_mask.clone()); + state.bm = Some(mask.clone()); } else { state.bm = Some(RoaringBitmap::new()); } @@ -548,8 +548,11 @@ impl SearchQuery { } } + // AND with mask + let mut results = state.bm.unwrap_or_default(); + results.bitand_assign(&mask); QueryResults { - results: state.bm.unwrap_or_default(), + results, comparators: self.comparators, } } @@ -954,4 +957,8 @@ impl SearchField { SearchField::AccountId | SearchField::DocumentId | SearchField::Id => false, } } + + pub(crate) fn is_json(&self) -> bool { + matches!(self, SearchField::Email(EmailSearchField::Headers)) + } } diff --git a/crates/utils/src/config/http.rs b/crates/utils/src/config/http.rs new file mode 100644 index 00000000..fa036dd0 --- /dev/null +++ b/crates/utils/src/config/http.rs @@ -0,0 +1,89 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::config::{Config, utils::AsKey}; +use base64::{Engine, engine::general_purpose}; +use reqwest::{ + Client, + header::{AUTHORIZATION, HeaderMap, HeaderName, HeaderValue, USER_AGENT}, +}; +use std::{str::FromStr, time::Duration}; + +pub fn build_http_client(config: &mut Config, prefix: impl AsKey) -> Option { + let mut headers = parse_http_headers(config, prefix.clone()); + headers.insert(USER_AGENT, "Stalwart/1.0.0".parse().unwrap()); + + let prefix = prefix.as_key(); + match Client::builder() + .connect_timeout( + config + .property_or_default::((&prefix, "timeout"), "30s") + .unwrap_or(Duration::from_secs(30)), + ) + .danger_accept_invalid_certs( + config + .property_or_default::((&prefix, "tls.allow-invalid-certs"), "false") + .unwrap_or(false), + ) + .default_headers(headers) + .build() + { + Ok(client) => Some(client), + Err(err) => { + config.new_build_error(&prefix, format!("Failed to build HTTP client: {err}")); + None + } + } +} + +pub fn parse_http_headers(config: &mut Config, prefix: impl AsKey) -> HeaderMap { + let prefix = prefix.as_key(); + let mut headers = HeaderMap::new(); + + for (header, value) in config + .values((&prefix, "headers")) + .map(|(_, v)| { + if let Some((k, v)) = v.split_once(':') { + Ok(( + HeaderName::from_str(k.trim()).map_err(|err| { + format!("Invalid header found in property \"{prefix}.headers\": {err}",) + })?, + HeaderValue::from_str(v.trim()).map_err(|err| { + format!("Invalid header found in property \"{prefix}.headers\": {err}",) + })?, + )) + } else { + Err(format!( + "Invalid header found in property \"{prefix}.headers\": {v}", + )) + } + }) + .collect::, String>>() + .map_err(|e| config.new_parse_error((&prefix, "headers"), e)) + .unwrap_or_default() + { + headers.insert(header, value); + } + + if let (Some(name), Some(secret)) = ( + config.value((&prefix, "auth.username")), + config.value((&prefix, "auth.secret")), + ) { + headers.insert( + AUTHORIZATION, + format!( + "Basic {}", + general_purpose::STANDARD.encode(format!("{}:{}", name, secret)) + ) + .parse() + .unwrap(), + ); + } else if let Some(token) = config.value((&prefix, "auth.token")) { + headers.insert(AUTHORIZATION, format!("Bearer {}", token).parse().unwrap()); + } + + headers +} diff --git a/crates/utils/src/config/mod.rs b/crates/utils/src/config/mod.rs index 1dbb864d..70d39467 100644 --- a/crates/utils/src/config/mod.rs +++ b/crates/utils/src/config/mod.rs @@ -5,6 +5,7 @@ */ pub mod cron; +pub mod http; pub mod ipmask; pub mod parser; pub mod utils;