From 3e1d2d54e41c66607aded7a94b115b24e3c7d17b Mon Sep 17 00:00:00 2001 From: mdecimus <11444311+mdecimus@users.noreply.github.com> Date: Thu, 30 Oct 2025 19:57:09 +0100 Subject: [PATCH] Database schema optimization - part 1 --- Cargo.lock | 12 +- crates/common/src/config/mod.rs | 10 +- crates/common/src/config/storage.rs | 6 +- crates/common/src/core.rs | 166 +++- crates/common/src/manager/backup.rs | 187 +--- crates/common/src/manager/console.rs | 20 +- crates/common/src/manager/reload.rs | 2 +- crates/common/src/manager/restore.rs | 87 +- crates/common/src/storage/index.rs | 82 +- crates/common/src/telemetry/tracers/store.rs | 1 + crates/dav/src/calendar/copy_move.rs | 18 +- crates/dav/src/calendar/delete.rs | 4 +- crates/dav/src/calendar/freebusy.rs | 2 +- crates/dav/src/calendar/get.rs | 2 +- crates/dav/src/calendar/mod.rs | 11 +- crates/dav/src/calendar/proppatch.rs | 2 +- crates/dav/src/calendar/scheduling.rs | 4 +- crates/dav/src/calendar/update.rs | 2 +- crates/dav/src/card/copy_move.rs | 18 +- crates/dav/src/card/delete.rs | 4 +- crates/dav/src/card/get.rs | 2 +- crates/dav/src/card/mod.rs | 11 +- crates/dav/src/card/proppatch.rs | 2 +- crates/dav/src/card/update.rs | 2 +- crates/dav/src/common/acl.rs | 4 +- crates/dav/src/common/lock.rs | 2 +- crates/dav/src/common/propfind.rs | 2 +- crates/dav/src/file/copy_move.rs | 22 +- crates/dav/src/file/get.rs | 2 +- crates/dav/src/file/mkcol.rs | 2 +- crates/dav/src/file/proppatch.rs | 2 +- crates/dav/src/file/update.rs | 6 +- .../directory/src/backend/internal/manage.rs | 22 +- crates/email/src/cache/email.rs | 4 +- crates/email/src/cache/mailbox.rs | 6 +- crates/email/src/mailbox/destroy.rs | 10 +- crates/email/src/mailbox/manage.rs | 4 +- crates/email/src/mailbox/mod.rs | 1 - crates/email/src/message/bayes.rs | 79 -- crates/email/src/message/copy.rs | 84 +- crates/email/src/message/delete.rs | 73 +- crates/email/src/message/index.rs | 623 ++++++------- crates/email/src/message/ingest.rs | 531 +++++------ crates/email/src/message/mod.rs | 1 - crates/email/src/sieve/delete.rs | 4 +- crates/email/src/sieve/ingest.rs | 14 +- crates/groupware/src/cache/calcard.rs | 214 +++-- crates/groupware/src/cache/file.rs | 64 +- crates/groupware/src/cache/mod.rs | 44 +- crates/groupware/src/calendar/index.rs | 99 +-- crates/groupware/src/calendar/itip.rs | 29 +- crates/groupware/src/calendar/storage.rs | 31 +- crates/groupware/src/contact/index.rs | 74 +- crates/groupware/src/contact/storage.rs | 18 +- crates/groupware/src/file/storage.rs | 10 +- crates/http/src/management/crypto.rs | 6 +- crates/http/src/management/stores.rs | 31 +- crates/imap-proto/Cargo.toml | 1 - crates/imap-proto/src/protocol/search.rs | 34 - crates/imap/src/core/mailbox.rs | 2 +- crates/imap/src/op/acl.rs | 4 +- crates/imap/src/op/copy_move.rs | 29 +- crates/imap/src/op/create.rs | 2 +- crates/imap/src/op/expunge.rs | 15 +- crates/imap/src/op/fetch.rs | 6 +- crates/imap/src/op/rename.rs | 6 +- crates/imap/src/op/search.rs | 841 ++++++++---------- crates/imap/src/op/status.rs | 33 +- crates/imap/src/op/store.rs | 18 +- crates/imap/src/op/subscribe.rs | 4 +- crates/imap/src/op/thread.rs | 14 +- crates/jmap-proto/Cargo.toml | 1 - crates/jmap-proto/src/method/query.rs | 78 +- crates/jmap-proto/src/object/email.rs | 17 - crates/jmap/src/addressbook/get.rs | 2 +- crates/jmap/src/addressbook/set.rs | 10 +- crates/jmap/src/api/acl.rs | 4 +- crates/jmap/src/api/mod.rs | 1 + crates/jmap/src/api/query.rs | 164 ++++ crates/jmap/src/blob/get.rs | 2 +- crates/jmap/src/calendar/get.rs | 2 +- crates/jmap/src/calendar/set.rs | 10 +- crates/jmap/src/calendar_event/copy.rs | 2 +- crates/jmap/src/calendar_event/get.rs | 2 +- crates/jmap/src/calendar_event/mod.rs | 11 +- crates/jmap/src/calendar_event/query.rs | 32 +- crates/jmap/src/calendar_event/set.rs | 6 +- .../src/calendar_event_notification/get.rs | 2 +- .../src/calendar_event_notification/query.rs | 14 +- .../src/calendar_event_notification/set.rs | 2 +- crates/jmap/src/contact/copy.rs | 2 +- crates/jmap/src/contact/get.rs | 2 +- crates/jmap/src/contact/mod.rs | 2 +- crates/jmap/src/contact/query.rs | 28 +- crates/jmap/src/contact/set.rs | 4 +- crates/jmap/src/email/get.rs | 2 +- crates/jmap/src/email/query.rs | 666 +++++++------- crates/jmap/src/email/set.rs | 14 +- crates/jmap/src/email/snippet.rs | 2 +- crates/jmap/src/file/get.rs | 2 +- crates/jmap/src/file/query.rs | 28 +- crates/jmap/src/file/set.rs | 2 +- crates/jmap/src/identity/get.rs | 15 +- crates/jmap/src/identity/set.rs | 15 +- crates/jmap/src/lib.rs | 194 ---- crates/jmap/src/mailbox/query.rs | 287 +++--- crates/jmap/src/mailbox/set.rs | 18 +- crates/jmap/src/participant_identity/get.rs | 6 +- crates/jmap/src/participant_identity/set.rs | 2 +- crates/jmap/src/principal/availability.rs | 2 +- crates/jmap/src/push/get.rs | 9 +- crates/jmap/src/push/set.rs | 14 +- crates/jmap/src/share_notification/query.rs | 2 +- crates/jmap/src/sieve/get.rs | 14 +- crates/jmap/src/sieve/query.rs | 14 +- crates/jmap/src/sieve/set.rs | 26 +- crates/jmap/src/submission/get.rs | 12 +- crates/jmap/src/submission/query.rs | 32 +- crates/jmap/src/submission/set.rs | 14 +- crates/jmap/src/thread/get.rs | 3 +- crates/jmap/src/vacation/get.rs | 6 +- crates/jmap/src/vacation/set.rs | 22 +- crates/managesieve/src/core/client.rs | 14 +- crates/managesieve/src/op/getscript.rs | 2 +- crates/managesieve/src/op/listscripts.rs | 9 +- crates/managesieve/src/op/putscript.rs | 24 +- crates/managesieve/src/op/renamescript.rs | 4 +- crates/managesieve/src/op/setactive.rs | 4 +- crates/migration/src/addressbook_v2.rs | 4 +- crates/migration/src/calendar_v2.rs | 4 +- crates/migration/src/contact_v2.rs | 4 +- crates/migration/src/email.rs | 8 +- crates/migration/src/encryption.rs | 2 +- crates/migration/src/event_v1.rs | 4 +- crates/migration/src/event_v2.rs | 8 +- crates/migration/src/identity.rs | 2 +- crates/migration/src/lib.rs | 12 +- crates/migration/src/mailbox.rs | 2 +- crates/migration/src/principal_v1.rs | 2 +- crates/migration/src/principal_v2.rs | 2 +- crates/migration/src/push_v1.rs | 2 +- crates/migration/src/push_v2.rs | 4 +- crates/migration/src/sieve_v1.rs | 4 +- crates/migration/src/sieve_v2.rs | 6 +- crates/migration/src/submission.rs | 2 +- crates/migration/src/tasks.rs | 11 +- crates/nlp/src/bayes/mod.rs | 4 +- crates/pop3/src/mailbox.rs | 27 +- crates/pop3/src/op/delete.rs | 2 +- crates/pop3/src/op/fetch.rs | 2 +- crates/services/Cargo.toml | 1 + crates/services/src/state_manager/push.rs | 12 +- crates/services/src/task_manager/alarm.rs | 6 +- crates/services/src/task_manager/bayes.rs | 14 +- crates/services/src/task_manager/fts.rs | 228 ++++- crates/services/src/task_manager/mod.rs | 78 +- .../src/backend/composite/read_replica.rs | 31 +- crates/store/src/backend/elastic/index.rs | 53 +- crates/store/src/backend/elastic/query.rs | 32 +- crates/store/src/backend/foundationdb/read.rs | 41 +- .../store/src/backend/foundationdb/write.rs | 10 - crates/store/src/backend/mysql/main.rs | 8 +- crates/store/src/backend/mysql/read.rs | 39 +- crates/store/src/backend/mysql/write.rs | 33 +- crates/store/src/backend/postgres/main.rs | 8 +- crates/store/src/backend/postgres/read.rs | 42 +- crates/store/src/backend/postgres/write.rs | 32 +- crates/store/src/backend/rocksdb/main.rs | 15 - crates/store/src/backend/rocksdb/read.rs | 39 +- crates/store/src/backend/rocksdb/write.rs | 10 - crates/store/src/backend/sqlite/main.rs | 8 +- crates/store/src/backend/sqlite/read.rs | 43 +- crates/store/src/backend/sqlite/write.rs | 35 +- crates/store/src/config.rs | 16 +- crates/store/src/dispatch/fts.rs | 73 -- crates/store/src/dispatch/mod.rs | 2 +- crates/store/src/dispatch/search.rs | 78 ++ crates/store/src/dispatch/store.rs | 158 +--- crates/store/src/fts/mod.rs | 200 ----- crates/store/src/fts/pdf.rs | 37 - crates/store/src/fts/postings.rs | 162 ---- crates/store/src/lib.rs | 36 +- crates/store/src/query/acl.rs | 2 +- crates/store/src/query/filter.rs | 340 ------- crates/store/src/query/mod.rs | 227 +---- crates/store/src/query/sort.rs | 415 --------- crates/store/src/{fts => search}/index.rs | 120 +-- crates/store/src/search/local.rs | 96 ++ crates/store/src/search/mod.rs | 358 ++++++++ crates/store/src/{fts => search}/query.rs | 31 +- crates/store/src/write/batch.rs | 64 +- crates/store/src/write/blob.rs | 2 +- crates/store/src/write/hash.rs | 48 - crates/store/src/write/key.rs | 187 +--- crates/store/src/write/mod.rs | 114 +-- crates/types/src/field.rs | 87 +- crates/utils/Cargo.toml | 3 + crates/utils/src/cheeky_hash.rs | 221 +++++ crates/utils/src/lib.rs | 1 + tests/src/cluster/stress.rs | 2 +- tests/src/jmap/calendar/identity.rs | 2 +- tests/src/jmap/mail/changes.rs | 12 +- tests/src/jmap/mail/query_changes.rs | 6 +- tests/src/jmap/mod.rs | 13 +- tests/src/store/blob.rs | 4 +- tests/src/store/ops.rs | 22 +- tests/src/webdav/mod.rs | 2 +- 207 files changed, 3856 insertions(+), 5885 deletions(-) delete mode 100644 crates/email/src/message/bayes.rs create mode 100644 crates/jmap/src/api/query.rs delete mode 100644 crates/store/src/dispatch/fts.rs create mode 100644 crates/store/src/dispatch/search.rs delete mode 100644 crates/store/src/fts/mod.rs delete mode 100644 crates/store/src/fts/pdf.rs delete mode 100644 crates/store/src/fts/postings.rs delete mode 100644 crates/store/src/query/filter.rs delete mode 100644 crates/store/src/query/sort.rs rename crates/store/src/{fts => search}/index.rs (71%) create mode 100644 crates/store/src/search/local.rs create mode 100644 crates/store/src/search/mod.rs rename crates/store/src/{fts => search}/query.rs (96%) delete mode 100644 crates/store/src/write/hash.rs create mode 100644 crates/utils/src/cheeky_hash.rs diff --git a/Cargo.lock b/Cargo.lock index c3b8bd3a..5c211b48 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3545,7 +3545,6 @@ dependencies = [ "compact_str", "hashify", "mail-parser", - "store", "tokio", "trc", "types", @@ -3920,7 +3919,6 @@ dependencies = [ "rkyv", "serde", "serde_json", - "store", "tokio", "trc", "types", @@ -4744,6 +4742,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0f889fb66f7acdf83442c35775764b51fed3c606ab9cee51500dbde2cf528ca" +[[package]] +name = "nohash-hasher" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" + [[package]] name = "nom" version = "7.1.3" @@ -7318,6 +7322,7 @@ dependencies = [ "sha2 0.10.9", "smtp", "smtp-proto", + "spam-filter", "store", "tokio", "trc", @@ -8790,12 +8795,14 @@ dependencies = [ "blake3", "chrono", "compact_str", + "farmhash", "fast-float", "form_urlencoded", "futures", "http-body-util", "mail-auth", "mail-send", + "nohash-hasher", "parking_lot", "pem", "privdrop", @@ -8818,6 +8825,7 @@ dependencies = [ "trc", "webpki-roots 1.0.3", "x509-parser", + "xxhash-rust", ] [[package]] diff --git a/crates/common/src/config/mod.rs b/crates/common/src/config/mod.rs index 2d717b86..c922682e 100644 --- a/crates/common/src/config/mod.rs +++ b/crates/common/src/config/mod.rs @@ -23,7 +23,7 @@ use hyper::{ use ring::signature::{EcdsaKeyPair, RsaKeyPair}; use spamfilter::SpamFilterConfig; use std::{str::FromStr, sync::Arc}; -use store::{BlobBackend, BlobStore, FtsStore, InMemoryStore, Store, Stores}; +use store::{BlobBackend, BlobStore, SearchStore, InMemoryStore, Store, Stores}; use telemetry::Metrics; use utils::config::{Config, utils::AsKey}; @@ -125,7 +125,7 @@ impl Core { .value_require("storage.fts") .map(|id| id.to_string()) .and_then(|id| { - if let Some(store) = stores.fts_stores.get(&id) { + if let Some(store) = stores.search_stores.get(&id) { store.clone().into() } else { config.new_parse_error( @@ -176,12 +176,12 @@ impl Core { if matches!(data, Store::None) || matches!(&blob.backend, BlobBackend::Store(Store::None)) || matches!(lookup, InMemoryStore::Store(Store::None)) - || matches!(fts, FtsStore::Store(Store::None)) + || matches!(fts, SearchStore::Store(Store::None)) { data = Store::default(); blob = BlobStore::default(); lookup = InMemoryStore::default(); - fts = FtsStore::default(); + fts = SearchStore::default(); config.new_build_error( "storage.*", "One or more stores are missing, disabling all stores", @@ -219,7 +219,7 @@ impl Core { stores: stores.stores, lookups: stores.in_memory_stores, blobs: stores.blob_stores, - ftss: stores.fts_stores, + ftss: stores.search_stores, }, } } diff --git a/crates/common/src/config/storage.rs b/crates/common/src/config/storage.rs index 52587965..9d2feeb4 100644 --- a/crates/common/src/config/storage.rs +++ b/crates/common/src/config/storage.rs @@ -8,7 +8,7 @@ use std::sync::Arc; use ahash::AHashMap; use directory::Directory; -use store::{BlobStore, FtsStore, InMemoryStore, PubSubStore, PurgeSchedule, Store}; +use store::{BlobStore, SearchStore, InMemoryStore, PubSubStore, PurgeSchedule, Store}; use crate::manager::config::ConfigManager; @@ -16,7 +16,7 @@ use crate::manager::config::ConfigManager; pub struct Storage { pub data: Store, pub blob: BlobStore, - pub fts: FtsStore, + pub fts: SearchStore, pub lookup: InMemoryStore, pub pubsub: PubSubStore, pub directory: Arc, @@ -27,5 +27,5 @@ pub struct Storage { pub stores: AHashMap, pub blobs: AHashMap, pub lookups: AHashMap, - pub ftss: AHashMap, + pub ftss: AHashMap, } diff --git a/crates/common/src/core.rs b/crates/common/src/core.rs index 41a11a85..c338aa93 100644 --- a/crates/common/src/core.rs +++ b/crates/common/src/core.rs @@ -24,13 +24,13 @@ use std::{ time::Duration, }; use store::{ - BitmapKey, BlobStore, Deserialize, FtsStore, InMemoryStore, IndexKey, IterateParams, Key, - LogKey, SUBSPACE_LOGS, SerializeInfallible, Store, U32_LEN, U64_LEN, ValueKey, + BlobStore, Deserialize, InMemoryStore, IndexKey, IndexKeyPrefix, IterateParams, Key, LogKey, + SUBSPACE_LOGS, SearchStore, SerializeInfallible, Store, U32_LEN, U64_LEN, ValueKey, dispatch::DocumentSet, roaring::RoaringBitmap, write::{ AlignedBytes, AnyClass, Archive, AssignedIds, BatchBuilder, BlobOp, DirectoryClass, - QueueClass, ValueClass, key::DeserializeBigEndian, now, + IndexPropertyClass, QueueClass, ValueClass, key::DeserializeBigEndian, now, }, }; use trc::AddContext; @@ -55,7 +55,7 @@ impl Server { } #[inline(always)] - pub fn fts_store(&self) -> &FtsStore { + pub fn search_store(&self) -> &SearchStore { &self.core.storage.fts } @@ -349,30 +349,28 @@ impl Server { self.store() .iterate( IterateParams::new( - IndexKey { + ValueKey { account_id, collection: Collection::Email.into(), document_id: 0, - field: EmailField::Size.into(), - key: 0u32.serialize(), + class: ValueClass::IndexProperty(IndexPropertyClass::Integer { + property: EmailField::Stats.into(), + value: 0, + }), }, - IndexKey { + ValueKey { account_id, collection: Collection::Email.into(), document_id: u32::MAX, - field: EmailField::Size.into(), - key: u32::MAX.serialize(), + class: ValueClass::IndexProperty(IndexPropertyClass::Integer { + property: EmailField::Stats.into(), + value: u64::MAX, + }), }, ) - .no_values() .ascending(), - |key, _| { - let value = key - .get(key.len() - (U32_LEN * 2)..key.len() - U32_LEN) - .ok_or_else(|| trc::Error::corrupted_key(key, None, trc::location!())) - .and_then(u32::deserialize)?; - - quota += value as i64; + |_, value| { + quota += value.deserialize_be_u32(0)? as i64; Ok(true) }, @@ -490,7 +488,7 @@ impl Server { } #[inline(always)] - pub async fn get_archive( + pub async fn archive( &self, account_id: u32, collection: Collection, @@ -515,7 +513,7 @@ impl Server { } #[inline(always)] - pub async fn get_archive_by_property( + pub async fn archive_by_property( &self, account_id: u32, collection: Collection, @@ -540,7 +538,7 @@ impl Server { }) } - pub async fn get_archives( + pub async fn archives( &self, account_id: u32, collection: Collection, @@ -589,22 +587,124 @@ impl Server { }) } - #[inline(always)] - pub async fn get_document_ids( + pub async fn document_ids( &self, account_id: u32, collection: Collection, - ) -> trc::Result> { - self.core - .storage - .data - .get_bitmap(BitmapKey::document_ids(account_id, collection)) + field: impl Into, + ) -> trc::Result { + let field = field.into(); + let mut results = RoaringBitmap::new(); + self.store() + .iterate( + IterateParams::new( + IndexKeyPrefix { + account_id, + collection: collection.into(), + field, + }, + IndexKeyPrefix { + account_id, + collection: collection.into(), + field: field + 1, + }, + ) + .no_values(), + |key, _| { + results.insert(key.deserialize_be_u32(key.len() - U32_LEN)?); + + Ok(true) + }, + ) .await - .add_context(|err| { - err.caused_by(trc::location!()) - .account_id(account_id) - .collection(collection) - }) + .caused_by(trc::location!()) + .map(|_| results) + } + + pub async fn document_exists( + &self, + account_id: u32, + collection: Collection, + field: impl Into, + filter: impl AsRef<[u8]>, + ) -> trc::Result { + let field = field.into(); + let mut exists = false; + let filter = filter.as_ref(); + let key_len = IndexKeyPrefix::len() + filter.len() + U32_LEN; + + self.store() + .iterate( + IterateParams::new( + IndexKey { + account_id, + collection: collection.into(), + document_id: 0, + field, + key: filter, + }, + IndexKey { + account_id, + collection: collection.into(), + document_id: u32::MAX, + field, + key: filter, + }, + ) + .no_values(), + |key, _| { + exists = key.len() == key_len; + + Ok(!exists) + }, + ) + .await + .caused_by(trc::location!()) + .map(|_| exists) + } + + pub async fn document_ids_matching( + &self, + account_id: u32, + collection: Collection, + field: impl Into, + filter: impl AsRef<[u8]>, + ) -> trc::Result { + let field = field.into(); + let filter = filter.as_ref(); + let key_len = IndexKeyPrefix::len() + filter.len() + U32_LEN; + let mut results = RoaringBitmap::new(); + + self.store() + .iterate( + IterateParams::new( + IndexKey { + account_id, + collection: collection.into(), + document_id: 0, + field, + key: filter, + }, + IndexKey { + account_id, + collection: collection.into(), + document_id: u32::MAX, + field, + key: filter, + }, + ) + .no_values(), + |key, _| { + if key.len() == key_len { + results.insert(key.deserialize_be_u32(key.len() - U32_LEN)?); + } + + Ok(true) + }, + ) + .await + .caused_by(trc::location!()) + .map(|_| results) } #[inline(always)] diff --git a/crates/common/src/manager/backup.rs b/crates/common/src/manager/backup.rs index 6d7d73e1..0848b20a 100644 --- a/crates/common/src/manager/backup.rs +++ b/crates/common/src/manager/backup.rs @@ -14,11 +14,10 @@ use std::{ sync::mpsc::{self, SyncSender}, }; use store::{ - BitmapKey, Deserialize, IndexKey, IterateParams, LogKey, SUBSPACE_BITMAP_ID, - SUBSPACE_BITMAP_TAG, SUBSPACE_BITMAP_TEXT, SerializeInfallible, U32_LEN, U64_LEN, ValueKey, + Deserialize, IndexKey, IterateParams, LogKey, SerializeInfallible, U32_LEN, U64_LEN, ValueKey, write::{ - AnyKey, BitmapClass, BitmapHash, BlobOp, DirectoryClass, InMemoryClass, QueueClass, - QueueEvent, TagValue, ValueClass, key::DeserializeBigEndian, + AnyKey, BlobOp, DirectoryClass, InMemoryClass, QueueClass, QueueEvent, ValueClass, + key::DeserializeBigEndian, }, }; use types::{ @@ -108,9 +107,6 @@ impl Core { params .has_family(Family::Index) .then(|| self.backup_index(¶ms.dest)), - params - .has_family(Family::Bitmap) - .then(|| self.backup_bitmaps(¶ms.dest)), params .has_family(Family::Log) .then(|| self.backup_logs(¶ms.dest)), @@ -240,7 +236,7 @@ impl Core { } fn backup_fts_index(&self, dest: &Path) -> TaskHandle { - let store = self.storage.data.clone(); + /*let store = self.storage.data.clone(); let (handle, writer) = spawn_writer(dest.join("fts_index")); ( tokio::spawn(async move { @@ -310,7 +306,8 @@ impl Core { .failed("Failed to iterate over data store"); }), handle, - ) + )*/ + todo!() } fn backup_acl(&self, dest: &Path) -> TaskHandle { @@ -855,178 +852,6 @@ impl Core { ) } - fn backup_bitmaps(&self, dest: &Path) -> TaskHandle { - let store = self.storage.data.clone(); - - let (handle, writer) = spawn_writer(dest.join("bitmap")); - ( - tokio::spawn(async move { - const BM_MARKER: u8 = 1 << 7; - - writer - .send(Op::Family(Family::Bitmap)) - .failed("Failed to send family"); - - let mut bitmaps: AHashMap<(u32, u8), AHashSet> = AHashMap::new(); - - for subspace in [ - SUBSPACE_BITMAP_ID, - SUBSPACE_BITMAP_TAG, - SUBSPACE_BITMAP_TEXT, - ] { - store - .iterate( - IterateParams::new( - AnyKey { - subspace, - key: vec![0u8], - }, - AnyKey { - subspace, - key: vec![u8::MAX; 10], - }, - ) - .no_values(), - |key, _| { - let account_id = key.deserialize_be_u32(0)?; - - let key = key.range(0..key.len() - U32_LEN)?; - - match subspace { - SUBSPACE_BITMAP_ID => { - let collection = key.deserialize_u8(U32_LEN)?; - bitmaps - .entry((account_id, collection)) - .or_default() - .insert(BitmapClass::DocumentIds); - } - SUBSPACE_BITMAP_TAG => { - let collection = key.deserialize_u8(U32_LEN)?; - let value = key.range(U32_LEN + 2..usize::MAX)?; - let (field, value) = match key - .deserialize_u8(U32_LEN + 1)? - { - field if field & BM_MARKER == 0 => { - (field, TagValue::Id(value.deserialize_leb128()?)) - } - field => { - (field & !BM_MARKER, TagValue::Text(value.to_vec())) - } - }; - - bitmaps - .entry((account_id, collection)) - .or_default() - .insert(BitmapClass::Tag { field, value }); - } - SUBSPACE_BITMAP_TEXT => { - let collection = key.deserialize_u8(key.len() - 2)?; - let mut hash = [0u8; 8]; - let (hash, len) = match key.len() - U32_LEN - 2 { - 9 => { - hash[..8].copy_from_slice( - key.range(U32_LEN..key.len() - 3)?, - ); - (hash, key.deserialize_u8(key.len() - 3)?) - } - len @ (1..=7) => { - hash[..len].copy_from_slice( - key.range(U32_LEN..key.len() - 2)?, - ); - (hash, len as u8) - } - _ => { - return Err(trc::Error::corrupted_key( - key, - None, - trc::location!(), - )); - } - }; - - bitmaps - .entry((account_id, collection)) - .or_default() - .insert(BitmapClass::Text { - field: key.deserialize_u8(key.len() - 1)?, - token: BitmapHash { hash, len }, - }); - } - _ => unreachable!(), - } - - Ok(true) - }, - ) - .await - .failed("Failed to iterate over data store"); - } - - for ((account_id, collection), classes) in bitmaps { - writer - .send(Op::AccountId(account_id)) - .failed("Failed to send account id"); - writer - .send(Op::Collection(collection)) - .failed("Failed to send collection"); - - for class in classes { - if let Some(bitmap) = store - .get_bitmap(BitmapKey { - account_id, - collection, - class: class.clone(), - document_id: 0, - }) - .await - .failed("Failed to get bitmap") - { - let key = match class { - BitmapClass::DocumentIds => { - vec![0u8] - } - BitmapClass::Tag { field, value } => { - let mut key = Vec::with_capacity(3); - - match value { - TagValue::Id(id) => { - key.push(1u8); - key.push(field); - key.extend_from_slice(&id.serialize()); - } - TagValue::Text(text) => { - key.push(2u8); - key.push(field); - key.extend_from_slice(&text); - } - } - - key - } - BitmapClass::Text { field, token } => { - let mut key = vec![4u8, field]; - key.push(token.len); - key.extend_from_slice(&token.hash); - key - } - }; - - let mut bytes = Vec::with_capacity(bitmap.serialized_size()); - bitmap - .serialize_into(&mut bytes) - .failed("Failed to serialize bitmap"); - - writer - .send(Op::KeyValue((key, bytes))) - .failed("Failed to send key value"); - } - } - } - }), - handle, - ) - } - fn backup_logs(&self, dest: &Path) -> TaskHandle { let store = self.storage.data.clone(); let (handle, writer) = spawn_writer(dest.join("log")); diff --git a/crates/common/src/manager/console.rs b/crates/common/src/manager/console.rs index c626fd4d..6fb6d81c 100644 --- a/crates/common/src/manager/console.rs +++ b/crates/common/src/manager/console.rs @@ -4,16 +4,12 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::env; -use std::io::{self, Write}; - use base64::Engine; use base64::engine::general_purpose; +use std::env; +use std::io::{self, Write}; use store::write::{AnyClass, AnyKey, BatchBuilder, ValueClass}; -use store::{ - Deserialize, IterateParams, SUBSPACE_BITMAP_ID, SUBSPACE_BITMAP_TAG, SUBSPACE_BITMAP_TEXT, - SUBSPACE_INDEXES, Store, -}; +use store::{Deserialize, IterateParams, SUBSPACE_INDEXES, Store}; const HELP: &str = concat!( "Stalwart Server v", @@ -76,15 +72,7 @@ pub async fn store_console(store: Store) { key: to_key.collect::>(), }, ) - .set_values( - ![ - SUBSPACE_INDEXES, - SUBSPACE_BITMAP_ID, - SUBSPACE_BITMAP_TAG, - SUBSPACE_BITMAP_TEXT, - ] - .contains(&from_subspace), - ), + .set_values(![SUBSPACE_INDEXES].contains(&from_subspace)), |key, value| { print!("{}", char::from(from_subspace)); print_escaped(key); diff --git a/crates/common/src/manager/reload.rs b/crates/common/src/manager/reload.rs index 9bf51364..f7692d2f 100644 --- a/crates/common/src/manager/reload.rs +++ b/crates/common/src/manager/reload.rs @@ -74,7 +74,7 @@ impl Server { let mut stores = Stores { stores: self.core.storage.stores.clone(), blob_stores: self.core.storage.blobs.clone(), - fts_stores: self.core.storage.ftss.clone(), + search_stores: self.core.storage.ftss.clone(), in_memory_stores: self.core.storage.lookups.clone(), pubsub_stores: Default::default(), purge_schedules: Default::default(), diff --git a/crates/common/src/manager/restore.rs b/crates/common/src/manager/restore.rs index ef978866..8b67fce1 100644 --- a/crates/common/src/manager/restore.rs +++ b/crates/common/src/manager/restore.rs @@ -12,10 +12,9 @@ use std::{ }; use store::{ BlobStore, Key, LogKey, SUBSPACE_LOGS, SerializeInfallible, Store, U32_LEN, - roaring::RoaringBitmap, write::{ - AnyClass, BatchBuilder, BitmapClass, BitmapHash, BlobOp, DirectoryClass, InMemoryClass, - Operation, TagValue, TaskQueueClass, ValueClass, ValueOp, key::DeserializeBigEndian, now, + AnyClass, BatchBuilder, BlobOp, DirectoryClass, InMemoryClass, Operation, TaskQueueClass, + ValueClass, ValueOp, key::DeserializeBigEndian, now, }, }; use store::{ @@ -88,7 +87,7 @@ async fn restore_file(store: Store, blob_store: BlobStore, path: &Path) { } Op::DocumentId(d) => { document_id = d; - batch.update_document(document_id); + batch.with_document(document_id); } Op::KeyValue((key, value)) => { batch_size += key.len() + value.len() + U32_LEN * 2; @@ -128,7 +127,7 @@ async fn restore_file(store: Store, blob_store: BlobStore, path: &Path) { } }; - batch.set(ValueClass::FtsIndex(BitmapHash { hash, len }), value); + //batch.set(ValueClass::FtsIndex(BitmapHash { hash, len }), value); } } Family::Acl => { @@ -147,9 +146,10 @@ async fn restore_file(store: Store, blob_store: BlobStore, path: &Path) { if account_id != u32::MAX && document_id != u32::MAX { if reader.version == 1 && collection == Collection::Email { batch.set( - ValueClass::TaskQueue(TaskQueueClass::IndexEmail { + ValueClass::TaskQueue(TaskQueueClass::UpdateIndex { due, - hash: hash.clone(), + collection: Collection::Email, + is_insert: true, }), 0u64.serialize(), ); @@ -274,76 +274,7 @@ async fn restore_file(store: Store, blob_store: BlobStore, path: &Path) { set: true, }); } - Family::Bitmap => { - let key = key.as_slice(); - let class: BitmapClass = - match key.first().expect("Failed to read bitmap class") { - 0 => BitmapClass::DocumentIds, - 1 => BitmapClass::Tag { - field: key.get(1).copied().expect("Failed to read field"), - value: TagValue::Id( - key.deserialize_be_u32(2).expect("Failed to read tag id"), - ), - }, - 2 => BitmapClass::Tag { - field: key.get(1).copied().expect("Failed to read field"), - value: TagValue::Text( - key.get(2..).expect("Failed to read tag text").to_vec(), - ), - }, - 3 => BitmapClass::Tag { - field: key.get(1).copied().expect("Failed to read field"), - value: TagValue::Id( - key.get(2) - .copied() - .expect("Failed to read tag static id") - .into(), - ), - }, - 4 => { - if reader.version == 1 && collection == Collection::Email { - continue; - } - - BitmapClass::Text { - field: key.get(1).copied().expect("Failed to read field"), - token: BitmapHash { - len: key - .get(2) - .copied() - .expect("Failed to read tag static id"), - hash: key - .get(3..11) - .expect("Failed to read tag static id") - .try_into() - .unwrap(), - }, - } - } - _ => failed("Invalid bitmap class"), - }; - let document_ids = RoaringBitmap::deserialize_from(&value[..]) - .expect("Failed to deserialize bitmap"); - - for document_id in document_ids { - batch.any_op(Operation::DocumentId { document_id }); - batch.any_op(Operation::Bitmap { - class: class.clone(), - set: true, - }); - - if batch.is_large_batch() { - store - .write(batch.build_all()) - .await - .failed("Failed to write batch"); - batch = BatchBuilder::new(); - batch - .with_account_id(account_id) - .with_collection(collection); - } - } - } + Family::Bitmap => {} Family::Log => { let change_id = key .as_slice() @@ -382,7 +313,7 @@ async fn restore_file(store: Store, blob_store: BlobStore, path: &Path) { batch .with_account_id(account_id) .with_collection(collection) - .update_document(document_id); + .with_document(document_id); batch_size = 0; } } diff --git a/crates/common/src/storage/index.rs b/crates/common/src/storage/index.rs index c6757b95..e8e37372 100644 --- a/crates/common/src/storage/index.rs +++ b/crates/common/src/storage/index.rs @@ -14,7 +14,10 @@ use rkyv::{ use std::{borrow::Cow, fmt::Debug}; use store::{ Serialize, SerializeInfallible, - write::{Archive, Archiver, BatchBuilder, BlobOp, DirectoryClass, IntoOperations}, + write::{ + Archive, Archiver, BatchBuilder, BlobOp, DirectoryClass, IntoOperations, TaskQueueClass, + ValueClass, now, + }, }; use types::{ acl::AclGrant, @@ -22,7 +25,7 @@ use types::{ collection::{Collection, SyncCollection}, field::Field, }; -use utils::{map::bitmap::Bitmap, snowflake::SnowflakeIdGenerator}; +use utils::{cheeky_hash::CheekyHash, map::bitmap::Bitmap, snowflake::SnowflakeIdGenerator}; #[derive(Debug, Clone, PartialEq, Eq)] pub enum IndexValue<'x> { @@ -30,9 +33,12 @@ pub enum IndexValue<'x> { field: Field, value: IndexItem<'x>, }, - IndexList { - field: Field, - value: Vec>, + Property { + field: ValueClass, + value: IndexItem<'x>, + }, + SearchIndex { + hashes: AHashSet, }, Blob { value: BlobHash, @@ -62,6 +68,7 @@ pub enum IndexItem<'x> { Slice(&'x [u8]), ShortInt([u8; std::mem::size_of::()]), LongInt([u8; std::mem::size_of::()]), + Hash(CheekyHash), None, } @@ -72,6 +79,7 @@ impl IndexItem<'_> { IndexItem::Slice(s) => s, IndexItem::ShortInt(s) => s, IndexItem::LongInt(s) => s, + IndexItem::Hash(h) => h.as_bytes(), IndexItem::None => &[], } } @@ -82,6 +90,7 @@ impl IndexItem<'_> { IndexItem::Slice(s) => s.to_vec(), IndexItem::ShortInt(s) => s.to_vec(), IndexItem::LongInt(s) => s.to_vec(), + IndexItem::Hash(h) => h.as_bytes().to_vec(), IndexItem::None => vec![], } } @@ -111,6 +120,7 @@ impl std::hash::Hash for IndexItem<'_> { IndexItem::Slice(s) => s.hash(state), IndexItem::ShortInt(s) => s.as_slice().hash(state), IndexItem::LongInt(s) => s.as_slice().hash(state), + IndexItem::Hash(h) => h.hash(state), IndexItem::None => 0.hash(state), } } @@ -366,13 +376,21 @@ fn build_index( } } } - IndexValue::IndexList { field, value } => { - for key in value { - if set { - batch.index(field, key.into_owned()); - } else { - batch.unindex(field, key.into_owned()); - } + IndexValue::SearchIndex { .. } => { + batch.set( + ValueClass::TaskQueue(TaskQueueClass::UpdateIndex { + due: now(), + collection: batch.last_collection().unwrap_or(Collection::None), + is_insert: set, + }), + vec![], + ); + } + IndexValue::Property { field, value } => { + if set { + batch.set(field, value.into_owned()); + } else { + batch.clear(field); } } IndexValue::Blob { value } => { @@ -491,24 +509,36 @@ fn merge_index( } } ( - IndexValue::IndexList { - field, + IndexValue::SearchIndex { hashes: old_hashes }, + IndexValue::SearchIndex { hashes: new_hashes }, + ) => { + if old_hashes != new_hashes { + batch.set( + ValueClass::TaskQueue(TaskQueueClass::UpdateIndex { + due: now(), + collection: batch.last_collection().unwrap_or(Collection::None), + is_insert: true, + }), + vec![], + ); + } + } + ( + IndexValue::Property { + field: old_field, value: old_value, }, - IndexValue::IndexList { - value: new_value, .. + IndexValue::Property { + field: new_field, + value: new_value, + .. }, ) => { - let mut remove_values = AHashSet::from_iter(old_value); - - for value in new_value { - if !remove_values.remove(&value) { - batch.index(field, value.into_owned()); - } - } - - for value in remove_values { - batch.unindex(field, value.into_owned()); + if old_field != new_field { + batch.clear(old_field); + batch.set(new_field, new_value.into_owned()); + } else if new_value != old_value { + batch.set(old_field, new_value.into_owned()); } } (IndexValue::Blob { value: old_hash }, IndexValue::Blob { value: new_hash }) => { diff --git a/crates/common/src/telemetry/tracers/store.rs b/crates/common/src/telemetry/tracers/store.rs index f87aaa4d..3ccff89d 100644 --- a/crates/common/src/telemetry/tracers/store.rs +++ b/crates/common/src/telemetry/tracers/store.rs @@ -183,6 +183,7 @@ impl TracingStore for Store { ) -> trc::Result> { let mut spans = SpanCollector::Empty; let num_params = params.len(); + let todo = "use FTS"; for (param_num, param) in params.iter().enumerate() { let (value, exact_len) = match param { diff --git a/crates/dav/src/calendar/copy_move.rs b/crates/dav/src/calendar/copy_move.rs index 1e6b88c3..0fb39e34 100644 --- a/crates/dav/src/calendar/copy_move.rs +++ b/crates/dav/src/calendar/copy_move.rs @@ -455,7 +455,7 @@ async fn copy_event( ) -> crate::Result { // Fetch event let event_ = server - .get_archive(from_account_id, Collection::CalendarEvent, from_document_id) + .archive(from_account_id, Collection::CalendarEvent, from_document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -523,7 +523,7 @@ async fn copy_event( let response = if let Some(to_document_id) = to_document_id { // Overwrite event on destination let event_ = server - .get_archive(to_account_id, Collection::CalendarEvent, to_document_id) + .archive(to_account_id, Collection::CalendarEvent, to_document_id) .await .caused_by(trc::location!())?; if let Some(event_) = event_ { @@ -573,7 +573,7 @@ async fn move_event( ) -> crate::Result { // Fetch event let event_ = server - .get_archive(from_account_id, Collection::CalendarEvent, from_document_id) + .archive(from_account_id, Collection::CalendarEvent, from_document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -682,7 +682,7 @@ async fn move_event( let response = if let Some(to_document_id) = to_document_id { // Overwrite event on destination let event_ = server - .get_archive(to_account_id, Collection::CalendarEvent, to_document_id) + .archive(to_account_id, Collection::CalendarEvent, to_document_id) .await .caused_by(trc::location!())?; if let Some(event_) = event_ { @@ -728,7 +728,7 @@ async fn rename_event( ) -> crate::Result { // Fetch event let event_ = server - .get_archive(account_id, Collection::CalendarEvent, document_id) + .archive(account_id, Collection::CalendarEvent, document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -776,7 +776,7 @@ async fn copy_container( ) -> crate::Result { // Fetch calendar let calendar_ = server - .get_archive(from_account_id, Collection::Calendar, from_document_id) + .archive(from_account_id, Collection::Calendar, from_document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -820,7 +820,7 @@ async fn copy_container( let to_document_id = if let Some(to_document_id) = to_document_id { // Overwrite destination let calendar_ = server - .get_archive(to_account_id, Collection::Calendar, to_document_id) + .archive(to_account_id, Collection::Calendar, to_document_id) .await .caused_by(trc::location!())?; if let Some(calendar_) = calendar_ { @@ -859,7 +859,7 @@ async fn copy_container( let mut required_space = 0; for from_child_document_id in from_children_ids { if let Some(event_) = server - .get_archive( + .archive( from_account_id, Collection::CalendarEvent, from_child_document_id, @@ -979,7 +979,7 @@ async fn rename_container( ) -> crate::Result { // Fetch calendar let calendar_ = server - .get_archive(account_id, Collection::Calendar, document_id) + .archive(account_id, Collection::Calendar, document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; diff --git a/crates/dav/src/calendar/delete.rs b/crates/dav/src/calendar/delete.rs index 4b1b541a..aee09d24 100644 --- a/crates/dav/src/calendar/delete.rs +++ b/crates/dav/src/calendar/delete.rs @@ -87,7 +87,7 @@ impl CalendarDeleteRequestHandler for Server { } let calendar_ = self - .get_archive(account_id, Collection::Calendar, document_id) + .archive(account_id, Collection::Calendar, document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -152,7 +152,7 @@ impl CalendarDeleteRequestHandler for Server { } let event_ = self - .get_archive(account_id, Collection::CalendarEvent, document_id) + .archive(account_id, Collection::CalendarEvent, document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; diff --git a/crates/dav/src/calendar/freebusy.rs b/crates/dav/src/calendar/freebusy.rs index 6c005f94..03e000ca 100644 --- a/crates/dav/src/calendar/freebusy.rs +++ b/crates/dav/src/calendar/freebusy.rs @@ -155,7 +155,7 @@ impl CalendarFreebusyRequestHandler for Server { for document_id in document_ids { let Some(archive) = self - .get_archive(account_id, Collection::CalendarEvent, document_id) + .archive(account_id, Collection::CalendarEvent, document_id) .await .caused_by(trc::location!())? else { diff --git a/crates/dav/src/calendar/get.rs b/crates/dav/src/calendar/get.rs index 428f671f..9ec08f46 100644 --- a/crates/dav/src/calendar/get.rs +++ b/crates/dav/src/calendar/get.rs @@ -73,7 +73,7 @@ impl CalendarGetRequestHandler for Server { // Fetch event let event_ = self - .get_archive( + .archive( account_id, Collection::CalendarEvent, resource.document_id(), diff --git a/crates/dav/src/calendar/mod.rs b/crates/dav/src/calendar/mod.rs index 3924b61f..33b51ad1 100644 --- a/crates/dav/src/calendar/mod.rs +++ b/crates/dav/src/calendar/mod.rs @@ -22,7 +22,6 @@ use dav_proto::schema::{ }; use groupware::scheduling::ItipError; use hyper::StatusCode; -use store::query::Filter; use trc::AddContext; use types::{collection::Collection, field::CalendarField}; @@ -97,18 +96,18 @@ pub(crate) async fn assert_is_unique_uid( ) -> crate::Result<()> { if let Some(uid) = uid { let hits = server - .store() - .filter( + .document_ids_matching( account_id, Collection::CalendarEvent, - vec![Filter::eq(CalendarField::Uid, uid.as_bytes().to_vec())], + CalendarField::Uid, + uid.as_bytes(), ) .await .caused_by(trc::location!())?; - if !hits.results.is_empty() { + if !hits.is_empty() { for path in resources.children(calendar_id) { - if hits.results.contains(path.document_id()) { + if hits.contains(path.document_id()) { return Err(DavError::Condition(DavErrorCondition::new( StatusCode::PRECONDITION_FAILED, CalCondition::NoUidConflict(resources.format_resource(path).into()), diff --git a/crates/dav/src/calendar/proppatch.rs b/crates/dav/src/calendar/proppatch.rs index 5eca74ad..3050bd69 100644 --- a/crates/dav/src/calendar/proppatch.rs +++ b/crates/dav/src/calendar/proppatch.rs @@ -112,7 +112,7 @@ impl CalendarPropPatchRequestHandler for Server { // Fetch archive let archive = self - .get_archive(account_id, collection, document_id) + .archive(account_id, collection, document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; diff --git a/crates/dav/src/calendar/scheduling.rs b/crates/dav/src/calendar/scheduling.rs index 7fc45edc..97a0c736 100644 --- a/crates/dav/src/calendar/scheduling.rs +++ b/crates/dav/src/calendar/scheduling.rs @@ -98,7 +98,7 @@ impl CalendarEventNotificationHandler for Server { // Fetch event let event_ = self - .get_archive( + .archive( account_id, Collection::CalendarEventNotification, resource.document_id(), @@ -181,7 +181,7 @@ impl CalendarEventNotificationHandler for Server { let document_id = resource.document_id(); let event_ = self - .get_archive( + .archive( account_id, Collection::CalendarEventNotification, document_id, diff --git a/crates/dav/src/calendar/update.rs b/crates/dav/src/calendar/update.rs index 1b487290..98556842 100644 --- a/crates/dav/src/calendar/update.rs +++ b/crates/dav/src/calendar/update.rs @@ -121,7 +121,7 @@ impl CalendarUpdateRequestHandler for Server { // Update let event_ = self - .get_archive(account_id, Collection::CalendarEvent, document_id) + .archive(account_id, Collection::CalendarEvent, document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; diff --git a/crates/dav/src/card/copy_move.rs b/crates/dav/src/card/copy_move.rs index d4c1e229..fb1e78ca 100644 --- a/crates/dav/src/card/copy_move.rs +++ b/crates/dav/src/card/copy_move.rs @@ -437,7 +437,7 @@ async fn copy_card( ) -> crate::Result { // Fetch card let card_ = server - .get_archive(from_account_id, Collection::ContactCard, from_document_id) + .archive(from_account_id, Collection::ContactCard, from_document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -498,7 +498,7 @@ async fn copy_card( let response = if let Some(to_document_id) = to_document_id { // Overwrite card on destination let card_ = server - .get_archive(to_account_id, Collection::ContactCard, to_document_id) + .archive(to_account_id, Collection::ContactCard, to_document_id) .await .caused_by(trc::location!())?; if let Some(card_) = card_ { @@ -546,7 +546,7 @@ async fn move_card( ) -> crate::Result { // Fetch card let card_ = server - .get_archive(from_account_id, Collection::ContactCard, from_document_id) + .archive(from_account_id, Collection::ContactCard, from_document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -640,7 +640,7 @@ async fn move_card( let response = if let Some(to_document_id) = to_document_id { // Overwrite card on destination let card_ = server - .get_archive(to_account_id, Collection::ContactCard, to_document_id) + .archive(to_account_id, Collection::ContactCard, to_document_id) .await .caused_by(trc::location!())?; if let Some(card_) = card_ { @@ -685,7 +685,7 @@ async fn rename_card( ) -> crate::Result { // Fetch card let card_ = server - .get_archive(account_id, Collection::ContactCard, document_id) + .archive(account_id, Collection::ContactCard, document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -733,7 +733,7 @@ async fn copy_container( ) -> crate::Result { // Fetch book let book_ = server - .get_archive(from_account_id, Collection::AddressBook, from_document_id) + .archive(from_account_id, Collection::AddressBook, from_document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -774,7 +774,7 @@ async fn copy_container( let to_document_id = if let Some(to_document_id) = to_document_id { // Overwrite destination let book_ = server - .get_archive(to_account_id, Collection::AddressBook, to_document_id) + .archive(to_account_id, Collection::AddressBook, to_document_id) .await .caused_by(trc::location!())?; if let Some(book_) = book_ { @@ -811,7 +811,7 @@ async fn copy_container( let mut required_space = 0; for from_child_document_id in from_children_ids { if let Some(card_) = server - .get_archive( + .archive( from_account_id, Collection::ContactCard, from_child_document_id, @@ -924,7 +924,7 @@ async fn rename_container( ) -> crate::Result { // Fetch book let book_ = server - .get_archive(account_id, Collection::AddressBook, document_id) + .archive(account_id, Collection::AddressBook, document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; diff --git a/crates/dav/src/card/delete.rs b/crates/dav/src/card/delete.rs index 39782124..8451f5ec 100644 --- a/crates/dav/src/card/delete.rs +++ b/crates/dav/src/card/delete.rs @@ -67,7 +67,7 @@ impl CardDeleteRequestHandler for Server { let mut batch = BatchBuilder::new(); if delete_resource.is_container() { let book_ = self - .get_archive(account_id, Collection::AddressBook, document_id) + .archive(account_id, Collection::AddressBook, document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -135,7 +135,7 @@ impl CardDeleteRequestHandler for Server { } let card_ = self - .get_archive(account_id, Collection::ContactCard, document_id) + .archive(account_id, Collection::ContactCard, document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; diff --git a/crates/dav/src/card/get.rs b/crates/dav/src/card/get.rs index 816f69d6..724ec3b0 100644 --- a/crates/dav/src/card/get.rs +++ b/crates/dav/src/card/get.rs @@ -73,7 +73,7 @@ impl CardGetRequestHandler for Server { // Fetch card let card_ = self - .get_archive(account_id, Collection::ContactCard, resource.document_id()) + .archive(account_id, Collection::ContactCard, resource.document_id()) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; diff --git a/crates/dav/src/card/mod.rs b/crates/dav/src/card/mod.rs index b966d2c3..5ef2204a 100644 --- a/crates/dav/src/card/mod.rs +++ b/crates/dav/src/card/mod.rs @@ -11,7 +11,6 @@ use dav_proto::schema::{ response::CardCondition, }; use hyper::StatusCode; -use store::query::Filter; use trc::AddContext; use types::{collection::Collection, field::ContactField}; @@ -81,17 +80,17 @@ pub(crate) async fn assert_is_unique_uid( ) -> crate::Result<()> { if let Some(uid) = uid { let hits = server - .store() - .filter( + .document_ids_matching( account_id, Collection::ContactCard, - vec![Filter::eq(ContactField::Uid, uid.as_bytes().to_vec())], + ContactField::Uid, + uid.as_bytes(), ) .await .caused_by(trc::location!())?; - if !hits.results.is_empty() { + if !hits.is_empty() { for path in resources.children(addressbook_id) { - if hits.results.contains(path.document_id()) { + if hits.contains(path.document_id()) { return Err(DavError::Condition(DavErrorCondition::new( StatusCode::PRECONDITION_FAILED, CardCondition::NoUidConflict(resources.format_resource(path).into()), diff --git a/crates/dav/src/card/proppatch.rs b/crates/dav/src/card/proppatch.rs index 114d5571..faa9aefe 100644 --- a/crates/dav/src/card/proppatch.rs +++ b/crates/dav/src/card/proppatch.rs @@ -109,7 +109,7 @@ impl CardPropPatchRequestHandler for Server { // Fetch archive let archive = self - .get_archive(account_id, collection, document_id) + .archive(account_id, collection, document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; diff --git a/crates/dav/src/card/update.rs b/crates/dav/src/card/update.rs index be2d3c32..2c55d42c 100644 --- a/crates/dav/src/card/update.rs +++ b/crates/dav/src/card/update.rs @@ -110,7 +110,7 @@ impl CardUpdateRequestHandler for Server { // Update let card_ = self - .get_archive(account_id, Collection::ContactCard, document_id) + .archive(account_id, Collection::ContactCard, document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; diff --git a/crates/dav/src/common/acl.rs b/crates/dav/src/common/acl.rs index 981e0753..9e6460c7 100644 --- a/crates/dav/src/common/acl.rs +++ b/crates/dav/src/common/acl.rs @@ -108,7 +108,7 @@ impl DavAclHandler for Server { // Fetch node let archive = self - .get_archive(account_id, collection, resource.document_id()) + .archive(account_id, collection, resource.document_id()) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -212,7 +212,7 @@ impl DavAclHandler for Server { } let archive = self - .get_archive(uri.account_id, uri.collection, uri.resource) + .archive(uri.account_id, uri.collection, uri.resource) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; diff --git a/crates/dav/src/common/lock.rs b/crates/dav/src/common/lock.rs index 5bcd74b9..e04f689f 100644 --- a/crates/dav/src/common/lock.rs +++ b/crates/dav/src/common/lock.rs @@ -526,7 +526,7 @@ impl LockRequestHandler for Server { if let Some(document_id) = resource_state.document_id.filter(|&id| id != u32::MAX) && let Some(archive) = self - .get_archive( + .archive( resource_state.account_id, resource_state.collection, document_id, diff --git a/crates/dav/src/common/propfind.rs b/crates/dav/src/common/propfind.rs index 15821e14..44b61cb9 100644 --- a/crates/dav/src/common/propfind.rs +++ b/crates/dav/src/common/propfind.rs @@ -431,7 +431,7 @@ impl PropFindRequestHandler for Server { item.document_id == SCHEDULE_INBOX_ID, ) } else if let Some(archive) = self - .get_archive(account_id, collection, document_id) + .archive(account_id, collection, document_id) .await .caused_by(trc::location!())? { diff --git a/crates/dav/src/file/copy_move.rs b/crates/dav/src/file/copy_move.rs index 27e4c571..93c6f9ca 100644 --- a/crates/dav/src/file/copy_move.rs +++ b/crates/dav/src/file/copy_move.rs @@ -358,7 +358,7 @@ async fn move_container( if from_account_id == to_account_id { let node_ = server - .get_archive(from_account_id, Collection::FileNode, from_document_id) + .archive(from_account_id, Collection::FileNode, from_document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -459,7 +459,7 @@ async fn copy_container( .caused_by(trc::location!())?; for (document_id, _) in copy_files.into_iter() { let node_ = server - .get_archive(from_account_id, Collection::FileNode, document_id) + .archive(from_account_id, Collection::FileNode, document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))? @@ -491,7 +491,7 @@ async fn copy_container( batch .with_account_id(to_account_id) .with_collection(Collection::FileNode) - .create_document(new_document_id) + .with_document(new_document_id) .custom( ObjectIndexBuilder::<(), _>::new() .with_changes(node) @@ -509,7 +509,7 @@ async fn copy_container( batch .with_account_id(from_account_id) .with_collection(Collection::FileNode) - .delete_document(document_id) + .with_document(document_id) .custom( ObjectIndexBuilder::<_, ()>::new() .with_access_token(access_token) @@ -550,7 +550,7 @@ async fn overwrite_and_delete_item( // dest_node is the current file at the destination let dest_node_ = server - .get_archive(to_account_id, Collection::FileNode, to_document_id) + .archive(to_account_id, Collection::FileNode, to_document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -561,7 +561,7 @@ async fn overwrite_and_delete_item( // source_node is the file to be copied let source_node__ = server - .get_archive(from_account_id, Collection::FileNode, from_document_id) + .archive(from_account_id, Collection::FileNode, from_document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -620,7 +620,7 @@ async fn overwrite_item( // dest_node is the current file at the destination let dest_node_ = server - .get_archive(to_account_id, Collection::FileNode, to_document_id) + .archive(to_account_id, Collection::FileNode, to_document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -631,7 +631,7 @@ async fn overwrite_item( // source_node is the file to be copied let mut source_node = server - .get_archive(from_account_id, Collection::FileNode, from_document_id) + .archive(from_account_id, Collection::FileNode, from_document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))? @@ -676,7 +676,7 @@ async fn move_item( let parent_id = destination.document_id.map(|id| id + 1).unwrap_or(0); let node_ = server - .get_archive(from_account_id, Collection::FileNode, from_document_id) + .archive(from_account_id, Collection::FileNode, from_document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -746,7 +746,7 @@ async fn copy_item( let parent_id = destination.document_id.map(|id| id + 1).unwrap_or(0); let mut node = server - .get_archive(from_account_id, Collection::FileNode, from_document_id) + .archive(from_account_id, Collection::FileNode, from_document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))? @@ -786,7 +786,7 @@ async fn rename_item( let from_document_id = from_resource.resource.document_id; let node_ = server - .get_archive(from_account_id, Collection::FileNode, from_document_id) + .archive(from_account_id, Collection::FileNode, from_document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; diff --git a/crates/dav/src/file/get.rs b/crates/dav/src/file/get.rs index 7c7aef92..edcf04e8 100644 --- a/crates/dav/src/file/get.rs +++ b/crates/dav/src/file/get.rs @@ -55,7 +55,7 @@ impl FileGetRequestHandler for Server { // Fetch node let node_ = self - .get_archive(account_id, Collection::FileNode, resource.resource) + .archive(account_id, Collection::FileNode, resource.resource) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; diff --git a/crates/dav/src/file/mkcol.rs b/crates/dav/src/file/mkcol.rs index 3c4027e8..dc3a9ece 100644 --- a/crates/dav/src/file/mkcol.rs +++ b/crates/dav/src/file/mkcol.rs @@ -121,7 +121,7 @@ impl FileMkColRequestHandler for Server { batch .with_account_id(account_id) .with_collection(Collection::FileNode) - .create_document(document_id) + .with_document(document_id) .custom(ObjectIndexBuilder::<(), _>::new().with_changes(node)) .caused_by(trc::location!())?; let etag = batch.etag(); diff --git a/crates/dav/src/file/proppatch.rs b/crates/dav/src/file/proppatch.rs index bcd95bb2..936254c9 100644 --- a/crates/dav/src/file/proppatch.rs +++ b/crates/dav/src/file/proppatch.rs @@ -75,7 +75,7 @@ impl FilePropPatchRequestHandler for Server { // Fetch node let node_ = self - .get_archive(account_id, Collection::FileNode, resource.resource) + .archive(account_id, Collection::FileNode, resource.resource) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; diff --git a/crates/dav/src/file/update.rs b/crates/dav/src/file/update.rs index 57ab90c5..6f4d39cb 100644 --- a/crates/dav/src/file/update.rs +++ b/crates/dav/src/file/update.rs @@ -74,7 +74,7 @@ impl FileUpdateRequestHandler for Server { { // Update let node_ = self - .get_archive(account_id, Collection::FileNode, document_id) + .archive(account_id, Collection::FileNode, document_id) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -183,7 +183,7 @@ impl FileUpdateRequestHandler for Server { batch .with_account_id(account_id) .with_collection(Collection::FileNode) - .update_document(document_id) + .with_document(document_id) .custom( ObjectIndexBuilder::new() .with_current(node) @@ -279,7 +279,7 @@ impl FileUpdateRequestHandler for Server { batch .with_account_id(account_id) .with_collection(Collection::FileNode) - .create_document(document_id) + .with_document(document_id) .custom( ObjectIndexBuilder::<(), _>::new() .with_changes(node) diff --git a/crates/directory/src/backend/internal/manage.rs b/crates/directory/src/backend/internal/manage.rs index de912eef..9fe2759b 100644 --- a/crates/directory/src/backend/internal/manage.rs +++ b/crates/directory/src/backend/internal/manage.rs @@ -26,7 +26,10 @@ use store::{ }, }; use trc::AddContext; -use types::collection::Collection; +use types::{ + collection::Collection, + field::{self, Field}, +}; use utils::{DomainPart, sanitize_email}; #[derive(Debug, Default, serde::Serialize, serde::Deserialize)] @@ -204,7 +207,8 @@ impl ManageDirectory for Store { .with_account_id(u32::MAX) .with_collection(Collection::Principal) .assert_value(name_key.clone(), ()) - .create_document(principal_id); + .with_document(principal_id) + .tag(Field::DOCUMENT_ID); build_search_index(&mut batch, principal_id, None, Some(&principal)); principal.sort(); batch @@ -592,7 +596,8 @@ impl ManageDirectory for Store { batch .with_account_id(u32::MAX) .with_collection(Collection::Principal) - .create_document(principal_id) + .with_document(principal_id) + .tag(Field::DOCUMENT_ID) .assert_value( ValueClass::Directory(DirectoryClass::NameToId( create_principal.name().as_bytes().to_vec(), @@ -687,7 +692,9 @@ impl ManageDirectory for Store { let typ = Type::from(&principal.typ); let mut batch = BatchBuilder::new(); - batch.with_account_id(u32::MAX); + batch + .with_account_id(u32::MAX) + .with_collection(Collection::Principal); let tenant = principal.data.iter().find_map(|data| { if let ArchivedPrincipalData::Tenant(tenant_id) = data { @@ -845,7 +852,8 @@ impl ManageDirectory for Store { // Delete principal batch - .delete_document(principal_id) + .with_document(principal_id) + .untag(Field::DOCUMENT_ID) .clear(DirectoryClass::NameToId(principal.name.as_bytes().to_vec())) .clear(DirectoryClass::Principal(principal_id)) .clear(DirectoryClass::UsedQuota(principal_id)); @@ -911,9 +919,7 @@ impl ManageDirectory for Store { // Delete push subscriptions if matches!(typ, Type::Individual) { - batch - .with_collection(Collection::PushSubscription) - .delete_document(principal_id); + batch.untag(field::PrincipalField::PushSubscriptions); } self.write(batch.build_all()) diff --git a/crates/email/src/cache/email.rs b/crates/email/src/cache/email.rs index 506e8ef4..40163dc2 100644 --- a/crates/email/src/cache/email.rs +++ b/crates/email/src/cache/email.rs @@ -35,7 +35,7 @@ pub(crate) async fn update_email_cache( for (document_id, is_update) in changed_ids { if *is_update && let Some(archive) = server - .get_archive(account_id, Collection::Email, *document_id) + .archive(account_id, Collection::Email, *document_id) .await .caused_by(trc::location!())? { @@ -78,7 +78,7 @@ pub(crate) async fn full_email_cache_build( }; server - .get_archives( + .archives( account_id, Collection::Email, &(), diff --git a/crates/email/src/cache/mailbox.rs b/crates/email/src/cache/mailbox.rs index bb8eacb2..ccb0fb45 100644 --- a/crates/email/src/cache/mailbox.rs +++ b/crates/email/src/cache/mailbox.rs @@ -34,7 +34,7 @@ pub(crate) async fn update_mailbox_cache( for (document_id, is_update) in changed_ids { if *is_update && let Some(archive) = server - .get_archive(account_id, Collection::Mailbox, *document_id) + .archive(account_id, Collection::Mailbox, *document_id) .await .caused_by(trc::location!())? { @@ -75,7 +75,7 @@ pub(crate) async fn full_mailbox_cache_build( }; server - .get_archives( + .archives( account_id, Collection::Mailbox, &(), @@ -93,7 +93,7 @@ pub(crate) async fn full_mailbox_cache_build( .await .caused_by(trc::location!())?; server - .get_archives( + .archives( account_id, Collection::Mailbox, &(), diff --git a/crates/email/src/mailbox/destroy.rs b/crates/email/src/mailbox/destroy.rs index 14368255..343921e6 100644 --- a/crates/email/src/mailbox/destroy.rs +++ b/crates/email/src/mailbox/destroy.rs @@ -78,7 +78,7 @@ impl MailboxDestroy for Server { let mut destroy_ids = RoaringBitmap::new(); - self.get_archives( + self.archives( account_id, Collection::Email, &message_ids, @@ -113,7 +113,7 @@ impl MailboxDestroy for Server { // Untag message from mailbox batch .with_collection(Collection::Email) - .update_document(message_id) + .with_document(message_id) .custom( ObjectIndexBuilder::new() .with_changes(new_message_data) @@ -129,7 +129,7 @@ impl MailboxDestroy for Server { // Bulk delete messages if !destroy_ids.is_empty() { - self.emails_tombstone(account_id, &mut batch, destroy_ids) + self.emails_delete(account_id, &mut batch, destroy_ids) .await?; } } else { @@ -139,7 +139,7 @@ impl MailboxDestroy for Server { // Obtain mailbox if let Some(mailbox_) = self - .get_archive(account_id, Collection::Mailbox, document_id) + .archive(account_id, Collection::Mailbox, document_id) .await .caused_by(trc::location!())? { @@ -157,7 +157,7 @@ impl MailboxDestroy for Server { batch .with_account_id(account_id) .with_collection(Collection::Mailbox) - .delete_document(document_id) + .with_document(document_id) .clear(MailboxField::UidCounter) .custom(ObjectIndexBuilder::<_, ()>::new().with_current(mailbox)) .caused_by(trc::location!())?; diff --git a/crates/email/src/mailbox/manage.rs b/crates/email/src/mailbox/manage.rs index d35e2608..3706a5d2 100644 --- a/crates/email/src/mailbox/manage.rs +++ b/crates/email/src/mailbox/manage.rs @@ -59,7 +59,7 @@ impl MailboxFnc for Server { object.add_subscriber(account_id); } batch - .create_document(document_id) + .with_document(document_id) .custom(ObjectIndexBuilder::<(), _>::new().with_changes(object)) .caused_by(trc::location!())?; } @@ -137,7 +137,7 @@ impl MailboxFnc for Server { batch .with_account_id(account_id) .with_collection(Collection::Mailbox) - .create_document(document_id) + .with_document(document_id) .custom( ObjectIndexBuilder::<(), _>::new() .with_changes(Mailbox::new(name).with_parent_id(next_parent_id)), diff --git a/crates/email/src/mailbox/mod.rs b/crates/email/src/mailbox/mod.rs index 88cc9c24..49658c25 100644 --- a/crates/email/src/mailbox/mod.rs +++ b/crates/email/src/mailbox/mod.rs @@ -16,7 +16,6 @@ pub const JUNK_ID: u32 = 2; pub const DRAFTS_ID: u32 = 3; pub const SENT_ID: u32 = 4; pub const ARCHIVE_ID: u32 = 5; -pub const TOMBSTONE_ID: u32 = u32::MAX - 1; #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)] #[rkyv(derive(Debug))] diff --git a/crates/email/src/message/bayes.rs b/crates/email/src/message/bayes.rs deleted file mode 100644 index 722f9cf6..00000000 --- a/crates/email/src/message/bayes.rs +++ /dev/null @@ -1,79 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use super::metadata::MessageMetadata; -use common::Server; -use mail_parser::Message; -use spam_filter::{ - SpamFilterInput, analysis::init::SpamFilterInit, modules::bayes::BayesClassifier, -}; -use std::future::Future; -use store::write::{TaskQueueClass, now}; -use trc::StoreEvent; -use types::{blob_hash::BlobHash, collection::Collection, field::EmailField}; - -pub trait EmailBayesTrain: Sync + Send { - fn email_bayes_train( - &self, - account_id: u32, - span_id: u64, - message: Message<'_>, - learn_spam: bool, - ) -> impl Future + Send; - - fn email_bayes_queue_task_build( - &self, - account_id: u32, - document_id: u32, - learn_spam: bool, - ) -> impl Future> + Send; -} - -impl EmailBayesTrain for Server { - async fn email_bayes_train( - &self, - account_id: u32, - span_id: u64, - message: Message<'_>, - learn_spam: bool, - ) { - self.bayes_train_if_balanced( - &self.spam_filter_init(SpamFilterInput::from_account_message( - &message, account_id, span_id, - )), - learn_spam, - ) - .await - } - - async fn email_bayes_queue_task_build( - &self, - account_id: u32, - document_id: u32, - learn_spam: bool, - ) -> trc::Result { - let metadata = self - .get_archive_by_property( - account_id, - Collection::Email, - document_id, - EmailField::Metadata.into(), - ) - .await? - .ok_or_else(|| { - StoreEvent::NotFound - .into_err() - .account_id(account_id) - .document_id(document_id) - })?; - - Ok(TaskQueueClass::BayesTrain { - due: now(), - hash: BlobHash::from(&metadata.unarchive::()?.blob_hash), - learn_spam, - }) - } -} diff --git a/crates/email/src/message/copy.rs b/crates/email/src/message/copy.rs index 0c15d6b6..4edfcfe1 100644 --- a/crates/email/src/message/copy.rs +++ b/crates/email/src/message/copy.rs @@ -5,14 +5,17 @@ */ use super::{ - index::{MAX_ID_LENGTH, MAX_SORT_FIELD_LENGTH, TrimTextValue, VisitText}, - ingest::{EmailIngest, IngestedEmail, ThreadResult}, + index::{MAX_SORT_FIELD_LENGTH, TrimTextValue, VisitText}, + ingest::{EmailIngest, IngestedEmail}, metadata::{MessageData, MessageMetadata}, }; -use crate::mailbox::UidMailbox; +use crate::{ + mailbox::UidMailbox, + message::ingest::{MergeThreadTask, ThreadInfo}, +}; use common::{Server, auth::ResourceToken, storage::index::ObjectIndexBuilder}; use mail_parser::{HeaderName, HeaderValue, parsers::fields::thread::thread_name}; -use store::write::{BatchBuilder, TaskQueueClass, ValueClass, now}; +use store::write::{BatchBuilder, IndexPropertyClass, TaskQueueClass, ValueClass, now}; use trc::AddContext; use types::{ blob::{BlobClass, BlobId}, @@ -20,6 +23,7 @@ use types::{ field::EmailField, keyword::Keyword, }; +use utils::cheeky_hash::{CheekyHash, CheekyHashMap}; pub enum CopyMessageError { NotFound, @@ -55,7 +59,7 @@ impl EmailCopy for Server { // Obtain metadata let account_id = resource_token.account_id; let mut metadata = if let Some(metadata) = self - .get_archive_by_property( + .archive_by_property( from_account_id, Collection::Email, from_message_id, @@ -94,23 +98,21 @@ impl EmailCopy for Server { } // Obtain threadId - let mut references = Vec::with_capacity(5); + let mut message_ids = CheekyHashMap::default(); let mut subject = ""; - let mut message_id = ""; for header in &metadata.contents[0].parts[0].headers { match &header.name { HeaderName::MessageId => { header.value.visit_text(|id| { - if !id.is_empty() && id.len() < MAX_ID_LENGTH { - references.push(id.as_bytes()); - message_id = id; + if !id.is_empty() { + message_ids.insert(CheekyHash::new(id.as_bytes()), true); } }); } HeaderName::InReplyTo | HeaderName::References | HeaderName::ResentMessageId => { header.value.visit_text(|id| { - if !id.is_empty() && id.len() < MAX_ID_LENGTH { - references.push(id.as_bytes()); + if !id.is_empty() { + message_ids.insert(CheekyHash::new(id.as_bytes()), false); } }); } @@ -129,21 +131,10 @@ impl EmailCopy for Server { } // Obtain threadId - let (is_new_thread, thread_id) = match self - .find_or_merge_thread(account_id, subject, references, None) + let thread_result = self + .find_thread_id(account_id, subject, &message_ids) .await - .caused_by(trc::location!())? - { - ThreadResult::Id(thread_id) => (false, thread_id), - ThreadResult::Create => ( - true, - self.store() - .assign_document_ids(account_id, Collection::Thread, 1) - .await - .caused_by(trc::location!())?, - ), - ThreadResult::Skip => unreachable!(), - }; + .caused_by(trc::location!())?; // Assign id let mut email = IngestedEmail { @@ -164,26 +155,31 @@ impl EmailCopy for Server { email.imap_uids.push(uid); } - // Prepare batch - let mut batch = BatchBuilder::new(); - batch.with_account_id(account_id); - - if is_new_thread { - batch - .with_collection(Collection::Thread) - .update_document(thread_id) - .log_container_insert(SyncCollection::Thread); - } - + // Obtain documentId let document_id = self .store() .assign_document_ids(account_id, Collection::Email, 1) .await .caused_by(trc::location!())?; + // Prepare batch + let mut batch = BatchBuilder::new(); + batch.with_account_id(account_id); + + // Determine thread id + let thread_id = if let Some(thread_id) = thread_result.thread_id { + thread_id + } else { + batch + .with_collection(Collection::Thread) + .with_document(document_id) + .log_container_insert(SyncCollection::Thread); + document_id + }; + batch .with_collection(Collection::Email) - .create_document(document_id) + .with_document(document_id) .custom( ObjectIndexBuilder::<(), _>::new().with_changes(MessageData { mailboxes: mailbox_ids, @@ -193,11 +189,15 @@ impl EmailCopy for Server { ) .caused_by(trc::location!())? .set( - ValueClass::TaskQueue(TaskQueueClass::IndexEmail { - due: now(), - hash: metadata.blob_hash.clone(), + ValueClass::IndexProperty(IndexPropertyClass::Hash { + property: EmailField::Threading.into(), + hash: thread_result.thread_hash, }), - vec![], + ThreadInfo::serialize(thread_id, &message_ids), + ) + .set( + ValueClass::TaskQueue(TaskQueueClass::IndexEmail { due: now() }), + MergeThreadTask::new(thread_result).serialize(), ); metadata .index( diff --git a/crates/email/src/message/delete.rs b/crates/email/src/message/delete.rs index 0b35a2e4..ca9afc9c 100644 --- a/crates/email/src/message/delete.rs +++ b/crates/email/src/message/delete.rs @@ -5,27 +5,24 @@ */ use super::metadata::MessageData; -use crate::{cache::MessageCacheFetch, mailbox::*, message::metadata::MessageMetadata}; +use crate::{cache::MessageCacheFetch, mailbox::*}; use common::{KV_LOCK_PURGE_ACCOUNT, Server, storage::index::ObjectIndexBuilder}; use groupware::calendar::storage::ItipAutoExpunge; use std::future::Future; use store::rand::prelude::SliceRandom; use store::write::key::DeserializeBigEndian; -use store::write::now; +use store::write::{IndexPropertyClass, TaskQueueClass, now}; +use store::{IterateParams, SerializeInfallible, U32_LEN, ValueKey}; use store::{ - BitmapKey, ValueKey, roaring::RoaringBitmap, - write::{AlignedBytes, Archive, BatchBuilder, BitmapClass, TagValue, ValueClass}, + write::{BatchBuilder, ValueClass}, }; -use store::{IndexKey, IterateParams, SerializeInfallible, U32_LEN}; use trc::AddContext; -#[cfg(feature = "enterprise")] -use types::blob_hash::BlobHash; use types::collection::{Collection, VanishedCollection}; -use types::field::EmailField; +use types::field::{EmailField, Field}; pub trait EmailDeletion: Sync + Send { - fn emails_tombstone( + fn emails_delete( &self, account_id: u32, batch: &mut BatchBuilder, @@ -41,26 +38,21 @@ pub trait EmailDeletion: Sync + Send { account_id: u32, hold_period: u64, ) -> impl Future> + Send; - - fn emails_purge_tombstoned( - &self, - account_id: u32, - ) -> impl Future> + Send; } impl EmailDeletion for Server { - async fn emails_tombstone( + async fn emails_delete( &self, account_id: u32, batch: &mut BatchBuilder, document_ids: RoaringBitmap, ) -> trc::Result { - // Tombstone message and untag it from the mailboxes + let due = now(); let mut deleted_ids = RoaringBitmap::new(); batch .with_account_id(account_id) .with_collection(Collection::Email); - self.get_archives( + self.archives( account_id, Collection::Email, &document_ids, @@ -76,10 +68,13 @@ impl EmailDeletion for Server { ); } batch - .update_document(document_id) + .with_document(document_id) .custom(ObjectIndexBuilder::<_, ()>::new().with_current(metadata)) .caused_by(trc::location!())? - .tag(EmailField::MailboxIds, TagValue::Id(TOMBSTONE_ID)) + .set( + ValueClass::TaskQueue(TaskQueueClass::UnindexEmail { due }), + 0u64.serialize(), + ) .commit_point(); deleted_ids.insert(document_id); @@ -100,7 +95,9 @@ impl EmailDeletion for Server { } async fn purge_accounts(&self, use_roles: bool) { - if let Ok(Some(account_ids)) = self.get_document_ids(u32::MAX, Collection::Principal).await + if let Ok(account_ids) = self + .document_ids(u32::MAX, Collection::Principal, Field::DOCUMENT_ID) + .await { let mut account_ids: Vec = account_ids .into_iter() @@ -167,14 +164,6 @@ impl EmailDeletion for Server { ); } - // Purge tombstoned messages - if let Err(err) = self.emails_purge_tombstoned(account_id).await { - trc::error!( - err.details("Failed to purge tombstoned messages.") - .account_id(account_id) - ); - } - // Purge changelogs if let Err(err) = self .delete_changes( @@ -224,23 +213,27 @@ impl EmailDeletion for Server { self.store() .iterate( IterateParams::new( - IndexKey { + ValueKey { account_id, collection: Collection::Email.into(), document_id: 0, - field: EmailField::ReceivedAt.into(), - key: 0u64.serialize(), + class: ValueClass::IndexProperty(IndexPropertyClass::Integer { + property: EmailField::Stats.into(), + value: 0, + }), }, - IndexKey { + ValueKey { account_id, collection: Collection::Email.into(), document_id: u32::MAX, - field: EmailField::ReceivedAt.into(), - key: now().saturating_sub(hold_period).serialize(), + class: ValueClass::IndexProperty(IndexPropertyClass::Integer { + property: EmailField::Stats.into(), + value: now().saturating_sub(hold_period), + }), }, ) - .no_values() - .ascending(), + .ascending() + .no_values(), |key, _| { let document_id = key .deserialize_be_u32(key.len() - U32_LEN) @@ -267,16 +260,16 @@ impl EmailDeletion for Server { Total = destroy_ids.len(), ); - // Tombstone messages + // Delete messages let mut batch = BatchBuilder::new(); - self.emails_tombstone(account_id, &mut batch, destroy_ids) + self.emails_delete(account_id, &mut batch, destroy_ids) .await?; self.commit_batch(batch).await?; Ok(()) } - async fn emails_purge_tombstoned(&self, account_id: u32) -> trc::Result<()> { + /*async fn emails_purge_tombstoned(&self, account_id: u32) -> trc::Result<()> { // Obtain tombstoned messages let tombstoned_ids = self .core @@ -383,5 +376,5 @@ impl EmailDeletion for Server { self.commit_batch(batch).await?; Ok(()) - } + }*/ } diff --git a/crates/email/src/message/index.rs b/crates/email/src/message/index.rs index 5c92b544..502492e3 100644 --- a/crates/email/src/message/index.rs +++ b/crates/email/src/message/index.rs @@ -24,8 +24,7 @@ use rkyv::option::ArchivedOption; use store::{ Serialize, SerializeInfallible, backend::MAX_TOKEN_LENGTH, - fts::{Field, index::FtsDocument}, - write::{Archiver, BatchBuilder, BlobOp, DirectoryClass}, + write::{Archiver, BatchBuilder, BlobOp, DirectoryClass, IndexPropertyClass, ValueClass}, }; use trc::AddContext; use types::{blob_hash::BlobHash, collection::SyncCollection, field::EmailField}; @@ -51,15 +50,21 @@ impl MessageMetadata { ) -> trc::Result<()> { if set { // Serialize metadata - batch - .index(EmailField::Size, self.size.serialize()) - .index(EmailField::ReceivedAt, (self.received_at).serialize()); + batch.set( + ValueClass::IndexProperty(IndexPropertyClass::Integer { + property: EmailField::Stats.into(), + value: self.received_at, + }), + self.size.serialize(), + ); } else { // Delete metadata batch .clear(EmailField::Metadata) - .unindex(EmailField::Size, self.size.serialize()) - .unindex(EmailField::ReceivedAt, (self.received_at).serialize()); + .clear(ValueClass::IndexProperty(IndexPropertyClass::Integer { + property: EmailField::Stats.into(), + value: self.received_at, + })); } // Index properties @@ -73,17 +78,6 @@ impl MessageMetadata { batch.add(DirectoryClass::UsedQuota(tenant_id), quota); } - if self.has_attachments { - if set { - batch.tag(EmailField::HasAttachment, ()); - } else { - batch.untag(EmailField::HasAttachment, ()); - } - } - - // Index headers - self.index_headers(batch, set); - // Link blob if set { batch.set( @@ -105,130 +99,154 @@ impl MessageMetadata { Ok(()) } - fn index_headers(&self, batch: &mut BatchBuilder, set: bool) { - let mut seen_headers = [false; 40]; - for header in self.root_part().headers.iter().rev() { - if matches!(header.name, HeaderName::Other(_)) { - continue; + /* + + if self.has_attachments { + if set { + batch.tag(EmailField::HasAttachment); + } else { + batch.untag(EmailField::HasAttachment); } + } - match header.name { - HeaderName::MessageId => { - header.value.visit_text(|id| { - // Add ids to inverted index - if id.len() < MAX_ID_LENGTH { - if set { - batch.index(EmailField::References, encode_message_id(id)); - } else { - batch.unindex(EmailField::References, encode_message_id(id)); - } - } - }); - } - HeaderName::InReplyTo | HeaderName::References | HeaderName::ResentMessageId => { - header.value.visit_text(|id| { - // Add ids to inverted index - if id.len() < MAX_ID_LENGTH { - if set { - batch.index(EmailField::References, id.serialize()); - } else { - batch.unindex(EmailField::References, id.serialize()); - } - } - }); - } - HeaderName::From | HeaderName::To | HeaderName::Cc | HeaderName::Bcc => { - if !seen_headers[header.name.id() as usize] { - let property = match &header.name { - HeaderName::From => EmailField::From, - HeaderName::To => EmailField::To, - HeaderName::Cc => EmailField::Cc, - HeaderName::Bcc => EmailField::Bcc, - _ => unreachable!(), - }; - let mut sort_text = SortedAddressBuilder::new(); - let mut found_addr = false; + // Index headers + self.index_headers(batch, set); - header.value.visit_addresses(|element, value| { - if !found_addr { - match element { - AddressElement::Name => { - found_addr = !sort_text.push(value); - } - AddressElement::Address => { - sort_text.push(value); - found_addr = true; - } - AddressElement::GroupName => (), + + metadata.index_headers(self, true); + + // Store and index hasAttachment property + if has_attachments { + self.tag(EmailField::HasAttachment); + } + + + fn index_headers(&self, batch: &mut BatchBuilder, set: bool) { + let mut seen_headers = [false; 40]; + for header in self.root_part().headers.iter().rev() { + if matches!(header.name, HeaderName::Other(_)) { + continue; + } + + match header.name { + HeaderName::MessageId => { + header.value.visit_text(|id| { + // Add ids to inverted index + if id.len() < MAX_ID_LENGTH { + if set { + batch.index(EmailField::References, encode_message_id(id)); + } else { + batch.unindex(EmailField::References, encode_message_id(id)); } } }); - - // Add address to inverted index - if set { - batch.index(property, sort_text.build()); - } else { - batch.unindex(property, sort_text.build()); - } - seen_headers[header.name.id() as usize] = true; } - } - HeaderName::Date => { - if !seen_headers[header.name.id() as usize] { - if let HeaderValue::DateTime(datetime) = &header.value { - let value = (datetime.to_timestamp() as u64).serialize(); + HeaderName::InReplyTo | HeaderName::References | HeaderName::ResentMessageId => { + header.value.visit_text(|id| { + // Add ids to inverted index + if id.len() < MAX_ID_LENGTH { + if set { + batch.index(EmailField::References, id.serialize()); + } else { + batch.unindex(EmailField::References, id.serialize()); + } + } + }); + } + HeaderName::From | HeaderName::To | HeaderName::Cc | HeaderName::Bcc => { + if !seen_headers[header.name.id() as usize] { + let property = match &header.name { + HeaderName::From => EmailField::From, + HeaderName::To => EmailField::To, + HeaderName::Cc => EmailField::Cc, + HeaderName::Bcc => EmailField::Bcc, + _ => unreachable!(), + }; + let mut sort_text = SortedAddressBuilder::new(); + let mut found_addr = false; + + header.value.visit_addresses(|element, value| { + if !found_addr { + match element { + AddressElement::Name => { + found_addr = !sort_text.push(value); + } + AddressElement::Address => { + sort_text.push(value); + found_addr = true; + } + AddressElement::GroupName => (), + } + } + }); + + // Add address to inverted index if set { - batch.index(EmailField::SentAt, value); + batch.index(property, sort_text.build()); } else { - batch.unindex(EmailField::SentAt, value); + batch.unindex(property, sort_text.build()); } + seen_headers[header.name.id() as usize] = true; } - seen_headers[header.name.id() as usize] = true; } - } - HeaderName::Subject => { - if !seen_headers[header.name.id() as usize] { - // Index subject - let subject = match &header.value { - HeaderValue::Text(text) => text.clone(), - HeaderValue::TextList(list) if !list.is_empty() => { - list.first().unwrap().clone() + HeaderName::Date => { + if !seen_headers[header.name.id() as usize] { + if let HeaderValue::DateTime(datetime) = &header.value { + let value = (datetime.to_timestamp() as u64).serialize(); + if set { + batch.index(EmailField::SentAt, value); + } else { + batch.unindex(EmailField::SentAt, value); + } } - _ => "".into(), - }; - - // Index thread name - let thread_name = thread_name(&subject); - let thread_name = if !thread_name.is_empty() { - thread_name.trim_text(MAX_SORT_FIELD_LENGTH) - } else { - "!" + seen_headers[header.name.id() as usize] = true; } - .serialize(); - - if set { - batch.index(EmailField::Subject, thread_name); - } else { - batch.unindex(EmailField::Subject, thread_name); - } - - seen_headers[header.name.id() as usize] = true; } - } + HeaderName::Subject => { + if !seen_headers[header.name.id() as usize] { + // Index subject + let subject = match &header.value { + HeaderValue::Text(text) => text.clone(), + HeaderValue::TextList(list) if !list.is_empty() => { + list.first().unwrap().clone() + } + _ => "".into(), + }; - _ => (), + // Index thread name + let thread_name = thread_name(&subject); + let thread_name = if !thread_name.is_empty() { + thread_name.trim_text(MAX_SORT_FIELD_LENGTH) + } else { + "!" + } + .serialize(); + + if set { + batch.index(EmailField::Subject, thread_name); + } else { + batch.unindex(EmailField::Subject, thread_name); + } + + seen_headers[header.name.id() as usize] = true; + } + } + + _ => (), + } + } + + // Add subject to index if missing + if !seen_headers[HeaderName::Subject.id() as usize] { + if set { + batch.index(EmailField::Subject, "!".serialize()); + } else { + batch.unindex(EmailField::Subject, "!".serialize()); + } } } - // Add subject to index if missing - if !seen_headers[HeaderName::Subject.id() as usize] { - if set { - batch.index(EmailField::Subject, "!".serialize()); - } else { - batch.unindex(EmailField::Subject, "!".serialize()); - } - } - } + */ } fn encode_message_id(message_id: &str) -> Vec { @@ -244,196 +262,198 @@ impl ArchivedMessageMetadata { &self.contents[0].parts[0] } - pub fn index( - &self, - batch: &mut BatchBuilder, - account_id: u32, - tenant_id: Option, - set: bool, - ) -> trc::Result<()> { - if set { - // Serialize metadata - batch - .index(EmailField::Size, u32::from(self.size).serialize()) - .index( - EmailField::ReceivedAt, - u64::from(self.received_at).serialize(), - ); - } else { - // Delete metadata - batch - .clear(EmailField::Metadata) - .unindex(EmailField::Size, u32::from(self.size).serialize()) - .unindex( - EmailField::ReceivedAt, - u64::from(self.received_at).serialize(), - ); - } - - // Index properties - let quota = if set { - u32::from(self.size) as i64 - } else { - -(u32::from(self.size) as i64) - }; - batch.add(DirectoryClass::UsedQuota(account_id), quota); - if let Some(tenant_id) = tenant_id { - batch.add(DirectoryClass::UsedQuota(tenant_id), quota); - } - - if self.has_attachments { + /*pub fn index( + &self, + batch: &mut BatchBuilder, + account_id: u32, + tenant_id: Option, + set: bool, + ) -> trc::Result<()> { if set { - batch.tag(EmailField::HasAttachment, ()); + // Serialize metadata + batch + .index(EmailField::Size, u32::from(self.size).serialize()) + .index( + EmailField::ReceivedAt, + u64::from(self.received_at).serialize(), + ); } else { - batch.untag(EmailField::HasAttachment, ()); - } - } - - // Index headers - self.index_headers(batch, set); - - // Link blob - let hash = BlobHash::from(&self.blob_hash); - if set { - batch.set(BlobOp::Link { hash }, Vec::new()); - } else { - batch.clear(BlobOp::Link { hash }); - } - - Ok(()) - } - - fn index_headers(&self, batch: &mut BatchBuilder, set: bool) { - let mut seen_headers = [false; 40]; - for header in self.root_part().headers.iter().rev() { - if matches!(header.name, ArchivedHeaderName::Other(_)) { - continue; + // Delete metadata + batch + .clear(EmailField::Metadata) + .unindex(EmailField::Size, u32::from(self.size).serialize()) + .unindex( + EmailField::ReceivedAt, + u64::from(self.received_at).serialize(), + ); } - match header.name { - ArchivedHeaderName::MessageId => { - header.value.visit_text(|id| { - // Add ids to inverted index - if id.len() < MAX_ID_LENGTH { - if set { - batch.index(EmailField::References, encode_message_id(id)); - } else { - batch.unindex(EmailField::References, encode_message_id(id)); - } - } - }); - } - ArchivedHeaderName::InReplyTo - | ArchivedHeaderName::References - | ArchivedHeaderName::ResentMessageId => { - header.value.visit_text(|id| { - // Add ids to inverted index - if id.len() < MAX_ID_LENGTH { - if set { - batch.index(EmailField::References, id.serialize()); - } else { - batch.unindex(EmailField::References, id.serialize()); - } - } - }); - } - ArchivedHeaderName::From - | ArchivedHeaderName::To - | ArchivedHeaderName::Cc - | ArchivedHeaderName::Bcc => { - if !seen_headers[header.name.id() as usize] { - let property = match &header.name { - ArchivedHeaderName::From => EmailField::From, - ArchivedHeaderName::To => EmailField::To, - ArchivedHeaderName::Cc => EmailField::Cc, - ArchivedHeaderName::Bcc => EmailField::Bcc, - _ => unreachable!(), - }; - let mut sort_text = SortedAddressBuilder::new(); - let mut found_addr = false; + // Index properties + let quota = if set { + u32::from(self.size) as i64 + } else { + -(u32::from(self.size) as i64) + }; + batch.add(DirectoryClass::UsedQuota(account_id), quota); + if let Some(tenant_id) = tenant_id { + batch.add(DirectoryClass::UsedQuota(tenant_id), quota); + } - header.value.visit_addresses(|element, value| { - if !found_addr { - match element { - AddressElement::Name => { - found_addr = !sort_text.push(value); - } - AddressElement::Address => { - sort_text.push(value); - found_addr = true; - } - AddressElement::GroupName => (), + if self.has_attachments { + if set { + batch.tag(EmailField::HasAttachment); + } else { + batch.untag(EmailField::HasAttachment); + } + } + + // Index headers + self.index_headers(batch, set); + + // Link blob + let hash = BlobHash::from(&self.blob_hash); + if set { + batch.set(BlobOp::Link { hash }, Vec::new()); + } else { + batch.clear(BlobOp::Link { hash }); + } + + Ok(()) + } + + fn index_headers(&self, batch: &mut BatchBuilder, set: bool) { + let mut seen_headers = [false; 40]; + for header in self.root_part().headers.iter().rev() { + if matches!(header.name, ArchivedHeaderName::Other(_)) { + continue; + } + + match header.name { + ArchivedHeaderName::MessageId => { + header.value.visit_text(|id| { + // Add ids to inverted index + if id.len() < MAX_ID_LENGTH { + if set { + batch.index(EmailField::References, encode_message_id(id)); + } else { + batch.unindex(EmailField::References, encode_message_id(id)); } } }); - - // Add address to inverted index - if set { - batch.index(property, sort_text.build()); - } else { - batch.unindex(property, sort_text.build()); - } - seen_headers[header.name.id() as usize] = true; } - } - ArchivedHeaderName::Date => { - if !seen_headers[header.name.id() as usize] { - if let ArchivedHeaderValue::DateTime(datetime) = &header.value { - let value = (mail_parser::DateTime::from(datetime).to_timestamp() - as u64) - .serialize(); + ArchivedHeaderName::InReplyTo + | ArchivedHeaderName::References + | ArchivedHeaderName::ResentMessageId => { + header.value.visit_text(|id| { + // Add ids to inverted index + if id.len() < MAX_ID_LENGTH { + if set { + batch.index(EmailField::References, id.serialize()); + } else { + batch.unindex(EmailField::References, id.serialize()); + } + } + }); + } + ArchivedHeaderName::From + | ArchivedHeaderName::To + | ArchivedHeaderName::Cc + | ArchivedHeaderName::Bcc => { + if !seen_headers[header.name.id() as usize] { + let property = match &header.name { + ArchivedHeaderName::From => EmailField::From, + ArchivedHeaderName::To => EmailField::To, + ArchivedHeaderName::Cc => EmailField::Cc, + ArchivedHeaderName::Bcc => EmailField::Bcc, + _ => unreachable!(), + }; + let mut sort_text = SortedAddressBuilder::new(); + let mut found_addr = false; + + header.value.visit_addresses(|element, value| { + if !found_addr { + match element { + AddressElement::Name => { + found_addr = !sort_text.push(value); + } + AddressElement::Address => { + sort_text.push(value); + found_addr = true; + } + AddressElement::GroupName => (), + } + } + }); + + // Add address to inverted index if set { - batch.index(EmailField::SentAt, value); + batch.index(property, sort_text.build()); } else { - batch.unindex(EmailField::SentAt, value); + batch.unindex(property, sort_text.build()); } + seen_headers[header.name.id() as usize] = true; } - seen_headers[header.name.id() as usize] = true; } - } - ArchivedHeaderName::Subject => { - if !seen_headers[header.name.id() as usize] { - // Index subject - let subject = match &header.value { - ArchivedHeaderValue::Text(text) => text.as_str(), - ArchivedHeaderValue::TextList(list) if !list.is_empty() => { - list.first().unwrap().as_str() + ArchivedHeaderName::Date => { + if !seen_headers[header.name.id() as usize] { + if let ArchivedHeaderValue::DateTime(datetime) = &header.value { + let value = (mail_parser::DateTime::from(datetime).to_timestamp() + as u64) + .serialize(); + if set { + batch.index(EmailField::SentAt, value); + } else { + batch.unindex(EmailField::SentAt, value); + } } - _ => "", - }; - - // Index thread name - let thread_name = thread_name(subject); - let thread_name = if !thread_name.is_empty() { - thread_name.trim_text(MAX_SORT_FIELD_LENGTH) - } else { - "!" + seen_headers[header.name.id() as usize] = true; } - .serialize(); - - if set { - batch.index(EmailField::Subject, thread_name); - } else { - batch.unindex(EmailField::Subject, thread_name); - } - - seen_headers[header.name.id() as usize] = true; } - } + ArchivedHeaderName::Subject => { + if !seen_headers[header.name.id() as usize] { + // Index subject + let subject = match &header.value { + ArchivedHeaderValue::Text(text) => text.as_str(), + ArchivedHeaderValue::TextList(list) if !list.is_empty() => { + list.first().unwrap().as_str() + } + _ => "", + }; - _ => (), + // Index thread name + let thread_name = thread_name(subject); + let thread_name = if !thread_name.is_empty() { + thread_name.trim_text(MAX_SORT_FIELD_LENGTH) + } else { + "!" + } + .serialize(); + + if set { + batch.index(EmailField::Subject, thread_name); + } else { + batch.unindex(EmailField::Subject, thread_name); + } + + seen_headers[header.name.id() as usize] = true; + } + } + + _ => (), + } + } + + // Add subject to index if missing + if !seen_headers[HeaderName::Subject.id() as usize] { + if set { + batch.index(EmailField::Subject, "!".serialize()); + } else { + batch.unindex(EmailField::Subject, "!".serialize()); + } } } - // Add subject to index if missing - if !seen_headers[HeaderName::Subject.id() as usize] { - if set { - batch.index(EmailField::Subject, "!".serialize()); - } else { - batch.unindex(EmailField::Subject, "!".serialize()); - } - } - } + */ } impl ArchivedMessageMetadataContents { @@ -475,8 +495,11 @@ impl IndexMessage for BatchBuilder { received_at: u64, ) -> trc::Result<&mut Self> { // Index size - self.index( - EmailField::Size, + self.set( + ValueClass::IndexProperty(IndexPropertyClass::Integer { + property: EmailField::Stats.into(), + value: received_at, + }), (message.raw_message.len() as u32).serialize(), ) .add( @@ -490,9 +513,6 @@ impl IndexMessage for BatchBuilder { ); } - // Index receivedAt - self.index(EmailField::ReceivedAt, received_at.serialize()); - let mut has_attachments = false; let mut preview = None; let preview_part_id = message @@ -556,12 +576,6 @@ impl IndexMessage for BatchBuilder { blob_hash, } .with_contents(message); - metadata.index_headers(self, true); - - // Store and index hasAttachment property - if has_attachments { - self.tag(EmailField::HasAttachment, ()); - } // Link blob self.set( @@ -635,7 +649,7 @@ pub trait IndexMessageText<'x>: Sized { fn index_message(self, message: &'x ArchivedMessageMetadata, raw_message: &'x [u8]) -> Self; } -impl<'x> IndexMessageText<'x> for FtsDocument<'x, mail_parser::HeaderName<'x>> { +/*impl<'x> IndexMessageText<'x> for FtsDocument<'x, mail_parser::HeaderName<'x>> { fn index_message( mut self, message: &'x ArchivedMessageMetadata, @@ -775,6 +789,7 @@ impl<'x> IndexMessageText<'x> for FtsDocument<'x, mail_parser::HeaderName<'x>> { self } } +*/ impl SortedAddressBuilder { pub fn new() -> Self { diff --git a/crates/email/src/message/ingest.rs b/crates/email/src/message/ingest.rs index 77193492..2a18d631 100644 --- a/crates/email/src/message/ingest.rs +++ b/crates/email/src/message/ingest.rs @@ -13,11 +13,11 @@ use crate::{ mailbox::{INBOX_ID, JUNK_ID, UidMailbox}, message::{ crypto::EncryptionParams, - index::{IndexMessage, MAX_ID_LENGTH, VisitText}, + index::{IndexMessage, VisitText}, metadata::MessageData, }, }; -use common::{Server, auth::AccessToken, storage::index::ObjectIndexBuilder}; +use common::{Server, auth::AccessToken}; use directory::Permission; use groupware::{ calendar::itip::{ItipIngest, ItipIngestError}, @@ -31,19 +31,15 @@ use spam_filter::{ SpamFilterInput, analysis::init::SpamFilterInit, modules::bayes::BayesClassifier, }; use std::future::Future; -use std::{ - borrow::Cow, - fmt::Write, - time::{Duration, Instant}, -}; +use std::{borrow::Cow, fmt::Write, time::Instant}; use store::{ - IndexKey, IndexKeyPrefix, IterateParams, U32_LEN, + IndexKeyPrefix, IterateParams, U32_LEN, ValueKey, ahash::AHashMap, - query::Filter, - roaring::RoaringBitmap, - write::{BatchBuilder, TaskQueueClass, ValueClass, key::DeserializeBigEndian, now}, + write::{ + BatchBuilder, IndexPropertyClass, TaskQueueClass, ValueClass, key::DeserializeBigEndian, + now, + }, }; -use store::{SerializeInfallible, rand::Rng}; use trc::{AddContext, MessageIngestEvent}; use types::{ blob::{BlobClass, BlobId}, @@ -51,7 +47,10 @@ use types::{ field::{ContactField, EmailField, MailboxField, PrincipalField}, keyword::Keyword, }; -use utils::sanitize_email; +use utils::{ + cheeky_hash::{CheekyHash, CheekyHashMap}, + sanitize_email, +}; #[derive(Default)] pub struct IngestedEmail { @@ -87,19 +86,16 @@ pub enum IngestSource<'x> { Restore, } -const MAX_RETRIES: u32 = 10; - pub trait EmailIngest: Sync + Send { fn email_ingest( &self, params: IngestEmail, ) -> impl Future> + Send; - fn find_or_merge_thread( + fn find_thread_id( &self, account_id: u32, thread_name: &str, - references: Vec<&[u8]>, - skip_duplicate: Option<(&[u8], u32)>, + message_ids: &CheekyHashMap, ) -> impl Future> + Send; fn assign_imap_uid( &self, @@ -109,10 +105,11 @@ pub trait EmailIngest: Sync + Send { fn email_bayes_can_train(&self, access_token: &AccessToken) -> bool; } -pub enum ThreadResult { - Id(u32), - Create, - Skip, +pub struct ThreadResult { + pub thread_id: Option, + pub thread_hash: CheekyHash, + pub merge_ids: Vec, + pub duplicate_ids: Vec, } impl EmailIngest for Server { @@ -214,16 +211,14 @@ impl EmailIngest for Server { && sender != deliver_to && is_sender_authenticated && !self - .store() - .filter( + .document_exists( account_id, Collection::ContactCard, - vec![Filter::eq(ContactField::Email, sender.into_bytes())], + ContactField::Email, + sender.as_bytes(), ) .await .caused_by(trc::location!())? - .results - .is_empty() { is_spam = false; if self @@ -403,26 +398,25 @@ impl EmailIngest for Server { // Obtain message references and thread name let mut message_id = None; - let mut log_thread_create = false; - let thread_id = { - let mut references = Vec::with_capacity(5); + let mut message_ids = CheekyHashMap::default(); + let thread_result = { let mut subject = ""; for header in message.root_part().headers().iter().rev() { match &header.name { HeaderName::MessageId => header.value.visit_text(|id| { - if !id.is_empty() && id.len() < MAX_ID_LENGTH { + if !id.is_empty() { if message_id.is_none() { message_id = id.to_string().into(); } - references.push(id.as_bytes()); + message_ids.insert(CheekyHash::new(id.as_bytes()), true); } }), HeaderName::InReplyTo | HeaderName::References | HeaderName::ResentMessageId => { header.value.visit_text(|id| { - if !id.is_empty() && id.len() < MAX_ID_LENGTH { - references.push(id.as_bytes()); + if !id.is_empty() { + message_ids.insert(CheekyHash::new(id.as_bytes()), false); } }); } @@ -440,49 +434,41 @@ impl EmailIngest for Server { } } - let skip_duplicate = if params.source.is_smtp() { - message_id.as_deref().map(|message_id| { - ( - message_id.as_bytes(), - params.mailbox_ids.first().copied().unwrap_or(INBOX_ID), - ) - }) - } else { - None - }; - match self - .find_or_merge_thread(account_id, subject, references, skip_duplicate) + self.find_thread_id(account_id, subject, &message_ids) .await? - { - ThreadResult::Id(thread_id) => thread_id, - ThreadResult::Create => { - log_thread_create = true; - self.store() - .assign_document_ids(account_id, Collection::Thread, 1) - .await - .caused_by(trc::location!())? - } - ThreadResult::Skip => { - // Duplicate message - trc::event!( - MessageIngest(MessageIngestEvent::Duplicate), - SpanId = params.session_id, - AccountId = account_id, - MessageId = message_id, - ); - - return Ok(IngestedEmail { - document_id: 0, - thread_id: 0, - change_id: u64::MAX, - blob_id: BlobId::default(), - imap_uids: Vec::new(), - size: 0, - }); - } - } }; + // Skip duplicate messages for SMTP ingestion + if !thread_result.duplicate_ids.is_empty() && params.source.is_smtp() { + // Fetch cached messages + let cache = self + .get_cached_messages(account_id) + .await + .caused_by(trc::location!())?; + + // Skip duplicate messages + if !cache + .in_mailbox(params.mailbox_ids.first().copied().unwrap_or(INBOX_ID)) + .any(|m| thread_result.duplicate_ids.contains(&m.document_id)) + { + trc::event!( + MessageIngest(MessageIngestEvent::Duplicate), + SpanId = params.session_id, + AccountId = account_id, + MessageId = message_id, + ); + + return Ok(IngestedEmail { + document_id: 0, + thread_id: 0, + change_id: u64::MAX, + blob_id: BlobId::default(), + imap_uids: Vec::new(), + size: 0, + }); + } + } + // Add additional headers to message if !extra_headers.is_empty() { let offset_start = extra_headers.len(); @@ -543,7 +529,7 @@ impl EmailIngest for Server { if do_encrypt && !message.is_encrypted() && let Some(encrypt_params_) = self - .get_archive_by_property( + .archive_by_property( account_id, Collection::Principal, 0, @@ -624,22 +610,29 @@ impl EmailIngest for Server { .collect::>(); batch.with_account_id(account_id); - if log_thread_create { - batch - .with_collection(Collection::Thread) - .update_document(thread_id) - .log_container_insert(SyncCollection::Thread); - } - - let due = now(); + // Obtain document ID let document_id = self .store() .assign_document_ids(account_id, Collection::Email, 1) .await .caused_by(trc::location!())?; + + // Determine thread id + let thread_id = if let Some(thread_id) = thread_result.thread_id { + thread_id + } else { + batch + .with_collection(Collection::Thread) + .with_document(document_id) + .log_container_insert(SyncCollection::Thread); + document_id + }; + + let due = now(); + batch .with_collection(Collection::Email) - .create_document(document_id) + .with_document(document_id) .index_message( account_id, tenant_id, @@ -654,21 +647,21 @@ impl EmailIngest for Server { ) .caused_by(trc::location!())? .set( - ValueClass::TaskQueue(TaskQueueClass::IndexEmail { - due, - hash: blob_id.hash.clone(), + ValueClass::IndexProperty(IndexPropertyClass::Hash { + property: EmailField::Threading.into(), + hash: thread_result.thread_hash, }), - vec![], + ThreadInfo::serialize(thread_id, &message_ids), + ) + .set( + ValueClass::TaskQueue(TaskQueueClass::IndexEmail { due }), + MergeThreadTask::new(thread_result).serialize(), ); // Request spam training if let Some(learn_spam) = train_spam { batch.set( - ValueClass::TaskQueue(TaskQueueClass::BayesTrain { - due, - hash: blob_id.hash.clone(), - learn_spam, - }), + ValueClass::TaskQueue(TaskQueueClass::BayesTrain { due, learn_spam }), vec![], ); } @@ -731,221 +724,105 @@ impl EmailIngest for Server { }) } - async fn find_or_merge_thread( + async fn find_thread_id( &self, account_id: u32, thread_name: &str, - mut references: Vec<&[u8]>, - skip_duplicate: Option<(&[u8], u32)>, + message_ids: &CheekyHashMap, ) -> trc::Result { - if references.is_empty() { - return Ok(ThreadResult::Create); + let mut result = ThreadResult { + thread_id: None, + thread_hash: CheekyHash::new(if !thread_name.is_empty() { + thread_name + } else { + "!" + }), + merge_ids: vec![], + duplicate_ids: vec![], + }; + + if message_ids.is_empty() { + return Ok(result); } - let mut try_count = 0; - let thread_name = if !thread_name.is_empty() { - thread_name - } else { - "!" - } - .serialize(); - - // Sort references ascending - references.sort_unstable(); - - loop { - // Find messages with a matching subject - let mut subj_results = RoaringBitmap::new(); - self.store() - .iterate( - IterateParams::new( - IndexKey { - account_id, - collection: Collection::Email.into(), - document_id: 0, - field: EmailField::Subject.into(), - key: thread_name.clone(), - }, - IndexKey { - account_id, - collection: Collection::Email.into(), - document_id: u32::MAX, - field: EmailField::Subject.into(), - key: thread_name.clone(), - }, - ) - .no_values() - .ascending(), - |key, _| { - let id_pos = key.len() - U32_LEN; - let value = key.get(IndexKeyPrefix::len()..id_pos).ok_or_else(|| { - trc::Error::corrupted_key(key, None, trc::location!()) - })?; - - if value == thread_name { - subj_results.insert(key.deserialize_be_u32(id_pos)?); - } - - Ok(true) + // Find thread ids + let key_len = IndexKeyPrefix::len() + result.thread_hash.len() + U32_LEN; + let document_id_pos = key_len - U32_LEN; + let mut thread_ids = AHashMap::>::with_capacity(16); + self.store() + .iterate( + IterateParams::new( + ValueKey { + account_id, + collection: Collection::Email.into(), + document_id: 0, + class: ValueClass::IndexProperty(IndexPropertyClass::Hash { + property: EmailField::Threading.into(), + hash: result.thread_hash, + }), + }, + ValueKey { + account_id, + collection: Collection::Email.into(), + document_id: u32::MAX, + class: ValueClass::IndexProperty(IndexPropertyClass::Hash { + property: EmailField::Threading.into(), + hash: result.thread_hash, + }), }, ) - .await - .caused_by(trc::location!())?; + .ascending(), + |key, value| { + if key.len() == key_len { + // Find matching references + let mut from_offset = U32_LEN; - // No matching subjects were found, skip early - if subj_results.is_empty() { - return Ok(ThreadResult::Create); - } - - // Find messages with matching references - let mut results = RoaringBitmap::new(); - let mut found_message_id = Vec::new(); - self.store() - .iterate( - IterateParams::new( - IndexKey { - account_id, - collection: Collection::Email.into(), - document_id: 0, - field: EmailField::References.into(), - key: references.first().unwrap().to_vec(), - }, - IndexKey { - account_id, - collection: Collection::Email.into(), - document_id: u32::MAX, - field: EmailField::References.into(), - key: references.last().unwrap().to_vec(), - }, - ) - .no_values() - .ascending(), - |key, _| { - let id_pos = key.len() - U32_LEN; - let mut value = - key.get(IndexKeyPrefix::len()..id_pos).ok_or_else(|| { - trc::Error::corrupted_key(key, None, trc::location!()) - })?; - let document_id = key.deserialize_be_u32(id_pos)?; - - if let Some(message_id) = value.strip_suffix(&[0]) { - value = message_id; - if skip_duplicate.is_some_and(|(message_id, _)| message_id == value) { - found_message_id.push(document_id); - } - } - - if subj_results.contains(document_id) - && references.binary_search(&value).is_ok() + while let Some(ref_hash) = + value.get(from_offset..).and_then(CheekyHash::deserialize) { - results.insert(document_id); + if let Some(is_message_id) = message_ids.get(&ref_hash) { + let document_id = key.deserialize_be_u32(document_id_pos)?; + let thread_id = value.deserialize_be_u32(0)?; - if subj_results.len() == results.len() { - return Ok(false); + if *is_message_id && from_offset == U32_LEN { + result.duplicate_ids.push(document_id); + } + + thread_ids.entry(thread_id).or_default().push(document_id); + + return Ok(true); } + + from_offset += ref_hash.len(); } - - Ok(true) - }, - ) - .await - .caused_by(trc::location!())?; - - // No matching messages - if results.is_empty() { - return Ok(ThreadResult::Create); - } - - // Fetch cached messages - let cache = self - .get_cached_messages(account_id) - .await - .caused_by(trc::location!())?; - - // Skip duplicate messages - if !found_message_id.is_empty() - && cache - .in_mailbox(skip_duplicate.unwrap().1) - .any(|m| found_message_id.contains(&m.document_id)) - { - return Ok(ThreadResult::Skip); - } - - // Find the most common threadId - let mut thread_counts = AHashMap::::with_capacity(16); - let mut thread_id = u32::MAX; - let mut thread_count = 0; - for item in &cache.emails.items { - if results.contains(item.document_id) { - let tc = thread_counts.entry(item.thread_id).or_default(); - *tc += 1; - if *tc > thread_count { - thread_count = *tc; - thread_id = item.thread_id; } - } + + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; + + match thread_ids.len() { + 0 => Ok(result), + 1 => { + // Happy path, only one thread id + result.thread_id = thread_ids.into_keys().next(); + Ok(result) } - - if thread_id == u32::MAX { - return Ok(ThreadResult::Create); - } else if thread_counts.len() == 1 { - return Ok(ThreadResult::Id(thread_id)); - } - - // Delete all but the most common threadId - let mut batch = BatchBuilder::new(); - batch - .with_account_id(account_id) - .with_collection(Collection::Thread); - for &delete_thread_id in thread_counts.keys() { - if delete_thread_id != thread_id { - batch - .update_document(delete_thread_id) - .log_container_delete(SyncCollection::Thread); - } - } - - // Move messages to the new threadId - batch.with_collection(Collection::Email); - - for item in &cache.emails.items { - if thread_id == item.thread_id || !thread_counts.contains_key(&item.thread_id) { - continue; - } - if let Some(data_) = self - .get_archive(account_id, Collection::Email, item.document_id) - .await - .caused_by(trc::location!())? - { - let data = data_ - .to_unarchived::() - .caused_by(trc::location!())?; - if data.inner.thread_id != item.thread_id { - continue; + _ => { + // Multiple thread ids that this message belongs to, merge them + let mut max_thread_id = u32::MAX; + let mut max_count = 0; + for (thread_id, ids) in thread_ids { + if ids.len() > max_count { + max_count = ids.len(); + max_thread_id = thread_id; } - let mut new_data = data.deserialize().caused_by(trc::location!())?; - new_data.thread_id = thread_id; - batch - .update_document(item.document_id) - .custom( - ObjectIndexBuilder::new() - .with_current(data) - .with_changes(new_data), - ) - .caused_by(trc::location!())?; - } - } - - match self.commit_batch(batch).await { - Ok(_) => return Ok(ThreadResult::Id(thread_id)), - Err(err) if err.is_assertion_failure() && try_count < MAX_RETRIES => { - let backoff = store::rand::rng().random_range(50..=300); - tokio::time::sleep(Duration::from_millis(backoff)).await; - try_count += 1; - } - Err(err) => { - return Err(err.caused_by(trc::location!())); + result.merge_ids.extend(ids); } + result.thread_id = Some(max_thread_id); + Ok(result) } } } @@ -956,7 +833,7 @@ impl EmailIngest for Server { batch .with_account_id(account_id) .with_collection(Collection::Mailbox) - .update_document(mailbox_id) + .with_document(mailbox_id) .add_and_get(MailboxField::UidCounter, 1); self.core .storage @@ -978,3 +855,73 @@ impl IngestSource<'_> { matches!(self, Self::Smtp { .. }) } } + +pub struct MergeThreadTask { + pub thread_hash: CheekyHash, + pub duplicate_ids: Vec, +} + +impl MergeThreadTask { + pub(crate) fn new(thread_result: ThreadResult) -> Self { + Self { + thread_hash: thread_result.thread_hash, + duplicate_ids: thread_result.duplicate_ids, + } + } + + pub(crate) fn serialize(&self) -> Vec { + if !self.duplicate_ids.is_empty() { + let mut buf = + Vec::with_capacity(self.thread_hash.len() + self.duplicate_ids.len() * U32_LEN); + buf.extend_from_slice(self.thread_hash.as_bytes()); + for id in &self.duplicate_ids { + buf.extend_from_slice(&id.to_be_bytes()); + } + buf + } else { + vec![] + } + } + + pub fn deserialize(bytes: &[u8]) -> Option { + if !bytes.is_empty() { + let thread_hash = CheekyHash::deserialize(bytes)?; + let mut duplicate_ids = Vec::new(); + let mut start_offset = thread_hash.len(); + + while let Some(id_bytes) = bytes.get(start_offset..start_offset + U32_LEN) { + duplicate_ids.push(u32::from_be_bytes(id_bytes.try_into().ok()?)); + start_offset += U32_LEN; + } + + Some(Self { + thread_hash, + duplicate_ids, + }) + } else { + None + } + } +} + +pub(crate) struct ThreadInfo; + +impl ThreadInfo { + pub fn serialize(thread_id: u32, ref_ids: &CheekyHashMap) -> Vec { + let mut buf = Vec::with_capacity(U32_LEN + ref_ids.len() * (1 + 16)); + buf.extend_from_slice(&thread_id.to_be_bytes()); + for (ref_id, is_message_id) in ref_ids { + if *is_message_id && buf.len() > U32_LEN { + // Place Message-id reference first + let mut new_buf = Vec::with_capacity(U32_LEN + ref_ids.len() * (1 + 16)); + new_buf.extend_from_slice(&thread_id.to_be_bytes()); + new_buf.extend_from_slice(ref_id.as_bytes()); + new_buf.extend_from_slice(&buf[U32_LEN..]); + buf = new_buf; + } else { + buf.extend_from_slice(ref_id.as_bytes()); + } + } + buf + } +} diff --git a/crates/email/src/message/mod.rs b/crates/email/src/message/mod.rs index ddafe2b5..e094c55f 100644 --- a/crates/email/src/message/mod.rs +++ b/crates/email/src/message/mod.rs @@ -4,7 +4,6 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -pub mod bayes; pub mod copy; pub mod crypto; pub mod delete; diff --git a/crates/email/src/sieve/delete.rs b/crates/email/src/sieve/delete.rs index f4ae430b..99d8b00a 100644 --- a/crates/email/src/sieve/delete.rs +++ b/crates/email/src/sieve/delete.rs @@ -30,14 +30,14 @@ impl SieveScriptDelete for Server { ) -> trc::Result { // Fetch record if let Some(obj_) = self - .get_archive(account_id, Collection::SieveScript, document_id) + .archive(account_id, Collection::SieveScript, document_id) .await? { // Delete record batch .with_account_id(account_id) .with_collection(Collection::SieveScript) - .delete_document(document_id) + .with_document(document_id) .clear(SieveField::Ids) .custom( ObjectIndexBuilder::<_, ()>::new() diff --git a/crates/email/src/sieve/ingest.rs b/crates/email/src/sieve/ingest.rs index d6f9d483..1ac9ef27 100644 --- a/crates/email/src/sieve/ingest.rs +++ b/crates/email/src/sieve/ingest.rs @@ -20,10 +20,9 @@ use sieve::{Envelope, Event, Input, Mailbox, Recipient, Sieve}; use std::{borrow::Cow, sync::Arc}; use std::{future::Future, str::FromStr}; use store::{ - Deserialize, Serialize, SerializeInfallible, ValueKey, + Deserialize, Serialize, ValueKey, ahash::AHashMap, dispatch::lookup::KeyValue, - query::Filter, write::{AlignedBytes, Archive, ArchiveVersion, Archiver, BatchBuilder, BlobOp, ValueClass}, }; use trc::{AddContext, SieveEvent}; @@ -608,15 +607,14 @@ impl SieveScriptIngest for Server { ) -> trc::Result> { // Find the script by name if let Some(document_id) = self - .store() - .filter( + .document_ids_matching( account_id, Collection::SieveScript, - vec![Filter::eq(SieveField::Name, name.serialize())], + SieveField::Name, + name.as_bytes(), ) .await .caused_by(trc::location!())? - .results .min() { self.sieve_script_compile(account_id, document_id) @@ -635,7 +633,7 @@ impl SieveScriptIngest for Server { ) -> trc::Result> { // Obtain script object let Some(script_object) = self - .get_archive(account_id, Collection::SieveScript, document_id) + .archive(account_id, Collection::SieveScript, document_id) .await? else { return Ok(None); @@ -710,7 +708,7 @@ impl SieveScriptIngest for Server { batch .with_account_id(account_id) .with_collection(Collection::SieveScript) - .update_document(document_id) + .with_document(document_id) .assert_value(SieveField::Archive, &script_object) .set( SieveField::Archive, diff --git a/crates/groupware/src/cache/calcard.rs b/crates/groupware/src/cache/calcard.rs index bc95be5c..cc2c596a 100644 --- a/crates/groupware/src/cache/calcard.rs +++ b/crates/groupware/src/cache/calcard.rs @@ -26,6 +26,7 @@ use trc::AddContext; use types::{ acl::AclGrant, collection::{Collection, SyncCollection}, + field::CalendarNotificationField, }; use utils::map::bitmap::Bitmap; @@ -38,59 +39,13 @@ pub(super) async fn build_calcard_resources( item_collection: Collection, update_lock: Arc, ) -> trc::Result { - let mut last_change_id = server - .core - .storage - .data - .get_last_change_id(account_id, sync_collection.into()) - .await - .caused_by(trc::location!())? - .unwrap_or_default(); - - // Create default folders let is_calendar = matches!(sync_collection, SyncCollection::Calendar); - let mut container_ids = server - .get_document_ids(account_id, container_collection) - .await - .caused_by(trc::location!())? - .unwrap_or_default(); let name = server .store() .get_principal_name(account_id) .await .caused_by(trc::location!())? .unwrap_or_else(|| format!("_{account_id}")); - if container_ids.is_empty() { - if is_calendar { - server - .create_default_calendar(access_token, account_id, &name) - .await?; - } else { - server - .create_default_addressbook(access_token, account_id, &name) - .await?; - } - last_change_id = server - .core - .storage - .data - .get_last_change_id(account_id, sync_collection.into()) - .await - .caused_by(trc::location!())? - .unwrap_or_default(); - - container_ids = server - .get_document_ids(account_id, container_collection) - .await - .caused_by(trc::location!())? - .unwrap_or_default(); - } - let item_ids = server - .get_document_ids(account_id, item_collection) - .await - .caused_by(trc::location!())? - .unwrap_or_default(); - let mut cache = DavResources { base_path: format!( "{}/{}/", @@ -102,77 +57,117 @@ pub(super) async fn build_calcard_resources( .base_path(), percent_encoding::utf8_percent_encode(&name, RFC_3986), ), - paths: AHashSet::with_capacity((container_ids.len() + item_ids.len()) as usize), - resources: Vec::with_capacity((container_ids.len() + item_ids.len()) as usize), - item_change_id: last_change_id, - container_change_id: last_change_id, - highest_change_id: last_change_id, + paths: AHashSet::with_capacity(16), + resources: Vec::with_capacity(16), + item_change_id: 0, + container_change_id: 0, + highest_change_id: 0, size: std::mem::size_of::() as u64, update_lock, }; - for document_id in container_ids { - if let Some(archive) = server - .get_archive(account_id, container_collection, document_id) + let mut is_first_check = true; + loop { + let last_change_id = server + .core + .storage + .data + .get_last_change_id(account_id, sync_collection.into()) .await .caused_by(trc::location!())? - { - let resource = if is_calendar { - resource_from_calendar(archive.unarchive::()?, document_id) - } else { - resource_from_addressbook(archive.unarchive::()?, document_id) - }; - let path = DavPath { - path: resource.container_name().unwrap().to_string(), - parent_id: None, - hierarchy_seq: 1, - resource_idx: cache.resources.len(), - }; + .unwrap_or_default(); + cache.item_change_id = last_change_id; + cache.container_change_id = last_change_id; + cache.highest_change_id = last_change_id; - cache.size += (std::mem::size_of::() - + std::mem::size_of::() - + (path.path.len()) * 2) as u64; - cache.paths.insert(path); - cache.resources.push(resource); - } - } - let parent_range = cache.resources.len(); - - for document_id in item_ids { - if let Some(archive) = server - .get_archive(account_id, item_collection, document_id) - .await - .caused_by(trc::location!())? - { - let resource = if is_calendar { - resource_from_event(archive.unarchive::()?, document_id) - } else { - resource_from_card(archive.unarchive::()?, document_id) - }; - let resource_idx = cache.resources.len(); - - for name in resource.child_names().unwrap_or_default().iter() { - if let Some(parent) = cache.resources.get(..parent_range).and_then(|resources| { - resources.iter().find(|r| r.document_id == name.parent_id) - }) { + server + .archives( + account_id, + container_collection, + &(), + |document_id, archive| { + let resource = if is_calendar { + resource_from_calendar(archive.unarchive::()?, document_id) + } else { + resource_from_addressbook(archive.unarchive::()?, document_id) + }; let path = DavPath { - path: format!("{}/{}", parent.container_name().unwrap(), name.name), - parent_id: Some(name.parent_id), - hierarchy_seq: 0, - resource_idx, + path: resource.container_name().unwrap().to_string(), + parent_id: None, + hierarchy_seq: 1, + resource_idx: cache.resources.len(), }; - cache.size += - (std::mem::size_of::() + name.name.len() + path.path.len()) as u64; + cache.size += (std::mem::size_of::() + + std::mem::size_of::() + + (path.path.len()) * 2) as u64; cache.paths.insert(path); - } - } - cache.size += std::mem::size_of::() as u64; - cache.resources.push(resource); - } - } + cache.resources.push(resource); - Ok(cache) + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; + + if cache.paths.is_empty() { + if !is_first_check { + if is_calendar { + server + .create_default_calendar(access_token, account_id, &name) + .await?; + } else { + server + .create_default_addressbook(access_token, account_id, &name) + .await?; + } + is_first_check = false; + continue; + } else { + return Ok(cache); + } + } + + let todo = "fix fdb range scan to support chunked reads"; + let parent_range = cache.resources.len(); + server + .archives(account_id, item_collection, &(), |document_id, archive| { + let resource = if is_calendar { + resource_from_event(archive.unarchive::()?, document_id) + } else { + resource_from_card(archive.unarchive::()?, document_id) + }; + let resource_idx = cache.resources.len(); + + for name in resource.child_names().unwrap_or_default().iter() { + if let Some(parent) = + cache.resources.get(..parent_range).and_then(|resources| { + resources.iter().find(|r| r.document_id == name.parent_id) + }) + { + let path = DavPath { + path: format!("{}/{}", parent.container_name().unwrap(), name.name), + parent_id: Some(name.parent_id), + hierarchy_seq: 0, + resource_idx, + }; + + cache.size += (std::mem::size_of::() + + name.name.len() + + path.path.len()) as u64; + cache.paths.insert(path); + } + } + cache.size += std::mem::size_of::() as u64; + cache.resources.push(resource); + + Ok(true) + }) + .await + .caused_by(trc::location!())?; + + return Ok(cache); + } } pub(super) async fn build_scheduling_resources( @@ -197,10 +192,13 @@ pub(super) async fn build_scheduling_resources( .unwrap_or_else(|| format!("_{account_id}")); let item_ids = server - .get_document_ids(account_id, Collection::CalendarEventNotification) + .document_ids( + account_id, + Collection::CalendarEventNotification, + CalendarNotificationField::CreatedToId, + ) .await - .caused_by(trc::location!())? - .unwrap_or_default(); + .caused_by(trc::location!())?; let mut cache = DavResources { base_path: format!( diff --git a/crates/groupware/src/cache/file.rs b/crates/groupware/src/cache/file.rs index 3d473d59..a64088be 100644 --- a/crates/groupware/src/cache/file.rs +++ b/crates/groupware/src/cache/file.rs @@ -11,17 +11,12 @@ use crate::{ use common::{DavPath, DavResource, DavResourceMetadata, DavResources, Server}; use directory::backend::internal::manage::ManageDirectory; use std::sync::Arc; -use store::{ - Deserialize, IterateParams, U32_LEN, ValueKey, - ahash::{AHashMap, AHashSet}, - write::{AlignedBytes, Archive, ValueClass, key::DeserializeBigEndian}, -}; +use store::ahash::{AHashMap, AHashSet}; use tokio::sync::Semaphore; use trc::AddContext; use types::{ acl::AclGrant, collection::{Collection, SyncCollection}, - field::Field, }; use utils::{map::bitmap::Bitmap, topological::TopologicalSort}; @@ -44,7 +39,25 @@ pub(super) async fn build_file_resources( .await .caused_by(trc::location!())? .unwrap_or_else(|| format!("_{account_id}")); - let resources = fetch_files(server, account_id).await?; + + let mut resources = Vec::with_capacity(16); + server + .archives( + account_id, + Collection::FileNode, + &(), + |document_id, archive| { + resources.push(resource_from_file( + archive.unarchive::()?, + document_id, + )); + + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; + let mut files = DavResources { base_path: format!( "{}/{}/", @@ -119,43 +132,6 @@ pub(super) fn build_nested_hierarchy(resources: &mut DavResources) { .collect(); } -async fn fetch_files(server: &Server, account_id: u32) -> trc::Result> { - let mut files = Vec::with_capacity(16); - - server - .store() - .iterate( - IterateParams::new( - ValueKey { - account_id, - collection: Collection::FileNode.into(), - document_id: 0, - class: ValueClass::from(Field::ARCHIVE), - }, - ValueKey { - account_id, - collection: Collection::FileNode.into(), - document_id: u32::MAX, - class: ValueClass::from(Field::ARCHIVE), - }, - ), - |key, value| { - let archive = as Deserialize>::deserialize(value)?; - - files.push(resource_from_file( - archive.unarchive::()?, - key.deserialize_be_u32(key.len() - U32_LEN)?, - )); - - Ok(true) - }, - ) - .await - .caused_by(trc::location!())?; - - Ok(files) -} - pub(super) fn resource_from_file(node: &ArchivedFileNode, document_id: u32) -> DavResource { let parent_id = node.parent_id.to_native(); DavResource { diff --git a/crates/groupware/src/cache/mod.rs b/crates/groupware/src/cache/mod.rs index 1973e638..a5a066b6 100644 --- a/crates/groupware/src/cache/mod.rs +++ b/crates/groupware/src/cache/mod.rs @@ -19,13 +19,17 @@ use common::{CacheSwap, DavResource, DavResources, Server, auth::AccessToken}; use file::{build_file_resources, build_nested_hierarchy, resource_from_file}; use std::{sync::Arc, time::Instant}; use store::{ + SerializeInfallible, ValueKey, ahash::AHashMap, query::log::{Change, Query}, - write::{AlignedBytes, Archive, BatchBuilder}, + write::{AlignedBytes, Archive, BatchBuilder, ValueClass}, }; use tokio::sync::Semaphore; use trc::{AddContext, StoreEvent}; -use types::collection::{Collection, SyncCollection}; +use types::{ + collection::{Collection, SyncCollection}, + field::PrincipalField, +}; pub mod calcard; pub mod file; @@ -56,7 +60,6 @@ pub trait GroupwareCache: Sync + Send { &self, access_token: &AccessToken, account_id: u32, - account_name: &str, ) -> impl Future>> + Send; fn cached_dav_resources( @@ -380,6 +383,13 @@ impl GroupwareCache for Server { ..Default::default() } .insert(access_token, account_id, document_id, &mut batch)?; + + // Set default calendar + batch + .with_collection(Collection::Principal) + .with_document(0) + .set(PrincipalField::DefaultCalendarId, document_id.serialize()); + self.commit_batch(batch).await?; Ok(Some(document_id)) } else { @@ -391,17 +401,23 @@ impl GroupwareCache for Server { &self, access_token: &AccessToken, account_id: u32, - account_name: &str, ) -> trc::Result> { - match self - .get_document_ids(account_id, Collection::Calendar) + let default_calendar_id = self + .store() + .get_value::(ValueKey { + account_id, + collection: Collection::Principal.into(), + document_id: 0, + class: ValueClass::Property(PrincipalField::DefaultCalendarId.into()), + }) .await - { - Ok(Some(ids)) if !ids.is_empty() => Ok(ids.iter().next()), - _ => { - self.create_default_calendar(access_token, account_id, account_name) - .await - } + .caused_by(trc::location!())?; + if default_calendar_id.is_some() { + Ok(default_calendar_id) + } else { + self.fetch_dav_resources(access_token, account_id, SyncCollection::Calendar) + .await + .map(|c| c.document_ids(true).next()) } } @@ -434,7 +450,7 @@ async fn process_changes( Change::InsertItem(id) | Change::UpdateItem(id) => { let document_id = id as u32; if let Some(archive) = server - .get_archive(account_id, collection.collection(false), document_id) + .archive(account_id, collection.collection(false), document_id) .await .caused_by(trc::location!())? { @@ -457,7 +473,7 @@ async fn process_changes( Change::InsertContainer(id) | Change::UpdateContainer(id) => { let document_id = id as u32; if let Some(archive) = server - .get_archive(account_id, collection.collection(true), document_id) + .archive(account_id, collection.collection(true), document_id) .await .caused_by(trc::location!())? { diff --git a/crates/groupware/src/calendar/index.rs b/crates/groupware/src/calendar/index.rs index b4ffd9b6..16621008 100644 --- a/crates/groupware/src/calendar/index.rs +++ b/crates/groupware/src/calendar/index.rs @@ -16,13 +16,12 @@ use calcard::icalendar::{ ArchivedICalendarParameterValue, ArchivedICalendarProperty, ArchivedICalendarValue, ICalendarParameterValue, ICalendarProperty, ICalendarValue, }; -use common::storage::index::{ - IndexItem, IndexValue, IndexableAndSerializableObject, IndexableObject, +use common::storage::index::{IndexValue, IndexableAndSerializableObject, IndexableObject}; +use store::{ + write::{IndexPropertyClass, ValueClass}, + xxhash_rust::xxh3, }; -use nlp::tokenizers::word::WordTokenizer; -use std::collections::HashSet; -use store::backend::MAX_TOKEN_LENGTH; -use types::{acl::AclGrant, collection::SyncCollection, field::CalendarField}; +use types::{acl::AclGrant, collection::SyncCollection, field::CalendarNotificationField}; impl IndexableObject for Calendar { fn index_values(&self) -> impl Iterator> { @@ -76,30 +75,8 @@ impl IndexableAndSerializableObject for Calendar { impl IndexableObject for CalendarEvent { fn index_values(&self) -> impl Iterator> { [ - IndexValue::Index { - field: CalendarField::Uid.into(), - value: self.data.event.uids().next().into(), - }, - IndexValue::Index { - field: CalendarField::Start.into(), - value: self.data.event_range_start().into(), - }, - IndexValue::Index { - field: CalendarField::Created.into(), - value: self.created.into(), - }, - IndexValue::Index { - field: CalendarField::Updated.into(), - value: self.modified.into(), - }, - IndexValue::IndexList { - field: CalendarField::Text.into(), - value: self - .text() - .map(Into::into) - .collect::>() - .into_iter() - .collect(), + IndexValue::SearchIndex { + hashes: self.hashes().collect(), }, IndexValue::Quota { used: self.dead_properties.size() as u32 @@ -120,30 +97,8 @@ impl IndexableObject for CalendarEvent { impl IndexableObject for &ArchivedCalendarEvent { fn index_values(&self) -> impl Iterator> { [ - IndexValue::Index { - field: CalendarField::Uid.into(), - value: self.data.event.uids().next().into(), - }, - IndexValue::Index { - field: CalendarField::Start.into(), - value: self.data.event_range_start().into(), - }, - IndexValue::Index { - field: CalendarField::Created.into(), - value: self.created.to_native().into(), - }, - IndexValue::Index { - field: CalendarField::Updated.into(), - value: self.modified.to_native().into(), - }, - IndexValue::IndexList { - field: CalendarField::Text.into(), - value: self - .text() - .map(Into::into) - .collect::>() - .into_iter() - .collect(), + IndexValue::SearchIndex { + hashes: self.hashes().collect(), }, IndexValue::Quota { used: self.dead_properties.size() as u32 @@ -171,12 +126,11 @@ impl IndexableObject for CalendarEventNotification { fn index_values(&self) -> impl Iterator> { [ IndexValue::Quota { used: self.size }, - IndexValue::Index { - field: CalendarField::Created.into(), - value: self.created.into(), - }, - IndexValue::Index { - field: CalendarField::EventId.into(), + IndexValue::Property { + field: ValueClass::IndexProperty(IndexPropertyClass::Integer { + property: CalendarNotificationField::CreatedToId.into(), + value: self.created as u64, + }), value: self.event_id.unwrap_or(u32::MAX).into(), }, IndexValue::LogItem { @@ -194,12 +148,11 @@ impl IndexableObject for &ArchivedCalendarEventNotification { IndexValue::Quota { used: self.size.to_native(), }, - IndexValue::Index { - field: CalendarField::Created.into(), - value: self.created.to_native().into(), - }, - IndexValue::Index { - field: CalendarField::EventId.into(), + IndexValue::Property { + field: ValueClass::IndexProperty(IndexPropertyClass::Integer { + property: CalendarNotificationField::CreatedToId.into(), + value: self.created.to_native() as u64, + }), value: self .event_id .as_ref() @@ -293,7 +246,7 @@ impl ArchivedDefaultAlert { } impl CalendarEvent { - pub fn text(&self) -> impl Iterator { + pub fn hashes(&self) -> impl Iterator { self.data .event .components @@ -327,15 +280,12 @@ impl CalendarEvent { _ => None, })) }) - .flat_map(|v| { - WordTokenizer::new(v.strip_prefix("mailto:").unwrap_or(v), MAX_TOKEN_LENGTH) - }) - .map(|t| t.word.into_owned()) + .map(|v| xxh3::xxh3_64(v.as_bytes())) } } impl ArchivedCalendarEvent { - pub fn text(&self) -> impl Iterator { + pub fn hashes(&self) -> impl Iterator { self.data .event .components @@ -369,9 +319,6 @@ impl ArchivedCalendarEvent { _ => None, })) }) - .flat_map(|v| { - WordTokenizer::new(v.strip_prefix("mailto:").unwrap_or(v), MAX_TOKEN_LENGTH) - }) - .map(|t| t.word.into_owned()) + .map(|v| xxh3::xxh3_64(v.as_bytes())) } } diff --git a/crates/groupware/src/calendar/itip.rs b/crates/groupware/src/calendar/itip.rs index c098481f..4448f770 100644 --- a/crates/groupware/src/calendar/itip.rs +++ b/crates/groupware/src/calendar/itip.rs @@ -34,14 +34,13 @@ use common::{ i18n, }; use store::{ - query::Filter, rand, write::{BatchBuilder, now}, }; use trc::AddContext; use types::{ collection::Collection, - field::{CalendarField, ContactField}, + field::{CalendarEventField, ContactField}, }; use utils::{template::Variables, url_params::UrlParams}; @@ -152,24 +151,20 @@ impl ItipIngest for Server { // Find event by UID let account_id = access_token.primary_id; let document_id = self - .store() - .filter( + .document_ids_matching( account_id, Collection::CalendarEvent, - vec![Filter::eq( - CalendarField::Uid, - itip_snapshots.uid.as_bytes().to_vec(), - )], + CalendarEventField::Uid, + itip_snapshots.uid.as_bytes(), ) .await .caused_by(trc::location!())? - .results .iter() .next(); if let Some(document_id) = document_id { if let Some(archive) = self - .get_archive(account_id, Collection::CalendarEvent, document_id) + .archive(account_id, Collection::CalendarEvent, document_id) .await .caused_by(trc::location!())? { @@ -280,17 +275,15 @@ impl ItipIngest for Server { // Verify that auto-adding invitations is allowed if !self.core.groupware.itip_auto_add && !matches!(changed_by, ChangedBy::PrincipalId(_)) - && self - .store() - .filter( + && !self + .document_exists( account_id, Collection::ContactCard, - vec![Filter::eq(ContactField::Email, sender.as_bytes().to_vec())], + ContactField::Email, + sender.as_bytes(), ) .await .caused_by(trc::location!())? - .results - .is_empty() { return Err(ItipIngestError::Message(ItipError::AutoAddDisabled)); } else if itip_method(&itip)? != &ICalendarMethod::Request { @@ -312,7 +305,7 @@ impl ItipIngest for Server { // Obtain parent calendar let Some(parent_id) = self - .get_or_create_default_calendar(access_token, account_id, &access_token.name) + .get_or_create_default_calendar(access_token, account_id) .await .caused_by(trc::location!())? else { @@ -410,7 +403,7 @@ impl ItipIngest for Server { async fn http_rsvp_handle(&self, query: &str, language: &str) -> trc::Result { let response = if let Some(rsvp) = decode_rsvp_response(self, query).await { if let Some(archive) = self - .get_archive(rsvp.account_id, Collection::CalendarEvent, rsvp.document_id) + .archive(rsvp.account_id, Collection::CalendarEvent, rsvp.document_id) .await .caused_by(trc::location!())? { diff --git a/crates/groupware/src/calendar/storage.rs b/crates/groupware/src/calendar/storage.rs index 333b1e51..0a45f006 100644 --- a/crates/groupware/src/calendar/storage.rs +++ b/crates/groupware/src/calendar/storage.rs @@ -29,7 +29,7 @@ use store::{ use trc::AddContext; use types::{ collection::{Collection, VanishedCollection}, - field::CalendarField, + field::CalendarNotificationField, }; pub trait ItipAutoExpunge: Sync + Send { @@ -42,7 +42,6 @@ pub trait ItipAutoExpunge: Sync + Send { impl ItipAutoExpunge for Server { async fn itip_auto_expunge(&self, account_id: u32, hold_period: u64) -> trc::Result<()> { - // Filter messages by received date let mut destroy_ids = RoaringBitmap::new(); self.store() .iterate( @@ -51,14 +50,14 @@ impl ItipAutoExpunge for Server { account_id, collection: Collection::CalendarEventNotification.into(), document_id: 0, - field: CalendarField::Created.into(), + field: CalendarNotificationField::CreatedToId.into(), key: 0u64.serialize(), }, IndexKey { account_id, collection: Collection::CalendarEventNotification.into(), document_id: u32::MAX, - field: CalendarField::Created.into(), + field: CalendarNotificationField::CreatedToId.into(), key: now().saturating_sub(hold_period).serialize(), }, ) @@ -97,7 +96,7 @@ impl ItipAutoExpunge for Server { for document_id in destroy_ids { // Fetch event if let Some(event_) = self - .get_archive( + .archive( account_id, Collection::CalendarEventNotification, document_id, @@ -138,7 +137,7 @@ impl CalendarEvent { batch .with_account_id(account_id) .with_collection(Collection::CalendarEvent) - .update_document(document_id) + .with_document(document_id) .custom( ObjectIndexBuilder::new() .with_current(event) @@ -166,7 +165,7 @@ impl CalendarEvent { batch .with_account_id(account_id) .with_collection(Collection::CalendarEvent) - .create_document(document_id) + .with_document(document_id) .custom( ObjectIndexBuilder::<(), _>::new() .with_changes(event) @@ -208,7 +207,7 @@ impl Calendar { batch .with_account_id(account_id) .with_collection(Collection::Calendar) - .create_document(document_id) + .with_document(document_id) .custom( ObjectIndexBuilder::<(), _>::new() .with_changes(calendar) @@ -233,7 +232,7 @@ impl Calendar { batch .with_account_id(account_id) .with_collection(Collection::Calendar) - .update_document(document_id) + .with_document(document_id) .custom( ObjectIndexBuilder::new() .with_current(calendar) @@ -262,7 +261,7 @@ impl CalendarEventNotification { batch .with_account_id(account_id) .with_collection(Collection::CalendarEventNotification) - .create_document(document_id) + .with_document(document_id) .custom( ObjectIndexBuilder::<(), _>::new() .with_changes(event) @@ -289,7 +288,7 @@ impl DestroyArchive> { let calendar_id = document_id; for document_id in children_ids { if let Some(event_) = server - .get_archive(account_id, Collection::CalendarEvent, document_id) + .archive(account_id, Collection::CalendarEvent, document_id) .await? { DestroyArchive( @@ -325,7 +324,7 @@ impl DestroyArchive> { batch .with_account_id(account_id) .with_collection(Collection::Calendar) - .delete_document(document_id) + .with_document(document_id) .custom( ObjectIndexBuilder::<_, ()>::new() .with_access_token(access_token) @@ -370,7 +369,7 @@ impl DestroyArchive> { batch .with_account_id(account_id) .with_collection(Collection::CalendarEvent) - .update_document(document_id) + .with_document(document_id) .custom( ObjectIndexBuilder::new() .with_access_token(access_token) @@ -406,7 +405,7 @@ impl DestroyArchive> { batch .with_account_id(account_id) .with_collection(Collection::CalendarEvent) - .delete_document(document_id); + .with_document(document_id); // Remove next alarm if it exists let now = now() as i64; @@ -457,7 +456,7 @@ impl DestroyArchive> { batch .with_account_id(account_id) .with_collection(Collection::CalendarEventNotification) - .delete_document(document_id) + .with_document(document_id) .custom( ObjectIndexBuilder::<_, ()>::new() .with_access_token(access_token) @@ -528,7 +527,7 @@ impl ArchivedCalendarEvent { ) -> trc::Result { for event_name in self.names.iter() { if let Some(calendar_) = server - .get_archive( + .archive( access_token.primary_id, Collection::Calendar, event_name.parent_id.to_native(), diff --git a/crates/groupware/src/contact/index.rs b/crates/groupware/src/contact/index.rs index a03d5e4e..19942f5b 100644 --- a/crates/groupware/src/contact/index.rs +++ b/crates/groupware/src/contact/index.rs @@ -6,12 +6,8 @@ use super::{AddressBook, ArchivedAddressBook, ArchivedContactCard, ContactCard}; use calcard::vcard::{ArchivedVCardProperty, VCardProperty}; -use common::storage::index::{ - IndexItem, IndexValue, IndexableAndSerializableObject, IndexableObject, -}; -use nlp::tokenizers::word::WordTokenizer; -use std::collections::HashSet; -use store::backend::MAX_TOKEN_LENGTH; +use common::storage::index::{IndexValue, IndexableAndSerializableObject, IndexableObject}; +use store::xxhash_rust::xxh3; use types::{acl::AclGrant, collection::SyncCollection, field::ContactField}; use utils::sanitize_email; @@ -86,30 +82,11 @@ impl IndexableObject for ContactCard { value: self.card.uid().into(), }, IndexValue::Index { - field: ContactField::Created.into(), - value: self.created.into(), - }, - IndexValue::Index { - field: ContactField::Updated.into(), - value: self.modified.into(), - }, - IndexValue::IndexList { - field: ContactField::Text.into(), - value: self - .text() - .map(Into::into) - .collect::>() - .into_iter() - .collect(), - }, - IndexValue::IndexList { field: ContactField::Email.into(), - value: self - .emails() - .map(Into::into) - .collect::>() - .into_iter() - .collect(), + value: self.emails().next().into(), + }, + IndexValue::SearchIndex { + hashes: self.hashes().collect(), }, IndexValue::Quota { used: self.dead_properties.size() as u32 @@ -134,30 +111,11 @@ impl IndexableObject for &ArchivedContactCard { value: self.card.uid().into(), }, IndexValue::Index { - field: ContactField::Created.into(), - value: self.created.to_native().into(), - }, - IndexValue::Index { - field: ContactField::Updated.into(), - value: self.modified.to_native().into(), - }, - IndexValue::IndexList { - field: ContactField::Text.into(), - value: self - .text() - .map(Into::into) - .collect::>() - .into_iter() - .collect(), - }, - IndexValue::IndexList { field: ContactField::Email.into(), - value: self - .emails() - .map(Into::into) - .collect::>() - .into_iter() - .collect(), + value: self.emails().next().into(), + }, + IndexValue::SearchIndex { + hashes: self.hashes().collect(), }, IndexValue::Quota { used: self.dead_properties.size() as u32 @@ -181,7 +139,7 @@ impl IndexableAndSerializableObject for ContactCard { } impl ContactCard { - pub fn text(&self) -> impl Iterator { + pub fn hashes(&self) -> impl Iterator { self.card .entries .iter() @@ -195,11 +153,11 @@ impl ContactCard { | VCardProperty::Org | VCardProperty::Note | VCardProperty::Nickname + | VCardProperty::Email ) }) .flat_map(|e| e.values.iter().filter_map(|v| v.as_text())) - .flat_map(|v| WordTokenizer::new(v, MAX_TOKEN_LENGTH)) - .map(|t| t.word.into_owned()) + .map(|v| xxh3::xxh3_64(v.as_bytes())) } pub fn emails(&self) -> impl Iterator { @@ -212,7 +170,7 @@ impl ContactCard { } impl ArchivedContactCard { - pub fn text(&self) -> impl Iterator { + pub fn hashes(&self) -> impl Iterator { self.card .entries .iter() @@ -226,11 +184,11 @@ impl ArchivedContactCard { | ArchivedVCardProperty::Org | ArchivedVCardProperty::Note | ArchivedVCardProperty::Nickname + | ArchivedVCardProperty::Email ) }) .flat_map(|e| e.values.iter().filter_map(|v| v.as_text())) - .flat_map(|v| WordTokenizer::new(v, MAX_TOKEN_LENGTH)) - .map(|t| t.word.into_owned()) + .map(|v| xxh3::xxh3_64(v.as_bytes())) } pub fn emails(&self) -> impl Iterator { diff --git a/crates/groupware/src/contact/storage.rs b/crates/groupware/src/contact/storage.rs index ceabe792..9a2c821f 100644 --- a/crates/groupware/src/contact/storage.rs +++ b/crates/groupware/src/contact/storage.rs @@ -29,7 +29,7 @@ impl ContactCard { batch .with_account_id(account_id) .with_collection(Collection::ContactCard) - .update_document(document_id) + .with_document(document_id) .custom( ObjectIndexBuilder::new() .with_current(card) @@ -56,7 +56,7 @@ impl ContactCard { batch .with_account_id(account_id) .with_collection(Collection::ContactCard) - .create_document(document_id) + .with_document(document_id) .custom( ObjectIndexBuilder::<(), _>::new() .with_changes(card) @@ -84,7 +84,7 @@ impl AddressBook { batch .with_account_id(account_id) .with_collection(Collection::AddressBook) - .create_document(document_id) + .with_document(document_id) .custom( ObjectIndexBuilder::<(), _>::new() .with_changes(book) @@ -109,7 +109,7 @@ impl AddressBook { batch .with_account_id(account_id) .with_collection(Collection::AddressBook) - .update_document(document_id) + .with_document(document_id) .custom( ObjectIndexBuilder::new() .with_current(book) @@ -136,7 +136,7 @@ impl DestroyArchive> { let addressbook_id = document_id; for document_id in children_ids { if let Some(card_) = server - .get_archive(account_id, Collection::ContactCard, document_id) + .archive(account_id, Collection::ContactCard, document_id) .await? { DestroyArchive( @@ -171,7 +171,7 @@ impl DestroyArchive> { batch .with_account_id(account_id) .with_collection(Collection::AddressBook) - .delete_document(document_id) + .with_document(document_id) .custom( ObjectIndexBuilder::<_, ()>::new() .with_access_token(access_token) @@ -217,7 +217,7 @@ impl DestroyArchive> { .caused_by(trc::location!())?; new_card.names.swap_remove(delete_idx); batch - .update_document(document_id) + .with_document(document_id) .custom( ObjectIndexBuilder::new() .with_access_token(access_token) @@ -228,7 +228,7 @@ impl DestroyArchive> { } else { // Delete card batch - .delete_document(document_id) + .with_document(document_id) .custom( ObjectIndexBuilder::<_, ()>::new() .with_access_token(access_token) @@ -257,7 +257,7 @@ impl DestroyArchive> { batch .with_account_id(account_id) .with_collection(Collection::ContactCard) - .delete_document(document_id) + .with_document(document_id) .custom( ObjectIndexBuilder::<_, ()>::new() .with_access_token(access_token) diff --git a/crates/groupware/src/file/storage.rs b/crates/groupware/src/file/storage.rs index 58b5b1b1..b18294c1 100644 --- a/crates/groupware/src/file/storage.rs +++ b/crates/groupware/src/file/storage.rs @@ -29,7 +29,7 @@ impl FileNode { batch .with_account_id(account_id) .with_collection(Collection::FileNode) - .create_document(document_id) + .with_document(document_id) .custom( ObjectIndexBuilder::<(), _>::new() .with_changes(node) @@ -51,7 +51,7 @@ impl FileNode { batch .with_account_id(account_id) .with_collection(Collection::FileNode) - .update_document(document_id) + .with_document(document_id) .custom( ObjectIndexBuilder::new() .with_current(node) @@ -75,7 +75,7 @@ impl DestroyArchive> { batch .with_account_id(account_id) .with_collection(Collection::FileNode) - .delete_document(document_id) + .with_document(document_id) .custom( ObjectIndexBuilder::<_, ()>::new() .with_current(self.0) @@ -124,12 +124,12 @@ impl DestroyArchive> { .with_collection(Collection::FileNode); for document_id in self.0 { if let Some(node) = server - .get_archive(account_id, Collection::FileNode, document_id) + .archive(account_id, Collection::FileNode, document_id) .await? { // Delete record batch - .delete_document(document_id) + .with_document(document_id) .custom( ObjectIndexBuilder::<_, ()>::new() .with_access_token(access_token) diff --git a/crates/http/src/management/crypto.rs b/crates/http/src/management/crypto.rs index 4c5e4c97..3feae081 100644 --- a/crates/http/src/management/crypto.rs +++ b/crates/http/src/management/crypto.rs @@ -38,7 +38,7 @@ pub trait CryptoHandler: Sync + Send { impl CryptoHandler for Server { async fn handle_crypto_get(&self, access_token: Arc) -> trc::Result { let ec = if let Some(params_) = self - .get_archive_by_property( + .archive_by_property( access_token.primary_id(), Collection::Principal, 0, @@ -94,7 +94,7 @@ impl CryptoHandler for Server { batch .with_account_id(access_token.primary_id()) .with_collection(Collection::Principal) - .update_document(0) + .with_document(0) .clear(PrincipalField::EncryptionKeys); self.core.storage.data.write(batch.build_all()).await?; return Ok(JsonResponse::new(json!({ @@ -144,7 +144,7 @@ impl CryptoHandler for Server { batch .with_account_id(access_token.primary_id()) .with_collection(Collection::Principal) - .update_document(0) + .with_document(0) .set(PrincipalField::EncryptionKeys, params); self.core.storage.data.write(batch.build_all()).await?; diff --git a/crates/http/src/management/stores.rs b/crates/http/src/management/stores.rs index cd77cd6d..9f6eb6b5 100644 --- a/crates/http/src/management/stores.rs +++ b/crates/http/src/management/stores.rs @@ -16,7 +16,10 @@ use directory::{ Permission, backend::internal::manage::{self, ManageDirectory}, }; -use email::message::{ingest::EmailIngest, metadata::MessageData}; +use email::{ + cache::MessageCacheFetch, + message::{ingest::EmailIngest, metadata::MessageData}, +}; use http_proto::{request::decode_path_element, *}; use hyper::Method; use serde_json::json; @@ -335,13 +338,14 @@ pub async fn reset_imap_uids(server: &Server, account_id: u32) -> trc::Result<(u let mut mailbox_count = 0; let mut email_count = 0; - for mailbox_id in server - .get_document_ids(account_id, Collection::Mailbox) - .await? - .unwrap_or_default() - { + let cache = server + .get_cached_messages(account_id) + .await + .caused_by(trc::location!())?; + + for &mailbox_id in cache.mailboxes.index.keys() { let mailbox = server - .get_archive(account_id, Collection::Mailbox, mailbox_id) + .archive(account_id, Collection::Mailbox, mailbox_id) .await .caused_by(trc::location!())? .ok_or_else(|| trc::ImapEvent::Error.into_err().caused_by(trc::location!()))? @@ -353,7 +357,7 @@ pub async fn reset_imap_uids(server: &Server, account_id: u32) -> trc::Result<(u batch .with_account_id(account_id) .with_collection(Collection::Mailbox) - .update_document(mailbox_id) + .with_document(mailbox_id) .custom( ObjectIndexBuilder::new() .with_current(mailbox) @@ -370,14 +374,9 @@ pub async fn reset_imap_uids(server: &Server, account_id: u32) -> trc::Result<(u } // Reset all UIDs - for message_id in server - .get_document_ids(account_id, Collection::Email) - .await - .caused_by(trc::location!())? - .unwrap_or_default() - { + for message_id in cache.emails.items.iter().map(|i| i.document_id) { let data = server - .get_archive(account_id, Collection::Email, message_id) + .archive(account_id, Collection::Email, message_id) .await .caused_by(trc::location!())?; let data_ = if let Some(data) = data { @@ -404,7 +403,7 @@ pub async fn reset_imap_uids(server: &Server, account_id: u32) -> trc::Result<(u batch .with_account_id(account_id) .with_collection(Collection::Email) - .update_document(message_id) + .with_document(message_id) .assert_value(ValueClass::Property(EmailField::Archive.into()), &data) .set( EmailField::Archive, diff --git a/crates/imap-proto/Cargo.toml b/crates/imap-proto/Cargo.toml index c1cbba08..20fb96d6 100644 --- a/crates/imap-proto/Cargo.toml +++ b/crates/imap-proto/Cargo.toml @@ -6,7 +6,6 @@ resolver = "2" [dependencies] types = { path = "../types" } -store = { path = "../store" } mail-parser = { version = "0.11", features = ["full_encoding", "rkyv"] } ahash = { version = "0.8" } chrono = { version = "0.4"} diff --git a/crates/imap-proto/src/protocol/search.rs b/crates/imap-proto/src/protocol/search.rs index c3586eab..5cdc71a6 100644 --- a/crates/imap-proto/src/protocol/search.rs +++ b/crates/imap-proto/src/protocol/search.rs @@ -4,8 +4,6 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use store::fts::{FilterItem, FilterType}; - use super::{Flag, Sequence, quoted_string, serialize_sequence}; #[derive(Debug, Clone, PartialEq, Eq)] @@ -114,38 +112,6 @@ pub enum Filter { ThreadId(String), } -impl FilterItem for Filter { - fn filter_type(&self) -> FilterType { - match self { - Filter::From(_) - | Filter::To(_) - | Filter::Cc(_) - | Filter::Bcc(_) - | Filter::Subject(_) - | Filter::Body(_) - | Filter::Text(_) - | Filter::Header(_, _) => FilterType::Fts, - Filter::And => FilterType::And, - Filter::Or => FilterType::Or, - Filter::Not => FilterType::Not, - Filter::End => FilterType::End, - _ => FilterType::Store, - } - } -} - -impl From for Filter { - fn from(value: FilterType) -> Self { - match value { - FilterType::And => Filter::And, - FilterType::Or => Filter::Or, - FilterType::Not => Filter::Not, - FilterType::End => Filter::End, - _ => unreachable!(), - } - } -} - #[derive(Debug, Clone, PartialEq, Eq)] pub enum ModSeqEntry { Shared(Flag), diff --git a/crates/imap/src/core/mailbox.rs b/crates/imap/src/core/mailbox.rs index e8431354..002414a1 100644 --- a/crates/imap/src/core/mailbox.rs +++ b/crates/imap/src/core/mailbox.rs @@ -396,7 +396,7 @@ impl SessionData { Ok(access_token.is_member(account_id) || self .server - .get_archive(account_id, Collection::Mailbox, document_id) + .archive(account_id, Collection::Mailbox, document_id) .await .and_then(|mailbox| { if let Some(mailbox) = mailbox { diff --git a/crates/imap/src/op/acl.rs b/crates/imap/src/op/acl.rs index 2fab8750..f35a5c1f 100644 --- a/crates/imap/src/op/acl.rs +++ b/crates/imap/src/op/acl.rs @@ -359,7 +359,7 @@ impl Session { batch .with_account_id(mailbox_id.account_id) .with_collection(Collection::Mailbox) - .update_document(mailbox_id.mailbox_id) + .with_document(mailbox_id.mailbox_id) .custom( ObjectIndexBuilder::new() .with_changes(mailbox) @@ -460,7 +460,7 @@ impl SessionData { if let Some(mailbox) = self.get_mailbox_by_name(&arguments.mailbox_name) { if let Some(values) = self .server - .get_archive(mailbox.account_id, Collection::Mailbox, mailbox.mailbox_id) + .archive(mailbox.account_id, Collection::Mailbox, mailbox.mailbox_id) .await .caused_by(trc::location!())? { diff --git a/crates/imap/src/op/copy_move.rs b/crates/imap/src/op/copy_move.rs index 50f34b2e..37e6d458 100644 --- a/crates/imap/src/op/copy_move.rs +++ b/crates/imap/src/op/copy_move.rs @@ -15,7 +15,6 @@ use email::{ cache::{MessageCacheFetch, email::MessageCacheAccess}, mailbox::{JUNK_ID, TRASH_ID, UidMailbox}, message::{ - bayes::EmailBayesTrain, copy::{CopyMessageError, EmailCopy}, ingest::EmailIngest, metadata::MessageData, @@ -28,7 +27,7 @@ use imap_proto::{ use std::{sync::Arc, time::Instant}; use store::{ roaring::RoaringBitmap, - write::{AlignedBytes, Archive, BatchBuilder, ValueClass}, + write::{AlignedBytes, Archive, BatchBuilder, TaskQueueClass, ValueClass, now}, }; use types::{ acl::Acl, @@ -252,7 +251,7 @@ impl SessionData { batch .with_account_id(account_id) .with_collection(Collection::Email) - .update_document(id) + .with_document(id) .custom( ObjectIndexBuilder::new() .with_current(data) @@ -299,7 +298,7 @@ impl SessionData { batch .with_account_id(account_id) .with_collection(Collection::Email) - .update_document(id) + .with_document(id) .custom( ObjectIndexBuilder::new() .with_current(data) @@ -317,12 +316,10 @@ impl SessionData { if can_spam_train { if dest_mailbox_id.mailbox_id == JUNK_ID { batch.set( - ValueClass::TaskQueue( - self.server - .email_bayes_queue_task_build(account_id, id, true) - .await - .imap_ctx(&arguments.tag, trc::location!())?, - ), + ValueClass::TaskQueue(TaskQueueClass::BayesTrain { + due: now(), + learn_spam: true, + }), vec![], ); has_spam_train_tasks = true; @@ -330,12 +327,10 @@ impl SessionData { && dest_mailbox_id.mailbox_id != TRASH_ID { batch.set( - ValueClass::TaskQueue( - self.server - .email_bayes_queue_task_build(account_id, id, false) - .await - .imap_ctx(&arguments.tag, trc::location!())?, - ), + ValueClass::TaskQueue(TaskQueueClass::BayesTrain { + due: now(), + learn_spam: false, + }), vec![], ); has_spam_train_tasks = true; @@ -562,7 +557,7 @@ impl SessionData { ) -> trc::Result>> { if let Some(data) = self .server - .get_archive(account_id, Collection::Email, id) + .archive(account_id, Collection::Email, id) .await? { Ok(Some(data)) diff --git a/crates/imap/src/op/create.rs b/crates/imap/src/op/create.rs index dbf46c77..f9f7951c 100644 --- a/crates/imap/src/op/create.rs +++ b/crates/imap/src/op/create.rs @@ -93,7 +93,7 @@ impl SessionData { batch .with_account_id(params.account_id) .with_collection(Collection::Mailbox) - .create_document(mailbox_id) + .with_document(mailbox_id) .custom(ObjectIndexBuilder::<(), _>::new().with_changes(mailbox)) .imap_ctx(&arguments.tag, trc::location!())? .commit_point(); diff --git a/crates/imap/src/op/expunge.rs b/crates/imap/src/op/expunge.rs index 8c3c2883..20f62913 100644 --- a/crates/imap/src/op/expunge.rs +++ b/crates/imap/src/op/expunge.rs @@ -11,7 +11,6 @@ use common::{listener::SessionStream, storage::index::ObjectIndexBuilder}; use directory::Permission; use email::{ cache::{MessageCacheFetch, email::MessageCacheAccess}, - mailbox::TOMBSTONE_ID, message::metadata::MessageData, }; use imap_proto::{ @@ -21,14 +20,14 @@ use imap_proto::{ }; use std::{sync::Arc, time::Instant}; use store::{ + SerializeInfallible, roaring::RoaringBitmap, - write::{BatchBuilder, TagValue}, + write::{BatchBuilder, TaskQueueClass, ValueClass, now}, }; use trc::AddContext; use types::{ acl::Acl, collection::{Collection, VanishedCollection}, - field::EmailField, keyword::Keyword, }; @@ -170,9 +169,10 @@ impl SessionData { batch .with_account_id(account_id) .with_collection(Collection::Email); + let due = now(); self.server - .get_archives( + .archives( account_id, Collection::Email, deleted_ids, @@ -183,7 +183,7 @@ impl SessionData { if let Some(message_uid) = metadata.inner.message_uid(mailbox_id) { // Add vanished items - batch.update_document(document_id); + batch.with_document(document_id); batch.log_vanished_item( VanishedCollection::Email, (mailbox_id, message_uid), @@ -194,7 +194,10 @@ impl SessionData { batch .custom(ObjectIndexBuilder::<_, ()>::new().with_current(metadata)) .caused_by(trc::location!())? - .tag(EmailField::MailboxIds, TagValue::Id(TOMBSTONE_ID)) + .set( + ValueClass::TaskQueue(TaskQueueClass::UnindexEmail { due }), + 0u64.serialize(), + ) .commit_point(); } else { // Untag message from this mailbox and remove Deleted flag diff --git a/crates/imap/src/op/fetch.rs b/crates/imap/src/op/fetch.rs index 2f02bf71..89926c5a 100644 --- a/crates/imap/src/op/fetch.rs +++ b/crates/imap/src/op/fetch.rs @@ -324,7 +324,7 @@ impl SessionData { // Obtain attributes and keywords let (metadata_, data) = if let (Some(email), Some(data)) = ( self.server - .get_archive_by_property( + .archive_by_property( account_id, Collection::Email, id, @@ -543,7 +543,7 @@ impl SessionData { if set_seen_flag && let Some(data_) = self .server - .get_archive(account_id, Collection::Email, id) + .archive(account_id, Collection::Email, id) .await .imap_ctx(&arguments.tag, trc::location!())? { @@ -558,7 +558,7 @@ impl SessionData { batch .with_account_id(account_id) .with_collection(Collection::Email) - .update_document(id) + .with_document(id) .custom( ObjectIndexBuilder::new() .with_current(data) diff --git a/crates/imap/src/op/rename.rs b/crates/imap/src/op/rename.rs index 45154ac8..daa4550d 100644 --- a/crates/imap/src/op/rename.rs +++ b/crates/imap/src/op/rename.rs @@ -85,7 +85,7 @@ impl SessionData { // Obtain mailbox let mailbox_ = self .server - .get_archive(params.account_id, Collection::Mailbox, mailbox_id) + .archive(params.account_id, Collection::Mailbox, mailbox_id) .await .imap_ctx(&arguments.tag, trc::location!())? .ok_or_else(|| { @@ -144,7 +144,7 @@ impl SessionData { batch .with_account_id(params.account_id) .with_collection(Collection::Mailbox) - .create_document(mailbox_id) + .with_document(mailbox_id) .custom(ObjectIndexBuilder::<(), _>::new().with_changes( email::mailbox::Mailbox::new(path_item).with_parent_id(parent_id), )) @@ -164,7 +164,7 @@ impl SessionData { batch .with_account_id(params.account_id) .with_collection(Collection::Mailbox) - .update_document(mailbox_id) + .with_document(mailbox_id) .custom( ObjectIndexBuilder::new() .with_current(mailbox) diff --git a/crates/imap/src/op/search.rs b/crates/imap/src/op/search.rs index 8c28e8ef..fc9f28ab 100644 --- a/crates/imap/src/op/search.rs +++ b/crates/imap/src/op/search.rs @@ -16,25 +16,23 @@ use imap_proto::{ Command, StatusResponse, protocol::{ Sequence, - search::{self, Arguments, Filter, Response, ResultOption}, + search::{self, Arguments, Comparator, Filter, Response, ResultOption}, }, receiver::Request, }; use mail_parser::HeaderName; use nlp::language::Language; -use std::{str::FromStr, sync::Arc, time::Instant}; +use std::{borrow::Cow, str::FromStr, sync::Arc, time::Instant}; use store::{ - SerializeInfallible, - fts::{Field, FilterGroup, FtsFilter, IntoFilterGroup}, - query::{self, ResultSet, log::Query, sort::Pagination}, + query::log::Query, roaring::RoaringBitmap, + search::{EmailSearchField, SearchComparator, SearchFilter}, write::now, }; use tokio::sync::watch; use trc::AddContext; use types::{ collection::{Collection, SyncCollection}, - field::EmailField, id::Id, keyword::Keyword, }; @@ -119,8 +117,14 @@ impl SessionData { op_start: Instant, ) -> trc::Result { // Run query + let is_sort = arguments.sort.is_some(); let (result_set, include_highest_modseq) = self - .query(arguments.filter, &mailbox, &prev_saved_search) + .query( + arguments.filter, + arguments.sort.unwrap_or_default(), + &mailbox, + &prev_saved_search, + ) .await?; // Obtain modseq @@ -137,78 +141,25 @@ impl SessionData { let mut min: Option<(u32, ImapId)> = None; let mut max: Option<(u32, ImapId)> = None; let mut total = 0; - let results_len = result_set.results.len() as usize; + let results_len = result_set.len(); let mut saved_results = if results_tx.is_some() { Some(Vec::with_capacity(results_len)) } else { None }; let mut imap_ids = Vec::with_capacity(results_len); - let is_sort = if let Some(sort) = arguments.sort { - mailbox.map_search_results( - self.server - .core - .storage - .data - .sort( - result_set, - sort.into_iter() - .map(|item| match item.sort { - search::Sort::Arrival => { - query::Comparator::field(EmailField::ReceivedAt, item.ascending) - } - search::Sort::Cc => { - query::Comparator::field(EmailField::Cc, item.ascending) - } - search::Sort::Date => { - query::Comparator::field(EmailField::SentAt, item.ascending) - } - search::Sort::From | search::Sort::DisplayFrom => { - query::Comparator::field(EmailField::From, item.ascending) - } - search::Sort::Size => { - query::Comparator::field(EmailField::Size, item.ascending) - } - search::Sort::Subject => { - query::Comparator::field(EmailField::Subject, item.ascending) - } - search::Sort::To | search::Sort::DisplayTo => { - query::Comparator::field(EmailField::To, item.ascending) - } - }) - .collect::>(), - Pagination::new(results_len, 0, None, 0), - ) - .await - .caused_by(trc::location!())? - .ids - .into_iter() - .map(|id| id.document_id()), - is_uid, - arguments.result_options.contains(&ResultOption::Min), - arguments.result_options.contains(&ResultOption::Max), - &mut min, - &mut max, - &mut total, - &mut imap_ids, - &mut saved_results, - ); - true - } else { - mailbox.map_search_results( - result_set.results.into_iter(), - is_uid, - arguments.result_options.contains(&ResultOption::Min), - arguments.result_options.contains(&ResultOption::Max), - &mut min, - &mut max, - &mut total, - &mut imap_ids, - &mut saved_results, - ); - imap_ids.sort_unstable(); - false - }; + mailbox.map_search_results( + result_set.into_iter(), + is_uid, + arguments.result_options.contains(&ResultOption::Min), + arguments.result_options.contains(&ResultOption::Max), + &mut min, + &mut max, + &mut total, + &mut imap_ids, + &mut saved_results, + ); + imap_ids.sort_unstable(); // Save results if let (Some(results_tx), Some(saved_results)) = (results_tx, saved_results) { @@ -258,9 +209,10 @@ impl SessionData { pub async fn query( &self, imap_filter: Vec, + imap_comparator: Vec, mailbox: &SelectedMailbox, prev_saved_search: &Option>>>, - ) -> trc::Result<(ResultSet, bool)> { + ) -> trc::Result<(Vec, bool)> { // Obtain message ids let mut filters = Vec::with_capacity(imap_filter.len() + 1); let cache = self @@ -274,412 +226,379 @@ impl SessionData { .map(|m| m.document_id), ); - filters.push(query::Filter::is_in_set(message_ids.clone())); + filters.push(SearchFilter::is_in_set(message_ids.clone())); // Convert query let mut include_highest_modseq = false; - for filter_group in imap_filter.into_filter_group() { - match filter_group { - FilterGroup::Fts(conds) => { - let mut fts_filters = Vec::with_capacity(filters.len()); - for cond in conds { - match cond { - search::Filter::Bcc(text) => { - fts_filters.push(FtsFilter::has_text( - Field::Header(HeaderName::Bcc), - text, - Language::None, - )); - } - search::Filter::Body(text) => { - fts_filters.push(FtsFilter::has_text_detect( - Field::Body, - text, - self.server.core.jmap.default_language, - )); - } - search::Filter::Cc(text) => { - fts_filters.push(FtsFilter::has_text( - Field::Header(HeaderName::Cc), - text, - Language::None, - )); - } - search::Filter::From(text) => { - fts_filters.push(FtsFilter::has_text( - Field::Header(HeaderName::From), - text, - Language::None, - )); - } - search::Filter::Header(header, value) => { - match HeaderName::parse(header) { - Some(HeaderName::Other(header_name)) => { - return Err(trc::ImapEvent::Error.into_err().details( - format!( - "Querying header '{header_name}' is not supported.", - ), - )); - } - Some(header_name) => { - if !value.is_empty() { - if matches!( - header_name, - HeaderName::MessageId - | HeaderName::InReplyTo - | HeaderName::References - | HeaderName::ResentMessageId - ) { - fts_filters.push(FtsFilter::has_keyword( - Field::Header(header_name), - value, - )); - } else { - fts_filters.push(FtsFilter::has_text( - Field::Header(header_name), - value, - Language::None, - )); - } - } else { - fts_filters.push(FtsFilter::has_keyword( - Field::Keyword, - header_name.as_str().to_lowercase(), - )); - } - } - None => (), + for filter in imap_filter { + match filter { + Filter::Sequence(sequence, uid_filter) => { + let mut set = RoaringBitmap::new(); + if let (Sequence::SavedSearch, Some(prev_saved_search)) = + (&sequence, &prev_saved_search) + { + if let Some(prev_saved_search) = prev_saved_search { + let state = mailbox.state.lock(); + for imap_id in prev_saved_search.iter() { + if let Some(id) = state.uid_to_id.get(&imap_id.uid) { + set.insert(*id); } } - search::Filter::Subject(text) => { - fts_filters.push(FtsFilter::has_text_detect( - Field::Header(HeaderName::Subject), - text, - self.server.core.jmap.default_language, - )); - } - search::Filter::Text(text) => { - fts_filters.push(FtsFilter::Or); - fts_filters.push(FtsFilter::has_text( - Field::Header(HeaderName::From), - text.as_str(), - Language::None, - )); - fts_filters.push(FtsFilter::has_text( - Field::Header(HeaderName::To), - text.as_str(), - Language::None, - )); - fts_filters.push(FtsFilter::has_text( - Field::Header(HeaderName::Cc), - text.as_str(), - Language::None, - )); - fts_filters.push(FtsFilter::has_text( - Field::Header(HeaderName::Bcc), - text.as_str(), - Language::None, - )); - fts_filters.push(FtsFilter::has_text_detect( - Field::Header(HeaderName::Subject), - text.as_str(), - self.server.core.jmap.default_language, - )); - fts_filters.push(FtsFilter::has_text_detect( - Field::Body, - text.as_str(), - self.server.core.jmap.default_language, - )); - fts_filters.push(FtsFilter::has_text_detect( - Field::Attachment, - text, - self.server.core.jmap.default_language, - )); - fts_filters.push(FtsFilter::End); - } - search::Filter::To(text) => { - fts_filters.push(FtsFilter::has_text( - Field::Header(HeaderName::To), - text, - Language::None, - )); - } - search::Filter::And => { - fts_filters.push(FtsFilter::And); - } - search::Filter::Or => { - fts_filters.push(FtsFilter::Or); - } - search::Filter::Not => { - fts_filters.push(FtsFilter::Not); - } - search::Filter::End => { - fts_filters.push(FtsFilter::End); - } - _ => (), + } else { + return Err(trc::ImapEvent::Error + .into_err() + .details("No saved search found.")); + } + } else { + for id in mailbox.sequence_to_ids(&sequence, uid_filter).await?.keys() { + set.insert(*id); } } - - filters.push(query::Filter::is_in_set( - self.server - .fts_store() - .query(mailbox.id.account_id, Collection::Email, fts_filters) - .await?, + filters.push(SearchFilter::is_in_set(set)); + } + Filter::All => { + filters.push(SearchFilter::is_in_set(message_ids.clone())); + } + Filter::Answered => { + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cache + .with_keyword(&Keyword::Answered) + .map(|m| m.document_id), + ))); + } + Filter::Before(date) => { + filters.push(SearchFilter::lt(EmailSearchField::ReceivedAt, date)); + } + Filter::Deleted => { + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cache.with_keyword(&Keyword::Deleted).map(|m| m.document_id), + ))); + } + Filter::Draft => { + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cache.with_keyword(&Keyword::Draft).map(|m| m.document_id), + ))); + } + Filter::Flagged => { + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cache.with_keyword(&Keyword::Flagged).map(|m| m.document_id), + ))); + } + Filter::Keyword(keyword) => { + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cache + .with_keyword(&Keyword::from(keyword)) + .map(|m| m.document_id), + ))); + } + Filter::Larger(size) => { + filters.push(SearchFilter::gt(EmailSearchField::Size, size)); + } + Filter::On(date) => { + filters.push(SearchFilter::And); + filters.push(SearchFilter::ge(EmailSearchField::ReceivedAt, date)); + filters.push(SearchFilter::lt(EmailSearchField::ReceivedAt, date + 86400)); + filters.push(SearchFilter::End); + } + Filter::Seen => { + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cache.with_keyword(&Keyword::Seen).map(|m| m.document_id), + ))); + } + Filter::SentBefore(date) => { + filters.push(SearchFilter::lt(EmailSearchField::SentAt, date)); + } + Filter::SentOn(date) => { + filters.push(SearchFilter::And); + filters.push(SearchFilter::ge(EmailSearchField::SentAt, date)); + filters.push(SearchFilter::lt(EmailSearchField::SentAt, date + 86400)); + filters.push(SearchFilter::End); + } + Filter::SentSince(date) => { + filters.push(SearchFilter::ge(EmailSearchField::SentAt, date)); + } + Filter::Since(date) => { + filters.push(SearchFilter::ge(EmailSearchField::ReceivedAt, date)); + } + Filter::Smaller(size) => { + filters.push(SearchFilter::lt(EmailSearchField::Size, size)); + } + Filter::Unanswered => { + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cache + .without_keyword(&Keyword::Answered) + .map(|m| m.document_id), + ))); + } + Filter::Undeleted => { + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cache + .without_keyword(&Keyword::Deleted) + .map(|m| m.document_id), + ))); + } + Filter::Undraft => { + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cache + .without_keyword(&Keyword::Draft) + .map(|m| m.document_id), + ))); + } + Filter::Unflagged => { + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cache + .without_keyword(&Keyword::Flagged) + .map(|m| m.document_id), + ))); + } + Filter::Unkeyword(keyword) => { + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cache + .without_keyword(&Keyword::from(keyword)) + .map(|m| m.document_id), + ))); + } + Filter::Unseen => { + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cache.without_keyword(&Keyword::Seen).map(|m| m.document_id), + ))); + } + Filter::Recent => { + //filters.push(SearchFilter::is_in_set(self.get_recent(&mailbox.id))); + } + Filter::New => { + /*filters.push(SearchFilter::And); + filters.push(SearchFilter::is_in_set(self.get_recent(&mailbox.id))); + filters.push(SearchFilter::Not); + filters.push(SearchFilter::is_in_bitmap( + EmailSearchField::Keywords, + Keyword::Seen, + )); + filters.push(SearchFilter::End); + filters.push(SearchFilter::End);*/ + } + Filter::Old => { + /*filters.push(SearchFilter::Not); + filters.push(SearchFilter::is_in_set(self.get_recent(&mailbox.id))); + filters.push(SearchFilter::End);*/ + } + Filter::Older(secs) => { + filters.push(SearchFilter::le( + EmailSearchField::ReceivedAt, + now().saturating_sub(secs as u64), )); } - FilterGroup::Store(cond) => match cond { - search::Filter::Sequence(sequence, uid_filter) => { - let mut set = RoaringBitmap::new(); - if let (Sequence::SavedSearch, Some(prev_saved_search)) = - (&sequence, &prev_saved_search) - { - if let Some(prev_saved_search) = prev_saved_search { - let state = mailbox.state.lock(); - for imap_id in prev_saved_search.iter() { - if let Some(id) = state.uid_to_id.get(&imap_id.uid) { - set.insert(*id); - } - } + Filter::Younger(secs) => { + filters.push(SearchFilter::ge( + EmailSearchField::ReceivedAt, + now().saturating_sub(secs as u64), + )); + } + Filter::ModSeq((modseq, _)) => { + let mut set = RoaringBitmap::new(); + for id in self + .server + .store() + .changes( + mailbox.id.account_id, + SyncCollection::Email.into(), + Query::from_modseq(modseq), + ) + .await? + .changes + .into_iter() + .filter_map(|change| change.try_unwrap_item_id()) + { + let id = (id & u32::MAX as u64) as u32; + if message_ids.contains(id) { + set.insert(id); + } + } + filters.push(SearchFilter::is_in_set(set)); + include_highest_modseq = true; + } + Filter::EmailId(id) => { + if let Ok(id) = Id::from_str(&id) { + filters.push(SearchFilter::is_in_set( + RoaringBitmap::from_sorted_iter([id.document_id()]).unwrap(), + )); + } else { + return Err(trc::ImapEvent::Error + .into_err() + .details(format!("Failed to parse email id '{id}'.",))); + } + } + Filter::ThreadId(id) => { + if let Ok(id) = Id::from_str(&id) { + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cache.in_thread(id.document_id()).map(|m| m.document_id), + ))); + } else { + return Err(trc::ImapEvent::Error + .into_err() + .details(format!("Failed to parse thread id '{id}'.",))); + } + } + Filter::Bcc(text) => { + filters.push(SearchFilter::has_text( + EmailSearchField::Bcc, + text, + Language::None, + )); + } + Filter::Body(text) => { + filters.push(SearchFilter::has_text_detect( + EmailSearchField::Body, + text, + self.server.core.jmap.default_language, + )); + } + Filter::Cc(text) => { + filters.push(SearchFilter::has_text( + EmailSearchField::Cc, + text, + Language::None, + )); + } + Filter::From(text) => { + filters.push(SearchFilter::has_text( + EmailSearchField::From, + text, + Language::None, + )); + } + Filter::Header(header, value) => { + if let Some(header) = HeaderName::parse(header) { + let is_id = matches!( + header, + HeaderName::MessageId + | HeaderName::InReplyTo + | HeaderName::References + | HeaderName::ResentMessageId + ); + let header = match header { + HeaderName::Other(value) => { + EmailSearchField::Header(Cow::Owned(value.to_ascii_lowercase())) + } + _ => EmailSearchField::Header(Cow::Borrowed(header.as_static_str())), + }; + + if !value.is_empty() { + if is_id { + filters.push(SearchFilter::eq(header, value)); } else { - return Err(trc::ImapEvent::Error - .into_err() - .details("No saved search found.")); + filters.push(SearchFilter::has_text(header, value, Language::None)); } } else { - for id in mailbox.sequence_to_ids(&sequence, uid_filter).await?.keys() { - set.insert(*id); - } - } - filters.push(query::Filter::is_in_set(set)); - } - search::Filter::All => { - filters.push(query::Filter::is_in_set(message_ids.clone())); - } - search::Filter::Answered => { - filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( - cache - .with_keyword(&Keyword::Answered) - .map(|m| m.document_id), - ))); - } - search::Filter::Before(date) => { - filters.push(query::Filter::lt( - EmailField::ReceivedAt, - (date as u64).serialize(), - )); - } - search::Filter::Deleted => { - filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( - cache.with_keyword(&Keyword::Deleted).map(|m| m.document_id), - ))); - } - search::Filter::Draft => { - filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( - cache.with_keyword(&Keyword::Draft).map(|m| m.document_id), - ))); - } - search::Filter::Flagged => { - filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( - cache.with_keyword(&Keyword::Flagged).map(|m| m.document_id), - ))); - } - search::Filter::Keyword(keyword) => { - filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( - cache - .with_keyword(&Keyword::from(keyword)) - .map(|m| m.document_id), - ))); - } - search::Filter::Larger(size) => { - filters.push(query::Filter::gt(EmailField::Size, size.serialize())); - } - search::Filter::On(date) => { - filters.push(query::Filter::And); - filters.push(query::Filter::ge( - EmailField::ReceivedAt, - (date as u64).serialize(), - )); - filters.push(query::Filter::lt( - EmailField::ReceivedAt, - ((date + 86400) as u64).serialize(), - )); - filters.push(query::Filter::End); - } - search::Filter::Seen => { - filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( - cache.with_keyword(&Keyword::Seen).map(|m| m.document_id), - ))); - } - search::Filter::SentBefore(date) => { - filters.push(query::Filter::lt( - EmailField::SentAt, - (date as u64).serialize(), - )); - } - search::Filter::SentOn(date) => { - filters.push(query::Filter::And); - filters.push(query::Filter::ge( - EmailField::SentAt, - (date as u64).serialize(), - )); - filters.push(query::Filter::lt( - EmailField::SentAt, - ((date + 86400) as u64).serialize(), - )); - filters.push(query::Filter::End); - } - search::Filter::SentSince(date) => { - filters.push(query::Filter::ge( - EmailField::SentAt, - (date as u64).serialize(), - )); - } - search::Filter::Since(date) => { - filters.push(query::Filter::ge( - EmailField::ReceivedAt, - (date as u64).serialize(), - )); - } - search::Filter::Smaller(size) => { - filters.push(query::Filter::lt(EmailField::Size, size.serialize())); - } - search::Filter::Unanswered => { - filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( - cache - .without_keyword(&Keyword::Answered) - .map(|m| m.document_id), - ))); - } - search::Filter::Undeleted => { - filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( - cache - .without_keyword(&Keyword::Deleted) - .map(|m| m.document_id), - ))); - } - search::Filter::Undraft => { - filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( - cache - .without_keyword(&Keyword::Draft) - .map(|m| m.document_id), - ))); - } - search::Filter::Unflagged => { - filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( - cache - .without_keyword(&Keyword::Flagged) - .map(|m| m.document_id), - ))); - } - search::Filter::Unkeyword(keyword) => { - filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( - cache - .without_keyword(&Keyword::from(keyword)) - .map(|m| m.document_id), - ))); - } - search::Filter::Unseen => { - filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( - cache.without_keyword(&Keyword::Seen).map(|m| m.document_id), - ))); - } - search::Filter::And => { - filters.push(query::Filter::And); - } - search::Filter::Or => { - filters.push(query::Filter::Or); - } - search::Filter::Not => { - filters.push(query::Filter::Not); - } - search::Filter::End => { - filters.push(query::Filter::End); - } - search::Filter::Recent => { - //filters.push(query::Filter::is_in_set(self.get_recent(&mailbox.id))); - } - search::Filter::New => { - /*filters.push(query::Filter::And); - filters.push(query::Filter::is_in_set(self.get_recent(&mailbox.id))); - filters.push(query::Filter::Not); - filters.push(query::Filter::is_in_bitmap( - EmailField::Keywords, - Keyword::Seen, - )); - filters.push(query::Filter::End); - filters.push(query::Filter::End);*/ - } - search::Filter::Old => { - /*filters.push(query::Filter::Not); - filters.push(query::Filter::is_in_set(self.get_recent(&mailbox.id))); - filters.push(query::Filter::End);*/ - } - search::Filter::Older(secs) => { - filters.push(query::Filter::le( - EmailField::ReceivedAt, - now().saturating_sub(secs as u64).serialize(), - )); - } - search::Filter::Younger(secs) => { - filters.push(query::Filter::ge( - EmailField::ReceivedAt, - now().saturating_sub(secs as u64).serialize(), - )); - } - search::Filter::ModSeq((modseq, _)) => { - let mut set = RoaringBitmap::new(); - for id in self - .server - .store() - .changes( - mailbox.id.account_id, - SyncCollection::Email.into(), - Query::from_modseq(modseq), - ) - .await? - .changes - .into_iter() - .filter_map(|change| change.try_unwrap_item_id()) - { - let id = (id & u32::MAX as u64) as u32; - if message_ids.contains(id) { - set.insert(id); - } - } - filters.push(query::Filter::is_in_set(set)); - include_highest_modseq = true; - } - search::Filter::EmailId(id) => { - if let Ok(id) = Id::from_str(&id) { - filters.push(query::Filter::is_in_set( - RoaringBitmap::from_sorted_iter([id.document_id()]).unwrap(), - )); - } else { - return Err(trc::ImapEvent::Error - .into_err() - .details(format!("Failed to parse email id '{id}'.",))); + filters.push(SearchFilter::exists(header)); } } - search::Filter::ThreadId(id) => { - if let Ok(id) = Id::from_str(&id) { - filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( - cache.in_thread(id.document_id()).map(|m| m.document_id), - ))); - } else { - return Err(trc::ImapEvent::Error - .into_err() - .details(format!("Failed to parse thread id '{id}'.",))); - } - } - _ => (), - }, + } + Filter::Subject(text) => { + filters.push(SearchFilter::has_text_detect( + EmailSearchField::Subject, + text, + self.server.core.jmap.default_language, + )); + } + Filter::Text(text) => { + filters.push(SearchFilter::Or); + filters.push(SearchFilter::has_text( + EmailSearchField::From, + text.as_str(), + Language::None, + )); + filters.push(SearchFilter::has_text( + EmailSearchField::To, + text.as_str(), + Language::None, + )); + filters.push(SearchFilter::has_text( + EmailSearchField::Cc, + text.as_str(), + Language::None, + )); + filters.push(SearchFilter::has_text( + EmailSearchField::Bcc, + text.as_str(), + Language::None, + )); + filters.push(SearchFilter::has_text_detect( + EmailSearchField::Subject, + text.as_str(), + self.server.core.jmap.default_language, + )); + filters.push(SearchFilter::has_text_detect( + EmailSearchField::Body, + text.as_str(), + self.server.core.jmap.default_language, + )); + filters.push(SearchFilter::has_text_detect( + EmailSearchField::Attachment, + text, + self.server.core.jmap.default_language, + )); + filters.push(SearchFilter::End); + } + Filter::To(text) => { + filters.push(SearchFilter::has_text( + EmailSearchField::To, + text, + Language::None, + )); + } + Filter::And => { + filters.push(SearchFilter::And); + } + Filter::Or => { + filters.push(SearchFilter::Or); + } + Filter::Not => { + filters.push(SearchFilter::Not); + } + Filter::End => { + filters.push(SearchFilter::End); + } } } + // Convert comparators + let mut comparators = Vec::with_capacity(imap_comparator.len()); + for comparator in imap_comparator { + comparators.push(match comparator.sort { + search::Sort::Arrival => { + SearchComparator::field(EmailSearchField::ReceivedAt, comparator.ascending) + } + search::Sort::Cc => { + SearchComparator::field(EmailSearchField::Cc, comparator.ascending) + } + search::Sort::Date => { + SearchComparator::field(EmailSearchField::SentAt, comparator.ascending) + } + search::Sort::From | search::Sort::DisplayFrom => { + SearchComparator::field(EmailSearchField::From, comparator.ascending) + } + search::Sort::Size => { + SearchComparator::field(EmailSearchField::Size, comparator.ascending) + } + search::Sort::Subject => { + SearchComparator::field(EmailSearchField::Subject, comparator.ascending) + } + search::Sort::To | search::Sort::DisplayTo => { + SearchComparator::field(EmailSearchField::To, comparator.ascending) + } + }); + } + // Run query self.server - .store() - .filter(mailbox.id.account_id, Collection::Email, filters) + .search_store() + .query( + mailbox.id.account_id, + Collection::Email, + filters, + comparators, + ) .await .map(|res| (res, include_highest_modseq)) .caused_by(trc::location!()) diff --git a/crates/imap/src/op/status.rs b/crates/imap/src/op/status.rs index f2b3d97e..2e5a80b2 100644 --- a/crates/imap/src/op/status.rs +++ b/crates/imap/src/op/status.rs @@ -20,9 +20,10 @@ use imap_proto::{ receiver::Request, }; use std::time::Instant; -use store::{Deserialize, U32_LEN}; +use store::{IterateParams, roaring::RoaringBitmap, write::key::DeserializeBigEndian}; use store::{ - IndexKeyPrefix, IterateParams, roaring::RoaringBitmap, write::key::DeserializeBigEndian, + U32_LEN, ValueKey, + write::{IndexPropertyClass, ValueClass}, }; use trc::AddContext; use types::{collection::Collection, field::EmailField, id::Id, keyword::Keyword}; @@ -299,30 +300,32 @@ impl SessionData { .data .iterate( IterateParams::new( - IndexKeyPrefix { + ValueKey { account_id, collection: Collection::Email.into(), - field: EmailField::Size.into(), + document_id: 0, + class: ValueClass::IndexProperty(IndexPropertyClass::Integer { + property: EmailField::Stats.into(), + value: 0, + }), }, - IndexKeyPrefix { + ValueKey { account_id, collection: Collection::Email.into(), - field: u8::from(EmailField::Size) + 1, + document_id: u32::MAX, + class: ValueClass::IndexProperty(IndexPropertyClass::Integer { + property: EmailField::Stats.into(), + value: u64::MAX, + }), }, ) - .ascending() - .no_values(), - |key, _| { + .ascending(), + |key, value| { let id_pos = key.len() - U32_LEN; let document_id = key.deserialize_be_u32(id_pos)?; if message_ids.contains(document_id) { - key.get(IndexKeyPrefix::len()..id_pos) - .ok_or_else(|| trc::Error::corrupted_key(key, None, trc::location!())) - .and_then(u32::deserialize) - .map(|size| { - total_size += size as u64; - })?; + total_size += value.deserialize_be_u32(0)? as u64; } Ok(true) }, diff --git a/crates/imap/src/op/store.rs b/crates/imap/src/op/store.rs index 61b51ef4..f5b659c1 100644 --- a/crates/imap/src/op/store.rs +++ b/crates/imap/src/op/store.rs @@ -12,7 +12,7 @@ use crate::{ use ahash::AHashSet; use common::{listener::SessionStream, storage::index::ObjectIndexBuilder}; use directory::Permission; -use email::message::{bayes::EmailBayesTrain, ingest::EmailIngest, metadata::MessageData}; +use email::message::{ingest::EmailIngest, metadata::MessageData}; use imap_proto::{ Command, ResponseCode, ResponseType, StatusResponse, protocol::{ @@ -25,7 +25,7 @@ use imap_proto::{ use std::{sync::Arc, time::Instant}; use store::{ query::log::{Change, Query}, - write::{BatchBuilder, ValueClass}, + write::{BatchBuilder, TaskQueueClass, ValueClass, now}, }; use trc::AddContext; use types::{ @@ -202,7 +202,7 @@ impl SessionData { // Obtain message data let data_ = if let Some(data) = self .server - .get_archive(account_id, Collection::Email, *id) + .archive(account_id, Collection::Email, *id) .await .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())? { @@ -292,7 +292,7 @@ impl SessionData { batch .with_account_id(account_id) .with_collection(Collection::Email) - .update_document(*id) + .with_document(*id) .custom( ObjectIndexBuilder::new() .with_current(data) @@ -303,12 +303,10 @@ impl SessionData { // Add spam train task if let Some(learn_spam) = train_spam { batch.set( - ValueClass::TaskQueue( - self.server - .email_bayes_queue_task_build(account_id, *id, learn_spam) - .await - .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())?, - ), + ValueClass::TaskQueue(TaskQueueClass::BayesTrain { + due: now(), + learn_spam, + }), vec![], ); has_spam_train_tasks = true; diff --git a/crates/imap/src/op/subscribe.rs b/crates/imap/src/op/subscribe.rs index 95cd88f9..3d99e364 100644 --- a/crates/imap/src/op/subscribe.rs +++ b/crates/imap/src/op/subscribe.rs @@ -92,7 +92,7 @@ impl SessionData { // Obtain mailbox let mailbox_ = self .server - .get_archive(account_id, Collection::Mailbox, mailbox_id) + .archive(account_id, Collection::Mailbox, mailbox_id) .await .imap_ctx(&tag, trc::location!())? .ok_or_else(|| { @@ -121,7 +121,7 @@ impl SessionData { batch .with_account_id(account_id) .with_collection(Collection::Mailbox) - .update_document(mailbox_id) + .with_document(mailbox_id) .custom( ObjectIndexBuilder::new() .with_current(mailbox) diff --git a/crates/imap/src/op/thread.rs b/crates/imap/src/op/thread.rs index 761ea968..22e258c4 100644 --- a/crates/imap/src/op/thread.rs +++ b/crates/imap/src/op/thread.rs @@ -11,7 +11,7 @@ use crate::{ use ahash::AHashMap; use common::listener::SessionStream; use directory::Permission; -use email::cache::MessageCacheFetch; +use email::cache::{MessageCacheFetch, email::MessageCacheAccess}; use imap_proto::{ Command, StatusResponse, protocol::{ @@ -64,10 +64,12 @@ impl SessionData { op_start: Instant, ) -> trc::Result { // Run query - let (result_set, _) = self.query(arguments.filter, &mailbox, &None).await?; + let (result_set, _) = self + .query(arguments.filter, vec![], &mailbox, &None) + .await?; // Synchronize mailbox - if !result_set.results.is_empty() { + if !result_set.is_empty() { self.synchronize_messages(&mailbox) .await .caused_by(trc::location!())?; @@ -88,9 +90,9 @@ impl SessionData { // Group messages by thread let mut threads: AHashMap> = AHashMap::new(); let state = mailbox.state.lock(); - for item in &cache.emails.items { - if result_set.results.contains(item.document_id) - && let Some((imap_id, _)) = state.map_result_id(item.document_id, is_uid) + for document_id in result_set { + if let Some(item) = cache.email_by_id(&document_id) + && let Some((imap_id, _)) = state.map_result_id(document_id, is_uid) { threads.entry(item.thread_id).or_default().push(imap_id); } diff --git a/crates/jmap-proto/Cargo.toml b/crates/jmap-proto/Cargo.toml index 1ce7cbb9..a4bdb867 100644 --- a/crates/jmap-proto/Cargo.toml +++ b/crates/jmap-proto/Cargo.toml @@ -5,7 +5,6 @@ edition = "2024" resolver = "2" [dependencies] -store = { path = "../store" } utils = { path = "../utils" } types = { path = "../types" } trc = { path = "../trc" } diff --git a/crates/jmap-proto/src/method/query.rs b/crates/jmap-proto/src/method/query.rs index fda327d8..ec6d1d65 100644 --- a/crates/jmap-proto/src/method/query.rs +++ b/crates/jmap-proto/src/method/query.rs @@ -15,9 +15,8 @@ use serde::{ }; use std::{ borrow::Cow, - fmt::{self, Display, Formatter}, + fmt::{self}, }; -use store::fts::{FilterItem, FilterType, FtsFilter}; use types::id::Id; #[derive(Debug, Clone)] @@ -339,51 +338,6 @@ where } } -impl From> for store::query::Filter -where - T: for<'de> DeserializeArguments<'de> + Default, -{ - fn from(value: Filter) -> Self { - match value { - Filter::And => Self::And, - Filter::Or => Self::Or, - Filter::Not => Self::Not, - Filter::Close => Self::End, - _ => unreachable!(), - } - } -} - -impl + Display + Clone + std::fmt::Debug, U> From> for FtsFilter -where - U: for<'de> DeserializeArguments<'de> + Default, -{ - fn from(value: Filter) -> Self { - match value { - Filter::And => Self::And, - Filter::Or => Self::Or, - Filter::Not => Self::Not, - Filter::Close => Self::End, - _ => unreachable!(), - } - } -} - -impl From for Filter -where - T: for<'de> DeserializeArguments<'de> + Default, -{ - fn from(value: FilterType) -> Self { - match value { - FilterType::And => Filter::And, - FilterType::Or => Filter::Or, - FilterType::Not => Filter::Not, - FilterType::End => Filter::Close, - _ => unreachable!(), - } - } -} - impl Comparator where T: for<'de> DeserializeArguments<'de> + Default, @@ -404,33 +358,3 @@ where } } } - -impl FilterItem for Filter -where - T: for<'de> DeserializeArguments<'de> + FilterItem + Default, -{ - fn filter_type(&self) -> FilterType { - match self { - Filter::And => FilterType::And, - Filter::Or => FilterType::Or, - Filter::Not => FilterType::Not, - Filter::Close => FilterType::End, - Filter::Property(p) => p.filter_type(), - } - } -} - -impl Display for Filter -where - T: for<'de> DeserializeArguments<'de> + Display + Default, -{ - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - match self { - Filter::And => write!(f, "and"), - Filter::Or => write!(f, "or"), - Filter::Not => write!(f, "not"), - Filter::Close => write!(f, "close"), - Filter::Property(p) => write!(f, "{}", p), - } - } -} diff --git a/crates/jmap-proto/src/object/email.rs b/crates/jmap-proto/src/object/email.rs index 9635dc21..44283a72 100644 --- a/crates/jmap-proto/src/object/email.rs +++ b/crates/jmap-proto/src/object/email.rs @@ -14,7 +14,6 @@ use jmap_tools::{Element, JsonPointer, JsonPointerItem, Key, Property}; use mail_parser::HeaderName; use serde::Serialize; use std::{borrow::Cow, fmt::Display, str::FromStr}; -use store::fts::{FilterItem, FilterType}; use types::{blob::BlobId, id::Id, keyword::Keyword}; #[derive(Debug, Clone, Default)] @@ -858,22 +857,6 @@ impl EmailComparator { } } -impl FilterItem for EmailFilter { - fn filter_type(&self) -> FilterType { - match self { - EmailFilter::From(_) - | EmailFilter::To(_) - | EmailFilter::Cc(_) - | EmailFilter::Bcc(_) - | EmailFilter::Subject(_) - | EmailFilter::Body(_) - | EmailFilter::Header(_) - | EmailFilter::Text(_) => FilterType::Fts, - _ => FilterType::Store, - } - } -} - impl JmapObjectId for EmailValue { fn as_id(&self) -> Option { if let EmailValue::Id(id) = self { diff --git a/crates/jmap/src/addressbook/get.rs b/crates/jmap/src/addressbook/get.rs index 4d9a9dc4..6196d4ac 100644 --- a/crates/jmap/src/addressbook/get.rs +++ b/crates/jmap/src/addressbook/get.rs @@ -95,7 +95,7 @@ impl AddressBookGet for Server { continue; } let _address_book = if let Some(address_book) = self - .get_archive(account_id, Collection::AddressBook, document_id) + .archive(account_id, Collection::AddressBook, document_id) .await? { address_book diff --git a/crates/jmap/src/addressbook/set.rs b/crates/jmap/src/addressbook/set.rs index 17b1d4fd..2d81455f 100644 --- a/crates/jmap/src/addressbook/set.rs +++ b/crates/jmap/src/addressbook/set.rs @@ -131,7 +131,7 @@ impl AddressBookSet for Server { // Obtain address book let document_id = id.document_id(); let address_book_ = if let Some(address_book_) = self - .get_archive(account_id, Collection::AddressBook, document_id) + .archive(account_id, Collection::AddressBook, document_id) .await? { address_book_ @@ -223,7 +223,7 @@ impl AddressBookSet for Server { }; let Some(address_book_) = self - .get_archive(account_id, Collection::AddressBook, document_id) + .archive(account_id, Collection::AddressBook, document_id) .await .caused_by(trc::location!())? else { @@ -278,7 +278,7 @@ impl AddressBookSet for Server { if !destroy_children.is_empty() { for document_id in destroy_children { if let Some(card_) = self - .get_archive(account_id, Collection::ContactCard, document_id) + .archive(account_id, Collection::ContactCard, document_id) .await? { let card = card_ @@ -331,7 +331,7 @@ impl AddressBookSet for Server { batch .with_account_id(account_id) .with_collection(Collection::Principal) - .update_document(0) + .with_document(0) .set( PrincipalField::DefaultAddressBookId, default_address_book_id.serialize(), @@ -341,7 +341,7 @@ impl AddressBookSet for Server { batch .with_account_id(account_id) .with_collection(Collection::Principal) - .update_document(0) + .with_document(0) .clear(PrincipalField::DefaultAddressBookId); } diff --git a/crates/jmap/src/api/acl.rs b/crates/jmap/src/api/acl.rs index d1466550..f95bb05e 100644 --- a/crates/jmap/src/api/acl.rs +++ b/crates/jmap/src/api/acl.rs @@ -13,6 +13,7 @@ use jmap_tools::{JsonPointerIter, Key, Map, Property, Value}; use types::{ acl::{Acl, AclGrant}, collection::Collection, + field::Field, id::Id, }; use utils::map::bitmap::Bitmap; @@ -240,9 +241,8 @@ impl JmapAcl for Server { } let principal_ids = self - .get_document_ids(u32::MAX, Collection::Principal) + .document_ids(u32::MAX, Collection::Principal, Field::DOCUMENT_ID) .await - .unwrap_or_default() .unwrap_or_default(); for grant in grants { diff --git a/crates/jmap/src/api/mod.rs b/crates/jmap/src/api/mod.rs index e034d990..42145972 100644 --- a/crates/jmap/src/api/mod.rs +++ b/crates/jmap/src/api/mod.rs @@ -21,6 +21,7 @@ use utils::map::vec_map::VecMap; pub mod acl; pub mod auth; pub mod event_source; +pub mod query; pub mod request; pub mod session; diff --git a/crates/jmap/src/api/query.rs b/crates/jmap/src/api/query.rs new file mode 100644 index 00000000..196abaa6 --- /dev/null +++ b/crates/jmap/src/api/query.rs @@ -0,0 +1,164 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use jmap_proto::{ + method::query::{QueryRequest, QueryResponse}, + object::JmapObject, + types::state::State, +}; +use types::id::Id; + +pub struct QueryResponseBuilder { + requested_position: i32, + position: i32, + pub limit: usize, + anchor: u32, + anchor_offset: i32, + has_anchor: bool, + anchor_found: bool, + + pub response: QueryResponse, +} + +impl QueryResponseBuilder { + pub fn new( + total_results: usize, + max_results: usize, + query_state: State, + request: &QueryRequest, + ) -> Self { + let (limit_total, limit) = if let Some(limit) = request.limit { + if limit > 0 { + let limit = std::cmp::min(limit, max_results); + (std::cmp::min(limit, total_results), limit) + } else { + (0, 0) + } + } else { + (std::cmp::min(max_results, total_results), max_results) + }; + + let (has_anchor, anchor) = request + .anchor + .map(|anchor| (true, anchor.document_id())) + .unwrap_or((false, 0)); + + QueryResponseBuilder { + requested_position: request.position.unwrap_or(0), + position: request.position.unwrap_or(0), + limit: limit_total, + anchor, + anchor_offset: request.anchor_offset.unwrap_or(0), + has_anchor, + anchor_found: false, + response: QueryResponse { + account_id: request.account_id, + query_state, + can_calculate_changes: true, + position: 0, + ids: vec![], + total: if request.calculate_total.unwrap_or(false) { + Some(total_results) + } else { + None + }, + limit: if total_results > limit { + Some(limit) + } else { + None + }, + }, + } + } + + #[inline(always)] + pub fn add(&mut self, prefix_id: u32, document_id: u32) -> bool { + self.add_id(Id::from_parts(prefix_id, document_id)) + } + + pub fn add_id(&mut self, id: Id) -> bool { + let document_id = id.document_id(); + + // Pagination + if !self.has_anchor { + if self.position >= 0 { + if self.position > 0 { + self.position -= 1; + } else { + self.response.ids.push(id); + if self.response.ids.len() == self.limit { + return false; + } + } + } else { + self.response.ids.push(id); + } + } else if self.anchor_offset >= 0 { + if !self.anchor_found { + if document_id != self.anchor { + return true; + } + self.anchor_found = true; + } + + if self.anchor_offset > 0 { + self.anchor_offset -= 1; + } else { + self.response.ids.push(id); + if self.response.ids.len() == self.limit { + return false; + } + } + } else { + self.anchor_found = document_id == self.anchor; + self.response.ids.push(id); + + if self.anchor_found { + self.position = self.anchor_offset; + return false; + } + } + + true + } + + pub fn is_full(&self) -> bool { + self.response.ids.len() == self.limit + } + + pub fn build(mut self) -> trc::Result { + if !self.has_anchor || self.anchor_found { + if !self.has_anchor && self.requested_position >= 0 { + self.response.position = if self.position == 0 { + self.requested_position + } else { + 0 + }; + } else if self.position >= 0 { + self.response.position = self.position; + } else { + let position = self.position.unsigned_abs() as usize; + let start_offset = if position < self.response.ids.len() { + self.response.ids.len() - position + } else { + 0 + }; + self.response.position = start_offset as i32; + let end_offset = if self.limit > 0 { + std::cmp::min(start_offset + self.limit, self.response.ids.len()) + } else { + self.response.ids.len() + }; + + self.response.ids = self.response.ids[start_offset..end_offset].to_vec() + } + + Ok(self.response) + } else { + Err(trc::JmapEvent::AnchorNotFound.into_err()) + } + } +} diff --git a/crates/jmap/src/blob/get.rs b/crates/jmap/src/blob/get.rs index 6303dc23..9facf38a 100644 --- a/crates/jmap/src/blob/get.rs +++ b/crates/jmap/src/blob/get.rs @@ -203,7 +203,7 @@ impl BlobOperations for Server { let collection = Collection::from(*collection); if collection == Collection::Email { if let Some(data_) = self - .get_archive(req_account_id, Collection::Email, *document_id) + .archive(req_account_id, Collection::Email, *document_id) .await? { let data = data_ diff --git a/crates/jmap/src/calendar/get.rs b/crates/jmap/src/calendar/get.rs index c5d66803..ceb4daf2 100644 --- a/crates/jmap/src/calendar/get.rs +++ b/crates/jmap/src/calendar/get.rs @@ -105,7 +105,7 @@ impl CalendarGet for Server { continue; } let _calendar = if let Some(calendar) = self - .get_archive(account_id, Collection::Calendar, document_id) + .archive(account_id, Collection::Calendar, document_id) .await? { calendar diff --git a/crates/jmap/src/calendar/set.rs b/crates/jmap/src/calendar/set.rs index 70759134..676db63a 100644 --- a/crates/jmap/src/calendar/set.rs +++ b/crates/jmap/src/calendar/set.rs @@ -136,7 +136,7 @@ impl CalendarSet for Server { // Obtain calendar let document_id = id.document_id(); let calendar_ = if let Some(calendar_) = self - .get_archive(account_id, Collection::Calendar, document_id) + .archive(account_id, Collection::Calendar, document_id) .await? { calendar_ @@ -214,7 +214,7 @@ impl CalendarSet for Server { }; let Some(calendar_) = self - .get_archive(account_id, Collection::Calendar, document_id) + .archive(account_id, Collection::Calendar, document_id) .await .caused_by(trc::location!())? else { @@ -269,7 +269,7 @@ impl CalendarSet for Server { if !destroy_children.is_empty() { for document_id in destroy_children { if let Some(event_) = self - .get_archive(account_id, Collection::CalendarEvent, document_id) + .archive(account_id, Collection::CalendarEvent, document_id) .await? { let event = event_ @@ -323,7 +323,7 @@ impl CalendarSet for Server { batch .with_account_id(account_id) .with_collection(Collection::Principal) - .update_document(0) + .with_document(0) .set( PrincipalField::DefaultCalendarId, default_calendar_id.serialize(), @@ -333,7 +333,7 @@ impl CalendarSet for Server { batch .with_account_id(account_id) .with_collection(Collection::Principal) - .update_document(0) + .with_document(0) .clear(PrincipalField::DefaultCalendarId); } diff --git a/crates/jmap/src/calendar_event/copy.rs b/crates/jmap/src/calendar_event/copy.rs index 3381732f..41ef258b 100644 --- a/crates/jmap/src/calendar_event/copy.rs +++ b/crates/jmap/src/calendar_event/copy.rs @@ -124,7 +124,7 @@ impl JmapCalendarEventCopy for Server { } let Some(_calendar_event) = self - .get_archive( + .archive( from_account_id, Collection::CalendarEvent, from_calendar_event_id, diff --git a/crates/jmap/src/calendar_event/get.rs b/crates/jmap/src/calendar_event/get.rs index fd4177dd..8fad0292 100644 --- a/crates/jmap/src/calendar_event/get.rs +++ b/crates/jmap/src/calendar_event/get.rs @@ -203,7 +203,7 @@ impl CalendarEventGet for Server { } let Some(_calendar_event) = self - .get_archive(account_id, Collection::CalendarEvent, document_id) + .archive(account_id, Collection::CalendarEvent, document_id) .await? else { response.not_found.push(id); diff --git a/crates/jmap/src/calendar_event/mod.rs b/crates/jmap/src/calendar_event/mod.rs index 1094a4d0..aa7e549c 100644 --- a/crates/jmap/src/calendar_event/mod.rs +++ b/crates/jmap/src/calendar_event/mod.rs @@ -7,7 +7,6 @@ use calcard::jscalendar::JSCalendarProperty; use common::Server; use jmap_proto::error::set::SetError; -use store::query::Filter; use trc::AddContext; use types::{collection::Collection, field::CalendarField, id::Id}; @@ -64,17 +63,15 @@ pub(super) async fn assert_is_unique_uid( uid: Option<&str>, ) -> trc::Result>>> { if let Some(uid) = uid - && !server - .store() - .filter( + && server + .document_exists( account_id, Collection::CalendarEvent, - vec![Filter::eq(CalendarField::Uid, uid.as_bytes().to_vec())], + CalendarField::Uid, + uid.as_bytes(), ) .await .caused_by(trc::location!())? - .results - .is_empty() { Ok(Err(SetError::invalid_properties() .with_property(JSCalendarProperty::Uid) diff --git a/crates/jmap/src/calendar_event/query.rs b/crates/jmap/src/calendar_event/query.rs index 1e697fad..b92f689b 100644 --- a/crates/jmap/src/calendar_event/query.rs +++ b/crates/jmap/src/calendar_event/query.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{JmapMethods, UpdateResults, changes::state::JmapCacheState}; +use crate::changes::state::JmapCacheState; use calcard::{common::timezone::Tz, jscalendar::JSCalendarDateTime}; use chrono::offset::TimeZone; use common::{Server, auth::AccessToken}; @@ -16,7 +16,7 @@ use jmap_proto::{ }; use nlp::tokenizers::word::WordTokenizer; use std::{cmp::Ordering, sync::Arc}; -use store::{backend::MAX_TOKEN_LENGTH, query, roaring::RoaringBitmap}; +use store::{backend::MAX_TOKEN_LENGTH, roaring::RoaringBitmap, search::SearchFilter}; use trc::AddContext; use types::{ TimeRange, @@ -68,16 +68,16 @@ impl CalendarEventQuery for Server { match cond { Filter::Property(cond) => match cond { CalendarEventFilter::InCalendar(MaybeInvalid::Value(id)) => { - filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache.children_ids(id.document_id()), ))) } CalendarEventFilter::Uid(uid) => { - filters.push(query::Filter::eq(CalendarField::Uid, uid.into_bytes())) + filters.push(SearchFilter::eq(CalendarField::Uid, uid.into_bytes())) } CalendarEventFilter::Text(value) => { for token in WordTokenizer::new(&value, MAX_TOKEN_LENGTH) { - filters.push(query::Filter::eq( + filters.push(SearchFilter::eq( CalendarField::Text, token.word.into_owned().into_bytes(), )); @@ -87,7 +87,7 @@ impl CalendarEventQuery for Server { if let Some(filter) = &filter && !did_filter_by_time { - filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache.resources.iter().filter_map(|r| { r.event_time_range().and_then(|(start, end)| { filter @@ -143,7 +143,7 @@ impl CalendarEventQuery for Server { for document_id in result_set.results { let Some(_calendar_event) = self - .get_archive(account_id, Collection::CalendarEvent, document_id) + .archive(account_id, Collection::CalendarEvent, document_id) .await? else { continue; @@ -230,19 +230,17 @@ impl CalendarEventQuery for Server { for comparator in comparators { comparators_.push(match &comparator.property { CalendarEventComparator::Uid => { - query::Comparator::field(CalendarField::Uid, comparator.is_ascending) + SearchComparator::field(CalendarField::Uid, comparator.is_ascending) } CalendarEventComparator::Start => { - query::Comparator::field(CalendarField::Start, comparator.is_ascending) + SearchComparator::field(CalendarField::Start, comparator.is_ascending) + } + CalendarEventComparator::Created => { + SearchComparator::field(CalendarField::Created, comparator.is_ascending) + } + CalendarEventComparator::Updated => { + SearchComparator::field(CalendarField::Updated, comparator.is_ascending) } - CalendarEventComparator::Created => query::Comparator::field( - CalendarField::Created, - comparator.is_ascending, - ), - CalendarEventComparator::Updated => query::Comparator::field( - CalendarField::Updated, - comparator.is_ascending, - ), unsupported => { return Err(trc::JmapEvent::UnsupportedSort .into_err() diff --git a/crates/jmap/src/calendar_event/set.rs b/crates/jmap/src/calendar_event/set.rs index fd567ae9..9d9c51b9 100644 --- a/crates/jmap/src/calendar_event/set.rs +++ b/crates/jmap/src/calendar_event/set.rs @@ -157,7 +157,7 @@ impl CalendarEventSet for Server { // Obtain calendar_event card let document_id = id.document_id(); let calendar_event_ = if let Some(calendar_event_) = self - .get_archive(account_id, Collection::CalendarEvent, document_id) + .archive(account_id, Collection::CalendarEvent, document_id) .await? { calendar_event_ @@ -444,7 +444,7 @@ impl CalendarEventSet for Server { } let Some(calendar_event_) = self - .get_archive(account_id, Collection::CalendarEvent, document_id) + .archive(account_id, Collection::CalendarEvent, document_id) .await .caused_by(trc::location!())? else { @@ -559,7 +559,7 @@ impl CalendarEventSet for Server { )))); } else if let Some(show_without_time) = use_default_alerts && let Some(_calendar) = self - .get_archive(account_id, Collection::Calendar, name.parent_id) + .archive(account_id, Collection::Calendar, name.parent_id) .await? { ical.components.extend( diff --git a/crates/jmap/src/calendar_event_notification/get.rs b/crates/jmap/src/calendar_event_notification/get.rs index 39f34857..5453a4e7 100644 --- a/crates/jmap/src/calendar_event_notification/get.rs +++ b/crates/jmap/src/calendar_event_notification/get.rs @@ -84,7 +84,7 @@ impl CalendarEventNotificationGet for Server { // Obtain the event object let document_id = id.document_id(); let _event = if let Some(event) = self - .get_archive( + .archive( account_id, Collection::CalendarEventNotification, document_id, diff --git a/crates/jmap/src/calendar_event_notification/query.rs b/crates/jmap/src/calendar_event_notification/query.rs index bc27349d..e7f652cb 100644 --- a/crates/jmap/src/calendar_event_notification/query.rs +++ b/crates/jmap/src/calendar_event_notification/query.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{JmapMethods, changes::state::JmapCacheState}; +use crate::{ changes::state::JmapCacheState}; use common::{Server, auth::AccessToken}; use groupware::cache::GroupwareCache; use jmap_proto::{ @@ -49,13 +49,13 @@ impl CalendarEventNotificationQuery for Server { match cond { Filter::Property(cond) => match cond { CalendarEventNotificationFilter::Before(before) => { - filters.push(query::Filter::lt( + filters.push(SearchFilter::lt( CalendarField::Created, (before.timestamp() as u64).serialize(), )) } CalendarEventNotificationFilter::After(after) => { - filters.push(query::Filter::gt( + filters.push(SearchFilter::gt( CalendarField::Created, (after.timestamp() as u64).serialize(), )) @@ -63,16 +63,16 @@ impl CalendarEventNotificationQuery for Server { CalendarEventNotificationFilter::CalendarEventIds(ids) => { let has_many = ids.len() > 1; if has_many { - filters.push(query::Filter::Or); + filters.push(SearchFilter::Or); } for id in ids.into_valid() { - filters.push(query::Filter::eq( + filters.push(SearchFilter::eq( CalendarField::EventId, id.document_id().serialize(), )); } if has_many { - filters.push(query::Filter::End); + filters.push(SearchFilter::End); } } unsupported => { @@ -110,7 +110,7 @@ impl CalendarEventNotificationQuery for Server { }) { comparators.push(match comparator.property { CalendarEventNotificationComparator::Created => { - query::Comparator::field(CalendarField::Created, comparator.is_ascending) + SearchComparator::field(CalendarField::Created, comparator.is_ascending) } CalendarEventNotificationComparator::_T(unsupported) => { return Err(trc::JmapEvent::UnsupportedSort diff --git a/crates/jmap/src/calendar_event_notification/set.rs b/crates/jmap/src/calendar_event_notification/set.rs index e6415af8..bb577404 100644 --- a/crates/jmap/src/calendar_event_notification/set.rs +++ b/crates/jmap/src/calendar_event_notification/set.rs @@ -72,7 +72,7 @@ impl CalendarEventNotificationSet for Server { }; let _event = if let Some(event) = self - .get_archive( + .archive( account_id, Collection::CalendarEventNotification, document_id, diff --git a/crates/jmap/src/contact/copy.rs b/crates/jmap/src/contact/copy.rs index 30a682ff..5d0d123a 100644 --- a/crates/jmap/src/contact/copy.rs +++ b/crates/jmap/src/contact/copy.rs @@ -107,7 +107,7 @@ impl JmapContactCardCopy for Server { } let Some(_contact) = self - .get_archive(from_account_id, Collection::ContactCard, from_contact_id) + .archive(from_account_id, Collection::ContactCard, from_contact_id) .await? else { response.not_created.append( diff --git a/crates/jmap/src/contact/get.rs b/crates/jmap/src/contact/get.rs index 19fac762..ed4d1bc7 100644 --- a/crates/jmap/src/contact/get.rs +++ b/crates/jmap/src/contact/get.rs @@ -98,7 +98,7 @@ impl ContactCardGet for Server { } let _contact = if let Some(contact) = self - .get_archive(account_id, Collection::ContactCard, document_id) + .archive(account_id, Collection::ContactCard, document_id) .await? { contact diff --git a/crates/jmap/src/contact/mod.rs b/crates/jmap/src/contact/mod.rs index 7a6f2006..52e6fde4 100644 --- a/crates/jmap/src/contact/mod.rs +++ b/crates/jmap/src/contact/mod.rs @@ -7,7 +7,7 @@ use calcard::jscontact::JSContactProperty; use common::{DavName, DavResources, Server}; use jmap_proto::error::set::SetError; -use store::query::Filter; +use store::SearchFilter; use trc::AddContext; use types::{collection::Collection, field::ContactField, id::Id}; diff --git a/crates/jmap/src/contact/query.rs b/crates/jmap/src/contact/query.rs index 01d41beb..4b2bff78 100644 --- a/crates/jmap/src/contact/query.rs +++ b/crates/jmap/src/contact/query.rs @@ -20,7 +20,7 @@ use types::{ }; use utils::sanitize_email; -use crate::{JmapMethods, changes::state::JmapCacheState}; +use crate::{ changes::state::JmapCacheState}; pub trait ContactCardQuery: Sync + Send { fn contact_card_query( @@ -48,38 +48,38 @@ impl ContactCardQuery for Server { match cond { Filter::Property(cond) => match cond { ContactCardFilter::InAddressBook(MaybeInvalid::Value(id)) => { - filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache.children_ids(id.document_id()), ))) } ContactCardFilter::Uid(uid) => { - filters.push(query::Filter::eq(ContactField::Uid, uid.into_bytes())) + filters.push(SearchFilter::eq(ContactField::Uid, uid.into_bytes())) } - ContactCardFilter::Email(email) => filters.push(query::Filter::eq( + ContactCardFilter::Email(email) => filters.push(SearchFilter::eq( ContactField::Email, sanitize_email(&email).unwrap_or(email).into_bytes(), )), ContactCardFilter::Text(value) => { for token in WordTokenizer::new(&value, MAX_TOKEN_LENGTH) { - filters.push(query::Filter::eq( + filters.push(SearchFilter::eq( ContactField::Text, token.word.into_owned().into_bytes(), )); } } - ContactCardFilter::CreatedBefore(before) => filters.push(query::Filter::lt( + ContactCardFilter::CreatedBefore(before) => filters.push(SearchFilter::lt( ContactField::Created, (before.timestamp() as u64).serialize(), )), - ContactCardFilter::CreatedAfter(after) => filters.push(query::Filter::gt( + ContactCardFilter::CreatedAfter(after) => filters.push(SearchFilter::gt( ContactField::Created, (after.timestamp() as u64).serialize(), )), - ContactCardFilter::UpdatedBefore(before) => filters.push(query::Filter::lt( + ContactCardFilter::UpdatedBefore(before) => filters.push(SearchFilter::lt( ContactField::Updated, (before.timestamp() as u64).serialize(), )), - ContactCardFilter::UpdatedAfter(after) => filters.push(query::Filter::gt( + ContactCardFilter::UpdatedAfter(after) => filters.push(SearchFilter::gt( ContactField::Updated, (after.timestamp() as u64).serialize(), )), @@ -105,7 +105,11 @@ impl ContactCardQuery for Server { } let (response, paginate) = self - .build_query_response(result_set.results.len() as usize, cache.get_state(false), &request) + .build_query_response( + result_set.results.len() as usize, + cache.get_state(false), + &request, + ) .await?; if let Some(paginate) = paginate { @@ -118,10 +122,10 @@ impl ContactCardQuery for Server { { comparators.push(match comparator.property { ContactCardComparator::Created => { - query::Comparator::field(ContactField::Created, comparator.is_ascending) + SearchComparator::field(ContactField::Created, comparator.is_ascending) } ContactCardComparator::Updated => { - query::Comparator::field(ContactField::Updated, comparator.is_ascending) + SearchComparator::field(ContactField::Updated, comparator.is_ascending) } unsupported => { return Err(trc::JmapEvent::UnsupportedSort diff --git a/crates/jmap/src/contact/set.rs b/crates/jmap/src/contact/set.rs index f90ba36d..5cad8aad 100644 --- a/crates/jmap/src/contact/set.rs +++ b/crates/jmap/src/contact/set.rs @@ -115,7 +115,7 @@ impl ContactCardSet for Server { // Obtain contact card let document_id = id.document_id(); let contact_card_ = if let Some(contact_card_) = self - .get_archive(account_id, Collection::ContactCard, document_id) + .archive(account_id, Collection::ContactCard, document_id) .await? { contact_card_ @@ -281,7 +281,7 @@ impl ContactCardSet for Server { }; let Some(contact_card_) = self - .get_archive(account_id, Collection::ContactCard, document_id) + .archive(account_id, Collection::ContactCard, document_id) .await .caused_by(trc::location!())? else { diff --git a/crates/jmap/src/email/get.rs b/crates/jmap/src/email/get.rs index 740c4d64..ff102953 100644 --- a/crates/jmap/src/email/get.rs +++ b/crates/jmap/src/email/get.rs @@ -149,7 +149,7 @@ impl EmailGet for Server { continue; } let metadata_ = match self - .get_archive_by_property( + .archive_by_property( account_id, Collection::Email, id.document_id(), diff --git a/crates/jmap/src/email/query.rs b/crates/jmap/src/email/query.rs index f3937aef..d9c7de87 100644 --- a/crates/jmap/src/email/query.rs +++ b/crates/jmap/src/email/query.rs @@ -4,25 +4,23 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{JmapMethods, changes::state::JmapCacheState}; +use crate::{api::query::QueryResponseBuilder, changes::state::JmapCacheState}; use common::{MessageStoreCache, Server, auth::AccessToken}; use email::cache::{MessageCacheFetch, email::MessageCacheAccess}; use jmap_proto::{ - method::query::{Comparator, Filter, QueryRequest, QueryResponse}, + method::query::{Filter, QueryRequest, QueryResponse}, object::email::{Email, EmailComparator, EmailFilter}, }; use mail_parser::HeaderName; use nlp::language::Language; -use std::future::Future; +use std::{borrow::Cow, future::Future}; use store::{ - SerializeInfallible, - ahash::AHashMap, - fts::{Field, FilterGroup, FtsFilter, IntoFilterGroup}, - query::{self}, + ahash::{AHashMap, AHashSet}, roaring::RoaringBitmap, + search::{EmailSearchField, SearchComparator, SearchFilter}, }; use trc::AddContext; -use types::{acl::Acl, collection::Collection, field::EmailField, keyword::Keyword}; +use types::{acl::Acl, collection::Collection, keyword::Keyword}; pub trait EmailQuery: Sync + Send { fn email_query( @@ -45,369 +43,339 @@ impl EmailQuery for Server { .await .caused_by(trc::location!())?; - for cond_group in std::mem::take(&mut request.filter).into_filter_group() { - match cond_group { - FilterGroup::Fts(conds) => { - let mut fts_filters = Vec::with_capacity(filters.len()); - for cond in conds { - match cond { - Filter::Property(cond) => match cond { - EmailFilter::Text(text) => { - fts_filters.push(FtsFilter::Or); - fts_filters.push(FtsFilter::has_text( - Field::Header(HeaderName::From), - &text, - Language::None, - )); - fts_filters.push(FtsFilter::has_text( - Field::Header(HeaderName::To), - &text, - Language::None, - )); - fts_filters.push(FtsFilter::has_text( - Field::Header(HeaderName::Cc), - &text, - Language::None, - )); - fts_filters.push(FtsFilter::has_text( - Field::Header(HeaderName::Bcc), - &text, - Language::None, - )); - fts_filters.push(FtsFilter::has_text_detect( - Field::Header(HeaderName::Subject), - &text, - self.core.jmap.default_language, - )); - fts_filters.push(FtsFilter::has_text_detect( - Field::Body, - &text, - self.core.jmap.default_language, - )); - fts_filters.push(FtsFilter::has_text_detect( - Field::Attachment, - text, - self.core.jmap.default_language, - )); - fts_filters.push(FtsFilter::End); - } - EmailFilter::From(text) => fts_filters.push(FtsFilter::has_text( - Field::Header(HeaderName::From), - text, - Language::None, - )), - EmailFilter::To(text) => fts_filters.push(FtsFilter::has_text( - Field::Header(HeaderName::To), - text, - Language::None, - )), - EmailFilter::Cc(text) => fts_filters.push(FtsFilter::has_text( - Field::Header(HeaderName::Cc), - text, - Language::None, - )), - EmailFilter::Bcc(text) => fts_filters.push(FtsFilter::has_text( - Field::Header(HeaderName::Bcc), - text, - Language::None, - )), - EmailFilter::Subject(text) => { - fts_filters.push(FtsFilter::has_text_detect( - Field::Header(HeaderName::Subject), - text, - self.core.jmap.default_language, - )) - } - EmailFilter::Body(text) => { - fts_filters.push(FtsFilter::has_text_detect( - Field::Body, - text, - self.core.jmap.default_language, - )) - } - EmailFilter::Header(header) => { - let mut header = header.into_iter(); - let header_name = header.next().ok_or_else(|| { - trc::JmapEvent::InvalidArguments - .into_err() - .details("Header name is missing.".to_string()) - })?; + for filter in std::mem::take(&mut request.filter) { + match filter { + Filter::Property(cond) => match cond { + EmailFilter::Text(text) => { + filters.push(SearchFilter::Or); + filters.push(SearchFilter::has_text( + EmailSearchField::From, + &text, + Language::None, + )); + filters.push(SearchFilter::has_text( + EmailSearchField::To, + &text, + Language::None, + )); + filters.push(SearchFilter::has_text( + EmailSearchField::Cc, + &text, + Language::None, + )); + filters.push(SearchFilter::has_text( + EmailSearchField::Bcc, + &text, + Language::None, + )); + filters.push(SearchFilter::has_text_detect( + EmailSearchField::Subject, + &text, + self.core.jmap.default_language, + )); + filters.push(SearchFilter::has_text_detect( + EmailSearchField::Body, + &text, + self.core.jmap.default_language, + )); + filters.push(SearchFilter::has_text_detect( + EmailSearchField::Attachment, + text, + self.core.jmap.default_language, + )); + filters.push(SearchFilter::End); + } + EmailFilter::From(text) => filters.push(SearchFilter::has_text( + EmailSearchField::From, + text, + Language::None, + )), + EmailFilter::To(text) => filters.push(SearchFilter::has_text( + EmailSearchField::To, + text, + Language::None, + )), + EmailFilter::Cc(text) => filters.push(SearchFilter::has_text( + EmailSearchField::Cc, + text, + Language::None, + )), + EmailFilter::Bcc(text) => filters.push(SearchFilter::has_text( + EmailSearchField::Bcc, + text, + Language::None, + )), + EmailFilter::Subject(text) => filters.push(SearchFilter::has_text_detect( + EmailSearchField::Subject, + text, + self.core.jmap.default_language, + )), + EmailFilter::Body(text) => filters.push(SearchFilter::has_text_detect( + EmailSearchField::Body, + text, + self.core.jmap.default_language, + )), + EmailFilter::Header(header) => { + let mut header = header.into_iter(); + let header_name = header.next().ok_or_else(|| { + trc::JmapEvent::InvalidArguments + .into_err() + .details("Header name is missing.".to_string()) + })?; - match HeaderName::parse(header_name) { - Some(HeaderName::Other(header_name)) => { - return Err(trc::JmapEvent::InvalidArguments - .into_err() - .details(format!( - "Querying header '{header_name}' is not supported.", - ))); - } - Some(header_name) => { - if let Some(header_value) = header.next() { - if matches!( - header_name, - HeaderName::MessageId - | HeaderName::InReplyTo - | HeaderName::References - | HeaderName::ResentMessageId - ) { - fts_filters.push(FtsFilter::has_keyword( - Field::Header(header_name), - header_value, - )); - } else { - fts_filters.push(FtsFilter::has_text( - Field::Header(header_name), - header_value, - Language::None, - )); - } - } else { - fts_filters.push(FtsFilter::has_keyword( - Field::Keyword, - header_name.as_str().to_lowercase(), - )); - } - } - None => (), - } + if let Some(header_name) = HeaderName::parse(header_name) { + let is_id = matches!( + header_name, + HeaderName::MessageId + | HeaderName::InReplyTo + | HeaderName::References + | HeaderName::ResentMessageId + ); + let header_name = match header_name { + HeaderName::Other(value) => { + EmailSearchField::Header(Cow::Owned(value.to_ascii_lowercase())) } - other => { - return Err(trc::JmapEvent::UnsupportedFilter - .into_err() - .details(other.to_string())); + _ => EmailSearchField::Header(Cow::Borrowed( + header_name.as_static_str(), + )), + }; + + if let Some(value) = header.next() { + if is_id { + filters.push(SearchFilter::eq(header_name, value)); + } else { + filters.push(SearchFilter::has_text( + header_name, + value, + Language::None, + )); } - }, - Filter::And | Filter::Or | Filter::Not | Filter::Close => { - fts_filters.push(cond.into()); + } else { + filters.push(SearchFilter::exists(header_name)); } } } - filters.push(query::Filter::is_in_set( - self.fts_filter(account_id, Collection::Email, fts_filters) - .await?, - )); - } - FilterGroup::Store(cond) => { - match cond { - Filter::Property(cond) => { - match cond { - EmailFilter::InMailbox(mailbox) => filters.push( - query::Filter::is_in_set(RoaringBitmap::from_iter( - cached_messages - .in_mailbox(mailbox.document_id()) - .map(|item| item.document_id), - )), - ), - EmailFilter::InMailboxOtherThan(mailboxes) => { - filters.push(query::Filter::Not); - filters.push(query::Filter::Or); - for mailbox in mailboxes { - filters.push(query::Filter::is_in_set( - RoaringBitmap::from_iter( - cached_messages - .in_mailbox(mailbox.document_id()) - .map(|item| item.document_id), - ), - )); - } - filters.push(query::Filter::End); - filters.push(query::Filter::End); - } - EmailFilter::Before(date) => filters.push(query::Filter::lt( - EmailField::ReceivedAt, - date.timestamp().serialize(), - )), - EmailFilter::After(date) => filters.push(query::Filter::gt( - EmailField::ReceivedAt, - date.timestamp().serialize(), - )), - EmailFilter::MinSize(size) => filters - .push(query::Filter::ge(EmailField::Size, size.serialize())), - EmailFilter::MaxSize(size) => filters - .push(query::Filter::lt(EmailField::Size, size.serialize())), - EmailFilter::AllInThreadHaveKeyword(keyword) => { - filters.push(query::Filter::is_in_set(thread_keywords( - &cached_messages, - keyword, - true, - ))) - } - EmailFilter::SomeInThreadHaveKeyword(keyword) => { - filters.push(query::Filter::is_in_set(thread_keywords( - &cached_messages, - keyword, - false, - ))) - } - EmailFilter::NoneInThreadHaveKeyword(keyword) => { - filters.push(query::Filter::Not); - filters.push(query::Filter::is_in_set(thread_keywords( - &cached_messages, - keyword, - false, - ))); - filters.push(query::Filter::End); - } - EmailFilter::HasKeyword(keyword) => { - filters.push(query::Filter::is_in_set( - RoaringBitmap::from_iter( - cached_messages - .with_keyword(&keyword) - .map(|item| item.document_id), - ), - )); - } - EmailFilter::NotKeyword(keyword) => { - filters.push(query::Filter::Not); - filters.push(query::Filter::is_in_set( - RoaringBitmap::from_iter( - cached_messages - .with_keyword(&keyword) - .map(|item| item.document_id), - ), - )); - filters.push(query::Filter::End); - } - EmailFilter::HasAttachment(has_attach) => { - if !has_attach { - filters.push(query::Filter::Not); - } - filters.push(query::Filter::is_in_bitmap( - EmailField::HasAttachment, - (), - )); - if !has_attach { - filters.push(query::Filter::End); - } - } - - // Non-standard - EmailFilter::Id(ids) => { - let mut set = RoaringBitmap::new(); - for id in ids { - set.insert(id.document_id()); - } - filters.push(query::Filter::is_in_set(set)); - } - EmailFilter::SentBefore(date) => filters.push(query::Filter::lt( - EmailField::SentAt, - date.timestamp().serialize(), - )), - EmailFilter::SentAfter(date) => filters.push(query::Filter::gt( - EmailField::SentAt, - date.timestamp().serialize(), - )), - EmailFilter::InThread(id) => filters.push( - query::Filter::is_in_set(RoaringBitmap::from_iter( - cached_messages - .in_thread(id.document_id()) - .map(|item| item.document_id), - )), - ), - other => { - return Err(trc::JmapEvent::UnsupportedFilter - .into_err() - .details(other.to_string())); - } - } - } - - Filter::And | Filter::Or | Filter::Not | Filter::Close => { - filters.push(cond.into()); - } + EmailFilter::InMailbox(mailbox) => { + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cached_messages + .in_mailbox(mailbox.document_id()) + .map(|item| item.document_id), + ))) } - } - } - } - - let mut result_set = self.filter(account_id, Collection::Email, filters).await?; - if access_token.is_shared(account_id) { - result_set.apply_mask(cached_messages.shared_messages(access_token, Acl::ReadItems)); - } - let (response, paginate) = self - .build_query_response( - result_set.results.len() as usize, - cached_messages.get_state(false), - &request, - ) - .await?; - - if let Some(paginate) = paginate { - // Parse sort criteria - let mut comparators = Vec::with_capacity(request.sort.as_ref().map_or(1, |s| s.len())); - for comparator in request - .sort - .filter(|s| !s.is_empty()) - .unwrap_or_else(|| vec![Comparator::descending(EmailComparator::ReceivedAt)]) - { - comparators.push(match comparator.property { - EmailComparator::ReceivedAt => { - query::Comparator::field(EmailField::ReceivedAt, comparator.is_ascending) + EmailFilter::InMailboxOtherThan(mailboxes) => { + let mailboxes = mailboxes + .into_iter() + .map(|m| m.document_id()) + .collect::>(); + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cached_messages.emails.items.iter().filter_map(|item| { + if item + .mailboxes + .iter() + .any(|mb| mailboxes.contains(&mb.mailbox_id)) + { + None + } else { + Some(item.document_id) + } + }), + ))); } - EmailComparator::Size => { - query::Comparator::field(EmailField::Size, comparator.is_ascending) + EmailFilter::Before(date) => filters.push(SearchFilter::lt( + EmailSearchField::ReceivedAt, + date.timestamp(), + )), + EmailFilter::After(date) => filters.push(SearchFilter::gt( + EmailSearchField::ReceivedAt, + date.timestamp(), + )), + EmailFilter::MinSize(size) => { + filters.push(SearchFilter::ge(EmailSearchField::Size, size)) } - EmailComparator::From => { - query::Comparator::field(EmailField::From, comparator.is_ascending) + EmailFilter::MaxSize(size) => { + filters.push(SearchFilter::lt(EmailSearchField::Size, size)) } - EmailComparator::To => { - query::Comparator::field(EmailField::To, comparator.is_ascending) + EmailFilter::AllInThreadHaveKeyword(keyword) => filters.push( + SearchFilter::is_in_set(thread_keywords(&cached_messages, keyword, true)), + ), + EmailFilter::SomeInThreadHaveKeyword(keyword) => filters.push( + SearchFilter::is_in_set(thread_keywords(&cached_messages, keyword, false)), + ), + EmailFilter::NoneInThreadHaveKeyword(keyword) => { + filters.push(SearchFilter::Not); + filters.push(SearchFilter::is_in_set(thread_keywords( + &cached_messages, + keyword, + false, + ))); + filters.push(SearchFilter::End); } - EmailComparator::Subject => { - query::Comparator::field(EmailField::Subject, comparator.is_ascending) - } - EmailComparator::SentAt => { - query::Comparator::field(EmailField::SentAt, comparator.is_ascending) - } - EmailComparator::HasKeyword(keyword) => query::Comparator::set( - RoaringBitmap::from_iter( + EmailFilter::HasKeyword(keyword) => { + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cached_messages .with_keyword(&keyword) .map(|item| item.document_id), - ), - comparator.is_ascending, - ), - EmailComparator::AllInThreadHaveKeyword(keyword) => query::Comparator::set( - thread_keywords(&cached_messages, keyword, true), - comparator.is_ascending, - ), - EmailComparator::SomeInThreadHaveKeyword(keyword) => query::Comparator::set( - thread_keywords(&cached_messages, keyword, false), - comparator.is_ascending, - ), - // Non-standard - EmailComparator::Cc => { - query::Comparator::field(EmailField::Cc, comparator.is_ascending) + ))); + } + EmailFilter::NotKeyword(keyword) => { + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cached_messages + .without_keyword(&keyword) + .map(|item| item.document_id), + ))); + } + EmailFilter::HasAttachment(has_attach) => { + filters.push(SearchFilter::eq( + EmailSearchField::HasAttachment, + has_attach, + )); } + // Non-standard + EmailFilter::Id(ids) => { + let mut set = RoaringBitmap::new(); + for id in ids { + set.insert(id.document_id()); + } + filters.push(SearchFilter::is_in_set(set)); + } + EmailFilter::SentBefore(date) => { + filters.push(SearchFilter::lt(EmailSearchField::SentAt, date.timestamp())) + } + EmailFilter::SentAfter(date) => { + filters.push(SearchFilter::gt(EmailSearchField::SentAt, date.timestamp())) + } + EmailFilter::InThread(id) => { + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( + cached_messages + .in_thread(id.document_id()) + .map(|item| item.document_id), + ))) + } other => { - return Err(trc::JmapEvent::UnsupportedSort + return Err(trc::JmapEvent::UnsupportedFilter .into_err() .details(other.to_string())); } - }); + }, + Filter::And => { + filters.push(SearchFilter::And); + } + Filter::Or => { + filters.push(SearchFilter::Or); + } + Filter::Not => { + filters.push(SearchFilter::Not); + } + Filter::Close => { + filters.push(SearchFilter::End); + } } - - // Sort results - self.sort( - result_set, - comparators, - paginate - .with_prefix_map( - &cached_messages - .emails - .items - .iter() - .map(|item| (item.document_id, item.thread_id)) - .collect(), - ) - .with_prefix_unique(request.arguments.collapse_threads.unwrap_or(false)), - response, - ) - .await - } else { - Ok(response) } + + // Parse sort criteria + let mut comparators = Vec::with_capacity(request.sort.as_ref().map_or(1, |s| s.len())); + for comparator in request + .sort + .take() + .filter(|s| !s.is_empty()) + .unwrap_or_default() + { + comparators.push(match comparator.property { + EmailComparator::ReceivedAt => { + SearchComparator::field(EmailSearchField::ReceivedAt, comparator.is_ascending) + } + EmailComparator::Size => { + SearchComparator::field(EmailSearchField::Size, comparator.is_ascending) + } + EmailComparator::From => { + SearchComparator::field(EmailSearchField::From, comparator.is_ascending) + } + EmailComparator::To => { + SearchComparator::field(EmailSearchField::To, comparator.is_ascending) + } + EmailComparator::Subject => { + SearchComparator::field(EmailSearchField::Subject, comparator.is_ascending) + } + EmailComparator::SentAt => { + SearchComparator::field(EmailSearchField::SentAt, comparator.is_ascending) + } + EmailComparator::HasKeyword(keyword) => SearchComparator::set( + RoaringBitmap::from_iter( + cached_messages + .with_keyword(&keyword) + .map(|item| item.document_id), + ), + comparator.is_ascending, + ), + EmailComparator::AllInThreadHaveKeyword(keyword) => SearchComparator::set( + thread_keywords(&cached_messages, keyword, true), + comparator.is_ascending, + ), + EmailComparator::SomeInThreadHaveKeyword(keyword) => SearchComparator::set( + thread_keywords(&cached_messages, keyword, false), + comparator.is_ascending, + ), + // Non-standard + EmailComparator::Cc => { + SearchComparator::field(EmailSearchField::Cc, comparator.is_ascending) + } + + other => { + return Err(trc::JmapEvent::UnsupportedSort + .into_err() + .details(other.to_string())); + } + }); + } + + let results = self + .search_store() + .query(account_id, Collection::Email, filters, comparators) + .await?; + + let mut response = QueryResponseBuilder::new( + results.len(), + self.core.jmap.query_max_results, + cached_messages.get_state(false), + &request, + ); + + if !results.is_empty() { + let filter_ids = if access_token.is_shared(account_id) { + cached_messages + .shared_messages(access_token, Acl::ReadItems) + .into() + } else { + None + }; + let collapse_threads = request.arguments.collapse_threads.unwrap_or(false); + let mut seen_thread_ids = AHashSet::new(); + + for document_id in results { + if filter_ids + .as_ref() + .is_some_and(|filter_ids| !filter_ids.contains(document_id)) + { + continue; + } + let Some(thread_id) = cached_messages + .email_by_id(&document_id) + .map(|email| email.thread_id) + else { + continue; + }; + if collapse_threads && !seen_thread_ids.insert(thread_id) { + continue; + } + + if !response.add(thread_id, document_id) { + break; + } + } + } + + response.build() } } diff --git a/crates/jmap/src/email/set.rs b/crates/jmap/src/email/set.rs index 494ce751..4d88c397 100644 --- a/crates/jmap/src/email/set.rs +++ b/crates/jmap/src/email/set.rs @@ -6,7 +6,6 @@ use super::headers::{BuildHeader, ValueToHeader}; use crate::{ - JmapMethods, blob::download::BlobDownload, changes::state::JmapCacheState, email::{PatchResult, handle_email_patch, ingested_into_object}, @@ -73,9 +72,8 @@ impl EmailSet for Server { // Prepare response let account_id = request.account_id.document_id(); let cache = self.get_cached_messages(account_id).await?; - let mut response = self - .prepare_set_response(&request, cache.assert_state(false, &request.if_in_state)?) - .await?; + let mut response = SetResponse::from_request(&request, self.core.jmap.set_max_objects)? + .with_state(cache.assert_state(false, &request.if_in_state)?); let can_train_spam = self.email_bayes_can_train(access_token); // Obtain mailboxIds @@ -797,7 +795,7 @@ impl EmailSet for Server { // Obtain message data let document_id = id.document_id(); let data_ = match self - .get_archive(account_id, Collection::Email, document_id) + .archive(account_id, Collection::Email, document_id) .await? { Some(data) => data, @@ -991,7 +989,7 @@ impl EmailSet for Server { batch .with_account_id(account_id) .with_collection(Collection::Email) - .update_document(document_id) + .with_document(document_id) .custom( ObjectIndexBuilder::new() .with_current(data) @@ -1072,10 +1070,10 @@ impl EmailSet for Server { } if !destroy_ids.is_empty() { - // Batch delete (tombstone) messages + // Batch delete messages let mut batch = BatchBuilder::new(); let not_destroyed = self - .emails_tombstone(account_id, &mut batch, destroy_ids) + .emails_delete(account_id, &mut batch, destroy_ids) .await?; if !batch.is_empty() { last_change_id = self diff --git a/crates/jmap/src/email/snippet.rs b/crates/jmap/src/email/snippet.rs index 2ebebe1a..ffa2783e 100644 --- a/crates/jmap/src/email/snippet.rs +++ b/crates/jmap/src/email/snippet.rs @@ -125,7 +125,7 @@ impl EmailSearchSnippet for Server { continue; } let metadata_ = match self - .get_archive_by_property( + .archive_by_property( account_id, Collection::Email, document_id, diff --git a/crates/jmap/src/file/get.rs b/crates/jmap/src/file/get.rs index 80e2dc95..92db2c9d 100644 --- a/crates/jmap/src/file/get.rs +++ b/crates/jmap/src/file/get.rs @@ -81,7 +81,7 @@ impl FileNodeGet for Server { continue; } let _file_node = if let Some(file_node) = self - .get_archive(account_id, Collection::FileNode, document_id) + .archive(account_id, Collection::FileNode, document_id) .await? { file_node diff --git a/crates/jmap/src/file/query.rs b/crates/jmap/src/file/query.rs index 1f8877ba..e785858e 100644 --- a/crates/jmap/src/file/query.rs +++ b/crates/jmap/src/file/query.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{JmapMethods, changes::state::JmapCacheState}; +use crate::{ changes::state::JmapCacheState}; use common::{Server, auth::AccessToken}; use groupware::cache::GroupwareCache; use jmap_proto::{ @@ -47,20 +47,20 @@ impl FileNodeQuery for Server { if let Some(resource) = cache.container_resource_path_by_id(id.document_id()) { - filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache.subtree(resource.path()).map(|r| r.document_id()), ))) } else { - filters.push(query::Filter::is_in_set(RoaringBitmap::new())); + filters.push(SearchFilter::is_in_set(RoaringBitmap::new())); } } FileNodeFilter::ParentId(MaybeInvalid::Value(id)) => { - filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache.children_ids(id.document_id()), ))); } FileNodeFilter::HasParentId(has_parent_id) => { - filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache.resources.iter().filter_map(|r| { if has_parent_id == r.parent_id().is_some() { Some(r.document_id) @@ -71,7 +71,7 @@ impl FileNodeQuery for Server { ))); } FileNodeFilter::Name(name) => { - filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache.resources.iter().filter_map(|r| { if r.container_name().is_some_and(|n| n == name) { Some(r.document_id) @@ -82,7 +82,7 @@ impl FileNodeQuery for Server { ))); } FileNodeFilter::NameMatch(name) => { - filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache.resources.iter().filter_map(|r| { if r.container_name().is_some_and(|n| name.matches(n)) { Some(r.document_id) @@ -94,7 +94,7 @@ impl FileNodeQuery for Server { } FileNodeFilter::MinSize(size) => { let size = size as u32; - filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache.resources.iter().filter_map(|r| { if r.size().is_some_and(|s| s >= size) { Some(r.document_id) @@ -106,7 +106,7 @@ impl FileNodeQuery for Server { } FileNodeFilter::MaxSize(size) => { let size = size as u32; - filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( cache.resources.iter().filter_map(|r| { if r.size().is_some_and(|s| s <= size) { Some(r.document_id) @@ -138,7 +138,11 @@ impl FileNodeQuery for Server { } let (response, paginate) = self - .build_query_response(result_set.results.len() as usize, cache.get_state(false), &request) + .build_query_response( + result_set.results.len() as usize, + cache.get_state(false), + &request, + ) .await?; if let Some(paginate) = paginate { @@ -151,10 +155,10 @@ impl FileNodeQuery for Server { { comparators.push(match comparator.property { FileNodeComparator::Created => { - query::Comparator::field(ContactField::Created, comparator.is_ascending) + SearchComparator::field(ContactField::Created, comparator.is_ascending) } FileNodeComparator::Updated => { - query::Comparator::field(ContactField::Updated, comparator.is_ascending) + SearchComparator::field(ContactField::Updated, comparator.is_ascending) } unsupported => { return Err(trc::JmapEvent::UnsupportedSort diff --git a/crates/jmap/src/file/set.rs b/crates/jmap/src/file/set.rs index 58fa3831..dcb12eed 100644 --- a/crates/jmap/src/file/set.rs +++ b/crates/jmap/src/file/set.rs @@ -187,7 +187,7 @@ impl FileNodeSet for Server { // Obtain file node let document_id = id.document_id(); let file_node_ = if let Some(file_node_) = self - .get_archive(account_id, Collection::FileNode, document_id) + .archive(account_id, Collection::FileNode, document_id) .await? { file_node_ diff --git a/crates/jmap/src/identity/get.rs b/crates/jmap/src/identity/get.rs index 92b8fe28..8a233073 100644 --- a/crates/jmap/src/identity/get.rs +++ b/crates/jmap/src/identity/get.rs @@ -20,7 +20,10 @@ use store::{ write::BatchBuilder, }; use trc::AddContext; -use types::collection::{Collection, SyncCollection}; +use types::{ + collection::{Collection, SyncCollection}, + field::Field, +}; use utils::sanitize_email; pub trait IdentityGet: Sync + Send { @@ -80,7 +83,7 @@ impl IdentityGet for Server { continue; } let _identity = if let Some(identity) = self - .get_archive(account_id, Collection::Identity, document_id) + .archive(account_id, Collection::Identity, document_id) .await? { identity @@ -142,9 +145,8 @@ impl IdentityGet for Server { async fn identity_get_or_create(&self, account_id: u32) -> trc::Result { let mut identity_ids = self - .get_document_ids(account_id, Collection::Identity) - .await? - .unwrap_or_default(); + .document_ids(account_id, Collection::Identity, Field::DOCUMENT_ID) + .await?; if !identity_ids.is_empty() { return Ok(identity_ids); } @@ -203,7 +205,8 @@ impl IdentityGet for Server { let document_id = next_document_id; next_document_id -= 1; batch - .create_document(document_id) + .with_document(document_id) + .tag(Field::DOCUMENT_ID) .custom(ObjectIndexBuilder::<(), _>::new().with_changes(Identity { name, email, diff --git a/crates/jmap/src/identity/set.rs b/crates/jmap/src/identity/set.rs index 7be6166b..7122aab6 100644 --- a/crates/jmap/src/identity/set.rs +++ b/crates/jmap/src/identity/set.rs @@ -41,9 +41,8 @@ impl IdentitySet for Server { ) -> trc::Result> { let account_id = request.account_id.document_id(); let identity_ids = self - .get_document_ids(account_id, Collection::Identity) - .await? - .unwrap_or_default(); + .document_ids(account_id, Collection::Identity, Field::DOCUMENT_ID) + .await?; let mut response = SetResponse::from_request(&request, self.core.jmap.set_max_objects)?; let will_destroy = request.unwrap_destroy().into_valid().collect::>(); @@ -111,7 +110,8 @@ impl IdentitySet for Server { batch .with_account_id(account_id) .with_collection(Collection::Identity) - .create_document(document_id) + .with_document(document_id) + .tag(Field::DOCUMENT_ID) .custom(ObjectIndexBuilder::<(), _>::new().with_changes(identity)) .caused_by(trc::location!())? .commit_point(); @@ -129,7 +129,7 @@ impl IdentitySet for Server { // Obtain identity let document_id = id.document_id(); let identity_ = if let Some(identity_) = self - .get_archive(account_id, Collection::Identity, document_id) + .archive(account_id, Collection::Identity, document_id) .await? { identity_ @@ -157,7 +157,7 @@ impl IdentitySet for Server { batch .with_account_id(account_id) .with_collection(Collection::Identity) - .update_document(document_id) + .with_document(document_id) .custom( ObjectIndexBuilder::new() .with_current(identity) @@ -176,7 +176,8 @@ impl IdentitySet for Server { batch .with_account_id(account_id) .with_collection(Collection::Identity) - .delete_document(document_id) + .with_document(document_id) + .untag(Field::DOCUMENT_ID) .clear(Field::ARCHIVE) .log_item_delete(SyncCollection::Identity, None) .commit_point(); diff --git a/crates/jmap/src/lib.rs b/crates/jmap/src/lib.rs index fc129931..441a1381 100644 --- a/crates/jmap/src/lib.rs +++ b/crates/jmap/src/lib.rs @@ -6,24 +6,6 @@ #![warn(clippy::large_futures)] -use common::Server; -use jmap_proto::{ - method::{ - query::{QueryRequest, QueryResponse}, - set::{SetRequest, SetResponse}, - }, - object::JmapObject, - types::state::State, -}; -use std::{fmt::Display, future::Future}; -use store::{ - fts::FtsFilter, - query::{Comparator, Filter, ResultSet, SortedResultSet, sort::Pagination}, - roaring::RoaringBitmap, -}; -use trc::AddContext; -use types::collection::Collection; - pub mod addressbook; pub mod api; pub mod blob; @@ -46,179 +28,3 @@ pub mod submission; pub mod thread; pub mod vacation; pub mod websocket; - -impl JmapMethods for Server { - async fn prepare_set_response( - &self, - request: &SetRequest<'_, T>, - asserted_state: State, - ) -> trc::Result> { - Ok( - SetResponse::from_request(request, self.core.jmap.set_max_objects)? - .with_state(asserted_state), - ) - } - - async fn filter( - &self, - account_id: u32, - collection: Collection, - filters: Vec, - ) -> trc::Result { - self.core - .storage - .data - .filter(account_id, collection, filters) - .await - .add_context(|err| { - err.caused_by(trc::location!()) - .account_id(account_id) - .collection(collection) - }) - } - - async fn fts_filter + Display + Clone + std::fmt::Debug + Sync + Send>( - &self, - account_id: u32, - collection: Collection, - filters: Vec>, - ) -> trc::Result { - self.core - .storage - .fts - .query(account_id, collection, filters) - .await - .add_context(|err| { - err.caused_by(trc::location!()) - .account_id(account_id) - .collection(collection) - }) - } - - async fn build_query_response( - &'_ self, - total: usize, - query_state: State, - request: &QueryRequest, - ) -> trc::Result<(QueryResponse, Option>)> { - let (limit_total, limit) = if let Some(limit) = request.limit { - if limit > 0 { - let limit = std::cmp::min(limit, self.core.jmap.query_max_results); - (std::cmp::min(limit, total), limit) - } else { - (0, 0) - } - } else { - ( - std::cmp::min(self.core.jmap.query_max_results, total), - self.core.jmap.query_max_results, - ) - }; - Ok(( - QueryResponse { - account_id: request.account_id, - query_state, - can_calculate_changes: true, - position: 0, - ids: vec![], - total: if request.calculate_total.unwrap_or(false) { - Some(total) - } else { - None - }, - limit: if total > limit { Some(limit) } else { None }, - }, - if limit_total > 0 { - Pagination::new( - limit_total, - request.position.unwrap_or(0), - request.anchor.map(|a| a.document_id()), - request.anchor_offset.unwrap_or(0), - ) - .into() - } else { - None - }, - )) - } - - async fn sort( - &self, - result_set: ResultSet, - comparators: Vec, - paginate: Pagination<'_>, - mut response: QueryResponse, - ) -> trc::Result { - // Sort results - let collection = result_set.collection; - let account_id = result_set.account_id; - response.update_results( - self.core - .storage - .data - .sort(result_set, comparators, paginate) - .await - .add_context(|err| { - err.caused_by(trc::location!()) - .account_id(account_id) - .collection(collection) - })?, - )?; - - Ok(response) - } -} - -pub trait JmapMethods: Sync + Send { - fn prepare_set_response( - &self, - request: &SetRequest, - asserted_state: State, - ) -> impl Future>> + Send; - - fn filter( - &self, - account_id: u32, - collection: Collection, - filters: Vec, - ) -> impl Future> + Send; - - fn fts_filter + Display + Clone + std::fmt::Debug + Sync + Send>( - &self, - account_id: u32, - collection: Collection, - filters: Vec>, - ) -> impl Future> + Send; - - fn build_query_response( - &'_ self, - total: usize, - query_state: State, - request: &QueryRequest, - ) -> impl Future>)>> + Send; - - fn sort( - &self, - result_set: ResultSet, - comparators: Vec, - paginate: Pagination, - response: QueryResponse, - ) -> impl Future> + Send; -} - -trait UpdateResults: Sized { - fn update_results(&mut self, sorted_results: SortedResultSet) -> trc::Result<()>; -} - -impl UpdateResults for QueryResponse { - fn update_results(&mut self, sorted_results: SortedResultSet) -> trc::Result<()> { - // Prepare response - if sorted_results.found_anchor { - self.position = sorted_results.position; - self.ids = sorted_results.ids; - Ok(()) - } else { - Err(trc::JmapEvent::AnchorNotFound.into_err()) - } - } -} diff --git a/crates/jmap/src/mailbox/query.rs b/crates/jmap/src/mailbox/query.rs index c9f0a0fe..99b2742d 100644 --- a/crates/jmap/src/mailbox/query.rs +++ b/crates/jmap/src/mailbox/query.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{JmapMethods, changes::state::JmapCacheState}; +use crate::{api::query::QueryResponseBuilder, changes::state::JmapCacheState}; use common::{Server, auth::AccessToken}; use email::cache::{MessageCacheFetch, mailbox::MailboxCacheAccess}; use jmap_proto::{ @@ -16,8 +16,8 @@ use std::{ future::Future, }; use store::{ - query::{self}, roaring::RoaringBitmap, + search::{SearchComparator, SearchFilter}, }; use types::{acl::Acl, collection::Collection, special_use::SpecialUse}; @@ -49,7 +49,7 @@ impl MailboxQuery for Server { let parent_id = parent_id .and_then(|id| id.try_unwrap().map(|id| id.document_id())) .unwrap_or(u32::MAX); - filters.push(query::Filter::is_in_set( + filters.push(SearchFilter::is_in_set( mailboxes .mailboxes .items @@ -68,7 +68,7 @@ impl MailboxQuery for Server { } } let name = name.to_lowercase(); - filters.push(query::Filter::is_in_set( + filters.push(SearchFilter::is_in_set( mailboxes .mailboxes .items @@ -80,7 +80,7 @@ impl MailboxQuery for Server { } MailboxFilter::Role(role) => { if let Some(role) = role { - filters.push(query::Filter::is_in_set( + filters.push(SearchFilter::is_in_set( mailboxes .mailboxes .items @@ -90,8 +90,7 @@ impl MailboxQuery for Server { .collect::(), )); } else { - filters.push(query::Filter::Not); - filters.push(query::Filter::is_in_set( + filters.push(SearchFilter::is_in_set( mailboxes .mailboxes .items @@ -100,44 +99,34 @@ impl MailboxQuery for Server { .map(|m| m.document_id) .collect::(), )); - filters.push(query::Filter::End); } } MailboxFilter::HasAnyRole(has_role) => { - if !has_role { - filters.push(query::Filter::Not); - } - filters.push(query::Filter::is_in_set( + filters.push(SearchFilter::is_in_set( mailboxes .mailboxes .items .iter() - .filter(|mailbox| !matches!(mailbox.role, SpecialUse::None)) + .filter(|mailbox| { + !matches!(mailbox.role, SpecialUse::None) == has_role + }) .map(|m| m.document_id) .collect::(), )); - if !has_role { - filters.push(query::Filter::End); - } } MailboxFilter::IsSubscribed(is_subscribed) => { - if !is_subscribed { - filters.push(query::Filter::Not); - } - filters.push(query::Filter::is_in_set( + filters.push(SearchFilter::is_in_set( mailboxes .mailboxes .items .iter() .filter(|mailbox| { mailbox.subscribers.contains(&access_token.primary_id) + == is_subscribed }) .map(|m| m.document_id) .collect::(), )); - if !is_subscribed { - filters.push(query::Filter::End); - } } MailboxFilter::_T(other) => { return Err(trc::JmapEvent::UnsupportedFilter @@ -146,141 +135,157 @@ impl MailboxQuery for Server { } } } - - Filter::And | Filter::Or | Filter::Not | Filter::Close => { - filters.push(cond.into()); + Filter::And => { + filters.push(SearchFilter::And); + } + Filter::Or => { + filters.push(SearchFilter::Or); + } + Filter::Not => { + filters.push(SearchFilter::Not); + } + Filter::Close => { + filters.push(SearchFilter::End); } } } - let mut result_set = self - .filter(account_id, Collection::Mailbox, filters) - .await?; - if access_token.is_shared(account_id) { - result_set.apply_mask(mailboxes.shared_mailboxes(access_token, Acl::Read)); + let mut comparators = Vec::with_capacity(request.sort.as_ref().map_or(1, |s| s.len())); + + // Sort as tree + if sort_as_tree { + let sorted_list = mailboxes + .mailboxes + .items + .iter() + .map(|mailbox| (mailbox.path.as_str(), mailbox.document_id)) + .collect::>(); + comparators.push(SearchComparator::sorted_list( + sorted_list.into_values().collect(), + true, + )); } - let (mut response, mut paginate) = self - .build_query_response( - result_set.results.len() as usize, - mailboxes.get_state(true), - &request, - ) + + // Parse sort criteria + for comparator in request + .sort + .take() + .filter(|s| !s.is_empty()) + .unwrap_or_else(|| vec![Comparator::ascending(MailboxComparator::ParentId)]) + { + comparators.push(match comparator.property { + MailboxComparator::Name => { + let sorted_list = mailboxes + .mailboxes + .items + .iter() + .map(|mailbox| (mailbox.name.as_str(), mailbox.document_id)) + .collect::>(); + + SearchComparator::sorted_list( + sorted_list.into_iter().map(|v| v.1).collect(), + comparator.is_ascending, + ) + } + MailboxComparator::SortOrder => { + let sorted_list = mailboxes + .mailboxes + .items + .iter() + .map(|mailbox| (mailbox.sort_order, mailbox.document_id)) + .collect::>(); + + SearchComparator::sorted_list( + sorted_list.into_iter().map(|v| v.1).collect(), + comparator.is_ascending, + ) + } + MailboxComparator::ParentId => { + let sorted_list = mailboxes + .mailboxes + .items + .iter() + .map(|mailbox| { + ( + mailbox.parent_id().map(|id| id + 1).unwrap_or_default(), + mailbox.document_id, + ) + }) + .collect::>(); + + SearchComparator::sorted_list( + sorted_list.into_iter().map(|v| v.1).collect(), + comparator.is_ascending, + ) + } + + MailboxComparator::_T(other) => { + return Err(trc::JmapEvent::UnsupportedSort.into_err().details(other)); + } + }); + } + + let results = self + .search_store() + .query(account_id, Collection::Mailbox, filters, comparators) .await?; - // Filter as tree - if filter_as_tree { - let mut filtered_ids = RoaringBitmap::new(); + let mut response = QueryResponseBuilder::new( + results.len(), + self.core.jmap.query_max_results, + mailboxes.get_state(true), + &request, + ); - for document_id in &result_set.results { - let mut check_id = document_id; - for _ in 0..self.core.jmap.mailbox_max_depth { - if let Some(mailbox) = mailboxes.mailbox_by_id(&check_id) { - if let Some(parent_id) = mailbox.parent_id() { - if result_set.results.contains(parent_id) { - check_id = parent_id; + if !results.is_empty() { + let filter_ids = if access_token.is_shared(account_id) { + mailboxes.shared_mailboxes(access_token, Acl::Read).into() + } else { + None + }; + + // Filter as tree + if filter_as_tree { + let mut total_filtered = 0; + let mut is_page_full = false; + + for document_id in &results { + let mut check_id = *document_id; + for _ in 0..self.core.jmap.mailbox_max_depth { + if let Some(mailbox) = mailboxes.mailbox_by_id(&check_id) { + if let Some(parent_id) = mailbox.parent_id() { + if results.contains(&parent_id) { + check_id = parent_id; + } else { + break; + } } else { - break; + total_filtered += 1; + if !is_page_full && !response.add(0, *document_id) { + is_page_full = true; + } } - } else { - filtered_ids.insert(document_id); } } } - } - if filtered_ids.len() != result_set.results.len() { - let total = filtered_ids.len() as usize; - if response.total.is_some() { - response.total = Some(total); + + if total_filtered != results.len() { + response.response.total = Some(total_filtered); } - if let Some(paginate) = &mut paginate - && paginate.limit > total - { - paginate.limit = total; + } else { + for document_id in results { + if filter_ids + .as_ref() + .is_some_and(|filter_ids| !filter_ids.contains(document_id)) + { + continue; + } else if !response.add(0, document_id) { + break; + } } - result_set.results = filtered_ids; } } - if let Some(paginate) = paginate { - let mut comparators = Vec::with_capacity(request.sort.as_ref().map_or(1, |s| s.len())); - - // Sort as tree - if sort_as_tree { - let sorted_list = mailboxes - .mailboxes - .items - .iter() - .map(|mailbox| (mailbox.path.as_str(), mailbox.document_id)) - .collect::>(); - comparators.push(query::Comparator::sorted_list( - sorted_list.into_values().collect(), - true, - )); - } - - // Parse sort criteria - for comparator in request - .sort - .filter(|s| !s.is_empty()) - .unwrap_or_else(|| vec![Comparator::ascending(MailboxComparator::ParentId)]) - { - comparators.push(match comparator.property { - MailboxComparator::Name => { - let sorted_list = mailboxes - .mailboxes - .items - .iter() - .map(|mailbox| (mailbox.name.as_str(), mailbox.document_id)) - .collect::>(); - - query::Comparator::sorted_list( - sorted_list.into_iter().map(|v| v.1).collect(), - comparator.is_ascending, - ) - } - MailboxComparator::SortOrder => { - let sorted_list = mailboxes - .mailboxes - .items - .iter() - .map(|mailbox| (mailbox.sort_order, mailbox.document_id)) - .collect::>(); - - query::Comparator::sorted_list( - sorted_list.into_iter().map(|v| v.1).collect(), - comparator.is_ascending, - ) - } - MailboxComparator::ParentId => { - let sorted_list = mailboxes - .mailboxes - .items - .iter() - .map(|mailbox| { - ( - mailbox.parent_id().map(|id| id + 1).unwrap_or_default(), - mailbox.document_id, - ) - }) - .collect::>(); - - query::Comparator::sorted_list( - sorted_list.into_iter().map(|v| v.1).collect(), - comparator.is_ascending, - ) - } - - MailboxComparator::_T(other) => { - return Err(trc::JmapEvent::UnsupportedSort.into_err().details(other)); - } - }); - } - - response = self - .sort(result_set, comparators, paginate, response) - .await?; - } - - Ok(response) + response.build() } } diff --git a/crates/jmap/src/mailbox/set.rs b/crates/jmap/src/mailbox/set.rs index db275c0a..f25febda 100644 --- a/crates/jmap/src/mailbox/set.rs +++ b/crates/jmap/src/mailbox/set.rs @@ -5,7 +5,6 @@ */ use crate::{ - JmapMethods, api::acl::{JmapAcl, JmapRights}, changes::state::JmapCacheState, }; @@ -83,9 +82,8 @@ impl MailboxSet for Server { account_id, is_shared: access_token.is_shared(account_id), access_token, - response: self - .prepare_set_response(&request, cache.assert_state(true, &request.if_in_state)?) - .await?, + response: SetResponse::from_request(&request, self.core.jmap.set_max_objects)? + .with_state(cache.assert_state(true, &request.if_in_state)?), mailbox_ids: RoaringBitmap::from_iter(cache.mailboxes.index.keys()), will_destroy: request.unwrap_destroy().into_valid().collect(), }; @@ -119,7 +117,7 @@ impl MailboxSet for Server { let parent_id = builder.changes().unwrap().parent_id; if parent_id > 0 { batch - .update_document(parent_id - 1) + .with_document(parent_id - 1) .assert_value(MailboxField::Archive, AssertValue::Some); } @@ -130,7 +128,7 @@ impl MailboxSet for Server { .caused_by(trc::location!())?; batch - .create_document(document_id) + .with_document(document_id) .custom(builder) .caused_by(trc::location!())? .commit_point(); @@ -172,7 +170,7 @@ impl MailboxSet for Server { // Obtain mailbox let document_id = id.document_id(); if let Some(mailbox) = self - .get_archive(account_id, Collection::Mailbox, document_id) + .archive(account_id, Collection::Mailbox, document_id) .await? { // Validate ACL @@ -213,12 +211,12 @@ impl MailboxSet for Server { let parent_id = builder.changes().unwrap().parent_id; if parent_id > 0 { batch - .update_document(parent_id - 1) + .with_document(parent_id - 1) .assert_value(MailboxField::Archive, AssertValue::Some); } batch - .update_document(document_id) + .with_document(document_id) .custom(builder) .caused_by(trc::location!())? .commit_point(); @@ -460,7 +458,7 @@ impl MailboxSet for Server { let parent_document_id = mailbox_parent_id - 1; if let Some(mailbox_) = self - .get_archive(ctx.account_id, Collection::Mailbox, parent_document_id) + .archive(ctx.account_id, Collection::Mailbox, parent_document_id) .await? { let mailbox = mailbox_ diff --git a/crates/jmap/src/participant_identity/get.rs b/crates/jmap/src/participant_identity/get.rs index 976d3181..edf6c8ab 100644 --- a/crates/jmap/src/participant_identity/get.rs +++ b/crates/jmap/src/participant_identity/get.rs @@ -116,7 +116,7 @@ impl ParticipantIdentityGet for Server { account_id: u32, ) -> trc::Result>> { if let Some(identities) = self - .get_archive_by_property( + .archive_by_property( account_id, Collection::Principal, 0, @@ -173,7 +173,7 @@ impl ParticipantIdentityGet for Server { batch .with_account_id(account_id) .with_collection(Collection::Principal) - .update_document(0) + .with_document(0) .set( PrincipalField::ParticipantIdentities, Archiver::new(identities) @@ -183,7 +183,7 @@ impl ParticipantIdentityGet for Server { self.commit_batch(batch).await.caused_by(trc::location!())?; - self.get_archive_by_property( + self.archive_by_property( account_id, Collection::Principal, 0, diff --git a/crates/jmap/src/participant_identity/set.rs b/crates/jmap/src/participant_identity/set.rs index 84e65f58..ea730e68 100644 --- a/crates/jmap/src/participant_identity/set.rs +++ b/crates/jmap/src/participant_identity/set.rs @@ -176,7 +176,7 @@ impl ParticipantIdentitySet for Server { batch .with_account_id(account_id) .with_collection(Collection::Principal) - .update_document(0); + .with_document(0); if let Some(archive) = identity_archive { batch.assert_value(PrincipalField::ParticipantIdentities, archive); } diff --git a/crates/jmap/src/principal/availability.rs b/crates/jmap/src/principal/availability.rs index 9b3865e6..3cd467c2 100644 --- a/crates/jmap/src/principal/availability.rs +++ b/crates/jmap/src/principal/availability.rs @@ -178,7 +178,7 @@ impl PrincipalGetAvailability for Server { // Fetch event let document_id = resource.document_id; let Some(archive) = self - .get_archive(account_id, Collection::CalendarEvent, document_id) + .archive(account_id, Collection::CalendarEvent, document_id) .await .caused_by(trc::location!())? else { diff --git a/crates/jmap/src/push/get.rs b/crates/jmap/src/push/get.rs index 19c986e2..8692b84f 100644 --- a/crates/jmap/src/push/get.rs +++ b/crates/jmap/src/push/get.rs @@ -54,7 +54,7 @@ impl PushSubscriptionFetch for Server { }; let Some(subscriptions_) = self - .get_archive_by_property( + .archive_by_property( account_id, Collection::Principal, 0, @@ -158,14 +158,15 @@ impl PushSubscriptionFetch for Server { if updated_subscriptions.subscriptions.is_empty() { batch .with_account_id(u32::MAX) - .with_collection(Collection::PushSubscription) - .delete_document(account_id); + .with_collection(Collection::Principal) + .with_account_id(account_id) + .tag(PrincipalField::PushSubscriptions); } batch .with_account_id(account_id) .with_collection(Collection::Principal) - .update_document(0) + .with_document(0) .assert_value(PrincipalField::PushSubscriptions, subscriptions); if !updated_subscriptions.subscriptions.is_empty() { diff --git a/crates/jmap/src/push/set.rs b/crates/jmap/src/push/set.rs index c86066a4..5970f7f9 100644 --- a/crates/jmap/src/push/set.rs +++ b/crates/jmap/src/push/set.rs @@ -47,7 +47,7 @@ impl PushSubscriptionSet for Server { // Load existing push subscriptions let account_id = access_token.primary_id(); let subscriptions_archive = self - .get_archive_by_property( + .archive_by_property( account_id, Collection::Principal, 0, @@ -206,19 +206,21 @@ impl PushSubscriptionSet for Server { if subscriptions_archive.is_none() { batch .with_account_id(u32::MAX) - .with_collection(Collection::PushSubscription) - .create_document(account_id); + .with_collection(Collection::Principal) + .with_document(account_id) + .tag(PrincipalField::PushSubscriptions); } else if subscriptions.subscriptions.is_empty() { batch .with_account_id(u32::MAX) - .with_collection(Collection::PushSubscription) - .delete_document(account_id); + .with_collection(Collection::Principal) + .with_document(account_id) + .untag(PrincipalField::PushSubscriptions); } batch .with_account_id(account_id) .with_collection(Collection::Principal) - .update_document(0); + .with_document(0); if let Some(subscriptions_archive) = subscriptions_archive { batch.assert_value(PrincipalField::PushSubscriptions, subscriptions_archive); diff --git a/crates/jmap/src/share_notification/query.rs b/crates/jmap/src/share_notification/query.rs index e4f413b3..0c2c39de 100644 --- a/crates/jmap/src/share_notification/query.rs +++ b/crates/jmap/src/share_notification/query.rs @@ -6,7 +6,7 @@ use std::time::Duration; -use crate::{JmapMethods, UpdateResults}; +use crate::{ UpdateResults}; use common::{Server, sharing::notification::ShareNotification}; use jmap_proto::{ method::query::{Filter, QueryRequest, QueryResponse}, diff --git a/crates/jmap/src/sieve/get.rs b/crates/jmap/src/sieve/get.rs index 949d244b..884fc086 100644 --- a/crates/jmap/src/sieve/get.rs +++ b/crates/jmap/src/sieve/get.rs @@ -17,6 +17,7 @@ use trc::AddContext; use types::{ blob::{BlobClass, BlobId, BlobSection}, collection::{Collection, SyncCollection}, + field::SieveField, }; pub trait SieveScriptGet: Sync + Send { @@ -39,14 +40,13 @@ impl SieveScriptGet for Server { SieveProperty::IsActive, ]); let account_id = request.account_id.document_id(); - let push_ids = self - .get_document_ids(account_id, Collection::SieveScript) - .await? - .unwrap_or_default(); + let script_ids = self + .document_ids(account_id, Collection::SieveScript, SieveField::Name) + .await?; let ids = if let Some(ids) = ids { ids } else { - push_ids + script_ids .iter() .take(self.core.jmap.get_max_objects) .map(Into::into) @@ -66,12 +66,12 @@ impl SieveScriptGet for Server { for id in ids { // Obtain the sieve script object let document_id = id.document_id(); - if !push_ids.contains(document_id) { + if !script_ids.contains(document_id) { response.not_found.push(id); continue; } let sieve_ = if let Some(sieve) = self - .get_archive(account_id, Collection::SieveScript, document_id) + .archive(account_id, Collection::SieveScript, document_id) .await? { sieve diff --git a/crates/jmap/src/sieve/query.rs b/crates/jmap/src/sieve/query.rs index 23ca2a38..6f981d88 100644 --- a/crates/jmap/src/sieve/query.rs +++ b/crates/jmap/src/sieve/query.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{JmapMethods, changes::state::StateManager}; +use crate::changes::state::StateManager; use common::Server; use email::sieve::ingest::SieveScriptIngest; use jmap_proto::{ @@ -52,17 +52,17 @@ impl SieveScriptQuery for Server { match cond { Filter::Property(cond) => match cond { SieveFilter::Name(name) => { - filters.push(query::Filter::contains(SieveField::Name, &name)) + filters.push(SearchFilter::contains(SieveField::Name, &name)) } SieveFilter::IsActive(is_active) => { if !is_active { - filters.push(query::Filter::Not); + filters.push(SearchFilter::Not); } - filters.push(query::Filter::is_in_set(RoaringBitmap::from_iter( + filters.push(SearchFilter::is_in_set(RoaringBitmap::from_iter( active_script_id, ))); if !is_active { - filters.push(query::Filter::End); + filters.push(SearchFilter::End); } } SieveFilter::_T(other) => { @@ -99,9 +99,9 @@ impl SieveScriptQuery for Server { { comparators.push(match comparator.property { SieveComparator::Name => { - query::Comparator::field(SieveField::Name, comparator.is_ascending) + SearchComparator::field(SieveField::Name, comparator.is_ascending) } - SieveComparator::IsActive => query::Comparator::set( + SieveComparator::IsActive => SearchComparator::set( RoaringBitmap::from_iter(active_script_id), comparator.is_ascending, ), diff --git a/crates/jmap/src/sieve/set.rs b/crates/jmap/src/sieve/set.rs index c681cf4d..ddcd682d 100644 --- a/crates/jmap/src/sieve/set.rs +++ b/crates/jmap/src/sieve/set.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{JmapMethods, blob::download::BlobDownload, changes::state::StateManager}; +use crate::{blob::download::BlobDownload, changes::state::StateManager}; use common::{ Server, auth::{AccessToken, ResourceToken}, @@ -28,7 +28,6 @@ use sieve::compiler::ErrorType; use std::future::Future; use store::{ Serialize, SerializeInfallible, - query::Filter, rand::{Rng, rng}, write::{Archive, Archiver, BatchBuilder}, }; @@ -83,23 +82,20 @@ impl SieveScriptSet for Server { ) -> trc::Result> { let account_id = request.account_id.document_id(); let sieve_ids = self - .get_document_ids(account_id, Collection::SieveScript) - .await? - .unwrap_or_default(); + .document_ids(account_id, Collection::SieveScript, SieveField::Name) + .await?; let mut ctx = SetContext { resource_token: self.get_resource_token(access_token, account_id).await?, access_token, - response: self - .prepare_set_response( - &request, + response: SetResponse::from_request(&request, self.core.jmap.set_max_objects)? + .with_state( self.assert_state( account_id, SyncCollection::SieveScript, &request.if_in_state, ) .await?, - ) - .await?, + ), }; let will_destroy = request.unwrap_destroy().into_valid().collect::>(); @@ -134,7 +130,7 @@ impl SieveScriptSet for Server { batch .with_account_id(account_id) .with_collection(Collection::SieveScript) - .create_document(document_id) + .with_document(document_id) .custom(builder.with_access_token(ctx.access_token)) .caused_by(trc::location!())? .commit_point(); @@ -200,7 +196,7 @@ impl SieveScriptSet for Server { // Obtain sieve script let document_id = id.document_id(); if let Some(sieve_) = self - .get_archive(account_id, Collection::SieveScript, document_id) + .archive(account_id, Collection::SieveScript, document_id) .await? { let sieve = sieve_ @@ -221,7 +217,7 @@ impl SieveScriptSet for Server { batch .with_account_id(account_id) .with_collection(Collection::SieveScript) - .update_document(document_id); + .with_document(document_id); let blob_id = if let Some(blob) = blob { // Store blob @@ -330,13 +326,13 @@ impl SieveScriptSet for Server { batch .with_account_id(account_id) .with_collection(Collection::Principal) - .update_document(0) + .with_document(0) .set(PrincipalField::ActiveScriptId, id.document_id().serialize()); } else if on_success_deactivate_script { batch .with_account_id(account_id) .with_collection(Collection::Principal) - .update_document(0) + .with_document(0) .clear(PrincipalField::ActiveScriptId); } } diff --git a/crates/jmap/src/submission/get.rs b/crates/jmap/src/submission/get.rs index 35a8b622..937e62a2 100644 --- a/crates/jmap/src/submission/get.rs +++ b/crates/jmap/src/submission/get.rs @@ -23,6 +23,7 @@ use store::rkyv::option::ArchivedOption; use trc::AddContext; use types::{ collection::{Collection, SyncCollection}, + field::EmailSubmissionField, id::Id, }; use utils::map::vec_map::VecMap; @@ -54,9 +55,12 @@ impl EmailSubmissionGet for Server { ]); let account_id = request.account_id.document_id(); let email_submission_ids = self - .get_document_ids(account_id, Collection::EmailSubmission) - .await? - .unwrap_or_default(); + .document_ids( + account_id, + Collection::EmailSubmission, + EmailSubmissionField::EmailId, + ) + .await?; let ids = if let Some(ids) = ids { ids } else { @@ -84,7 +88,7 @@ impl EmailSubmissionGet for Server { continue; } let submission_ = if let Some(submission) = self - .get_archive(account_id, Collection::EmailSubmission, document_id) + .archive(account_id, Collection::EmailSubmission, document_id) .await? { submission diff --git a/crates/jmap/src/submission/query.rs b/crates/jmap/src/submission/query.rs index e057cca5..ecc52b76 100644 --- a/crates/jmap/src/submission/query.rs +++ b/crates/jmap/src/submission/query.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{JmapMethods, changes::state::StateManager}; +use crate::changes::state::StateManager; use common::Server; use email::submission::UndoStatus; use jmap_proto::{ @@ -41,37 +41,37 @@ impl EmailSubmissionQuery for Server { match cond { Filter::Property(cond) => match cond { EmailSubmissionFilter::IdentityIds(ids) => { - filters.push(query::Filter::Or); + filters.push(SearchFilter::Or); for id in ids.into_valid() { - filters.push(query::Filter::eq( + filters.push(SearchFilter::eq( EmailSubmissionField::IdentityId, id.document_id().serialize(), )); } - filters.push(query::Filter::End); + filters.push(SearchFilter::End); } EmailSubmissionFilter::EmailIds(ids) => { - filters.push(query::Filter::Or); + filters.push(SearchFilter::Or); for id in ids.into_valid() { - filters.push(query::Filter::eq( + filters.push(SearchFilter::eq( EmailSubmissionField::EmailId, id.id().serialize(), )); } - filters.push(query::Filter::End); + filters.push(SearchFilter::End); } EmailSubmissionFilter::ThreadIds(ids) => { - filters.push(query::Filter::Or); + filters.push(SearchFilter::Or); for id in ids.into_valid() { - filters.push(query::Filter::eq( + filters.push(SearchFilter::eq( EmailSubmissionField::ThreadId, id.document_id().serialize(), )); } - filters.push(query::Filter::End); + filters.push(SearchFilter::End); } EmailSubmissionFilter::UndoStatus(undo_status) => { - filters.push(query::Filter::eq( + filters.push(SearchFilter::eq( EmailSubmissionField::UndoStatus, match undo_status { email_submission::UndoStatus::Pending => UndoStatus::Pending, @@ -82,11 +82,11 @@ impl EmailSubmissionQuery for Server { .serialize(), )) } - EmailSubmissionFilter::Before(before) => filters.push(query::Filter::lt( + EmailSubmissionFilter::Before(before) => filters.push(SearchFilter::lt( EmailSubmissionField::SendAt, (before.timestamp() as u64).serialize(), )), - EmailSubmissionFilter::After(after) => filters.push(query::Filter::gt( + EmailSubmissionFilter::After(after) => filters.push(SearchFilter::gt( EmailSubmissionField::SendAt, (after.timestamp() as u64).serialize(), )), @@ -124,15 +124,15 @@ impl EmailSubmissionQuery for Server { .unwrap_or_else(|| vec![Comparator::descending(EmailSubmissionComparator::SentAt)]) { comparators.push(match comparator.property { - EmailSubmissionComparator::EmailId => query::Comparator::field( + EmailSubmissionComparator::EmailId => SearchComparator::field( EmailSubmissionField::EmailId, comparator.is_ascending, ), - EmailSubmissionComparator::ThreadId => query::Comparator::field( + EmailSubmissionComparator::ThreadId => SearchComparator::field( EmailSubmissionField::ThreadId, comparator.is_ascending, ), - EmailSubmissionComparator::SentAt => query::Comparator::field( + EmailSubmissionComparator::SentAt => SearchComparator::field( EmailSubmissionField::SendAt, comparator.is_ascending, ), diff --git a/crates/jmap/src/submission/set.rs b/crates/jmap/src/submission/set.rs index 721be31a..e327ffa5 100644 --- a/crates/jmap/src/submission/set.rs +++ b/crates/jmap/src/submission/set.rs @@ -95,7 +95,7 @@ impl EmailSubmissionSet for Server { batch .with_account_id(account_id) .with_collection(Collection::EmailSubmission) - .create_document(document_id) + .with_document(document_id) .custom(ObjectIndexBuilder::<(), _>::new().with_changes(submission)) .caused_by(trc::location!())? .commit_point(); @@ -118,7 +118,7 @@ impl EmailSubmissionSet for Server { // Obtain submission let document_id = id.document_id(); let submission = if let Some(submission) = self - .get_archive(account_id, Collection::EmailSubmission, document_id) + .archive(account_id, Collection::EmailSubmission, document_id) .await? { submission @@ -171,7 +171,7 @@ impl EmailSubmissionSet for Server { batch .with_account_id(account_id) .with_collection(Collection::EmailSubmission) - .update_document(document_id) + .with_document(document_id) .custom( ObjectIndexBuilder::new() .with_current(submission) @@ -211,14 +211,14 @@ impl EmailSubmissionSet for Server { for id in will_destroy { let document_id = id.document_id(); if let Some(submission) = self - .get_archive(account_id, Collection::EmailSubmission, document_id) + .archive(account_id, Collection::EmailSubmission, document_id) .await? { // Update record batch .with_account_id(account_id) .with_collection(Collection::EmailSubmission) - .delete_document(document_id) + .with_document(document_id) .custom( ObjectIndexBuilder::<_, ()>::new().with_current( submission @@ -456,7 +456,7 @@ impl EmailSubmissionSet for Server { // Fetch identity's mailFrom let identity_mail_from = if let Some(identity) = self - .get_archive(account_id, Collection::Identity, submission.identity_id) + .archive(account_id, Collection::Identity, submission.identity_id) .await? { identity @@ -492,7 +492,7 @@ impl EmailSubmissionSet for Server { // Obtain message metadata let metadata_ = if let Some(metadata) = self - .get_archive_by_property( + .archive_by_property( account_id, Collection::Email, submission.email_id, diff --git a/crates/jmap/src/thread/get.rs b/crates/jmap/src/thread/get.rs index f371beb2..343a0d26 100644 --- a/crates/jmap/src/thread/get.rs +++ b/crates/jmap/src/thread/get.rs @@ -84,6 +84,7 @@ impl ThreadGet for Server { Map::with_capacity(2).with_key_value(ThreadProperty::Id, id); if add_email_ids { let doc_count = document_ids.len() as usize; + let todo = " sorted as vec![Comparator::ascending(EmailField::ReceivedAt)],"; thread.insert_unchecked( ThreadProperty::EmailIds, self.core @@ -91,7 +92,7 @@ impl ThreadGet for Server { .data .sort( ResultSet::new(account_id, Collection::Email, document_ids), - vec![Comparator::ascending(EmailField::ReceivedAt)], + vec![], Pagination::new(doc_count, 0, None, 0), ) .await diff --git a/crates/jmap/src/vacation/get.rs b/crates/jmap/src/vacation/get.rs index 8a725f01..5c348e47 100644 --- a/crates/jmap/src/vacation/get.rs +++ b/crates/jmap/src/vacation/get.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{JmapMethods, changes::state::StateManager}; +use crate::{ changes::state::StateManager}; use common::Server; use email::sieve::{SieveScript, ingest::SieveScriptIngest}; use jmap_proto::{ @@ -17,7 +17,7 @@ use jmap_proto::{ }; use jmap_tools::{Map, Value}; use std::future::Future; -use store::query::Filter; +use store::SearchFilter; use trc::AddContext; use types::{ collection::{Collection, SyncCollection}, @@ -82,7 +82,7 @@ impl VacationResponseGet for Server { if do_get { if let Some(document_id) = self.get_vacation_sieve_script_id(account_id).await? { if let Some(sieve_) = self - .get_archive(account_id, Collection::SieveScript, document_id) + .archive(account_id, Collection::SieveScript, document_id) .await? { let active_script_id = self.sieve_script_get_active_id(account_id).await?; diff --git a/crates/jmap/src/vacation/set.rs b/crates/jmap/src/vacation/set.rs index e88d9e20..6b2b4738 100644 --- a/crates/jmap/src/vacation/set.rs +++ b/crates/jmap/src/vacation/set.rs @@ -5,7 +5,7 @@ */ use super::get::VacationResponseGet; -use crate::{JmapMethods, changes::state::StateManager}; +use crate::changes::state::StateManager; use common::{Server, auth::AccessToken, storage::index::ObjectIndexBuilder}; use email::sieve::{ SieveScript, VacationResponse, delete::SieveScriptDelete, ingest::SieveScriptIngest, @@ -51,17 +51,15 @@ impl VacationResponseSet for Server { access_token: &AccessToken, ) -> trc::Result> { let account_id = request.account_id.document_id(); - let mut response = self - .prepare_set_response( - &request, + let mut response = SetResponse::from_request(&request, self.core.jmap.set_max_objects)? + .with_state( self.assert_state( account_id, SyncCollection::SieveScript, &request.if_in_state, ) .await?, - ) - .await?; + ); let will_destroy = request.unwrap_destroy().into_valid().collect::>(); // Process set or update requests @@ -134,7 +132,7 @@ impl VacationResponseSet for Server { let (mut sieve, prev_sieve) = if let Some(document_id) = document_id { let prev_sieve = self - .get_archive(account_id, Collection::SieveScript, document_id) + .archive(account_id, Collection::SieveScript, document_id) .await? .ok_or_else(|| { trc::StoreEvent::NotFound @@ -261,7 +259,7 @@ impl VacationResponseSet for Server { // Update id let document_id = if let Some(document_id) = document_id { - batch.update_document(document_id); + batch.with_document(document_id); document_id } else { let document_id = self @@ -269,7 +267,7 @@ impl VacationResponseSet for Server { .assign_document_ids(account_id, Collection::SieveScript, 1) .await .caused_by(trc::location!())?; - batch.create_document(document_id); + batch.with_document(document_id); document_id }; @@ -293,13 +291,13 @@ impl VacationResponseSet for Server { if !was_active { batch .with_collection(Collection::Principal) - .update_document(0) + .with_document(0) .set(PrincipalField::ActiveScriptId, document_id.serialize()); } } else if was_active { batch .with_collection(Collection::Principal) - .update_document(0) + .with_document(0) .clear(PrincipalField::ActiveScriptId); } @@ -335,7 +333,7 @@ impl VacationResponseSet for Server { if active_script_id == Some(document_id) { batch .with_collection(Collection::Principal) - .update_document(0) + .with_document(0) .clear(PrincipalField::ActiveScriptId); } diff --git a/crates/managesieve/src/core/client.rs b/crates/managesieve/src/core/client.rs index 61bcbe49..ce7a74ee 100644 --- a/crates/managesieve/src/core/client.rs +++ b/crates/managesieve/src/core/client.rs @@ -10,7 +10,6 @@ use common::{ listener::{SessionResult, SessionStream}, }; use imap_proto::receiver::{self, Request}; -use store::query::Filter; use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; use trc::{AddContext, SecurityEvent}; use types::{collection::Collection, field::SieveField}; @@ -271,21 +270,16 @@ impl Session { impl Session { pub async fn get_script_id(&self, account_id: u32, name: &str) -> trc::Result { self.server - .core - .storage - .data - .filter( + .document_ids_matching( account_id, Collection::SieveScript, - vec![Filter::eq( - SieveField::Name, - name.to_lowercase().into_bytes(), - )], + SieveField::Name, + name.to_lowercase().as_bytes(), ) .await .caused_by(trc::location!()) .and_then(|results| { - results.results.min().ok_or_else(|| { + results.min().ok_or_else(|| { trc::ManageSieveEvent::Error .into_err() .code(ResponseCode::NonExistent) diff --git a/crates/managesieve/src/op/getscript.rs b/crates/managesieve/src/op/getscript.rs index 111e6029..26d0bd20 100644 --- a/crates/managesieve/src/op/getscript.rs +++ b/crates/managesieve/src/op/getscript.rs @@ -33,7 +33,7 @@ impl Session { let document_id = self.get_script_id(account_id, &name).await?; let sieve_ = self .server - .get_archive(account_id, Collection::SieveScript, document_id) + .archive(account_id, Collection::SieveScript, document_id) .await .caused_by(trc::location!())? .ok_or_else(|| { diff --git a/crates/managesieve/src/op/listscripts.rs b/crates/managesieve/src/op/listscripts.rs index c6a6d63b..f01692de 100644 --- a/crates/managesieve/src/op/listscripts.rs +++ b/crates/managesieve/src/op/listscripts.rs @@ -10,7 +10,7 @@ use directory::Permission; use email::sieve::{SieveScript, ingest::SieveScriptIngest}; use std::time::Instant; use trc::AddContext; -use types::collection::Collection; +use types::{collection::Collection, field::SieveField}; impl Session { pub async fn handle_listscripts(&mut self) -> trc::Result> { @@ -21,10 +21,9 @@ impl Session { let account_id = self.state.access_token().primary_id(); let document_ids = self .server - .get_document_ids(account_id, Collection::SieveScript) + .document_ids(account_id, Collection::SieveScript, SieveField::Name) .await - .caused_by(trc::location!())? - .unwrap_or_default(); + .caused_by(trc::location!())?; if document_ids.is_empty() { return Ok(StatusResponse::ok("").into_bytes()); @@ -37,7 +36,7 @@ impl Session { for document_id in document_ids { if let Some(script_) = self .server - .get_archive(account_id, Collection::SieveScript, document_id) + .archive(account_id, Collection::SieveScript, document_id) .await .caused_by(trc::location!())? { diff --git a/crates/managesieve/src/op/putscript.rs b/crates/managesieve/src/op/putscript.rs index c87be914..c27b5d0e 100644 --- a/crates/managesieve/src/op/putscript.rs +++ b/crates/managesieve/src/op/putscript.rs @@ -13,7 +13,6 @@ use sieve::compiler::ErrorType; use std::time::Instant; use store::{ Serialize, - query::Filter, write::{Archiver, BatchBuilder}, }; use trc::AddContext; @@ -56,12 +55,11 @@ impl Session { if self .server - .get_document_ids(account_id, Collection::SieveScript) + .document_ids(account_id, Collection::SieveScript, SieveField::Name) .await .caused_by(trc::location!())? - .map(|ids| ids.len() as usize) - .unwrap_or(0) - > access_token.object_quota(Collection::SieveScript) as usize + .len() + > access_token.object_quota(Collection::SieveScript) as u64 { return Err(trc::ManageSieveEvent::Error .into_err() @@ -104,7 +102,7 @@ impl Session { // Obtain script values let script_ = self .server - .get_archive(account_id, Collection::SieveScript, document_id) + .archive(account_id, Collection::SieveScript, document_id) .await .caused_by(trc::location!())? .ok_or_else(|| { @@ -130,7 +128,7 @@ impl Session { batch .with_account_id(account_id) .with_collection(Collection::SieveScript) - .update_document(document_id) + .with_document(document_id) .custom( ObjectIndexBuilder::new() .with_changes( @@ -177,7 +175,7 @@ impl Session { batch .with_account_id(account_id) .with_collection(Collection::SieveScript) - .create_document(document_id) + .with_document(document_id) .custom( ObjectIndexBuilder::<(), _>::new() .with_changes( @@ -221,18 +219,14 @@ impl Session { } else { Ok(self .server - .store() - .filter( + .document_ids_matching( account_id, Collection::SieveScript, - vec![Filter::eq( - SieveField::Name, - name.to_lowercase().into_bytes(), - )], + SieveField::Name, + name.to_lowercase().as_bytes(), ) .await .caused_by(trc::location!())? - .results .min()) } } diff --git a/crates/managesieve/src/op/renamescript.rs b/crates/managesieve/src/op/renamescript.rs index c2358a51..cd4bf293 100644 --- a/crates/managesieve/src/op/renamescript.rs +++ b/crates/managesieve/src/op/renamescript.rs @@ -58,7 +58,7 @@ impl Session { // Obtain script values let script = self .server - .get_archive(account_id, Collection::SieveScript, document_id) + .archive(account_id, Collection::SieveScript, document_id) .await .caused_by(trc::location!())? .ok_or_else(|| { @@ -75,7 +75,7 @@ impl Session { batch .with_account_id(account_id) .with_collection(Collection::SieveScript) - .update_document(document_id) + .with_document(document_id) .custom( ObjectIndexBuilder::new() .with_changes(script.inner.clone().with_name(new_name.clone())) diff --git a/crates/managesieve/src/op/setactive.rs b/crates/managesieve/src/op/setactive.rs index ba8d9f36..325f0e35 100644 --- a/crates/managesieve/src/op/setactive.rs +++ b/crates/managesieve/src/op/setactive.rs @@ -40,13 +40,13 @@ impl Session { batch .with_account_id(account_id) .with_collection(Collection::Principal) - .update_document(0) + .with_document(0) .set(PrincipalField::ActiveScriptId, document_id.serialize()); } else { batch .with_account_id(account_id) .with_collection(Collection::Principal) - .update_document(0) + .with_document(0) .clear(PrincipalField::ActiveScriptId); } self.server diff --git a/crates/migration/src/addressbook_v2.rs b/crates/migration/src/addressbook_v2.rs index d5b34e4a..e94b300f 100644 --- a/crates/migration/src/addressbook_v2.rs +++ b/crates/migration/src/addressbook_v2.rs @@ -43,7 +43,7 @@ pub(crate) async fn migrate_addressbook_v013(server: &Server, account_id: u32) - for document_id in document_ids.iter() { let Some(archive) = server - .get_archive(account_id, Collection::AddressBook, document_id) + .archive(account_id, Collection::AddressBook, document_id) .await .caused_by(trc::location!())? else { @@ -74,7 +74,7 @@ pub(crate) async fn migrate_addressbook_v013(server: &Server, account_id: u32) - batch .with_account_id(account_id) .with_collection(Collection::AddressBook) - .update_document(document_id) + .with_document(document_id) .set( Field::ARCHIVE, Archiver::new(new_book) diff --git a/crates/migration/src/calendar_v2.rs b/crates/migration/src/calendar_v2.rs index cf291fdf..129a69a0 100644 --- a/crates/migration/src/calendar_v2.rs +++ b/crates/migration/src/calendar_v2.rs @@ -74,7 +74,7 @@ pub(crate) async fn migrate_calendar_v013(server: &Server, account_id: u32) -> t for document_id in document_ids.iter() { let Some(archive) = server - .get_archive(account_id, Collection::Calendar, document_id) + .archive(account_id, Collection::Calendar, document_id) .await .caused_by(trc::location!())? else { @@ -117,7 +117,7 @@ pub(crate) async fn migrate_calendar_v013(server: &Server, account_id: u32) -> t batch .with_account_id(account_id) .with_collection(Collection::Calendar) - .update_document(document_id) + .with_document(document_id) .set( Field::ARCHIVE, Archiver::new(new_calendar) diff --git a/crates/migration/src/contact_v2.rs b/crates/migration/src/contact_v2.rs index 39ed80e1..5eaa3f45 100644 --- a/crates/migration/src/contact_v2.rs +++ b/crates/migration/src/contact_v2.rs @@ -37,7 +37,7 @@ pub(crate) async fn migrate_contacts_v013(server: &Server, account_id: u32) -> t for document_id in document_ids.iter() { let Some(archive) = server - .get_archive(account_id, Collection::ContactCard, document_id) + .archive(account_id, Collection::ContactCard, document_id) .await .caused_by(trc::location!())? else { @@ -62,7 +62,7 @@ pub(crate) async fn migrate_contacts_v013(server: &Server, account_id: u32) -> t batch .with_account_id(account_id) .with_collection(Collection::ContactCard) - .update_document(document_id) + .with_document(document_id) .set( Field::ARCHIVE, Archiver::new(new_contact) diff --git a/crates/migration/src/email.rs b/crates/migration/src/email.rs index 0e2b387f..6d57cadd 100644 --- a/crates/migration/src/email.rs +++ b/crates/migration/src/email.rs @@ -7,7 +7,7 @@ use super::{LegacyBincode, get_properties}; use common::Server; use email::{ - mailbox::{TOMBSTONE_ID, UidMailbox}, + mailbox::UidMailbox, message::{ index::{MAX_ID_LENGTH, VisitText}, metadata::{ @@ -63,7 +63,7 @@ pub(crate) async fn migrate_emails(server: &Server, account_id: u32) -> trc::Res collection: Collection::Email.into(), class: BitmapClass::Tag { field: FIELD_MAILBOX_IDS, - value: TagValue::Id(TOMBSTONE_ID), + value: TagValue::Id(u32::MAX - 1), }, document_id: 0, }) @@ -114,7 +114,7 @@ pub(crate) async fn migrate_emails(server: &Server, account_id: u32) -> trc::Res batch .with_account_id(account_id) .with_collection(Collection::Email) - .update_document(message_id); + .with_document(message_id); for mailbox in &data.mailboxes { batch.untag(EmailField::MailboxIds, TagValue::Id(mailbox.mailbox_id)); @@ -155,7 +155,7 @@ pub(crate) async fn migrate_emails(server: &Server, account_id: u32) -> trc::Res batch .with_account_id(account_id) .with_collection(Collection::Email) - .update_document(message_id); + .with_document(message_id); for header in metadata.root_part().headers.iter().rev() { if matches!(header.name, HeaderName::MessageId) { diff --git a/crates/migration/src/encryption.rs b/crates/migration/src/encryption.rs index ea7955f0..211698a6 100644 --- a/crates/migration/src/encryption.rs +++ b/crates/migration/src/encryption.rs @@ -32,7 +32,7 @@ pub(crate) async fn migrate_encryption_params( batch .with_account_id(account_id) .with_collection(Collection::Principal) - .update_document(0) + .with_document(0) .set( PrincipalField::EncryptionKeys, Archiver::new(legacy.0) diff --git a/crates/migration/src/event_v1.rs b/crates/migration/src/event_v1.rs index 302f8c0b..7e94ce79 100644 --- a/crates/migration/src/event_v1.rs +++ b/crates/migration/src/event_v1.rs @@ -89,7 +89,7 @@ pub(crate) async fn migrate_calendar_events_v012(server: &Server) -> trc::Result for document_id in document_ids.iter() { let Some(archive) = server - .get_archive(account_id, Collection::CalendarEvent, document_id) + .archive(account_id, Collection::CalendarEvent, document_id) .await .caused_by(trc::location!())? else { @@ -121,7 +121,7 @@ pub(crate) async fn migrate_calendar_events_v012(server: &Server) -> trc::Result batch .with_account_id(account_id) .with_collection(Collection::CalendarEvent) - .update_document(document_id) + .with_document(document_id) .set( Field::ARCHIVE, Archiver::new(new_event) diff --git a/crates/migration/src/event_v2.rs b/crates/migration/src/event_v2.rs index e5e78cdc..5ce20f40 100644 --- a/crates/migration/src/event_v2.rs +++ b/crates/migration/src/event_v2.rs @@ -77,7 +77,7 @@ pub(crate) async fn migrate_calendar_events_v013( for document_id in document_ids.iter() { let Some(archive) = server - .get_archive(account_id, Collection::CalendarEvent, document_id) + .archive(account_id, Collection::CalendarEvent, document_id) .await .caused_by(trc::location!())? else { @@ -111,7 +111,7 @@ pub(crate) async fn migrate_calendar_events_v013( batch .with_account_id(account_id) .with_collection(Collection::CalendarEvent) - .update_document(document_id) + .with_document(document_id) .set( Field::ARCHIVE, Archiver::new(new_event) @@ -151,7 +151,7 @@ pub(crate) async fn migrate_calendar_scheduling_v013( for document_id in document_ids.iter() { let Some(archive) = server - .get_archive( + .archive( account_id, Collection::CalendarEventNotification, document_id, @@ -179,7 +179,7 @@ pub(crate) async fn migrate_calendar_scheduling_v013( batch .with_account_id(account_id) .with_collection(Collection::CalendarEventNotification) - .update_document(document_id) + .with_document(document_id) .set( Field::ARCHIVE, Archiver::new(new_event) diff --git a/crates/migration/src/identity.rs b/crates/migration/src/identity.rs index ff0a3e30..04f9f740 100644 --- a/crates/migration/src/identity.rs +++ b/crates/migration/src/identity.rs @@ -44,7 +44,7 @@ pub(crate) async fn migrate_identities(server: &Server, account_id: u32) -> trc: batch .with_account_id(account_id) .with_collection(Collection::Identity) - .update_document(identity_id) + .with_document(identity_id) .set( Field::ARCHIVE, Archiver::new(Identity::from_legacy(legacy)) diff --git a/crates/migration/src/lib.rs b/crates/migration/src/lib.rs index 525904bd..6ab5795c 100644 --- a/crates/migration/src/lib.rs +++ b/crates/migration/src/lib.rs @@ -4,12 +4,12 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{ +/*use crate::{ queue::{migrate_queue_v011, migrate_queue_v012}, v011::migrate_v0_11, v012::migrate_v0_12, v013::migrate_v0_13, -}; +};*/ use common::{DATABASE_SCHEMA_VERSION, Server, manager::boot::DEFAULT_SETTINGS}; use std::time::Duration; use store::{ @@ -21,7 +21,7 @@ use store::{ use trc::AddContext; use types::collection::Collection; -pub mod addressbook_v2; +/*pub mod addressbook_v2; pub mod calendar_v2; pub mod changelog; pub mod contact_v2; @@ -45,14 +45,14 @@ pub mod tasks; pub mod threads; pub mod v011; pub mod v012; -pub mod v013; +pub mod v013;*/ const LOCK_WAIT_TIME_ACCOUNT: u64 = 3 * 60; const LOCK_WAIT_TIME_CORE: u64 = 5 * 60; const LOCK_RETRY_TIME: Duration = Duration::from_secs(30); pub async fn try_migrate(server: &Server) -> trc::Result<()> { - if let Some(version) = std::env::var("FORCE_MIGRATE_QUEUE") + /*if let Some(version) = std::env::var("FORCE_MIGRATE_QUEUE") .ok() .and_then(|s| s.parse::().ok()) { @@ -170,7 +170,7 @@ pub async fn try_migrate(server: &Server) -> trc::Result<()> { .store() .write(batch.build_all()) .await - .caused_by(trc::location!())?; + .caused_by(trc::location!())?;*/ Ok(()) } diff --git a/crates/migration/src/mailbox.rs b/crates/migration/src/mailbox.rs index 465da868..29bea264 100644 --- a/crates/migration/src/mailbox.rs +++ b/crates/migration/src/mailbox.rs @@ -48,7 +48,7 @@ pub(crate) async fn migrate_mailboxes(server: &Server, account_id: u32) -> trc:: batch .with_account_id(account_id) .with_collection(Collection::Mailbox) - .update_document(mailbox_id) + .with_document(mailbox_id) .set( Field::ARCHIVE, Archiver::new(Mailbox::from_legacy(legacy)) diff --git a/crates/migration/src/principal_v1.rs b/crates/migration/src/principal_v1.rs index e25473ee..1caafb48 100644 --- a/crates/migration/src/principal_v1.rs +++ b/crates/migration/src/principal_v1.rs @@ -58,7 +58,7 @@ pub(crate) async fn migrate_principals_v0_11(server: &Server) -> trc::Result trc::Result match legacy.deserialize_untrusted::() { @@ -88,7 +88,7 @@ pub(crate) async fn migrate_push_subscriptions_v013( .create_document(account_id) .with_account_id(account_id) .with_collection(Collection::Principal) - .update_document(0) + .with_document(0) .set( PrincipalField::PushSubscriptions, Archiver::new(PushSubscriptions { subscriptions }) diff --git a/crates/migration/src/sieve_v1.rs b/crates/migration/src/sieve_v1.rs index 1d3b3edd..c5288a2c 100644 --- a/crates/migration/src/sieve_v1.rs +++ b/crates/migration/src/sieve_v1.rs @@ -81,7 +81,7 @@ pub(crate) async fn migrate_sieve_v011(server: &Server, account_id: u32) -> trc: batch .with_account_id(account_id) .with_collection(Collection::SieveScript) - .update_document(script_id) + .with_document(script_id) .index(SieveField::Name, script.name.to_lowercase()) .set( Field::ARCHIVE, @@ -93,7 +93,7 @@ pub(crate) async fn migrate_sieve_v011(server: &Server, account_id: u32) -> trc: if is_active { batch .with_collection(Collection::Principal) - .update_document(0) + .with_document(0) .set(PrincipalField::ActiveScriptId, script_id.serialize()); } diff --git a/crates/migration/src/sieve_v2.rs b/crates/migration/src/sieve_v2.rs index ab9014aa..2036176d 100644 --- a/crates/migration/src/sieve_v2.rs +++ b/crates/migration/src/sieve_v2.rs @@ -32,7 +32,7 @@ pub(crate) async fn migrate_sieve_v013(server: &Server, account_id: u32) -> trc: for script_id in &script_ids { match server - .get_archive(account_id, Collection::SieveScript, script_id) + .archive(account_id, Collection::SieveScript, script_id) .await { Ok(Some(legacy)) => match legacy.deserialize_untrusted::() { @@ -48,7 +48,7 @@ pub(crate) async fn migrate_sieve_v013(server: &Server, account_id: u32) -> trc: batch .with_account_id(account_id) .with_collection(Collection::SieveScript) - .update_document(script_id) + .with_document(script_id) .unindex(Field::new(0u8), vec![u8::from(old_sieve.is_active)]) .set( Field::ARCHIVE, @@ -61,7 +61,7 @@ pub(crate) async fn migrate_sieve_v013(server: &Server, account_id: u32) -> trc: batch .with_account_id(account_id) .with_collection(Collection::Principal) - .update_document(0) + .with_document(0) .set(PrincipalField::ActiveScriptId, script_id.serialize()); } num_migrated += 1; diff --git a/crates/migration/src/submission.rs b/crates/migration/src/submission.rs index 8d30db39..a773c4d5 100644 --- a/crates/migration/src/submission.rs +++ b/crates/migration/src/submission.rs @@ -82,7 +82,7 @@ pub(crate) async fn migrate_email_submissions( batch .with_account_id(account_id) .with_collection(Collection::EmailSubmission) - .update_document(email_submission_id) + .with_document(email_submission_id) .index(EmailSubmissionField::UndoStatus, es.undo_status.as_index()) .index(EmailSubmissionField::EmailId, es.email_id.serialize()) .index(EmailSubmissionField::ThreadId, es.thread_id.serialize()) diff --git a/crates/migration/src/tasks.rs b/crates/migration/src/tasks.rs index 0ad610b1..06d61bf0 100644 --- a/crates/migration/src/tasks.rs +++ b/crates/migration/src/tasks.rs @@ -10,26 +10,19 @@ use store::{ write::{AnyClass, BatchBuilder, TaskQueueClass, ValueClass, key::DeserializeBigEndian, now}, }; use trc::AddContext; -use types::blob_hash::BlobHash; pub(crate) async fn migrate_tasks_v011(server: &Server) -> trc::Result<()> { let from_key = ValueKey:: { account_id: 0, collection: 0, document_id: 0, - class: ValueClass::TaskQueue(TaskQueueClass::IndexEmail { - due: 0, - hash: BlobHash::default(), - }), + class: ValueClass::TaskQueue(TaskQueueClass::IndexEmail { due: 0 }), }; let to_key = ValueKey:: { account_id: u32::MAX, collection: u8::MAX, document_id: u32::MAX, - class: ValueClass::TaskQueue(TaskQueueClass::IndexEmail { - due: u64::MAX, - hash: BlobHash::default(), - }), + class: ValueClass::TaskQueue(TaskQueueClass::IndexEmail { due: u64::MAX }), }; let now = now(); diff --git a/crates/nlp/src/bayes/mod.rs b/crates/nlp/src/bayes/mod.rs index 11f84e78..f9bad387 100644 --- a/crates/nlp/src/bayes/mod.rs +++ b/crates/nlp/src/bayes/mod.rs @@ -76,7 +76,7 @@ impl From> for TokenHash { hash.len = t1.len() as u8; } else { let h1 = xxhash_rust::xxh3::xxh3_64(t1).to_be_bytes(); - let h2 = farmhash::hash64(t1).to_be_bytes(); + let h2 = farmhash::fingerprint64(t1).to_be_bytes(); hash.hash[..std::mem::size_of::()].copy_from_slice(&h1); hash.hash[std::mem::size_of::()..].copy_from_slice(&h2); hash.len = HASH_LEN as u8; @@ -99,7 +99,7 @@ impl From> for TokenHash { t1.iter() .copied() .chain(xxhash_rust::xxh3::xxh3_64(t2).to_be_bytes().into_iter()) - .chain(farmhash::hash64(t2).to_be_bytes().into_iter()), + .chain(farmhash::fingerprint64(t2).to_be_bytes().into_iter()), ) { *h = b; } diff --git a/crates/pop3/src/mailbox.rs b/crates/pop3/src/mailbox.rs index 79dd1f1d..defa4058 100644 --- a/crates/pop3/src/mailbox.rs +++ b/crates/pop3/src/mailbox.rs @@ -12,8 +12,9 @@ use email::{ }; use std::collections::BTreeMap; use store::{ - IndexKey, IterateParams, SerializeInfallible, U32_LEN, ahash::AHashMap, - write::key::DeserializeBigEndian, + IterateParams, U32_LEN, ValueKey, + ahash::AHashMap, + write::{IndexPropertyClass, ValueClass, key::DeserializeBigEndian}, }; use trc::AddContext; use types::{collection::Collection, field::EmailField, special_use::SpecialUse}; @@ -60,26 +61,30 @@ impl Session { .data .iterate( IterateParams::new( - IndexKey { + ValueKey { account_id, collection: Collection::Email.into(), document_id: 0, - field: EmailField::Size.into(), - key: 0u32.serialize(), + class: ValueClass::IndexProperty(IndexPropertyClass::Integer { + property: EmailField::Stats.into(), + value: 0, + }), }, - IndexKey { + ValueKey { account_id, collection: Collection::Email.into(), document_id: u32::MAX, - field: EmailField::Size.into(), - key: u32::MAX.serialize(), + class: ValueClass::IndexProperty(IndexPropertyClass::Integer { + property: EmailField::Stats.into(), + value: u64::MAX, + }), }, ) - .no_values(), - |key, _| { + .ascending(), + |key, value| { message_sizes.insert( key.deserialize_be_u32(key.len() - U32_LEN)?, - key.deserialize_be_u32(key.len() - (U32_LEN * 2))?, + value.deserialize_be_u32(0)?, ); Ok(true) diff --git a/crates/pop3/src/op/delete.rs b/crates/pop3/src/op/delete.rs index b44cb6bc..8dd4f85a 100644 --- a/crates/pop3/src/op/delete.rs +++ b/crates/pop3/src/op/delete.rs @@ -89,7 +89,7 @@ impl Session { let mut batch = BatchBuilder::new(); let not_deleted = self .server - .emails_tombstone(mailbox.account_id, &mut batch, deleted) + .emails_delete(mailbox.account_id, &mut batch, deleted) .await .caused_by(trc::location!())?; diff --git a/crates/pop3/src/op/fetch.rs b/crates/pop3/src/op/fetch.rs index 850a2257..4fa57c19 100644 --- a/crates/pop3/src/op/fetch.rs +++ b/crates/pop3/src/op/fetch.rs @@ -24,7 +24,7 @@ impl Session { if let Some(message) = mailbox.messages.get(msg.saturating_sub(1) as usize) { if let Some(metadata_) = self .server - .get_archive_by_property( + .archive_by_property( mailbox.account_id, Collection::Email, message.id, diff --git a/crates/services/Cargo.toml b/crates/services/Cargo.toml index 8cfb4f9e..a9fe39c3 100644 --- a/crates/services/Cargo.toml +++ b/crates/services/Cargo.toml @@ -12,6 +12,7 @@ trc = { path = "../trc" } email = { path = "../email" } smtp = { path = "../smtp" } groupware = { path = "../groupware" } +spam-filter = { path = "../spam-filter" } types = { path = "../types" } jmap_proto = { path = "../jmap-proto" } directory = { path = "../directory" } diff --git a/crates/services/src/state_manager/push.rs b/crates/services/src/state_manager/push.rs index 84fb335a..17015d94 100644 --- a/crates/services/src/state_manager/push.rs +++ b/crates/services/src/state_manager/push.rs @@ -42,11 +42,16 @@ pub fn spawn_push_manager(inner: Arc) -> mpsc::Sender { // Load active subscriptions on startup { let server = inner.build_server(); + match server - .get_document_ids(u32::MAX, Collection::PushSubscription) + .document_ids( + u32::MAX, + Collection::Principal, + PrincipalField::PushSubscriptions, + ) .await { - Ok(Some(account_ids)) => { + Ok(account_ids) => { for account_id in account_ids { if server .core @@ -93,7 +98,6 @@ pub fn spawn_push_manager(inner: Arc) -> mpsc::Sender { } } } - Ok(None) => {} Err(err) => { trc::error!(err.caused_by(trc::location!())); } @@ -484,7 +488,7 @@ async fn load_push_subscriptions( .collect::>(); if let Some(push_subscriptions) = server - .get_archive_by_property( + .archive_by_property( account_id, Collection::Principal, 0, diff --git a/crates/services/src/task_manager/alarm.rs b/crates/services/src/task_manager/alarm.rs index 2ed43911..42ce0f5d 100644 --- a/crates/services/src/task_manager/alarm.rs +++ b/crates/services/src/task_manager/alarm.rs @@ -121,7 +121,7 @@ async fn send_email_alarm( // Fetch event let Some(event_) = server - .get_archive(account_id, Collection::CalendarEvent, document_id) + .archive(account_id, Collection::CalendarEvent, document_id) .await .caused_by(trc::location!())? else { @@ -310,7 +310,7 @@ async fn send_display_alarm( ) -> trc::Result { // Fetch event let Some(event_) = server - .get_archive(account_id, Collection::CalendarEvent, document_id) + .archive(account_id, Collection::CalendarEvent, document_id) .await .caused_by(trc::location!())? else { @@ -400,7 +400,7 @@ async fn write_next_alarm( batch .with_account_id(account_id) .with_collection(Collection::CalendarEvent) - .update_document(document_id); + .with_document(document_id); next_alarm.write_task(&mut batch); server .store() diff --git a/crates/services/src/task_manager/bayes.rs b/crates/services/src/task_manager/bayes.rs index e37ded20..f57570b9 100644 --- a/crates/services/src/task_manager/bayes.rs +++ b/crates/services/src/task_manager/bayes.rs @@ -5,8 +5,10 @@ */ use common::Server; -use email::message::bayes::EmailBayesTrain; use mail_parser::MessageParser; +use spam_filter::{ + SpamFilterInput, analysis::init::SpamFilterInit, modules::bayes::BayesClassifier, +}; use std::time::Instant; use trc::{SpamEvent, TaskQueueEvent}; use types::{blob_hash::BlobHash, collection::Collection}; @@ -37,10 +39,12 @@ impl BayesTrainTask for Server { .await { // Train bayes classifier for account - self.email_bayes_train( - account_id, - 0, - MessageParser::new().parse(&raw_message).unwrap_or_default(), + self.bayes_train_if_balanced( + &self.spam_filter_init(SpamFilterInput::from_account_message( + &MessageParser::new().parse(&raw_message).unwrap_or_default(), + account_id, + 0, + )), learn_spam, ) .await; diff --git a/crates/services/src/task_manager/fts.rs b/crates/services/src/task_manager/fts.rs index a17540a5..0014a98d 100644 --- a/crates/services/src/task_manager/fts.rs +++ b/crates/services/src/task_manager/fts.rs @@ -11,7 +11,6 @@ use std::time::Instant; use store::{ IterateParams, SerializeInfallible, U32_LEN, ValueKey, ahash::AHashMap, - fts::index::FtsDocument, roaring::RoaringBitmap, write::{BatchBuilder, BlobOp, TaskQueueClass, ValueClass, key::DeserializeBigEndian, now}, }; @@ -38,6 +37,206 @@ pub trait FtsIndexTask: Sync + Send { impl FtsIndexTask for Server { async fn fts_index(&self, account_id: u32, document_id: u32, hash: &BlobHash) -> bool { + let todo = "merge threads"; + let todo = "combine task with bayes train if needed"; + let todo = "delete Threading field on delete"; + + /*loop { + // Find messages with a matching subject + let mut subj_results = RoaringBitmap::new(); + self.store() + .iterate( + IterateParams::new( + IndexKey { + account_id, + collection: Collection::Email.into(), + document_id: 0, + field: EmailField::Subject.into(), + key: thread_name.clone(), + }, + IndexKey { + account_id, + collection: Collection::Email.into(), + document_id: u32::MAX, + field: EmailField::Subject.into(), + key: thread_name.clone(), + }, + ) + .no_values() + .ascending(), + |key, _| { + let id_pos = key.len() - U32_LEN; + let value = key.get(IndexKeyPrefix::len()..id_pos).ok_or_else(|| { + trc::Error::corrupted_key(key, None, trc::location!()) + })?; + + if value == thread_name { + subj_results.insert(key.deserialize_be_u32(id_pos)?); + } + + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; + + // No matching subjects were found, skip early + if subj_results.is_empty() { + return Ok(ThreadResult::Id(None)); + } + + // Find messages with matching references + let mut results = RoaringBitmap::new(); + let mut found_message_id = Vec::new(); + self.store() + .iterate( + IterateParams::new( + IndexKey { + account_id, + collection: Collection::Email.into(), + document_id: 0, + field: EmailField::References.into(), + key: references.first().unwrap().to_vec(), + }, + IndexKey { + account_id, + collection: Collection::Email.into(), + document_id: u32::MAX, + field: EmailField::References.into(), + key: references.last().unwrap().to_vec(), + }, + ) + .no_values() + .ascending(), + |key, _| { + let id_pos = key.len() - U32_LEN; + let mut value = + key.get(IndexKeyPrefix::len()..id_pos).ok_or_else(|| { + trc::Error::corrupted_key(key, None, trc::location!()) + })?; + let document_id = key.deserialize_be_u32(id_pos)?; + + if let Some(message_id) = value.strip_suffix(&[0]) { + value = message_id; + if skip_duplicate.is_some_and(|(message_id, _)| message_id == value) { + found_message_id.push(document_id); + } + } + + if subj_results.contains(document_id) + && references.binary_search(&value).is_ok() + { + results.insert(document_id); + + if subj_results.len() == results.len() { + return Ok(false); + } + } + + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; + + // No matching messages + if results.is_empty() { + return Ok(ThreadResult::Id(None)); + } + + // Fetch cached messages + let cache = self + .get_cached_messages(account_id) + .await + .caused_by(trc::location!())?; + + // Skip duplicate messages + if !found_message_id.is_empty() + && cache + .in_mailbox(skip_duplicate.unwrap().1) + .any(|m| found_message_id.contains(&m.document_id)) + { + return Ok(ThreadResult::Skip); + } + + // Find the most common threadId + let mut thread_counts = AHashMap::::with_capacity(16); + let mut thread_id = u32::MAX; + let mut thread_count = 0; + for item in &cache.emails.items { + if results.contains(item.document_id) { + let tc = thread_counts.entry(item.thread_id).or_default(); + *tc += 1; + if *tc > thread_count { + thread_count = *tc; + thread_id = item.thread_id; + } + } + } + + if thread_id == u32::MAX { + return Ok(ThreadResult::Id(None)); + } else if thread_counts.len() == 1 { + return Ok(ThreadResult::Id(Some(thread_id))); + } + + // Delete all but the most common threadId + let mut batch = BatchBuilder::new(); + batch + .with_account_id(account_id) + .with_collection(Collection::Thread); + for &delete_thread_id in thread_counts.keys() { + if delete_thread_id != thread_id { + batch + .with_document(delete_thread_id) + .log_container_delete(SyncCollection::Thread); + } + } + + // Move messages to the new threadId + batch.with_collection(Collection::Email); + + for item in &cache.emails.items { + if thread_id == item.thread_id || !thread_counts.contains_key(&item.thread_id) { + continue; + } + if let Some(data_) = self + .archive(account_id, Collection::Email, item.document_id) + .await + .caused_by(trc::location!())? + { + let data = data_ + .to_unarchived::() + .caused_by(trc::location!())?; + if data.inner.thread_id != item.thread_id { + continue; + } + let mut new_data = data.deserialize().caused_by(trc::location!())?; + new_data.thread_id = thread_id; + batch + .with_document(item.document_id) + .custom( + ObjectIndexBuilder::new() + .with_current(data) + .with_changes(new_data), + ) + .caused_by(trc::location!())?; + } + } + + match self.commit_batch(batch).await { + Ok(_) => return Ok(ThreadResult::Id(Some(thread_id))), + Err(err) if err.is_assertion_failure() && try_count < MAX_RETRIES => { + let backoff = store::rand::rng().random_range(50..=300); + tokio::time::sleep(Duration::from_millis(backoff)).await; + try_count += 1; + } + Err(err) => { + return Err(err.caused_by(trc::location!())); + } + } + }*/ + // Obtain raw message let op_start = Instant::now(); let raw_message = if let Ok(Some(raw_message)) = self @@ -57,7 +256,7 @@ impl FtsIndexTask for Server { }; match self - .get_archive_by_property( + .archive_by_property( account_id, Collection::Email, document_id, @@ -69,7 +268,7 @@ impl FtsIndexTask for Server { match metadata_.unarchive::() { Ok(metadata) if metadata.blob_hash.0.as_slice() == hash.as_slice() => { // Index message - let document = + /*let document = FtsDocument::with_default_language(self.core.jmap.default_language) .with_account_id(account_id) .with_collection(Collection::Email) @@ -83,7 +282,7 @@ impl FtsIndexTask for Server { ); return false; - } + }*/ trc::event!( MessageIngest(MessageIngestEvent::FtsIndex), @@ -184,7 +383,7 @@ impl FtsIndexTask for Server { hash: BlobHash::new_max(), }), }; - let mut hashes: AHashMap> = AHashMap::new(); + let mut document_ids: AHashMap> = AHashMap::new(); self.core .storage .data @@ -197,17 +396,10 @@ impl FtsIndexTask for Server { .ok_or_else(|| trc::Error::corrupted_key(key, None, trc::location!()))?; if accounts.contains(account_id) && collection == Collection::Email as u8 { - let hash = - BlobHash::try_from_hash_slice(key.get(0..BLOB_HASH_LEN).ok_or_else( - || trc::Error::corrupted_key(key, None, trc::location!()), - )?) - .unwrap(); - let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; - - hashes + document_ids .entry(account_id) .or_default() - .push((document_id, hash)); + .push(key.deserialize_be_u32(key.len() - U32_LEN)?); } Ok(true) @@ -218,15 +410,15 @@ impl FtsIndexTask for Server { let due = now(); - for (account_id, hashes) in hashes { + for (account_id, document_ids) in document_ids { let mut batch = BatchBuilder::new(); batch .with_account_id(account_id) .with_collection(Collection::Email); - for (document_id, hash) in hashes { - batch.update_document(document_id).set( - ValueClass::TaskQueue(TaskQueueClass::IndexEmail { hash, due }), + for document_id in document_ids { + batch.with_document(document_id).set( + ValueClass::TaskQueue(TaskQueueClass::IndexEmail { due }), 0u64.serialize(), ); diff --git a/crates/services/src/task_manager/mod.rs b/crates/services/src/task_manager/mod.rs index 5950b844..55c94c59 100644 --- a/crates/services/src/task_manager/mod.rs +++ b/crates/services/src/task_manager/mod.rs @@ -6,13 +6,11 @@ use crate::task_manager::imip::SendImipTask; use alarm::SendAlarmTask; -use bayes::BayesTrainTask; use common::IPC_CHANNEL_BUFFER; use common::config::server::ServerProtocol; use common::listener::limiter::ConcurrencyLimiter; use common::listener::{ServerInstance, TcpAcceptor}; use common::{Inner, KV_LOCK_TASK, Server, core::BuildServer}; -use fts::FtsIndexTask; use groupware::calendar::alarm::{CalendarAlarm, CalendarAlarmType}; use std::collections::hash_map::Entry; use std::future::Future; @@ -49,13 +47,14 @@ pub struct Task { #[derive(Debug, Clone, Hash, PartialEq, Eq)] pub enum TaskAction { - Index { hash: BlobHash }, - BayesTrain { hash: BlobHash, learn_spam: bool }, + Index, + Unindex, + BayesTrain { learn_spam: bool }, SendAlarm { alarm: CalendarAlarm }, SendImip, } -const FTS_LOCK_EXPIRY: u64 = 60 * 5; // 5 minutes +const INDEX_EXPIRY: u64 = 60 * 5; // 5 minutes const BAYES_LOCK_EXPIRY: u64 = 60 * 30; // 30 minutes const ALARM_EXPIRY: u64 = 60 * 2; // 2 minutes const QUEUE_REFRESH_INTERVAL: u64 = 60 * 5; // 5 minutes @@ -103,15 +102,23 @@ pub fn spawn_task_manager(inner: Arc) { // Lock task if server.try_lock_task(&task).await { let success = match &task.action { - TaskAction::Index { hash } => { - server - .fts_index(task.account_id, task.document_id, hash) - .await + TaskAction::Index => { + let todo = "implement"; + /*server + .fts_index(task.account_id, task.document_id, hash) + .await*/ + true } - TaskAction::BayesTrain { hash, learn_spam } => { - server - .bayes_train(task.account_id, task.document_id, hash, *learn_spam) - .await + TaskAction::Unindex => { + let todo = "implement"; + true + } + TaskAction::BayesTrain { learn_spam } => { + let todo = "implement"; + /*server + .bayes_train(task.account_id, task.document_id, hash, *learn_spam) + .await*/ + true } TaskAction::SendAlarm { alarm } => { if server.core.groupware.alarms_enabled { @@ -148,7 +155,7 @@ pub fn spawn_task_manager(inner: Arc) { let mut batch = BatchBuilder::new(); batch .with_account_id(task.account_id) - .update_document(task.document_id); + .with_document(task.document_id); for value in task.value_classes() { batch.clear(value); @@ -204,10 +211,7 @@ impl TaskQueueManager for Server { account_id: 0, collection: 0, document_id: 0, - class: ValueClass::TaskQueue(TaskQueueClass::IndexEmail { - due: 0, - hash: BlobHash::default(), - }), + class: ValueClass::TaskQueue(TaskQueueClass::IndexEmail { due: 0 }), }; let to_key = ValueKey:: { account_id: u32::MAX, @@ -215,7 +219,6 @@ impl TaskQueueManager for Server { document_id: u32::MAX, class: ValueClass::TaskQueue(TaskQueueClass::IndexEmail { due: now_timestamp + QUEUE_REFRESH_INTERVAL, - hash: BlobHash::default(), }), }; @@ -376,7 +379,7 @@ impl Task { fn lock_key(&self) -> Vec { match &self.action { - TaskAction::Index { .. } => KeySerializer::new((U32_LEN * 2) + U64_LEN + 1) + TaskAction::Index => KeySerializer::new((U32_LEN * 2) + U64_LEN + 1) .write(0u8) .write(self.due) .write_leb128(self.account_id) @@ -399,12 +402,18 @@ impl Task { .write_leb128(self.account_id) .write_leb128(self.document_id) .finalize(), + TaskAction::Unindex => KeySerializer::new((U32_LEN * 2) + U64_LEN + 1) + .write(4u8) + .write(self.due) + .write_leb128(self.account_id) + .write_leb128(self.document_id) + .finalize(), } } fn lock_expiry(&self) -> u64 { match self.action { - TaskAction::Index { .. } => FTS_LOCK_EXPIRY, + TaskAction::Index | TaskAction::Unindex => INDEX_EXPIRY, TaskAction::BayesTrain { .. } => BAYES_LOCK_EXPIRY, TaskAction::SendAlarm { .. } | TaskAction::SendImip => ALARM_EXPIRY, } @@ -413,12 +422,9 @@ impl Task { fn value_classes(&self) -> impl Iterator { [ Some(ValueClass::TaskQueue(match &self.action { - TaskAction::Index { hash } => TaskQueueClass::IndexEmail { - hash: hash.clone(), - due: self.due, - }, - TaskAction::BayesTrain { hash, learn_spam } => TaskQueueClass::BayesTrain { - hash: hash.clone(), + TaskAction::Index => TaskQueueClass::IndexEmail { due: self.due }, + TaskAction::Unindex => TaskQueueClass::UnindexEmail { due: self.due }, + TaskAction::BayesTrain { learn_spam } => TaskQueueClass::BayesTrain { due: self.due, learn_spam: *learn_spam, }, @@ -450,24 +456,10 @@ impl Task { account_id: key.deserialize_be_u32(U64_LEN)?, document_id: key.deserialize_be_u32(U64_LEN + U32_LEN + 1)?, action: match key.get(U64_LEN + U32_LEN) { - Some(0) => TaskAction::Index { - hash: key - .get( - U64_LEN + U32_LEN + U32_LEN + 1 - ..U64_LEN + U32_LEN + U32_LEN + BLOB_HASH_LEN + 1, - ) - .and_then(|bytes| BlobHash::try_from_hash_slice(bytes).ok()) - .ok_or_else(|| trc::Error::corrupted_key(key, None, trc::location!()))?, - }, + Some(0) => TaskAction::Index, + Some(7) => TaskAction::Unindex, Some(v @ (1 | 2)) => TaskAction::BayesTrain { learn_spam: *v == 1, - hash: key - .get( - U64_LEN + U32_LEN + U32_LEN + 1 - ..U64_LEN + U32_LEN + U32_LEN + BLOB_HASH_LEN + 1, - ) - .and_then(|bytes| BlobHash::try_from_hash_slice(bytes).ok()) - .ok_or_else(|| trc::Error::corrupted_key(key, None, trc::location!()))?, }, Some(3) => TaskAction::SendAlarm { alarm: CalendarAlarm { diff --git a/crates/store/src/backend/composite/read_replica.rs b/crates/store/src/backend/composite/read_replica.rs index e1f59694..1e90a854 100644 --- a/crates/store/src/backend/composite/read_replica.rs +++ b/crates/store/src/backend/composite/read_replica.rs @@ -8,20 +8,17 @@ * */ +use crate::{ + Deserialize, IterateParams, Key, Store, Stores, ValueKey, + write::{AssignedIds, Batch, ValueClass}, +}; use std::{ future::Future, ops::Range, sync::atomic::{AtomicUsize, Ordering}, }; - -use roaring::RoaringBitmap; use utils::config::{Config, utils::AsKey}; -use crate::{ - BitmapKey, Deserialize, IterateParams, Key, Store, Stores, ValueKey, - write::{AssignedIds, Batch, BitmapClass, ValueClass}, -}; - pub struct SQLReadReplica { primary: Store, replicas: Vec, @@ -192,26 +189,6 @@ impl SQLReadReplica { .await } - pub async fn get_bitmap( - &self, - key: BitmapKey, - ) -> trc::Result> { - self.run_op(move |store| { - let key = key.clone(); - - async move { - match store { - #[cfg(feature = "postgres")] - Store::PostgreSQL(store) => store.get_bitmap(key).await, - #[cfg(feature = "mysql")] - Store::MySQL(store) => store.get_bitmap(key).await, - _ => panic!("Invalid store type"), - } - } - }) - .await - } - pub async fn iterate( &self, params: IterateParams, diff --git a/crates/store/src/backend/elastic/index.rs b/crates/store/src/backend/elastic/index.rs index 6fa8b297..3915b513 100644 --- a/crates/store/src/backend/elastic/index.rs +++ b/crates/store/src/backend/elastic/index.rs @@ -4,21 +4,14 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{borrow::Cow, fmt::Display}; - +use super::{ElasticSearchStore, assert_success}; +use crate::{backend::elastic::INDEX_NAMES, dispatch::DocumentSet, search::IndexDocument}; use elasticsearch::{DeleteByQueryParts, IndexParts}; use serde::{Deserialize, Serialize}; use serde_json::json; +use std::{borrow::Cow, fmt::Display}; use types::collection::Collection; -use crate::{ - backend::elastic::INDEX_NAMES, - dispatch::DocumentSet, - fts::{Field, index::FtsDocument}, -}; - -use super::{ElasticSearchStore, assert_success}; - #[derive(Serialize, Deserialize, Default)] struct Document<'x> { document_id: u32, @@ -36,11 +29,9 @@ struct Header<'x> { } impl ElasticSearchStore { - pub async fn fts_index + Display + Clone + std::fmt::Debug>( - &self, - document: FtsDocument<'_, T>, - ) -> trc::Result<()> { - assert_success( + pub async fn index_insert(&self, document: IndexDocument) -> trc::Result<()> { + todo!() + /*assert_success( self.index .index(IndexParts::Index(INDEX_NAMES[document.collection as usize])) .body(Document::from(document)) @@ -48,10 +39,10 @@ impl ElasticSearchStore { .await, ) .await - .map(|_| ()) + .map(|_| ())*/ } - pub async fn fts_remove( + pub async fn index_remove( &self, account_id: u32, collection: Collection, @@ -81,7 +72,7 @@ impl ElasticSearchStore { .map(|_| ()) } - pub async fn fts_remove_all(&self, account_id: u32) -> trc::Result<()> { + pub async fn index_remove_all(&self, account_id: u32) -> trc::Result<()> { assert_success( self.index .delete_by_query(DeleteByQueryParts::Index(INDEX_NAMES)) @@ -101,29 +92,3 @@ impl ElasticSearchStore { .map(|_| ()) } } - -impl<'x, T: Into + Display + Clone + std::fmt::Debug> From> - for Document<'x> -{ - fn from(value: FtsDocument<'x, T>) -> Self { - let mut document = Document { - account_id: value.account_id, - document_id: value.document_id, - ..Default::default() - }; - - for part in value.parts { - match part.field { - Field::Header(name) => document.header.push(Header { - name: name.to_string().into(), - value: part.text, - }), - Field::Body => document.body.push(part.text), - Field::Attachment => document.attachments.push(part.text), - Field::Keyword => document.keywords.push(part.text), - } - } - - document - } -} diff --git a/crates/store/src/backend/elastic/query.rs b/crates/store/src/backend/elastic/query.rs index fa5dab09..a1e7100f 100644 --- a/crates/store/src/backend/elastic/query.rs +++ b/crates/store/src/backend/elastic/query.rs @@ -4,24 +4,25 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{borrow::Cow, fmt::Display}; +use crate::search::{SearchComparator, SearchFilter}; +use super::{ElasticSearchStore, INDEX_NAMES, assert_success}; use elasticsearch::SearchParts; use roaring::RoaringBitmap; use serde_json::{Value, json}; - -use crate::fts::{Field, FtsFilter}; - -use super::{ElasticSearchStore, INDEX_NAMES, assert_success}; +use std::{borrow::Cow, fmt::Display}; impl ElasticSearchStore { - pub async fn fts_query + Display + Clone + std::fmt::Debug>( + pub async fn index_query( &self, account_id: u32, collection: impl Into, - filters: Vec>, - ) -> trc::Result { - let mut stack: Vec<(FtsFilter, Vec)> = vec![]; + filters: Vec, + comparators: Vec, + ) -> trc::Result> { + todo!() + + /*let mut stack: Vec<(FtsFilter, Vec)> = vec![]; let mut conditions = vec![json!({ "match": { "account_id": account_id } })]; let mut logical_op = FtsFilter::And; @@ -118,17 +119,6 @@ impl ElasticSearchStore { })? as u32); } - Ok(results) - } -} - -impl + Display + Clone + std::fmt::Debug> Field { - pub fn name(&self) -> Cow<'static, str> { - match self { - Field::Header(name) => format!("header.{name}").into(), - Field::Body => "body".into(), - Field::Attachment => "attachment".into(), - Field::Keyword => "keyword".into(), - } + Ok(results)*/ } } diff --git a/crates/store/src/backend/foundationdb/read.rs b/crates/store/src/backend/foundationdb/read.rs index 39518eb9..3a71b4ff 100644 --- a/crates/store/src/backend/foundationdb/read.rs +++ b/crates/store/src/backend/foundationdb/read.rs @@ -6,20 +6,16 @@ use super::{FdbStore, MAX_VALUE_SIZE, ReadVersion, into_error}; use crate::{ - BitmapKey, Deserialize, IterateParams, Key, U32_LEN, ValueKey, WITH_SUBSPACE, + Deserialize, IterateParams, Key, ValueKey, WITH_SUBSPACE, backend::deserialize_i64_le, - write::{ - BitmapClass, ValueClass, - key::{DeserializeBigEndian, KeySerializer}, - }, + write::{ValueClass, key::KeySerializer}, }; use foundationdb::{ KeySelector, RangeOption, Transaction, future::FdbSlice, - options::{self, StreamingMode}, + options::{self}, }; use futures::TryStreamExt; -use roaring::RoaringBitmap; #[allow(dead_code)] pub(crate) enum ChunkedValue { @@ -43,37 +39,6 @@ impl FdbStore { } } - pub(crate) async fn get_bitmap( - &self, - mut key: BitmapKey, - ) -> trc::Result> { - let mut bm = RoaringBitmap::new(); - let begin = key.serialize(WITH_SUBSPACE); - key.document_id = u32::MAX; - let end = key.serialize(WITH_SUBSPACE); - let key_len = begin.len(); - let trx = self.read_trx().await?; - let mut values = trx.get_ranges_keyvalues( - RangeOption { - begin: KeySelector::first_greater_or_equal(begin), - end: KeySelector::first_greater_or_equal(end), - mode: StreamingMode::WantAll, - reverse: false, - ..RangeOption::default() - }, - true, - ); - - while let Some(value) = values.try_next().await.map_err(into_error)? { - let key = value.key(); - if key.len() == key_len { - bm.insert(key.deserialize_be_u32(key.len() - U32_LEN)?); - } - } - - Ok(if !bm.is_empty() { Some(bm) } else { None }) - } - pub(crate) async fn iterate( &self, params: IterateParams, diff --git a/crates/store/src/backend/foundationdb/write.rs b/crates/store/src/backend/foundationdb/write.rs index b3c8eddb..8809ac80 100644 --- a/crates/store/src/backend/foundationdb/write.rs +++ b/crates/store/src/backend/foundationdb/write.rs @@ -180,16 +180,6 @@ impl FdbStore { trx.clear(&key); } } - Operation::Bitmap { class, set } => { - let key = - class.serialize(account_id, collection, document_id, WITH_SUBSPACE); - - if *set { - trx.set(&key, &[]); - } else { - trx.clear(&key); - } - } Operation::Log { collection, set } => { let key = LogKey { account_id, diff --git a/crates/store/src/backend/mysql/main.rs b/crates/store/src/backend/mysql/main.rs index e0f24a2f..471a78ab 100644 --- a/crates/store/src/backend/mysql/main.rs +++ b/crates/store/src/backend/mysql/main.rs @@ -101,7 +101,6 @@ impl MysqlStore { SUBSPACE_QUEUE_EVENT, SUBSPACE_REPORT_OUT, SUBSPACE_REPORT_IN, - SUBSPACE_FTS_INDEX, SUBSPACE_LOGS, SUBSPACE_TELEMETRY_SPAN, SUBSPACE_TELEMETRY_METRIC, @@ -130,12 +129,7 @@ impl MysqlStore { .await .map_err(into_error)?; - for table in [ - SUBSPACE_INDEXES, - SUBSPACE_BITMAP_ID, - SUBSPACE_BITMAP_TAG, - SUBSPACE_BITMAP_TEXT, - ] { + for table in [SUBSPACE_INDEXES] { let table = char::from(table); conn.query_drop(format!( "CREATE TABLE IF NOT EXISTS {table} ( diff --git a/crates/store/src/backend/mysql/read.rs b/crates/store/src/backend/mysql/read.rs index 4d69d233..07cf25d1 100644 --- a/crates/store/src/backend/mysql/read.rs +++ b/crates/store/src/backend/mysql/read.rs @@ -4,16 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use super::{MysqlStore, into_error}; +use crate::{Deserialize, IterateParams, Key, ValueKey, write::ValueClass}; use futures::TryStreamExt; use mysql_async::{Row, prelude::Queryable}; -use roaring::RoaringBitmap; - -use crate::{ - BitmapKey, Deserialize, IterateParams, Key, U32_LEN, ValueKey, - write::{BitmapClass, ValueClass, key::DeserializeBigEndian}, -}; - -use super::{MysqlStore, into_error}; impl MysqlStore { pub(crate) async fn get_value(&self, key: impl Key) -> trc::Result> @@ -41,35 +35,6 @@ impl MysqlStore { }) } - pub(crate) async fn get_bitmap( - &self, - mut key: BitmapKey, - ) -> trc::Result> { - let begin = key.serialize(0); - key.document_id = u32::MAX; - let key_len = begin.len(); - let end = key.serialize(0); - let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?; - let table = char::from(key.subspace()); - - let mut bm = RoaringBitmap::new(); - let s = conn - .prep(format!("SELECT k FROM {table} WHERE k >= ? AND k <= ?")) - .await - .map_err(into_error)?; - let mut rows = conn - .exec_stream::, _, _>(&s, (begin, end)) - .await - .map_err(into_error)?; - - while let Some(key) = rows.try_next().await.map_err(into_error)? { - if key.len() == key_len { - bm.insert(key.as_slice().deserialize_be_u32(key.len() - U32_LEN)?); - } - } - Ok(if !bm.is_empty() { Some(bm) } else { None }) - } - pub(crate) async fn iterate( &self, params: IterateParams, diff --git a/crates/store/src/backend/mysql/write.rs b/crates/store/src/backend/mysql/write.rs index 13854c8a..e5879aba 100644 --- a/crates/store/src/backend/mysql/write.rs +++ b/crates/store/src/backend/mysql/write.rs @@ -8,8 +8,7 @@ use super::{MysqlStore, into_error}; use crate::{ IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_QUOTA, U64_LEN, write::{ - AssignedIds, Batch, BitmapClass, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, Operation, - ValueClass, ValueOp, + AssignedIds, Batch, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, Operation, ValueClass, ValueOp, }, }; use ahash::AHashMap; @@ -281,36 +280,6 @@ impl MysqlStore { }; trx.exec_drop(&s, (key,)).await?; } - Operation::Bitmap { class, set } => { - let is_document_id = matches!(class, BitmapClass::DocumentIds); - let key = class.serialize(account_id, collection, document_id, 0); - let table = char::from(class.subspace()); - - let s = if *set { - if is_document_id { - trx.prep("INSERT INTO b (k) VALUES (?)").await? - } else { - trx.prep(format!("INSERT IGNORE INTO {} (k) VALUES (?)", table)) - .await? - } - } else { - trx.prep(format!("DELETE FROM {} WHERE k = ?", table)) - .await? - }; - - if let Err(err) = trx.exec_drop(&s, (key,)).await { - trx.rollback().await?; - return Err( - if is_document_id - && matches!(&err, Error::Server(err) if [1062, 1213].contains(&err.code)) - { - CommitError::Retry - } else { - CommitError::Mysql(err) - }, - ); - } - } Operation::Log { collection, set } => { let key = LogKey { account_id, diff --git a/crates/store/src/backend/postgres/main.rs b/crates/store/src/backend/postgres/main.rs index b827c4e7..e674e9b9 100644 --- a/crates/store/src/backend/postgres/main.rs +++ b/crates/store/src/backend/postgres/main.rs @@ -88,7 +88,6 @@ impl PostgresStore { SUBSPACE_QUEUE_EVENT, SUBSPACE_REPORT_OUT, SUBSPACE_REPORT_IN, - SUBSPACE_FTS_INDEX, SUBSPACE_LOGS, SUBSPACE_BLOBS, SUBSPACE_TELEMETRY_SPAN, @@ -109,12 +108,7 @@ impl PostgresStore { .map_err(into_error)?; } - for table in [ - SUBSPACE_INDEXES, - SUBSPACE_BITMAP_ID, - SUBSPACE_BITMAP_TAG, - SUBSPACE_BITMAP_TEXT, - ] { + for table in [SUBSPACE_INDEXES] { let table = char::from(table); conn.execute( &format!( diff --git a/crates/store/src/backend/postgres/read.rs b/crates/store/src/backend/postgres/read.rs index 0d97a91b..d137c7e7 100644 --- a/crates/store/src/backend/postgres/read.rs +++ b/crates/store/src/backend/postgres/read.rs @@ -4,15 +4,9 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use futures::{TryStreamExt, pin_mut}; -use roaring::RoaringBitmap; - -use crate::{ - BitmapKey, Deserialize, IterateParams, Key, U32_LEN, ValueKey, - write::{BitmapClass, ValueClass, key::DeserializeBigEndian}, -}; - use super::{PostgresStore, into_error}; +use crate::{Deserialize, IterateParams, Key, ValueKey, write::ValueClass}; +use futures::{TryStreamExt, pin_mut}; impl PostgresStore { pub(crate) async fn get_value(&self, key: impl Key) -> trc::Result> @@ -40,38 +34,6 @@ impl PostgresStore { }) } - pub(crate) async fn get_bitmap( - &self, - mut key: BitmapKey, - ) -> trc::Result> { - let begin = key.serialize(0); - key.document_id = u32::MAX; - let key_len = begin.len(); - let end = key.serialize(0); - let conn = self.conn_pool.get().await.map_err(into_error)?; - let table = char::from(key.subspace()); - - let mut bm = RoaringBitmap::new(); - let s = conn - .prepare_cached(&format!("SELECT k FROM {table} WHERE k >= $1 AND k <= $2")) - .await - .map_err(into_error)?; - let rows = conn - .query_raw(&s, &[&begin, &end]) - .await - .map_err(into_error)?; - - pin_mut!(rows); - - while let Some(row) = rows.try_next().await.map_err(into_error)? { - let key: &[u8] = row.try_get(0).map_err(into_error)?; - if key.len() == key_len { - bm.insert(key.deserialize_be_u32(key.len() - U32_LEN)?); - } - } - Ok(if !bm.is_empty() { Some(bm) } else { None }) - } - pub(crate) async fn iterate( &self, params: IterateParams, diff --git a/crates/store/src/backend/postgres/write.rs b/crates/store/src/backend/postgres/write.rs index 11baccf7..c19c5c58 100644 --- a/crates/store/src/backend/postgres/write.rs +++ b/crates/store/src/backend/postgres/write.rs @@ -8,8 +8,7 @@ use super::{PostgresStore, into_error}; use crate::{ IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_QUOTA, U64_LEN, write::{ - AssignedIds, Batch, BitmapClass, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, Operation, - ValueClass, ValueOp, + AssignedIds, Batch, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, Operation, ValueClass, ValueOp, }, }; use ahash::AHashMap; @@ -283,35 +282,6 @@ impl PostgresStore { }; trx.execute(&s, &[&key]).await?; } - Operation::Bitmap { class, set } => { - let is_document_id = matches!(class, BitmapClass::DocumentIds); - let key = class.serialize(account_id, collection, document_id, 0); - let table = char::from(class.subspace()); - - let s = if *set { - if is_document_id { - trx.prepare_cached("INSERT INTO b (k) VALUES ($1)").await? - } else { - trx.prepare_cached(&format!( - "INSERT INTO {} (k) VALUES ($1) ON CONFLICT (k) DO NOTHING", - table - )) - .await? - } - } else { - trx.prepare_cached(&format!("DELETE FROM {} WHERE k = $1", table)) - .await? - }; - - trx.execute(&s, &[&key]).await.map_err(|err| { - if is_document_id && matches!(err.code(), Some(&SqlState::UNIQUE_VIOLATION)) - { - CommitError::Retry - } else { - CommitError::Postgres(err) - } - })?; - } Operation::Log { collection, set } => { let key = LogKey { account_id, diff --git a/crates/store/src/backend/rocksdb/main.rs b/crates/store/src/backend/rocksdb/main.rs index 540f2835..feaf7ede 100644 --- a/crates/store/src/backend/rocksdb/main.rs +++ b/crates/store/src/backend/rocksdb/main.rs @@ -35,20 +35,6 @@ impl RocksDbStore { let mut cfs = Vec::new(); - // Bitmaps - for subspace in [ - SUBSPACE_BITMAP_ID, - SUBSPACE_BITMAP_TAG, - SUBSPACE_BITMAP_TEXT, - ] { - let mut cf_opts = Options::default(); - cf_opts.set_max_write_buffer_number(16); - cfs.push(ColumnFamilyDescriptor::new( - std::str::from_utf8(&[subspace]).unwrap(), - cf_opts, - )); - } - // Counters for subspace in [SUBSPACE_COUNTER, SUBSPACE_QUOTA, SUBSPACE_IN_MEMORY_COUNTER] { let mut cf_opts = Options::default(); @@ -84,7 +70,6 @@ impl RocksDbStore { SUBSPACE_QUEUE_EVENT, SUBSPACE_REPORT_OUT, SUBSPACE_REPORT_IN, - SUBSPACE_FTS_INDEX, SUBSPACE_LOGS, SUBSPACE_BLOBS, SUBSPACE_TELEMETRY_SPAN, diff --git a/crates/store/src/backend/rocksdb/read.rs b/crates/store/src/backend/rocksdb/read.rs index 9bf8e4d3..e3bf7570 100644 --- a/crates/store/src/backend/rocksdb/read.rs +++ b/crates/store/src/backend/rocksdb/read.rs @@ -4,16 +4,11 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use roaring::RoaringBitmap; -use rocksdb::{Direction, IteratorMode}; - use super::{RocksDbStore, into_error}; - use crate::{ - BitmapKey, Deserialize, IterateParams, Key, U32_LEN, ValueKey, - backend::rocksdb::CfHandle, - write::{BitmapClass, ValueClass, key::DeserializeBigEndian}, + Deserialize, IterateParams, Key, ValueKey, backend::rocksdb::CfHandle, write::ValueClass, }; +use rocksdb::{Direction, IteratorMode}; impl RocksDbStore { pub(crate) async fn get_value(&self, key: impl Key) -> trc::Result> @@ -39,36 +34,6 @@ impl RocksDbStore { .await } - pub(crate) async fn get_bitmap( - &self, - mut key: BitmapKey, - ) -> trc::Result> { - let db = self.db.clone(); - self.spawn_worker(move || { - let mut bm = RoaringBitmap::new(); - let subspace = key.subspace(); - let begin = key.serialize(0); - key.document_id = u32::MAX; - let end = key.serialize(0); - let key_len = begin.len(); - for row in db.iterator_cf( - &db.subspace_handle(subspace), - IteratorMode::From(&begin, Direction::Forward), - ) { - let (key, _) = row.map_err(into_error)?; - let key = key.as_ref(); - if key.len() == key_len && key >= begin.as_slice() && key <= end.as_slice() { - bm.insert(key.deserialize_be_u32(key.len() - U32_LEN)?); - } else { - break; - } - } - - Ok(if !bm.is_empty() { Some(bm) } else { None }) - }) - .await - } - pub(crate) async fn iterate( &self, params: IterateParams, diff --git a/crates/store/src/backend/rocksdb/write.rs b/crates/store/src/backend/rocksdb/write.rs index 3d249954..1fc2fa85 100644 --- a/crates/store/src/backend/rocksdb/write.rs +++ b/crates/store/src/backend/rocksdb/write.rs @@ -248,16 +248,6 @@ impl RocksDBTransaction<'_, '_> { txn.delete_cf(&self.cf_indexes, &key)?; } } - Operation::Bitmap { class, set } => { - let cf = self.db.subspace_handle(class.subspace()); - let key = class.serialize(account_id, collection, document_id, 0); - - if *set { - txn.put_cf(&cf, &key, [])?; - } else { - txn.delete_cf(&cf, &key)?; - } - } Operation::Log { collection, set } => { let key = LogKey { account_id, diff --git a/crates/store/src/backend/sqlite/main.rs b/crates/store/src/backend/sqlite/main.rs index e10b28a0..e85def3d 100644 --- a/crates/store/src/backend/sqlite/main.rs +++ b/crates/store/src/backend/sqlite/main.rs @@ -101,7 +101,6 @@ impl SqliteStore { SUBSPACE_QUEUE_EVENT, SUBSPACE_REPORT_OUT, SUBSPACE_REPORT_IN, - SUBSPACE_FTS_INDEX, SUBSPACE_LOGS, SUBSPACE_BLOBS, SUBSPACE_TELEMETRY_SPAN, @@ -121,12 +120,7 @@ impl SqliteStore { .map_err(into_error)?; } - for table in [ - SUBSPACE_INDEXES, - SUBSPACE_BITMAP_ID, - SUBSPACE_BITMAP_TAG, - SUBSPACE_BITMAP_TEXT, - ] { + for table in [SUBSPACE_INDEXES] { let table = char::from(table); conn.execute( &format!( diff --git a/crates/store/src/backend/sqlite/read.rs b/crates/store/src/backend/sqlite/read.rs index 79d0522c..906bb3dc 100644 --- a/crates/store/src/backend/sqlite/read.rs +++ b/crates/store/src/backend/sqlite/read.rs @@ -4,15 +4,9 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use roaring::RoaringBitmap; -use rusqlite::OptionalExtension; - -use crate::{ - BitmapKey, Deserialize, IterateParams, Key, U32_LEN, ValueKey, - write::{BitmapClass, ValueClass, key::DeserializeBigEndian}, -}; - use super::{SqliteStore, into_error}; +use crate::{Deserialize, IterateParams, Key, ValueKey, write::ValueClass}; +use rusqlite::OptionalExtension; impl SqliteStore { pub(crate) async fn get_value(&self, key: impl Key) -> trc::Result> @@ -39,39 +33,6 @@ impl SqliteStore { .await } - pub(crate) async fn get_bitmap( - &self, - mut key: BitmapKey, - ) -> trc::Result> { - let begin = key.serialize(0); - key.document_id = u32::MAX; - let key_len = begin.len(); - let end = key.serialize(0); - let conn = self.conn_pool.get().map_err(into_error)?; - let table = char::from(key.subspace()); - - self.spawn_worker(move || { - let mut bm = RoaringBitmap::new(); - let mut query = conn - .prepare_cached(&format!("SELECT k FROM {table} WHERE k >= ? AND k <= ?")) - .map_err(into_error)?; - let mut rows = query.query([&begin, &end]).map_err(into_error)?; - - while let Some(row) = rows.next().map_err(into_error)? { - let key = row - .get_ref(0) - .map_err(into_error)? - .as_bytes() - .map_err(into_error)?; - if key.len() == key_len { - bm.insert(key.deserialize_be_u32(key.len() - U32_LEN)?); - } - } - Ok(if !bm.is_empty() { Some(bm) } else { None }) - }) - .await - } - pub(crate) async fn iterate( &self, params: IterateParams, diff --git a/crates/store/src/backend/sqlite/write.rs b/crates/store/src/backend/sqlite/write.rs index a7632631..6f68738c 100644 --- a/crates/store/src/backend/sqlite/write.rs +++ b/crates/store/src/backend/sqlite/write.rs @@ -7,7 +7,7 @@ use super::{SqliteStore, into_error}; use crate::{ IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_QUOTA, U64_LEN, - write::{AssignedIds, Batch, BitmapClass, Operation, ValueClass, ValueOp}, + write::{AssignedIds, Batch, Operation, ValueClass, ValueOp}, }; use rusqlite::{OptionalExtension, TransactionBehavior, params}; use trc::AddContext; @@ -194,39 +194,6 @@ impl SqliteStore { .caused_by(trc::location!())?; } } - Operation::Bitmap { class, set } => { - let is_document_id = matches!(class, BitmapClass::DocumentIds); - let key = class.serialize(account_id, collection, document_id, 0); - let table = char::from(class.subspace()); - - if *set { - if is_document_id { - trx.prepare_cached("INSERT INTO b (k) VALUES (?)") - .map_err(into_error) - .caused_by(trc::location!())? - .execute(params![&key]) - .map_err(into_error) - .caused_by(trc::location!())?; - } else { - trx.prepare_cached(&format!( - "INSERT OR IGNORE INTO {} (k) VALUES (?)", - table - )) - .map_err(into_error) - .caused_by(trc::location!())? - .execute(params![&key]) - .map_err(into_error) - .caused_by(trc::location!())?; - } - } else { - trx.prepare_cached(&format!("DELETE FROM {} WHERE k = ?", table)) - .map_err(into_error) - .caused_by(trc::location!())? - .execute(params![&key]) - .map_err(into_error) - .caused_by(trc::location!())?; - }; - } Operation::Log { collection, set } => { let key = LogKey { account_id, diff --git a/crates/store/src/config.rs b/crates/store/src/config.rs index 30957e2e..a9378f3f 100644 --- a/crates/store/src/config.rs +++ b/crates/store/src/config.rs @@ -85,7 +85,7 @@ impl Stores { .map(Store::from) { self.stores.insert(store_id.clone(), db.clone()); - self.fts_stores.insert(store_id.clone(), db.clone().into()); + self.search_stores.insert(store_id.clone(), db.clone().into()); self.blob_stores.insert( store_id.clone(), BlobStore::from(db.clone()).with_compression(compression_algo), @@ -110,7 +110,7 @@ impl Stores { .map(Store::from) { self.stores.insert(store_id.clone(), db.clone()); - self.fts_stores.insert(store_id.clone(), db.clone().into()); + self.search_stores.insert(store_id.clone(), db.clone().into()); self.blob_stores.insert( store_id.clone(), BlobStore::from(db.clone()).with_compression(compression_algo), @@ -129,7 +129,7 @@ impl Stores { .map(Store::from) { self.stores.insert(store_id.clone(), db.clone()); - self.fts_stores.insert(store_id.clone(), db.clone().into()); + self.search_stores.insert(store_id.clone(), db.clone().into()); self.blob_stores.insert( store_id.clone(), BlobStore::from(db.clone()).with_compression(compression_algo), @@ -148,7 +148,7 @@ impl Stores { .map(Store::from) { self.stores.insert(store_id.clone(), db.clone()); - self.fts_stores.insert(store_id.clone(), db.clone().into()); + self.search_stores.insert(store_id.clone(), db.clone().into()); self.blob_stores.insert( store_id.clone(), BlobStore::from(db.clone()).with_compression(compression_algo), @@ -172,7 +172,7 @@ impl Stores { crate::backend::sqlite::SqliteStore::open(config, prefix).map(Store::from) { self.stores.insert(store_id.clone(), db.clone()); - self.fts_stores.insert(store_id.clone(), db.clone().into()); + self.search_stores.insert(store_id.clone(), db.clone().into()); self.blob_stores.insert( store_id.clone(), BlobStore::from(db.clone()).with_compression(compression_algo), @@ -201,9 +201,9 @@ impl Stores { if let Some(db) = crate::backend::elastic::ElasticSearchStore::open(config, prefix) .await - .map(crate::FtsStore::from) + .map(crate::SearchStore::from) { - self.fts_stores.insert(store_id, db); + self.search_stores.insert(store_id, db); } } #[cfg(feature = "redis")] @@ -303,7 +303,7 @@ impl Stores { { let db = Store::SQLReadReplica(db.into()); self.stores.insert(id.to_string(), db.clone()); - self.fts_stores.insert(id.to_string(), db.clone().into()); + self.search_stores.insert(id.to_string(), db.clone().into()); self.blob_stores.insert( id.to_string(), BlobStore::from(db.clone()).with_compression( diff --git a/crates/store/src/dispatch/fts.rs b/crates/store/src/dispatch/fts.rs deleted file mode 100644 index 6ea403bc..00000000 --- a/crates/store/src/dispatch/fts.rs +++ /dev/null @@ -1,73 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use std::fmt::Display; - -use roaring::RoaringBitmap; -use trc::AddContext; -use types::collection::Collection; - -use crate::{ - FtsStore, - fts::{FtsFilter, index::FtsDocument}, -}; - -use super::DocumentSet; - -impl FtsStore { - pub async fn index + Display + Clone + std::fmt::Debug>( - &self, - document: FtsDocument<'_, T>, - ) -> trc::Result<()> { - match self { - FtsStore::Store(store) => store.fts_index(document).await, - #[cfg(feature = "elastic")] - FtsStore::ElasticSearch(store) => store.fts_index(document).await, - } - .caused_by(trc::location!()) - } - - pub async fn query + Display + Clone + std::fmt::Debug>( - &self, - account_id: u32, - collection: Collection, - filters: Vec>, - ) -> trc::Result { - match self { - FtsStore::Store(store) => store.fts_query(account_id, collection, filters).await, - #[cfg(feature = "elastic")] - FtsStore::ElasticSearch(store) => { - store.fts_query(account_id, collection, filters).await - } - } - .caused_by(trc::location!()) - } - - pub async fn remove( - &self, - account_id: u32, - collection: Collection, - document_ids: &impl DocumentSet, - ) -> trc::Result<()> { - match self { - FtsStore::Store(store) => store.fts_remove(account_id, collection, document_ids).await, - #[cfg(feature = "elastic")] - FtsStore::ElasticSearch(store) => { - store.fts_remove(account_id, collection, document_ids).await - } - } - .caused_by(trc::location!()) - } - - pub async fn remove_all(&self, account_id: u32) -> trc::Result<()> { - match self { - FtsStore::Store(store) => store.fts_remove_all(account_id).await, - #[cfg(feature = "elastic")] - FtsStore::ElasticSearch(store) => store.fts_remove_all(account_id).await, - } - .caused_by(trc::location!()) - } -} diff --git a/crates/store/src/dispatch/mod.rs b/crates/store/src/dispatch/mod.rs index 77d1cde6..5c9948e8 100644 --- a/crates/store/src/dispatch/mod.rs +++ b/crates/store/src/dispatch/mod.rs @@ -9,9 +9,9 @@ use roaring::RoaringBitmap; use crate::Store; pub mod blob; -pub mod fts; pub mod lookup; pub mod pubsub; +pub mod search; pub mod store; impl Store { diff --git a/crates/store/src/dispatch/search.rs b/crates/store/src/dispatch/search.rs new file mode 100644 index 00000000..4ac2f178 --- /dev/null +++ b/crates/store/src/dispatch/search.rs @@ -0,0 +1,78 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use super::DocumentSet; +use crate::{ + SearchStore, + search::{IndexDocument, SearchComparator, SearchFilter}, +}; +use trc::AddContext; +use types::collection::Collection; + +impl SearchStore { + pub async fn index(&self, document: IndexDocument) -> trc::Result<()> { + match self { + SearchStore::Store(store) => store.index_insert(document).await, + #[cfg(feature = "elastic")] + SearchStore::ElasticSearch(store) => store.index_insert(document).await, + } + .caused_by(trc::location!()) + } + + pub async fn query( + &self, + account_id: u32, + collection: Collection, + filters: Vec, + comparators: Vec, + ) -> trc::Result> { + match self { + SearchStore::Store(store) => { + store + .index_query(account_id, collection, filters, comparators) + .await + } + #[cfg(feature = "elastic")] + SearchStore::ElasticSearch(store) => { + store + .index_query(account_id, collection, filters, comparators) + .await + } + } + .caused_by(trc::location!()) + } + + pub async fn remove( + &self, + account_id: u32, + collection: Collection, + document_ids: &impl DocumentSet, + ) -> trc::Result<()> { + match self { + SearchStore::Store(store) => { + store + .index_remove(account_id, collection, document_ids) + .await + } + #[cfg(feature = "elastic")] + SearchStore::ElasticSearch(store) => { + store + .index_remove(account_id, collection, document_ids) + .await + } + } + .caused_by(trc::location!()) + } + + pub async fn remove_all(&self, account_id: u32) -> trc::Result<()> { + match self { + SearchStore::Store(store) => store.index_remove_all(account_id).await, + #[cfg(feature = "elastic")] + SearchStore::ElasticSearch(store) => store.index_remove_all(account_id).await, + } + .caused_by(trc::location!()) + } +} diff --git a/crates/store/src/dispatch/store.rs b/crates/store/src/dispatch/store.rs index ed2284b3..5d2a5f60 100644 --- a/crates/store/src/dispatch/store.rs +++ b/crates/store/src/dispatch/store.rs @@ -4,29 +4,21 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{ - ops::{BitAndAssign, Range}, - time::Instant, -}; - -use compact_str::ToCompactString; -use roaring::RoaringBitmap; -use trc::{AddContext, StoreEvent}; -use types::collection::Collection; - +use super::DocumentSet; use crate::{ - BitmapKey, Deserialize, IterateParams, Key, QueryResult, SUBSPACE_BITMAP_ID, - SUBSPACE_BITMAP_TAG, SUBSPACE_BITMAP_TEXT, SUBSPACE_COUNTER, SUBSPACE_INDEXES, SUBSPACE_LOGS, - Store, U32_LEN, Value, ValueKey, + Deserialize, IterateParams, Key, QueryResult, SUBSPACE_COUNTER, SUBSPACE_INDEXES, + SUBSPACE_LOGS, Store, U32_LEN, Value, ValueKey, write::{ - AnyClass, AnyKey, AssignedIds, Batch, BatchBuilder, BitmapClass, BitmapHash, Operation, - ReportClass, ValueClass, ValueOp, + AnyClass, AnyKey, AssignedIds, Batch, BatchBuilder, Operation, ReportClass, ValueClass, + ValueOp, key::{DeserializeBigEndian, KeySerializer}, now, }, }; - -use super::DocumentSet; +use compact_str::ToCompactString; +use std::{ops::Range, time::Instant}; +use trc::{AddContext, StoreEvent}; +use types::collection::Collection; #[cfg(feature = "test_mode")] #[allow(clippy::type_complexity)] @@ -65,54 +57,6 @@ impl Store { .caused_by(trc::location!()) } - pub async fn get_bitmap( - &self, - key: BitmapKey, - ) -> trc::Result> { - match self { - #[cfg(feature = "sqlite")] - Self::SQLite(store) => store.get_bitmap(key).await, - #[cfg(feature = "foundation")] - Self::FoundationDb(store) => store.get_bitmap(key).await, - #[cfg(feature = "postgres")] - Self::PostgreSQL(store) => store.get_bitmap(key).await, - #[cfg(feature = "mysql")] - Self::MySQL(store) => store.get_bitmap(key).await, - #[cfg(feature = "rocks")] - Self::RocksDb(store) => store.get_bitmap(key).await, - // SPDX-SnippetBegin - // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - // SPDX-License-Identifier: LicenseRef-SEL - #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] - Self::SQLReadReplica(store) => store.get_bitmap(key).await, - // SPDX-SnippetEnd - Self::None => Err(trc::StoreEvent::NotConfigured.into()), - } - .caused_by(trc::location!()) - } - - pub async fn get_bitmaps_intersection( - &self, - keys: Vec>, - ) -> trc::Result> { - let mut result: Option = None; - for key in keys { - if let Some(bitmap) = self.get_bitmap(key).await.caused_by(trc::location!())? { - if let Some(result) = &mut result { - result.bitand_assign(&bitmap); - if result.is_empty() { - break; - } - } else { - result = Some(bitmap); - } - } else { - return Ok(None); - } - } - Ok(result) - } - pub async fn iterate( &self, params: IterateParams, @@ -410,14 +354,7 @@ impl Store { } pub async fn danger_destroy_account(&self, account_id: u32) -> trc::Result<()> { - for subspace in [ - SUBSPACE_BITMAP_ID, - SUBSPACE_BITMAP_TAG, - SUBSPACE_BITMAP_TEXT, - SUBSPACE_LOGS, - SUBSPACE_INDEXES, - SUBSPACE_COUNTER, - ] { + for subspace in [SUBSPACE_LOGS, SUBSPACE_INDEXES, SUBSPACE_COUNTER] { self.delete_range( AnyKey { subspace, @@ -435,16 +372,6 @@ impl Store { for (from_class, to_class) in [ (ValueClass::Acl(account_id), ValueClass::Acl(account_id + 1)), (ValueClass::Property(0), ValueClass::Property(0)), - ( - ValueClass::FtsIndex(BitmapHash { - hash: [0u8; 8], - len: 0, - }), - ValueClass::FtsIndex(BitmapHash { - hash: [u8::MAX; 8], - len: u8::MAX, - }), - ), ] { self.delete_range( ValueKey { @@ -542,9 +469,6 @@ impl Store { for subspace in [ SUBSPACE_ACL, - SUBSPACE_BITMAP_ID, - SUBSPACE_BITMAP_TAG, - SUBSPACE_BITMAP_TEXT, SUBSPACE_DIRECTORY, SUBSPACE_TASK_QUEUE, SUBSPACE_INDEXES, @@ -562,7 +486,6 @@ impl Store { SUBSPACE_QUOTA, SUBSPACE_REPORT_OUT, SUBSPACE_REPORT_IN, - SUBSPACE_FTS_INDEX, SUBSPACE_TELEMETRY_SPAN, SUBSPACE_TELEMETRY_METRIC, SUBSPACE_TELEMETRY_INDEX, @@ -713,8 +636,6 @@ impl Store { #[cfg(feature = "test_mode")] #[allow(unused_variables)] pub async fn assert_is_empty(&self, blob_store: crate::BlobStore) { - use utils::codec::leb128::Leb128Iterator; - use crate::*; self.blob_expire_all().await; @@ -737,16 +658,12 @@ impl Store { (SUBSPACE_QUEUE_EVENT, true), (SUBSPACE_REPORT_OUT, true), (SUBSPACE_REPORT_IN, true), - (SUBSPACE_FTS_INDEX, true), (SUBSPACE_BLOB_RESERVE, true), (SUBSPACE_BLOB_LINK, true), (SUBSPACE_BLOBS, true), (SUBSPACE_COUNTER, false), (SUBSPACE_QUOTA, false), (SUBSPACE_BLOBS, true), - (SUBSPACE_BITMAP_ID, false), - (SUBSPACE_BITMAP_TAG, false), - (SUBSPACE_BITMAP_TEXT, false), (SUBSPACE_INDEXES, false), (SUBSPACE_TELEMETRY_SPAN, true), (SUBSPACE_TELEMETRY_METRIC, true), @@ -765,61 +682,6 @@ impl Store { IterateParams::new(from_key, to_key).set_values(with_values), |key, value| { match subspace { - SUBSPACE_BITMAP_ID | SUBSPACE_BITMAP_TAG | SUBSPACE_BITMAP_TEXT => { - if key.get(0..4).unwrap_or_default() == u32::MAX.to_be_bytes() { - return Ok(true); - } - - const BM_DOCUMENT_IDS: u8 = 0; - const BM_TAG: u8 = 1 << 6; - const BM_TEXT: u8 = 1 << 7; - const TAG_TEXT: u8 = 1 << 0; - const TAG_STATIC: u8 = 1 << 1; - - match key[5] { - BM_DOCUMENT_IDS => { - print!("Found document ids bitmap"); - } - BM_TAG => { - print!( - "Found tagged id {} bitmap", - key[7..].iter().next_leb128::().unwrap() - ); - } - TAG_TEXT => { - print!( - "Found tagged text {:?} bitmap", - String::from_utf8_lossy(&key[7..]) - ); - } - TAG_STATIC => { - print!("Found tagged static {} bitmap", key[7]); - } - other => { - if other & BM_TEXT == BM_TEXT { - print!( - "Found text hash {:?} bitmap", - String::from_utf8_lossy(&key[7..]) - ); - } else { - print!("Found unknown bitmap"); - } - } - } - - println!( - concat!( - ", account {}, collection {},", - " family {}, field {}, key {:?}: {:?}" - ), - u32::from_be_bytes(key[0..4].try_into().unwrap()), - key[4], - key[5], - key[6], - key, - value - ); - } SUBSPACE_COUNTER if key.len() == U32_LEN + 1 || key.len() == U32_LEN => { // Message ID and change ID counters return Ok(true); diff --git a/crates/store/src/fts/mod.rs b/crates/store/src/fts/mod.rs deleted file mode 100644 index eadc70bd..00000000 --- a/crates/store/src/fts/mod.rs +++ /dev/null @@ -1,200 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use std::fmt::Display; - -use nlp::language::Language; - -pub mod index; -pub mod postings; -pub mod query; - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum Field + Display + std::fmt::Debug> { - Header(T), - Body, - Attachment, - Keyword, -} - -#[derive(Debug, PartialEq, Eq)] -pub enum FtsFilter + Display + std::fmt::Debug> { - Exact { - field: Field, - text: String, - language: Language, - }, - Contains { - field: Field, - text: String, - language: Language, - }, - Keyword { - field: Field, - text: String, - }, - And, - Or, - Not, - End, -} - -impl + Display + std::fmt::Debug> FtsFilter { - pub fn has_text_detect( - field: Field, - text: impl Into, - default_language: Language, - ) -> Self { - let (text, language) = Language::detect(text.into(), default_language); - Self::has_text(field, text, language) - } - - pub fn has_text(field: Field, text: impl Into, language: Language) -> Self { - let text = text.into(); - let (is_exact, text) = if let Some(text) = text - .strip_prefix('"') - .and_then(|t| t.strip_suffix('"')) - .or_else(|| text.strip_prefix('\'').and_then(|t| t.strip_suffix('\''))) - { - (true, text.to_string()) - } else { - (false, text) - }; - - if !matches!(language, Language::None) && is_exact { - FtsFilter::Exact { - field, - text: text.to_string(), - language, - } - } else { - FtsFilter::Contains { - field, - text, - language, - } - } - } - - pub fn has_keyword(field: Field, text: impl Into) -> Self { - FtsFilter::Keyword { - field, - text: text.into(), - } - } - - pub fn has_english_text(field: Field, text: impl Into) -> Self { - Self::has_text(field, text, Language::English) - } -} - -#[derive(Clone, Copy)] -pub enum FilterType { - And, - Or, - Not, - End, - Store, - Fts, -} - -pub enum FilterGroup { - Fts(Vec), - Store(T), -} - -pub trait FilterItem: Clone { - fn filter_type(&self) -> FilterType; -} - -pub trait IntoFilterGroup> { - fn into_filter_group(self) -> Vec>; -} - -impl> IntoFilterGroup for Vec { - fn into_filter_group(self) -> Vec> { - let mut filter = Vec::with_capacity(self.len()); - let mut iter = self.into_iter(); - let mut logical_op = None; - - while let Some(item) = iter.next() { - if matches!(item.filter_type(), FilterType::Fts) { - let mut store_item = None; - let mut depth = 0; - let mut fts = Vec::with_capacity(5); - - // Add the logical operator if there is one - let in_logical_op = if let Some(op) = logical_op.take() { - fts.push(op); - true - } else { - false - }; - fts.push(item); - - for item in iter.by_ref() { - match item.filter_type() { - FilterType::And | FilterType::Or | FilterType::Not => { - depth += 1; - fts.push(item); - } - FilterType::End if depth > 0 => { - depth -= 1; - fts.push(item); - } - FilterType::Fts => { - fts.push(item); - } - _ => { - store_item = Some(item); - break; - } - } - } - - if in_logical_op { - fts.push(T::from(FilterType::End)); - } - - if depth > 0 { - let mut store = Vec::with_capacity(depth * 2); - while depth > 0 { - let item = fts.pop().unwrap(); - if matches!( - item.filter_type(), - FilterType::And | FilterType::Or | FilterType::Not - ) { - depth -= 1; - } - store.push(FilterGroup::Store(item)); - } - - filter.push(FilterGroup::Fts(fts)); - filter.extend(store); - } else { - filter.push(FilterGroup::Fts(fts)); - } - - if let Some(item) = store_item { - filter.push(FilterGroup::Store(item)); - } - } else { - match item.filter_type() { - FilterType::And | FilterType::Or => { - logical_op = Some(item.clone()); - } - FilterType::Not => { - logical_op = Some(T::from(FilterType::And)); - } - _ => {} - } - filter.push(FilterGroup::Store(item)); - } - } - - filter - } -} diff --git a/crates/store/src/fts/pdf.rs b/crates/store/src/fts/pdf.rs deleted file mode 100644 index c4dd30ea..00000000 --- a/crates/store/src/fts/pdf.rs +++ /dev/null @@ -1,37 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use std::panic; - -use lopdf::Document; - -pub fn extract_pdf(bytes: &[u8]) -> Option { - panic::catch_unwind(|| { - let mut buf = Vec::::new(); - let mut out = PlainTextOutput::new(&mut buf as &mut dyn std::io::Write); - - output_doc(&Document::load_mem(bytes).ok()?, &mut out).ok()?; - - match String::from_utf8(buf) { - Ok(result) => result, - Err(err) => String::from_utf8_lossy(err.as_bytes()).into_owned(), - } - .into() - }) - .ok()? -} - -/* -#[cfg(test)] -mod tests { - - #[test] - fn extract_pdf() { - let bytes = include_bytes!("/tmp/pdf/files/ep.pdf"); - let text = super::extract_pdf(bytes); - } -} -*/ diff --git a/crates/store/src/fts/postings.rs b/crates/store/src/fts/postings.rs deleted file mode 100644 index b0e0bbfe..00000000 --- a/crates/store/src/fts/postings.rs +++ /dev/null @@ -1,162 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use std::cmp::Ordering; - -use ahash::AHashSet; -use utils::codec::leb128::Leb128Reader; - -use crate::{ - SerializeInfallible, - write::{bitpack::BitpackIterator, key::KeySerializer}, -}; - -#[derive(Default)] -pub(super) struct Postings { - fields: AHashSet, - postings: Vec, -} - -#[derive(Default)] -pub(super) struct SerializedPostings> { - bytes: T, -} - -impl Postings { - pub fn insert(&mut self, field: u8, posting: u32) { - self.fields.insert(field); - self.postings.push(posting); - } - - pub fn insert_keyword(&mut self, field: u8) { - self.fields.insert(field); - } -} - -impl> SerializedPostings { - pub fn new(bytes: T) -> Self { - SerializedPostings { bytes } - } - - pub fn has_field(&self, field: u8) -> bool { - for byte in self.bytes.as_ref() { - match byte { - 0xFF => return false, - _ if *byte == field => return true, - _ => {} - } - } - - false - } - - pub fn positions(&self) -> Vec { - self.into_iter().collect() - } - - pub fn matches_positions(&self, positions: &[u32], offset: u32) -> bool { - let mut next_pos = self.into_iter().peekable(); - - for expect_pos in positions.iter().map(|pos| *pos + offset) { - while let Some(pos) = next_pos.peek() { - match pos.cmp(&expect_pos) { - Ordering::Less => { - next_pos.next(); - } - Ordering::Equal => { - return true; - } - Ordering::Greater => { - break; - } - } - } - } - - false - } -} - -impl<'x, T: AsRef<[u8]>> IntoIterator for &'x SerializedPostings { - type Item = u32; - type IntoIter = BitpackIterator<'x>; - - fn into_iter(self) -> Self::IntoIter { - let bytes = self.bytes.as_ref(); - - for (bytes_offset, byte) in bytes.iter().enumerate() { - if *byte == 0xFF { - if let Some((items_left, bytes_read)) = bytes - .get(bytes_offset + 1..) - .and_then(|bytes| bytes.read_leb128::()) - { - return BitpackIterator { - bytes, - bytes_offset: bytes_offset + bytes_read + 1, - items_left, - ..Default::default() - }; - } - - break; - } - } - - BitpackIterator::default() - } -} - -impl SerializeInfallible for Postings { - fn serialize(&self) -> Vec { - // Serialize fields - let mut serializer = - KeySerializer::new((self.fields.len() + 1) + (self.postings.len() * 2)); - for field in &self.fields { - serializer = serializer.write(*field); - } - serializer = serializer.write(u8::MAX); - - // Compress postings - if !self.postings.is_empty() { - serializer.bitpack_sorted(&self.postings).finalize() - } else { - serializer.finalize() - } - } -} - -#[cfg(test)] -mod tests { - - use super::*; - use ahash::AHashMap; - - #[test] - fn postings_match_positions() { - let mut maps: AHashMap<&str, Postings> = AHashMap::new(); - let tokens = [ - "the", "quick", "brown", "fox", "jumps", "over", "the", "lazy", "dog", - ]; - - for (pos, word) in tokens.into_iter().enumerate() { - maps.entry(word).or_default().insert(0, pos as u32); - } - - let maps: AHashMap<&str, SerializedPostings>> = maps - .into_iter() - .map(|(k, v)| (k, SerializedPostings::new(v.serialize()))) - .collect(); - - let mut positions = Vec::new(); - for (pos, word) in tokens.into_iter().enumerate() { - if pos > 0 { - assert!(maps[word].matches_positions(&positions, pos as u32)); - } else { - positions = maps[word].positions(); - } - } - } -} diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index 36eb22df..7d91ea65 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -4,13 +4,11 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{borrow::Cow, sync::Arc}; - pub mod backend; pub mod config; pub mod dispatch; -pub mod fts; pub mod query; +pub mod search; pub mod write; pub use ahash; @@ -23,8 +21,9 @@ pub use xxhash_rust; use ahash::AHashMap; use backend::{fs::FsStore, http::HttpStore, memory::StaticMemoryStore}; +use std::{borrow::Cow, sync::Arc}; use utils::config::cron::SimpleCron; -use write::{BitmapClass, ValueClass}; +use write::ValueClass; pub trait Deserialize: Sized + Sync + Send { fn deserialize(bytes: &[u8]) -> trc::Result; @@ -49,14 +48,6 @@ pub trait Key: Sync + Send + Clone { fn subspace(&self) -> u8; } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct BitmapKey> { - pub account_id: u32, - pub collection: u8, - pub class: T, - pub document_id: u32, -} - #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct IndexKey> { pub account_id: u32, @@ -93,9 +84,6 @@ pub const U32_LEN: usize = std::mem::size_of::(); pub const U16_LEN: usize = std::mem::size_of::(); pub const SUBSPACE_ACL: u8 = b'a'; -pub const SUBSPACE_BITMAP_ID: u8 = b'b'; -pub const SUBSPACE_BITMAP_TAG: u8 = b'c'; -pub const SUBSPACE_BITMAP_TEXT: u8 = b'v'; pub const SUBSPACE_DIRECTORY: u8 = b'd'; pub const SUBSPACE_TASK_QUEUE: u8 = b'f'; pub const SUBSPACE_INDEXES: u8 = b'i'; @@ -113,12 +101,18 @@ pub const SUBSPACE_QUEUE_EVENT: u8 = b'q'; pub const SUBSPACE_QUOTA: u8 = b'u'; pub const SUBSPACE_REPORT_OUT: u8 = b'h'; pub const SUBSPACE_REPORT_IN: u8 = b'r'; -pub const SUBSPACE_FTS_INDEX: u8 = b'g'; pub const SUBSPACE_TELEMETRY_SPAN: u8 = b'o'; pub const SUBSPACE_TELEMETRY_INDEX: u8 = b'w'; pub const SUBSPACE_TELEMETRY_METRIC: u8 = b'x'; pub const SUBSPACE_RESERVED_2: u8 = b'z'; +/* +pub const SUBSPACE_BITMAP_ID: u8 = b'b'; +pub const SUBSPACE_BITMAP_TAG: u8 = b'c'; +pub const SUBSPACE_BITMAP_TEXT: u8 = b'v'; +pub const SUBSPACE_FTS_INDEX: u8 = b'g'; + +*/ #[derive(Clone)] pub struct IterateParams { @@ -133,7 +127,7 @@ pub struct IterateParams { pub struct Stores { pub stores: AHashMap, pub blob_stores: AHashMap, - pub fts_stores: AHashMap, + pub search_stores: AHashMap, pub in_memory_stores: AHashMap, pub pubsub_stores: AHashMap, pub purge_schedules: Vec, @@ -190,7 +184,7 @@ pub enum BlobBackend { } #[derive(Clone)] -pub enum FtsStore { +pub enum SearchStore { Store(Store), #[cfg(feature = "elastic")] ElasticSearch(Arc), @@ -290,7 +284,7 @@ impl From for BlobStore { } #[cfg(feature = "elastic")] -impl From for FtsStore { +impl From for SearchStore { fn from(store: backend::elastic::ElasticSearchStore) -> Self { Self::ElasticSearch(Arc::new(store)) } @@ -303,7 +297,7 @@ impl From for InMemoryStore { } } -impl From for FtsStore { +impl From for SearchStore { fn from(store: Store) -> Self { Self::Store(store) } @@ -339,7 +333,7 @@ impl Default for InMemoryStore { } } -impl Default for FtsStore { +impl Default for SearchStore { fn default() -> Self { Self::Store(Store::None) } diff --git a/crates/store/src/query/acl.rs b/crates/store/src/query/acl.rs index 4fbe123b..9b74aa81 100644 --- a/crates/store/src/query/acl.rs +++ b/crates/store/src/query/acl.rs @@ -130,7 +130,7 @@ impl Store { last_collection = acl_item.to_collection; } batch - .update_document(acl_item.to_document_id) + .with_document(acl_item.to_document_id) .acl_revoke(revoke_account_id); } if !batch.is_empty() { diff --git a/crates/store/src/query/filter.rs b/crates/store/src/query/filter.rs deleted file mode 100644 index 2fd00e50..00000000 --- a/crates/store/src/query/filter.rs +++ /dev/null @@ -1,340 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use std::ops::{BitAndAssign, BitOrAssign, BitXorAssign}; - -use ahash::HashSet; -use nlp::tokenizers::word::WordTokenizer; -use roaring::RoaringBitmap; -use trc::AddContext; -use types::collection::Collection; - -use crate::{ - BitmapKey, IndexKey, IndexKeyPrefix, IterateParams, Key, Store, U32_LEN, - backend::MAX_TOKEN_LENGTH, write::key::DeserializeBigEndian, -}; - -use super::{Filter, Operator, ResultSet}; - -struct State { - pub op: Filter, - pub bm: Option, -} - -impl Store { - pub async fn filter( - &self, - account_id: u32, - collection_: Collection, - filters: Vec, - ) -> trc::Result { - let collection = u8::from(collection_); - - if filters.is_empty() { - return Ok(ResultSet { - account_id, - collection: collection_, - results: self - .get_bitmap(BitmapKey::document_ids(account_id, collection)) - .await - .caused_by(trc::location!())? - .unwrap_or_else(RoaringBitmap::new), - }); - } - - let mut state: State = Filter::And.into(); - let mut stack = Vec::new(); - let mut filters = filters.into_iter().peekable(); - - let mut not_mask = RoaringBitmap::new(); - let mut not_fetch = false; - - while let Some(filter) = filters.next() { - let mut result = match filter { - Filter::MatchValue { field, op, value } => self - .range_to_bitmap(account_id, collection, field, &value, op) - .await - .caused_by(trc::location!())?, - Filter::HasText { - field, - text, - tokenize, - } => { - if tokenize { - self.get_bitmaps_intersection( - WordTokenizer::new(&text, MAX_TOKEN_LENGTH) - .map(|token| token.word.into_owned()) - .collect::>() - .into_iter() - .map(|word| { - BitmapKey::text_token(account_id, collection, field, word) - }) - .collect(), - ) - .await - .caused_by(trc::location!())? - } else { - self.get_bitmap(BitmapKey::text_token(account_id, collection, field, text)) - .await - .caused_by(trc::location!())? - } - } - Filter::InBitmap(class) => self - .get_bitmap(BitmapKey { - account_id, - collection, - class, - document_id: 0, - }) - .await - .caused_by(trc::location!())?, - Filter::DocumentSet(set) => Some(set), - op @ (Filter::And | Filter::Or | Filter::Not) => { - stack.push(state); - state = op.into(); - continue; - } - Filter::End => { - if let Some(prev_state) = stack.pop() { - let bm = state.bm; - state = prev_state; - bm - } else { - break; - } - } - }; - - // Only fetch not mask if we need it - if matches!(state.op, Filter::Not) && !not_fetch { - not_mask = self - .get_bitmap(BitmapKey::document_ids(account_id, collection)) - .await - .caused_by(trc::location!())? - .unwrap_or_else(RoaringBitmap::new); - not_fetch = true; - } - - // Apply logical operation - if let Some(dest) = &mut state.bm { - match state.op { - Filter::And => { - if let Some(result) = result { - dest.bitand_assign(result); - } else { - dest.clear(); - } - } - Filter::Or => { - if let Some(result) = result { - dest.bitor_assign(result); - } - } - Filter::Not => { - if let Some(mut result) = result { - result.bitxor_assign(¬_mask); - dest.bitand_assign(result); - } - } - _ => unreachable!(), - } - } else if let Some(ref mut result_) = result { - if let Filter::Not = state.op { - result_.bitxor_assign(¬_mask); - } - state.bm = result; - } else if let Filter::Not = state.op { - state.bm = Some(not_mask.clone()); - } else { - state.bm = Some(RoaringBitmap::new()); - } - - // And short-circuit - if matches!(state.op, Filter::And) && state.bm.as_ref().unwrap().is_empty() { - while let Some(filter) = filters.peek() { - if matches!(filter, Filter::End) { - break; - } else { - filters.next(); - } - } - } - } - - Ok(ResultSet { - account_id, - collection: collection_, - results: state.bm.unwrap_or_default(), - }) - } - - async fn range_to_bitmap( - &self, - account_id: u32, - collection: u8, - field: u8, - match_value: &[u8], - op: Operator, - ) -> trc::Result> { - let mut finder = None; - let (begin, end) = match op { - Operator::LowerThan => ( - IndexKey { - account_id, - collection, - document_id: 0, - field, - key: &[][..], - }, - IndexKey { - account_id, - collection, - document_id: 0, - field, - key: match_value, - }, - ), - Operator::LowerEqualThan => ( - IndexKey { - account_id, - collection, - document_id: 0, - field, - key: &[][..], - }, - IndexKey { - account_id, - collection, - document_id: u32::MAX, - field, - key: match_value, - }, - ), - Operator::GreaterThan => ( - IndexKey { - account_id, - collection, - document_id: u32::MAX, - field, - key: match_value, - }, - IndexKey { - account_id, - collection, - document_id: u32::MAX, - field: field + 1, - key: &[][..], - }, - ), - Operator::GreaterEqualThan => ( - IndexKey { - account_id, - collection, - document_id: 0, - field, - key: match_value, - }, - IndexKey { - account_id, - collection, - document_id: u32::MAX, - field: field + 1, - key: &[][..], - }, - ), - Operator::Equal => ( - IndexKey { - account_id, - collection, - document_id: 0, - field, - key: match_value, - }, - IndexKey { - account_id, - collection, - document_id: u32::MAX, - field, - key: match_value, - }, - ), - Operator::Contains => { - finder = memchr::memmem::Finder::new(match_value).into(); - - ( - IndexKey { - account_id, - collection, - document_id: 0, - field, - key: &[][..], - }, - IndexKey { - account_id, - collection, - document_id: u32::MAX, - field: field + 1, - key: &[u8::MAX, u8::MAX, u8::MAX, u8::MAX][..], - }, - ) - } - }; - - let mut bm = RoaringBitmap::new(); - let prefix = IndexKeyPrefix { - account_id, - collection, - field, - } - .serialize(0); - - self.iterate( - IterateParams::new(begin, end).no_values().ascending(), - |key, _| { - if !key.starts_with(&prefix) { - return Ok(false); - } - - let id_pos = key.len() - U32_LEN; - let value = key - .get(IndexKeyPrefix::len()..id_pos) - .ok_or_else(|| trc::Error::corrupted_key(key, None, trc::location!()))?; - - let matches = match op { - Operator::LowerThan => value < match_value, - Operator::LowerEqualThan => value <= match_value, - Operator::GreaterThan => value > match_value, - Operator::GreaterEqualThan => value >= match_value, - Operator::Equal => value == match_value, - Operator::Contains => finder.as_ref().unwrap().find(value).is_some(), - }; - - if matches { - bm.insert(key.deserialize_be_u32(id_pos)?); - } - - Ok(true) - }, - ) - .await - .caused_by(trc::location!())?; - - if !bm.is_empty() { - Ok(Some(bm)) - } else { - Ok(None) - } - } -} - -impl From for State { - fn from(value: Filter) -> Self { - Self { - op: value, - bm: None, - } - } -} diff --git a/crates/store/src/query/mod.rs b/crates/store/src/query/mod.rs index 8345c2c8..4e4fd4e7 100644 --- a/crates/store/src/query/mod.rs +++ b/crates/store/src/query/mod.rs @@ -5,234 +5,9 @@ */ pub mod acl; -pub mod filter; pub mod log; -pub mod sort; -use roaring::RoaringBitmap; -use types::{collection::Collection, id::Id}; - -use crate::{ - BitmapKey, IterateParams, Key, - write::{BitmapClass, BitmapHash, TagValue}, -}; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum Operator { - LowerThan, - LowerEqualThan, - GreaterThan, - GreaterEqualThan, - Equal, - Contains, -} - -#[derive(Debug)] -pub enum Filter { - MatchValue { - field: u8, - op: Operator, - value: Vec, - }, - HasText { - field: u8, - text: String, - tokenize: bool, - }, - InBitmap(BitmapClass), - DocumentSet(RoaringBitmap), - And, - Or, - Not, - End, -} - -#[derive(Debug)] -pub enum Comparator { - Field { field: u8, ascending: bool }, - DocumentSet { set: RoaringBitmap, ascending: bool }, - SortedList { list: Vec, ascending: bool }, -} - -#[derive(Debug)] -pub struct ResultSet { - pub account_id: u32, - pub collection: Collection, - pub results: RoaringBitmap, -} - -pub struct SortedResultSet { - pub position: i32, - pub ids: Vec, - pub found_anchor: bool, -} - -impl ResultSet { - pub fn new(account_id: u32, collection: Collection, results: RoaringBitmap) -> Self { - ResultSet { - account_id, - collection, - results, - } - } - - pub fn apply_mask(&mut self, mask: RoaringBitmap) { - self.results &= mask; - } -} - -impl Filter { - pub fn cond(field: impl Into, op: Operator, value: Vec) -> Self { - Filter::MatchValue { - field: field.into(), - op, - value, - } - } - - pub fn eq(field: impl Into, value: Vec) -> Self { - Filter::MatchValue { - field: field.into(), - op: Operator::Equal, - value, - } - } - - pub fn lt(field: impl Into, value: Vec) -> Self { - Filter::MatchValue { - field: field.into(), - op: Operator::LowerThan, - value, - } - } - - pub fn le(field: impl Into, value: Vec) -> Self { - Filter::MatchValue { - field: field.into(), - op: Operator::LowerEqualThan, - value, - } - } - - pub fn gt(field: impl Into, value: Vec) -> Self { - Filter::MatchValue { - field: field.into(), - op: Operator::GreaterThan, - value, - } - } - - pub fn ge(field: impl Into, value: Vec) -> Self { - Filter::MatchValue { - field: field.into(), - op: Operator::GreaterEqualThan, - value, - } - } - - pub fn contains(field: impl Into, value: &str) -> Self { - Filter::MatchValue { - field: field.into(), - op: Operator::Contains, - value: value.to_lowercase().into_bytes(), - } - } - - pub fn has_text(field: impl Into, text: impl Into) -> Self { - Filter::HasText { - field: field.into(), - text: text.into(), - tokenize: true, - } - } - - pub fn is_in_bitmap(field: impl Into, value: impl Into) -> Self { - Self::InBitmap(BitmapClass::Tag { - field: field.into(), - value: value.into(), - }) - } - - pub fn is_in_set(set: RoaringBitmap) -> Self { - Filter::DocumentSet(set) - } -} - -impl Comparator { - pub fn field(field: impl Into, ascending: bool) -> Self { - Self::Field { - field: field.into(), - ascending, - } - } - - pub fn set(set: RoaringBitmap, ascending: bool) -> Self { - Self::DocumentSet { set, ascending } - } - - pub fn sorted_list(list: Vec, ascending: bool) -> Self { - Self::SortedList { list, ascending } - } - - pub fn ascending(field: impl Into) -> Self { - Self::Field { - field: field.into(), - ascending: true, - } - } - - pub fn descending(field: impl Into) -> Self { - Self::Field { - field: field.into(), - ascending: false, - } - } -} - -impl BitmapKey { - pub fn document_ids(account_id: u32, collection: impl Into) -> Self { - BitmapKey { - account_id, - collection: collection.into(), - class: BitmapClass::DocumentIds, - document_id: 0, - } - } - - pub fn text_token( - account_id: u32, - collection: impl Into, - field: impl Into, - token: impl AsRef<[u8]>, - ) -> Self { - BitmapKey { - account_id, - collection: collection.into(), - class: BitmapClass::Text { - field: field.into(), - token: BitmapHash::new(token), - }, - document_id: 0, - } - } - - pub fn tag( - account_id: u32, - collection: impl Into, - field: impl Into, - value: impl Into, - ) -> Self { - BitmapKey { - account_id, - collection: collection.into(), - class: BitmapClass::Tag { - field: field.into(), - value: value.into(), - }, - document_id: 0, - } - } -} +use crate::{IterateParams, Key}; impl IterateParams { pub fn new(begin: T, end: T) -> Self { diff --git a/crates/store/src/query/sort.rs b/crates/store/src/query/sort.rs deleted file mode 100644 index b3497075..00000000 --- a/crates/store/src/query/sort.rs +++ /dev/null @@ -1,415 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use super::{Comparator, ResultSet, SortedResultSet}; -use crate::{IndexKeyPrefix, IterateParams, Store, U32_LEN, write::key::DeserializeBigEndian}; -use ahash::{AHashMap, AHashSet}; -use std::cmp::Ordering; -use trc::AddContext; -use types::id::Id; - -#[derive(Debug)] -pub struct Pagination<'x> { - requested_position: i32, - position: i32, - pub limit: usize, - anchor: u32, - anchor_offset: i32, - has_anchor: bool, - anchor_found: bool, - pub ids: Vec, - prefix_map: Option<&'x AHashMap>, - prefix_unique: bool, -} - -impl Store { - pub async fn sort( - &self, - result_set: ResultSet, - mut comparators: Vec, - mut paginate: Pagination<'_>, - ) -> trc::Result { - paginate.limit = match (result_set.results.len(), paginate.limit) { - (0, _) => { - return Ok(SortedResultSet { - position: paginate.position, - ids: vec![], - found_anchor: true, - }); - } - (_, 0) => result_set.results.len() as usize, - (a, b) => std::cmp::min(a as usize, b), - }; - - if comparators.len() == 1 && !paginate.prefix_unique { - match comparators.pop().unwrap() { - Comparator::Field { field, ascending } => { - let mut results = result_set.results; - let collection = u8::from(result_set.collection); - - self.iterate( - IterateParams::new( - IndexKeyPrefix { - account_id: result_set.account_id, - collection, - field, - }, - IndexKeyPrefix { - account_id: result_set.account_id, - collection, - field: field + 1, - }, - ) - .no_values() - .set_ascending(ascending), - |key, _| { - let document_id = key.deserialize_be_u32(key.len() - U32_LEN)?; - - Ok(!results.remove(document_id) || paginate.add(0, document_id)) - }, - ) - .await - .caused_by(trc::location!())?; - - // Add remaining items not present in the index - if !results.is_empty() && !paginate.is_full() { - for document_id in results { - if !paginate.add(0, document_id) { - break; - } - } - } - } - Comparator::DocumentSet { set, ascending } => { - let in_set = &result_set.results & &set; - let not_in_set = &result_set.results ^ &in_set; - let sets = if ascending { - [in_set, not_in_set] - } else { - [not_in_set, in_set] - }; - 'outer: for set in sets { - for document_id in set { - if !paginate.add(0, document_id) { - break 'outer; - } - } - } - } - Comparator::SortedList { list, ascending } => { - if ascending { - for document_id in list { - if result_set.results.contains(document_id) - && !paginate.add(0, document_id) - { - break; - } - } - } else { - for document_id in list.into_iter().rev() { - if result_set.results.contains(document_id) - && !paginate.add(0, document_id) - { - break; - } - } - } - } - } - - // Obtain prefixes - let prefix_map = paginate.prefix_map.take(); - let mut sorted_results = paginate.build(); - if let Some(prefix_map) = prefix_map { - for id in sorted_results.ids.iter_mut() { - let document_id = id.document_id(); - if let Some(prefix_id) = prefix_map.get(&document_id) { - *id = Id::from_parts(*prefix_id, document_id); - } - } - } - - Ok(sorted_results) - } else if comparators.len() > 1 { - //TODO improve this algorithm, avoid re-sorting in memory. - let mut sorted_ids = AHashMap::with_capacity(paginate.limit); - - for (pos, comparator) in comparators.into_iter().take(4).enumerate() { - match comparator { - Comparator::Field { field, ascending } => { - let mut results = result_set.results.clone(); - let mut prev_data = vec![]; - let mut has_grouped_ids = false; - let mut idx = 0; - let collection = u8::from(result_set.collection); - - self.iterate( - IterateParams::new( - IndexKeyPrefix { - account_id: result_set.account_id, - collection, - field, - }, - IndexKeyPrefix { - account_id: result_set.account_id, - collection, - field: field + 1, - }, - ) - .no_values() - .set_ascending(ascending), - |key, _| { - let id_pos = key.len() - U32_LEN; - let document_id = key.deserialize_be_u32(id_pos)?; - - Ok(if results.remove(document_id) { - let data = key.get(IndexKeyPrefix::len()..id_pos).ok_or_else( - || trc::Error::corrupted_key(key, None, trc::location!()), - )?; - debug_assert!(!data.is_empty()); - - if data != prev_data { - idx += 1; - prev_data = data.to_vec(); - } else { - has_grouped_ids = true; - } - - sorted_ids.entry(document_id).or_insert([0u32; 4])[pos] = idx; - - !results.is_empty() - } else { - true - }) - }, - ) - .await - .caused_by(trc::location!())?; - - // Add remaining items not present in the index - if !results.is_empty() { - idx += 1; - for document_id in results { - sorted_ids.entry(document_id).or_insert([0u32; 4])[pos] = idx; - } - } else if !has_grouped_ids { - // If we are sorting by multiple fields and we don't have grouped ids, we can - // stop here - break; - } - } - Comparator::DocumentSet { set, ascending } => { - let in_set = &result_set.results & &set; - let not_in_set = &result_set.results ^ &in_set; - let sets = if ascending { - [(in_set, 0), (not_in_set, 1)] - } else { - [(not_in_set, 0), (in_set, 1)] - }; - - for (document_ids, idx) in sets { - for document_id in document_ids { - sorted_ids.entry(document_id).or_insert([0u32; 4])[pos] = idx; - } - } - } - Comparator::SortedList { list, ascending } => { - if ascending { - for (idx, document_id) in list.into_iter().enumerate() { - if result_set.results.contains(document_id) { - sorted_ids.entry(document_id).or_insert([0u32; 4])[pos] = - idx as u32; - } - } - } else { - for (idx, document_id) in list.into_iter().rev().enumerate() { - if result_set.results.contains(document_id) { - sorted_ids.entry(document_id).or_insert([0u32; 4])[pos] = - idx as u32; - } - } - } - } - } - } - - let mut seen_prefixes = AHashSet::new(); - let mut sorted_ids = sorted_ids.into_iter().collect::>(); - sorted_ids.sort_by(|a, b| match a.1.cmp(&b.1) { - Ordering::Equal => a.0.cmp(&b.0), - other => other, - }); - for (document_id, _) in sorted_ids { - // Obtain document prefixId - let prefix_id = if let Some(prefix_map) = paginate.prefix_map { - if let Some(prefix_id) = prefix_map.get(&document_id) { - if paginate.prefix_unique && !seen_prefixes.insert(*prefix_id) { - continue; - } - *prefix_id - } else { - // Document no longer exists? - continue; - } - } else { - 0 - }; - - // Add document to results - if !paginate.add(prefix_id, document_id) { - break; - } - } - - Ok(paginate.build()) - } else { - let mut seen_prefixes = AHashSet::new(); - for document_id in result_set.results { - // Obtain document prefixId - let prefix_id = if let Some(prefix_map) = paginate.prefix_map { - if let Some(prefix_id) = prefix_map.get(&document_id) { - if paginate.prefix_unique && !seen_prefixes.insert(*prefix_id) { - continue; - } - *prefix_id - } else { - // Document no longer exists? - continue; - } - } else { - 0 - }; - - // Add document to results - if !paginate.add(prefix_id, document_id) { - break; - } - } - Ok(paginate.build()) - } - } -} - -impl<'x> Pagination<'x> { - pub fn new(limit: usize, position: i32, anchor: Option, anchor_offset: i32) -> Self { - let (has_anchor, anchor) = anchor.map(|anchor| (true, anchor)).unwrap_or((false, 0)); - - Self { - requested_position: position, - position, - limit, - anchor, - anchor_offset, - has_anchor, - anchor_found: false, - ids: Vec::with_capacity(limit), - prefix_map: None, - prefix_unique: false, - } - } - - pub fn with_prefix_map(mut self, prefix_map: &'x AHashMap) -> Self { - self.prefix_map = Some(prefix_map); - self - } - - pub fn with_prefix_unique(mut self, prefix_unique: bool) -> Self { - self.prefix_unique = prefix_unique; - self - } - - #[inline(always)] - pub fn add(&mut self, prefix_id: u32, document_id: u32) -> bool { - self.add_id(Id::from_parts(prefix_id, document_id)) - } - - pub fn add_id(&mut self, id: Id) -> bool { - let document_id = id.document_id(); - - // Pagination - if !self.has_anchor { - if self.position >= 0 { - if self.position > 0 { - self.position -= 1; - } else { - self.ids.push(id); - if self.ids.len() == self.limit { - return false; - } - } - } else { - self.ids.push(id); - } - } else if self.anchor_offset >= 0 { - if !self.anchor_found { - if document_id != self.anchor { - return true; - } - self.anchor_found = true; - } - - if self.anchor_offset > 0 { - self.anchor_offset -= 1; - } else { - self.ids.push(id); - if self.ids.len() == self.limit { - return false; - } - } - } else { - self.anchor_found = document_id == self.anchor; - self.ids.push(id); - - if self.anchor_found { - self.position = self.anchor_offset; - return false; - } - } - - true - } - - pub fn is_full(&self) -> bool { - self.ids.len() == self.limit - } - - pub fn build(self) -> SortedResultSet { - let mut result = SortedResultSet { - ids: self.ids, - position: 0, - found_anchor: !self.has_anchor || self.anchor_found, - }; - - if result.found_anchor { - if !self.has_anchor && self.requested_position >= 0 { - result.position = if self.position == 0 { - self.requested_position - } else { - 0 - }; - } else if self.position >= 0 { - result.position = self.position; - } else { - let position = self.position.unsigned_abs() as usize; - let start_offset = if position < result.ids.len() { - result.ids.len() - position - } else { - 0 - }; - result.position = start_offset as i32; - let end_offset = if self.limit > 0 { - std::cmp::min(start_offset + self.limit, result.ids.len()) - } else { - result.ids.len() - }; - - result.ids = result.ids[start_offset..end_offset].to_vec() - } - } - - result - } -} diff --git a/crates/store/src/fts/index.rs b/crates/store/src/search/index.rs similarity index 71% rename from crates/store/src/fts/index.rs rename to crates/store/src/search/index.rs index 30a1d7d8..377debec 100644 --- a/crates/store/src/fts/index.rs +++ b/crates/store/src/search/index.rs @@ -22,109 +22,15 @@ use crate::{ IterateParams, SerializeInfallible, Store, U32_LEN, ValueKey, backend::MAX_TOKEN_LENGTH, dispatch::DocumentSet, - write::{ - BatchBuilder, BitmapHash, Operation, ValueClass, ValueOp, hash::TokenType, - key::DeserializeBigEndian, - }, + search::IndexDocument, + write::{BatchBuilder, Operation, ValueClass, ValueOp, key::DeserializeBigEndian}, }; -use super::{Field, postings::Postings}; pub const TERM_INDEX_VERSION: u8 = 1; -#[derive(Debug)] -pub(crate) struct Text<'x, T: Into + Display + std::fmt::Debug> { - pub field: Field, - pub text: Cow<'x, str>, - pub typ: Type, -} - -#[derive(Debug)] -pub(crate) enum Type { - Text(Language), - Tokenize, - Keyword, -} - -#[derive(Debug)] -pub struct FtsDocument<'x, T: Into + Display + std::fmt::Debug> { - pub(crate) parts: Vec>, - pub(crate) default_language: Language, - pub(crate) account_id: u32, - pub(crate) collection: Collection, - pub(crate) document_id: u32, -} - -impl<'x, T: Into + Display + std::fmt::Debug> FtsDocument<'x, T> { - pub fn with_default_language(default_language: Language) -> FtsDocument<'x, T> { - FtsDocument { - parts: vec![], - default_language, - account_id: 0, - document_id: 0, - collection: Collection::None, - } - } - - pub fn with_account_id(mut self, account_id: u32) -> Self { - self.account_id = account_id; - self - } - - pub fn with_document_id(mut self, document_id: u32) -> Self { - self.document_id = document_id; - self - } - - pub fn with_collection(mut self, collection: Collection) -> Self { - self.collection = collection; - self - } - - pub fn index(&mut self, field: Field, text: impl Into>, language: Language) { - self.parts.push(Text { - field, - text: text.into(), - typ: Type::Text(language), - }); - } - - pub fn index_tokenized(&mut self, field: Field, text: impl Into>) { - self.parts.push(Text { - field, - text: text.into(), - typ: Type::Tokenize, - }); - } - - pub fn index_keyword(&mut self, field: Field, text: impl Into>) { - let text = text.into(); - if !text.is_empty() { - self.parts.push(Text { - field, - text, - typ: Type::Keyword, - }); - } - } -} - -impl + Display + std::fmt::Debug> From> for u8 { - fn from(value: Field) -> Self { - match value { - Field::Body => 0, - Field::Attachment => 1, - Field::Keyword => 2, - Field::Header(value) => 3 + value.into(), - } - } -} - impl Store { - pub async fn fts_index + Display + std::fmt::Debug>( - &self, - document: FtsDocument<'_, T>, - ) -> trc::Result<()> { - let mut detect = LanguageDetector::new(); + pub async fn index_insert(&self, document: IndexDocument) -> trc::Result<()> { + /*let mut detect = LanguageDetector::new(); let mut tokens: AHashMap = AHashMap::new(); let mut parts = Vec::new(); let mut position = 0; @@ -215,7 +121,7 @@ impl Store { batch .with_account_id(document.account_id) .with_collection(document.collection) - .update_document(document.document_id); + .with_document(document.document_id); for key in keys.into_iter() { if batch.is_large_batch() { @@ -224,26 +130,26 @@ impl Store { batch .with_account_id(document.account_id) .with_collection(document.collection) - .update_document(document.document_id); + .with_document(document.document_id); } batch.any_op(key); } if !batch.is_empty() { self.write(batch.build_all()).await?; - } + }*/ Ok(()) } - pub async fn fts_remove( + pub async fn index_remove( &self, account_id: u32, collection: Collection, document_ids: &impl DocumentSet, ) -> trc::Result<()> { // Find keys to delete - let mut delete_keys: AHashMap> = AHashMap::new(); + /*let mut delete_keys: AHashMap> = AHashMap::new(); self.iterate( IterateParams::new( ValueKey { @@ -309,7 +215,7 @@ impl Store { .with_collection(collection); for (document_id, keys) in delete_keys { - batch.update_document(document_id); + batch.with_document(document_id); for key in keys { if batch.is_large_batch() { @@ -320,7 +226,7 @@ impl Store { batch .with_account_id(account_id) .with_collection(collection) - .update_document(document_id); + .with_document(document_id); } batch.any_op(Operation::Value { class: key, @@ -333,12 +239,12 @@ impl Store { self.write(batch.build_all()) .await .caused_by(trc::location!())?; - } + }*/ Ok(()) } - pub async fn fts_remove_all(&self, _: u32) -> trc::Result<()> { + pub async fn index_remove_all(&self, _: u32) -> trc::Result<()> { // No-op // Term indexes are stored in the same key range as the document diff --git a/crates/store/src/search/local.rs b/crates/store/src/search/local.rs new file mode 100644 index 00000000..d18e63ef --- /dev/null +++ b/crates/store/src/search/local.rs @@ -0,0 +1,96 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use crate::search::SearchFilter; + +/*pub enum FilterGroup { + Fts(Vec), + Store(T), +} + +fn split_local_remote(filter: Vec) -> Vec> { + let mut filter = Vec::with_capacity(self.len()); + let mut iter = self.into_iter(); + let mut logical_op = None; + + while let Some(item) = iter.next() { + if matches!(item.filter_type(), FilterType::Fts) { + let mut store_item = None; + let mut depth = 0; + let mut fts = Vec::with_capacity(5); + + // Add the logical operator if there is one + let in_logical_op = if let Some(op) = logical_op.take() { + fts.push(op); + true + } else { + false + }; + fts.push(item); + + for item in iter.by_ref() { + match item.filter_type() { + FilterType::And | FilterType::Or | FilterType::Not => { + depth += 1; + fts.push(item); + } + FilterType::End if depth > 0 => { + depth -= 1; + fts.push(item); + } + FilterType::Fts => { + fts.push(item); + } + _ => { + store_item = Some(item); + break; + } + } + } + + if in_logical_op { + fts.push(T::from(FilterType::End)); + } + + if depth > 0 { + let mut store = Vec::with_capacity(depth * 2); + while depth > 0 { + let item = fts.pop().unwrap(); + if matches!( + item.filter_type(), + FilterType::And | FilterType::Or | FilterType::Not + ) { + depth -= 1; + } + store.push(FilterGroup::Store(item)); + } + + filter.push(FilterGroup::Fts(fts)); + filter.extend(store); + } else { + filter.push(FilterGroup::Fts(fts)); + } + + if let Some(item) = store_item { + filter.push(FilterGroup::Store(item)); + } + } else { + match item.filter_type() { + FilterType::And | FilterType::Or => { + logical_op = Some(item.clone()); + } + FilterType::Not => { + logical_op = Some(T::from(FilterType::And)); + } + _ => {} + } + filter.push(FilterGroup::Store(item)); + } + } + + filter +} +*/ diff --git a/crates/store/src/search/mod.rs b/crates/store/src/search/mod.rs new file mode 100644 index 00000000..aacfe3e7 --- /dev/null +++ b/crates/store/src/search/mod.rs @@ -0,0 +1,358 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +pub mod index; +pub mod local; +pub mod query; + +use nlp::language::Language; +use roaring::RoaringBitmap; +use std::borrow::Cow; +use types::collection::Collection; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SearchOperator { + LowerThan, + LowerEqualThan, + GreaterThan, + GreaterEqualThan, + Equal, + Contains, + Exists, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SearchField { + Email(EmailSearchField), + Calendar(CalendarSearchField), + Contact(ContactSearchField), + File(FileSearchField), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum EmailSearchField { + From, + To, + Cc, + Bcc, + Subject, + Body, + Attachment, + ReceivedAt, + SentAt, + Size, + HasAttachment, + Header(Cow<'static, str>), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum CalendarSearchField { + Summary, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ContactSearchField { + Name, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum FileSearchField { + Name, + Content, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SearchValue { + Text { value: String, language: Language }, + Number(i64), + Boolean(bool), +} + +#[derive(Debug)] +pub enum SearchFilter { + Operator { + field: SearchField, + op: SearchOperator, + value: SearchValue, + }, + DocumentSet(RoaringBitmap), + And, + Or, + Not, + End, +} + +#[derive(Debug)] +pub enum SearchComparator { + Field { field: SearchField, ascending: bool }, + DocumentSet { set: RoaringBitmap, ascending: bool }, + SortedList { list: Vec, ascending: bool }, +} + +#[derive(Debug)] +pub struct IndexDocument { + pub(crate) account_id: u32, + pub(crate) collection: Collection, + pub(crate) document_id: u32, + pub(crate) fields: Vec, + pub(crate) default_language: Language, +} + +#[derive(Debug)] +pub struct IndexField { + pub(crate) field: SearchField, + pub(crate) value: SearchValue, +} + +impl SearchFilter { + pub fn cond( + field: impl Into, + op: SearchOperator, + value: impl Into, + ) -> Self { + SearchFilter::Operator { + field: field.into(), + op, + value: value.into(), + } + } + + pub fn exists(field: impl Into) -> Self { + SearchFilter::Operator { + field: field.into(), + op: SearchOperator::Exists, + value: SearchValue::Boolean(true), + } + } + + pub fn eq(field: impl Into, value: impl Into) -> Self { + SearchFilter::Operator { + field: field.into(), + op: SearchOperator::Equal, + value: value.into(), + } + } + + pub fn lt(field: impl Into, value: impl Into) -> Self { + SearchFilter::Operator { + field: field.into(), + op: SearchOperator::LowerThan, + value: value.into(), + } + } + + pub fn le(field: impl Into, value: impl Into) -> Self { + SearchFilter::Operator { + field: field.into(), + op: SearchOperator::LowerEqualThan, + value: value.into(), + } + } + + pub fn gt(field: impl Into, value: impl Into) -> Self { + SearchFilter::Operator { + field: field.into(), + op: SearchOperator::GreaterThan, + value: value.into(), + } + } + + pub fn ge(field: impl Into, value: impl Into) -> Self { + SearchFilter::Operator { + field: field.into(), + op: SearchOperator::GreaterEqualThan, + value: value.into(), + } + } + + pub fn has_text_detect( + field: impl Into, + text: impl Into, + default_language: Language, + ) -> Self { + let (text, language) = Language::detect(text.into(), default_language); + Self::has_text(field, text, language) + } + + pub fn has_text( + field: impl Into, + text: impl Into, + language: Language, + ) -> Self { + let text = text.into(); + let (is_exact, text) = if let Some(text) = text + .strip_prefix('"') + .and_then(|t| t.strip_suffix('"')) + .or_else(|| text.strip_prefix('\'').and_then(|t| t.strip_suffix('\''))) + { + (true, text.to_string()) + } else { + (false, text) + }; + + if !matches!(language, Language::None) && is_exact { + SearchFilter::Operator { + field: field.into(), + op: SearchOperator::Equal, + value: SearchValue::Text { + value: text, + language, + }, + } + } else { + SearchFilter::Operator { + field: field.into(), + op: SearchOperator::Contains, + value: SearchValue::Text { + value: text, + language, + }, + } + } + } + + pub fn has_english_text(field: impl Into, text: impl Into) -> Self { + Self::has_text(field, text, Language::English) + } + + pub fn is_in_set(set: RoaringBitmap) -> Self { + SearchFilter::DocumentSet(set) + } +} + +impl SearchComparator { + pub fn field(field: impl Into, ascending: bool) -> Self { + Self::Field { + field: field.into(), + ascending, + } + } + + pub fn set(set: RoaringBitmap, ascending: bool) -> Self { + Self::DocumentSet { set, ascending } + } + + pub fn sorted_list(list: Vec, ascending: bool) -> Self { + Self::SortedList { list, ascending } + } + + pub fn ascending(field: impl Into) -> Self { + Self::Field { + field: field.into(), + ascending: true, + } + } + + pub fn descending(field: impl Into) -> Self { + Self::Field { + field: field.into(), + ascending: false, + } + } +} + +impl IndexDocument { + pub fn with_default_language(default_language: Language) -> Self { + Self { + fields: vec![], + default_language, + account_id: 0, + document_id: 0, + collection: Collection::None, + } + } + + pub fn with_account_id(mut self, account_id: u32) -> Self { + self.account_id = account_id; + self + } + + pub fn with_document_id(mut self, document_id: u32) -> Self { + self.document_id = document_id; + self + } + + pub fn with_collection(mut self, collection: Collection) -> Self { + self.collection = collection; + self + } + + pub fn index(&mut self, field: impl Into, value: impl Into) { + self.fields.push(IndexField { + field: field.into(), + value: value.into(), + }); + } +} + +impl From for SearchField { + fn from(field: EmailSearchField) -> Self { + SearchField::Email(field) + } +} + +impl From for SearchField { + fn from(field: CalendarSearchField) -> Self { + SearchField::Calendar(field) + } +} + +impl From for SearchField { + fn from(field: ContactSearchField) -> Self { + SearchField::Contact(field) + } +} + +impl From for SearchField { + fn from(field: FileSearchField) -> Self { + SearchField::File(field) + } +} + +impl From for SearchValue { + fn from(value: u64) -> Self { + SearchValue::Number(value as i64) + } +} + +impl From for SearchValue { + fn from(value: i64) -> Self { + SearchValue::Number(value) + } +} + +impl From for SearchValue { + fn from(value: u32) -> Self { + SearchValue::Number(value as i64) + } +} + +impl From for SearchValue { + fn from(value: i32) -> Self { + SearchValue::Number(value as i64) + } +} + +impl From for SearchValue { + fn from(value: usize) -> Self { + SearchValue::Number(value as i64) + } +} + +impl From for SearchValue { + fn from(value: bool) -> Self { + SearchValue::Boolean(value) + } +} + +impl From for SearchValue { + fn from(value: String) -> Self { + SearchValue::Text { + value, + language: Language::None, + } + } +} diff --git a/crates/store/src/fts/query.rs b/crates/store/src/search/query.rs similarity index 96% rename from crates/store/src/fts/query.rs rename to crates/store/src/search/query.rs index 2a3ef84a..a8df1e23 100644 --- a/crates/store/src/fts/query.rs +++ b/crates/store/src/search/query.rs @@ -4,12 +4,11 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::postings::SerializedPostings; use crate::{ - BitmapKey, IterateParams, Store, U32_LEN, ValueKey, + IterateParams, Store, U32_LEN, ValueKey, backend::MAX_TOKEN_LENGTH, - fts::FtsFilter, - write::{BitmapHash, ValueClass, hash::TokenType, key::DeserializeBigEndian}, + search::{SearchComparator, SearchFilter}, + write::{ValueClass, key::DeserializeBigEndian}, }; use ahash::AHashMap; use nlp::language::stemmer::Stemmer; @@ -21,7 +20,7 @@ use std::{ use trc::AddContext; use types::collection::Collection; -struct State { +/*struct State { pub op: FtsTokenized, pub bm: Option, } @@ -42,17 +41,19 @@ enum FtsTokenized { Or, Not, End, -} +}*/ impl Store { - pub async fn fts_query + Display + Clone + std::fmt::Debug>( + pub async fn index_query( &self, account_id: u32, collection: Collection, - filters: Vec>, - ) -> trc::Result { + filters: Vec, + comparators: Vec, + ) -> trc::Result> { + todo!() // Tokenize text - let mut tokenized_filters = Vec::with_capacity(filters.len()); + /*let mut tokenized_filters = Vec::with_capacity(filters.len()); let mut token_count = AHashMap::new(); for filter in filters { let filter = match filter { @@ -256,10 +257,10 @@ impl Store { } } - Ok(state.bm.unwrap_or_default()) + Ok(state.bm.unwrap_or_default())*/ } - async fn get_postings( + /*async fn get_postings( &self, account_id: u32, collection: u8, @@ -392,13 +393,15 @@ impl Store { None }) } + + */ } -impl From for State { +/*impl From for State { fn from(value: FtsTokenized) -> Self { Self { op: value, bm: None, } } -} +}*/ diff --git a/crates/store/src/write/batch.rs b/crates/store/src/write/batch.rs index 0c20c367..d631537b 100644 --- a/crates/store/src/write/batch.rs +++ b/crates/store/src/write/batch.rs @@ -5,8 +5,8 @@ */ use super::{ - Batch, BatchBuilder, BitmapClass, ChangedCollection, IntoOperations, Operation, TagValue, - ValueClass, ValueOp, assert::ToAssertValue, log::VanishedItem, + Batch, BatchBuilder, ChangedCollection, IntoOperations, Operation, ValueClass, ValueOp, + assert::ToAssertValue, log::VanishedItem, }; use crate::{ SerializeInfallible, U32_LEN, @@ -54,39 +54,13 @@ impl BatchBuilder { self } - pub fn create_document(&mut self, document_id: u32) -> &mut Self { - self.ops.push(Operation::DocumentId { document_id }); - self.ops.push(Operation::Bitmap { - class: BitmapClass::DocumentIds, - set: true, - }); - self.current_document_id = Some(document_id); - self.batch_size += U32_LEN * 3; - self.batch_ops += 1; - self.has_assertions = false; - self - } - - pub fn update_document(&mut self, document_id: u32) -> &mut Self { + pub fn with_document(&mut self, document_id: u32) -> &mut Self { self.ops.push(Operation::DocumentId { document_id }); self.current_document_id = Some(document_id); self.has_assertions = false; self } - pub fn delete_document(&mut self, document_id: u32) -> &mut Self { - self.ops.push(Operation::DocumentId { document_id }); - self.ops.push(Operation::Bitmap { - class: BitmapClass::DocumentIds, - set: false, - }); - self.current_document_id = Some(document_id); - self.batch_size += U32_LEN * 3; - self.batch_ops += 1; - self.has_assertions = false; - self - } - pub fn assert_value( &mut self, class: impl Into, @@ -131,34 +105,14 @@ impl BatchBuilder { self } - pub fn tag(&mut self, field: impl FieldType, value: impl Into) -> &mut Self { - let value = value.into(); - let value_len = value.serialized_size(); - self.ops.push(Operation::Bitmap { - class: BitmapClass::Tag { - field: field.into(), - value, - }, - set: true, - }); - self.batch_size += (U32_LEN * 3) + value_len; - self.batch_ops += 1; - self + #[inline(always)] + pub fn tag(&mut self, field: impl FieldType) -> &mut Self { + self.index(field, vec![]) } - pub fn untag(&mut self, field: impl FieldType, value: impl Into) -> &mut Self { - let value = value.into(); - let value_len = value.serialized_size(); - self.ops.push(Operation::Bitmap { - class: BitmapClass::Tag { - field: field.into(), - value, - }, - set: false, - }); - self.batch_size += (U32_LEN * 3) + value_len; - self.batch_ops += 1; - self + #[inline(always)] + pub fn untag(&mut self, field: impl FieldType) -> &mut Self { + self.unindex(field, vec![]) } pub fn add(&mut self, class: impl Into, value: i64) -> &mut Self { diff --git a/crates/store/src/write/blob.rs b/crates/store/src/write/blob.rs index bfbeb946..5cf50bd5 100644 --- a/crates/store/src/write/blob.rs +++ b/crates/store/src/write/blob.rs @@ -303,7 +303,7 @@ impl Store { batch.with_collection(collection); last_collection = collection; } - batch.update_document(document_id); + batch.with_document(document_id); batch.any_op(Operation::Value { class: ValueClass::Blob(op), op: ValueOp::Clear, diff --git a/crates/store/src/write/hash.rs b/crates/store/src/write/hash.rs deleted file mode 100644 index b10f8407..00000000 --- a/crates/store/src/write/hash.rs +++ /dev/null @@ -1,48 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC - * - * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL - */ - -use super::BitmapHash; -use crate::backend::MAX_TOKEN_LENGTH; - -impl BitmapHash { - pub fn new(item: impl AsRef<[u8]>) -> Self { - Self { - len: std::cmp::min(item.as_ref().len(), MAX_TOKEN_LENGTH) as u8, - hash: hash(item), - } - } - - pub fn to_u64(&self) -> u64 { - u64::from_be_bytes(self.hash) - } -} - -fn hash(item: impl AsRef<[u8]>) -> [u8; 8] { - let item = item.as_ref(); - let mut result = [0u8; 8]; - - if item.len() <= 8 { - result[..item.len()].copy_from_slice(item); - } else { - result[..4].copy_from_slice(&xxhash_rust::xxh3::xxh3_64(item).to_le_bytes()[..4]); - result[4..8].copy_from_slice(&farmhash::hash64(item).to_le_bytes()[..4]); - } - - result -} - -#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)] -pub struct TokenType {} - -impl TokenType { - pub fn word(field: u8) -> u8 { - field - } - - pub fn stemmed(field: u8) -> u8 { - (1 << 7) | field - } -} diff --git a/crates/store/src/write/key.rs b/crates/store/src/write/key.rs index 15c3369e..fe6f2945 100644 --- a/crates/store/src/write/key.rs +++ b/crates/store/src/write/key.rs @@ -4,26 +4,23 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use super::{ + AnyKey, BlobOp, DirectoryClass, InMemoryClass, QueueClass, ReportClass, ReportEvent, + TaskQueueClass, TelemetryClass, ValueClass, +}; +use crate::{ + Deserialize, IndexKey, IndexKeyPrefix, Key, LogKey, SUBSPACE_ACL, SUBSPACE_BLOB_LINK, + SUBSPACE_BLOB_RESERVE, SUBSPACE_COUNTER, SUBSPACE_DIRECTORY, SUBSPACE_IN_MEMORY_COUNTER, + SUBSPACE_IN_MEMORY_VALUE, SUBSPACE_INDEXES, SUBSPACE_LOGS, SUBSPACE_PROPERTY, + SUBSPACE_QUEUE_EVENT, SUBSPACE_QUEUE_MESSAGE, SUBSPACE_QUOTA, SUBSPACE_REPORT_IN, + SUBSPACE_REPORT_OUT, SUBSPACE_SETTINGS, SUBSPACE_TASK_QUEUE, SUBSPACE_TELEMETRY_INDEX, + SUBSPACE_TELEMETRY_METRIC, SUBSPACE_TELEMETRY_SPAN, U16_LEN, U32_LEN, U64_LEN, ValueKey, + WITH_SUBSPACE, write::IndexPropertyClass, +}; use std::convert::TryInto; use types::{blob_hash::BLOB_HASH_LEN, collection::SyncCollection}; use utils::codec::leb128::Leb128_; -use crate::{ - BitmapKey, Deserialize, IndexKey, IndexKeyPrefix, Key, LogKey, SUBSPACE_ACL, - SUBSPACE_BITMAP_ID, SUBSPACE_BITMAP_TAG, SUBSPACE_BITMAP_TEXT, SUBSPACE_BLOB_LINK, - SUBSPACE_BLOB_RESERVE, SUBSPACE_COUNTER, SUBSPACE_DIRECTORY, SUBSPACE_FTS_INDEX, - SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_IN_MEMORY_VALUE, SUBSPACE_INDEXES, SUBSPACE_LOGS, - SUBSPACE_PROPERTY, SUBSPACE_QUEUE_EVENT, SUBSPACE_QUEUE_MESSAGE, SUBSPACE_QUOTA, - SUBSPACE_REPORT_IN, SUBSPACE_REPORT_OUT, SUBSPACE_SETTINGS, SUBSPACE_TASK_QUEUE, - SUBSPACE_TELEMETRY_INDEX, SUBSPACE_TELEMETRY_METRIC, SUBSPACE_TELEMETRY_SPAN, U16_LEN, U32_LEN, - U64_LEN, ValueKey, WITH_SUBSPACE, -}; - -use super::{ - AnyKey, BitmapClass, BlobOp, DirectoryClass, InMemoryClass, QueueClass, ReportClass, - ReportEvent, TagValue, TaskQueueClass, TelemetryClass, ValueClass, -}; - pub struct KeySerializer { pub buf: Vec, } @@ -256,48 +253,46 @@ impl ValueClass { }; match self { - ValueClass::Property(field) => serializer + ValueClass::Property(property) => serializer .write(account_id) .write(collection) - .write(*field) + .write(*property) .write(document_id), - ValueClass::FtsIndex(hash) => { - let serializer = serializer.write(account_id).write( - hash.hash - .get(0..std::cmp::min(hash.len as usize, 8)) - .unwrap_or_default(), - ); - - if hash.len >= 8 { - serializer.write(hash.len) - } else { - serializer - } - .write(collection) - .write(document_id) - } + ValueClass::IndexProperty(property) => match property { + IndexPropertyClass::Hash { property, hash } => serializer + .write(account_id) + .write(collection) + .write(*property) + .write(hash.as_bytes()) + .write(document_id), + IndexPropertyClass::Integer { property, value } => serializer + .write(account_id) + .write(collection) + .write(*property) + .write(*value) + .write(document_id), + }, ValueClass::Acl(grant_account_id) => serializer .write(*grant_account_id) .write(account_id) .write(collection) .write(document_id), ValueClass::TaskQueue(task) => match task { - TaskQueueClass::IndexEmail { due, hash } => serializer - .write(*due) - .write(account_id) - .write(0u8) - .write(document_id) - .write::<&[u8]>(hash.as_ref()), - TaskQueueClass::BayesTrain { + TaskQueueClass::UpdateIndex { + collection, + is_insert, due, - hash, - learn_spam, } => serializer + .write(*due) + .write(account_id) + .write(if *is_insert { 7u8 } else { 8u8 }) + .write(u8::from(*collection)) + .write(document_id), + TaskQueueClass::BayesTrain { due, learn_spam } => serializer .write(*due) .write(account_id) .write(if *learn_spam { 1u8 } else { 2u8 }) - .write(document_id) - .write::<&[u8]>(hash.as_ref()), + .write(document_id), TaskQueueClass::SendAlarm { due, event_id, @@ -473,92 +468,6 @@ impl + Sync + Send + Clone> Key for IndexKey { } } -impl + Sync + Send + Clone> Key for BitmapKey { - fn subspace(&self) -> u8 { - self.class.as_ref().subspace() - } - - fn serialize(&self, flags: u32) -> Vec { - self.class - .as_ref() - .serialize(self.account_id, self.collection, self.document_id, flags) - } -} - -impl BitmapClass { - pub fn subspace(&self) -> u8 { - match self { - BitmapClass::DocumentIds => SUBSPACE_BITMAP_ID, - BitmapClass::Tag { .. } => SUBSPACE_BITMAP_TAG, - BitmapClass::Text { .. } => SUBSPACE_BITMAP_TEXT, - } - } - - pub fn serialize( - &self, - account_id: u32, - collection: u8, - document_id: u32, - flags: u32, - ) -> Vec { - const BM_MARKER: u8 = 1 << 7; - - match self { - BitmapClass::DocumentIds => if (flags & WITH_SUBSPACE) != 0 { - KeySerializer::new(U32_LEN + 2).write(SUBSPACE_BITMAP_ID) - } else { - KeySerializer::new(U32_LEN + 1) - } - .write(account_id) - .write(collection), - BitmapClass::Tag { field, value } => match value { - TagValue::Id(id) => if (flags & WITH_SUBSPACE) != 0 { - KeySerializer::new((U32_LEN * 2) + 4).write(SUBSPACE_BITMAP_TAG) - } else { - KeySerializer::new((U32_LEN * 2) + 3) - } - .write(account_id) - .write(collection) - .write(*field) - .write_leb128(*id), - TagValue::Text(text) => if (flags & WITH_SUBSPACE) != 0 { - KeySerializer::new(U32_LEN + 4 + text.len()).write(SUBSPACE_BITMAP_TAG) - } else { - KeySerializer::new(U32_LEN + 3 + text.len()) - } - .write(account_id) - .write(collection) - .write(*field | BM_MARKER) - .write(text.as_slice()), - }, - BitmapClass::Text { field, token } => { - let serializer = if (flags & WITH_SUBSPACE) != 0 { - KeySerializer::new(U32_LEN + 16 + 3 + 1).write(SUBSPACE_BITMAP_TEXT) - } else { - KeySerializer::new(U32_LEN + 16 + 3) - } - .write(account_id) - .write( - token - .hash - .get(0..std::cmp::min(token.len as usize, 8)) - .unwrap(), - ); - - if token.len >= 8 { - serializer.write(token.len) - } else { - serializer - } - .write(collection) - .write(*field) - } - } - .write(document_id) - .finalize() - } -} - impl + Sync + Send + Clone> Key for AnyKey { fn serialize(&self, flags: u32) -> Vec { let key = self.key.as_ref(); @@ -580,13 +489,10 @@ impl ValueClass { pub fn serialized_size(&self) -> usize { match self { ValueClass::Property(_) => U32_LEN * 2 + 3, - ValueClass::FtsIndex(hash) => { - if hash.len >= 8 { - U32_LEN * 2 + 10 - } else { - hash.len as usize + U32_LEN * 2 + 1 - } - } + ValueClass::IndexProperty(p) => match p { + IndexPropertyClass::Hash { hash, .. } => U32_LEN * 2 + 3 + hash.len(), + IndexPropertyClass::Integer { .. } => U32_LEN * 2 + 3 + U64_LEN, + }, ValueClass::Acl(_) => U32_LEN * 3 + 2, ValueClass::InMemory(InMemoryClass::Counter(v) | InMemoryClass::Key(v)) | ValueClass::Config(v) => v.len(), @@ -603,9 +509,8 @@ impl ValueClass { } }, ValueClass::TaskQueue(e) => match e { - TaskQueueClass::IndexEmail { .. } | TaskQueueClass::BayesTrain { .. } => { - (BLOB_HASH_LEN + U64_LEN * 2) + 1 - } + TaskQueueClass::UpdateIndex { .. } => (U64_LEN * 2) + 2, + TaskQueueClass::BayesTrain { .. } => (U64_LEN * 2) + 1, TaskQueueClass::SendAlarm { .. } => U64_LEN + (U32_LEN * 3) + 1, TaskQueueClass::SendImip { is_payload, .. } => { if *is_payload { @@ -648,8 +553,8 @@ impl ValueClass { SUBSPACE_PROPERTY } } + ValueClass::IndexProperty { .. } => SUBSPACE_PROPERTY, ValueClass::Acl(_) => SUBSPACE_ACL, - ValueClass::FtsIndex(_) => SUBSPACE_FTS_INDEX, ValueClass::TaskQueue { .. } => SUBSPACE_TASK_QUEUE, ValueClass::Blob(op) => match op { BlobOp::Reserve { .. } => SUBSPACE_BLOB_RESERVE, diff --git a/crates/store/src/write/mod.rs b/crates/store/src/write/mod.rs index 5f9dd211..02bdde16 100644 --- a/crates/store/src/write/mod.rs +++ b/crates/store/src/write/mod.rs @@ -17,17 +17,19 @@ use types::{ blob_hash::BlobHash, collection::{Collection, SyncCollection, VanishedCollection}, field::{ - CalendarField, ContactField, EmailField, EmailSubmissionField, Field, MailboxField, - PrincipalField, SieveField, + CalendarEventField, CalendarNotificationField, ContactField, EmailField, + EmailSubmissionField, Field, MailboxField, PrincipalField, SieveField, }, }; -use utils::map::{bitmap::Bitmap, vec_map::VecMap}; +use utils::{ + cheeky_hash::CheekyHash, + map::{bitmap::Bitmap, vec_map::VecMap}, +}; pub mod assert; pub mod batch; pub mod bitpack; pub mod blob; -pub mod hash; pub mod key; pub mod log; pub mod serialize; @@ -146,10 +148,6 @@ pub enum Operation { key: Vec, set: bool, }, - Bitmap { - class: BitmapClass, - set: bool, - }, Log { collection: LogCollection, set: Vec, @@ -162,31 +160,12 @@ pub enum LogCollection { Vanished(VanishedCollection), } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub enum BitmapClass { - DocumentIds, - Tag { field: u8, value: TagValue }, - Text { field: u8, token: BitmapHash }, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct BitmapHash { - pub hash: [u8; 8], - pub len: u8, -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub enum TagValue { - Id(u32), - Text(Vec), -} - #[derive(Debug, PartialEq, Clone, Eq, Hash)] pub enum ValueClass { Property(u8), + IndexProperty(IndexPropertyClass), Acl(u32), InMemory(InMemoryClass), - FtsIndex(BitmapHash), TaskQueue(TaskQueueClass), Directory(DirectoryClass), Blob(BlobOp), @@ -203,15 +182,21 @@ pub enum ValueClass { ChangeId, } +#[derive(Debug, PartialEq, Clone, Eq, Hash)] +pub enum IndexPropertyClass { + Hash { property: u8, hash: CheekyHash }, + Integer { property: u8, value: u64 }, +} + #[derive(Debug, PartialEq, Clone, Eq, Hash)] pub enum TaskQueueClass { - IndexEmail { + UpdateIndex { due: u64, - hash: BlobHash, + collection: Collection, + is_insert: bool, }, BayesTrain { due: u64, - hash: BlobHash, learn_spam: bool, }, SendAlarm { @@ -332,36 +317,6 @@ pub struct AnyKey> { pub key: T, } -impl From for TagValue { - fn from(value: u32) -> Self { - TagValue::Id(value) - } -} - -impl From> for TagValue { - fn from(value: Vec) -> Self { - TagValue::Text(value) - } -} - -impl From for TagValue { - fn from(value: String) -> Self { - TagValue::Text(value.into_bytes()) - } -} - -impl From for TagValue { - fn from(value: u8) -> Self { - TagValue::Id(value as u32) - } -} - -impl From<()> for TagValue { - fn from(_: ()) -> Self { - TagValue::Text(vec![]) - } -} - pub trait TokenizeText { fn tokenize_into(&self, tokens: &mut HashSet); fn to_tokens(&self) -> HashSet; @@ -398,24 +353,6 @@ impl AsRef for ValueClass { } } -impl AsRef for BitmapClass { - fn as_ref(&self) -> &BitmapClass { - self - } -} - -impl BitmapClass { - pub fn tag_id(property: impl Into, id: u32) -> Self - where - TagValue: From, - { - BitmapClass::Tag { - field: property.into(), - value: id.into(), - } - } -} - impl AssignedIds { pub fn push_counter_id(&mut self, id: i64) { self.ids.push(AssignedId::Counter(id)); @@ -477,15 +414,6 @@ impl> AsRef<[u8]> for Archive { } } -impl TagValue { - pub fn serialized_size(&self) -> usize { - match self { - TagValue::Id(_) => std::mem::size_of::(), - TagValue::Text(items) => items.len(), - } - } -} - impl ArchiveVersion { pub fn hash(&self) -> Option { match self { @@ -540,8 +468,14 @@ impl From for ValueClass { } } -impl From for ValueClass { - fn from(value: CalendarField) -> Self { +impl From for ValueClass { + fn from(value: CalendarEventField) -> Self { + ValueClass::Property(value.into()) + } +} + +impl From for ValueClass { + fn from(value: CalendarNotificationField) -> Self { ValueClass::Property(value.into()) } } diff --git a/crates/types/src/field.rs b/crates/types/src/field.rs index e9843b99..1cddbcd0 100644 --- a/crates/types/src/field.rs +++ b/crates/types/src/field.rs @@ -5,6 +5,7 @@ */ const ARCHIVE_FIELD: u8 = 50; +const DOCUMENT_ID_FIELD: u8 = 51; pub trait FieldType: Into + Copy + std::fmt::Debug + PartialEq + Eq {} @@ -17,21 +18,20 @@ pub struct Field(u8); pub enum ContactField { Uid, Email, - Created, - Updated, - Text, Archive, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] -pub enum CalendarField { +pub enum CalendarEventField { Uid, - Created, - Updated, - Start, - Text, - EventId, + Archive, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u8)] +pub enum CalendarNotificationField { + CreatedToId, Archive, } @@ -40,17 +40,8 @@ pub enum CalendarField { pub enum EmailField { Archive, Metadata, - Size, - Subject, - References, - MailboxIds, - ReceivedAt, - SentAt, - HasAttachment, - From, - To, - Cc, - Bcc, + Stats, + Threading, } #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -96,24 +87,25 @@ impl From for u8 { match value { ContactField::Uid => 0, ContactField::Email => 1, - ContactField::Created => 2, - ContactField::Updated => 3, - ContactField::Text => 4, ContactField::Archive => ARCHIVE_FIELD, } } } -impl From for u8 { - fn from(value: CalendarField) -> Self { +impl From for u8 { + fn from(value: CalendarEventField) -> Self { match value { - CalendarField::Uid => 0, - CalendarField::Text => 1, - CalendarField::Created => 2, - CalendarField::Updated => 3, - CalendarField::Start => 4, - CalendarField::EventId => 5, - CalendarField::Archive => ARCHIVE_FIELD, + CalendarEventField::Uid => 0, + CalendarEventField::Archive => ARCHIVE_FIELD, + } + } +} + +impl From for u8 { + fn from(value: CalendarNotificationField) -> Self { + match value { + CalendarNotificationField::CreatedToId => 0, + CalendarNotificationField::Archive => ARCHIVE_FIELD, } } } @@ -121,18 +113,9 @@ impl From for u8 { impl From for u8 { fn from(value: EmailField) -> Self { match value { - EmailField::From => 87, - EmailField::To => 35, - EmailField::Cc => 74, - EmailField::Bcc => 69, - EmailField::Subject => 29, - EmailField::Size => 27, EmailField::Metadata => 71, - EmailField::References => 20, - EmailField::MailboxIds => 7, - EmailField::ReceivedAt => 19, - EmailField::SentAt => 26, - EmailField::HasAttachment => 89, + EmailField::Threading => 90, + EmailField::Stats => 91, EmailField::Archive => ARCHIVE_FIELD, } } @@ -196,8 +179,14 @@ impl From for Field { } } -impl From for Field { - fn from(value: CalendarField) -> Self { +impl From for Field { + fn from(value: CalendarEventField) -> Self { + Field(u8::from(value)) + } +} + +impl From for Field { + fn from(value: CalendarNotificationField) -> Self { Field(u8::from(value)) } } @@ -234,15 +223,21 @@ impl From for Field { impl Field { pub const ARCHIVE: Field = Field(ARCHIVE_FIELD); + pub const DOCUMENT_ID: Field = Field(DOCUMENT_ID_FIELD); pub fn new(value: u8) -> Self { Field(value) } + + pub fn inner(&self) -> u8 { + self.0 + } } impl FieldType for Field {} impl FieldType for ContactField {} -impl FieldType for CalendarField {} +impl FieldType for CalendarEventField {} +impl FieldType for CalendarNotificationField {} impl FieldType for EmailField {} impl FieldType for MailboxField {} impl FieldType for PrincipalField {} diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index 908860e9..bcfd8666 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -37,6 +37,9 @@ quick_cache = "0.6.9" fast-float = "0.2.0" rkyv = { version = "0.8.10", features = ["little_endian"] } compact_str = "0.9.0" +xxhash-rust = { version = "0.8.5", features = ["xxh3"] } +farmhash = "1.1.5" +nohash-hasher = "0.2.0" [target.'cfg(unix)'.dependencies] privdrop = "0.5.3" diff --git a/crates/utils/src/cheeky_hash.rs b/crates/utils/src/cheeky_hash.rs new file mode 100644 index 00000000..53c54da8 --- /dev/null +++ b/crates/utils/src/cheeky_hash.rs @@ -0,0 +1,221 @@ +/* + * SPDX-FileCopyrightText: 2020 Stalwart Labs LLC + * + * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL + */ + +use nohash_hasher::IsEnabled; +use std::{ + collections::{HashMap, HashSet}, + hash::Hash, +}; + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[repr(transparent)] +pub struct CheekyHash([u8; HASH_SIZE]); +const HASH_SIZE: usize = std::mem::size_of::() * 2; +const HASH_PAYLOAD: usize = HASH_SIZE - 1; + +pub type CheekyHashSet = HashSet>; +pub type CheekyHashMap = HashMap>; + +impl CheekyHash { + pub fn new(bytes: impl AsRef<[u8]>) -> Self { + let mut hash = [0u8; HASH_SIZE]; + let bytes = bytes.as_ref(); + + if bytes.len() < HASH_PAYLOAD { + hash[0] = bytes.len() as u8; + hash[1..1 + bytes.len()].copy_from_slice(bytes); + } else { + let h1 = xxhash_rust::xxh3::xxh3_64(bytes).to_be_bytes(); + let h2 = farmhash::fingerprint64(bytes).to_be_bytes(); + hash[0] = bytes.len().min(u8::MAX as usize) as u8; + hash[1..1 + std::mem::size_of::()].copy_from_slice(&h1); + hash[1 + std::mem::size_of::()..] + .copy_from_slice(&h2[..std::mem::size_of::() - 1]); + } + + CheekyHash(hash) + } + + pub fn deserialize(bytes: &[u8]) -> Option { + let len = *bytes.first()?; + let mut hash = [len; HASH_SIZE]; + let hash_len = 1 + (len as usize).min(HASH_PAYLOAD); + + hash[1..hash_len].copy_from_slice(bytes.get(1..hash_len)?); + Some(CheekyHash(hash)) + } + + #[allow(clippy::len_without_is_empty)] + #[inline(always)] + pub fn len(&self) -> usize { + (self.0[0] as usize).min(HASH_PAYLOAD) + 1 + } + + #[inline(always)] + pub fn as_bytes(&self) -> &[u8] { + &self.0[..self.len()] + } +} + +impl AsRef<[u8]> for CheekyHash { + fn as_ref(&self) -> &[u8] { + self.as_bytes() + } +} + +impl Hash for CheekyHash { + fn hash(&self, state: &mut H) { + let len = self.0[0] as usize; + if len < HASH_PAYLOAD { + state.write_u64(xxhash_rust::xxh3::xxh3_64(&self.0[1..1 + len])); + } else { + state.write_u64(u64::from_be_bytes( + self.0[1..1 + std::mem::size_of::()] + .try_into() + .unwrap(), + )); + } + } +} + +impl IsEnabled for CheekyHash {} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_cheeky_hash_all() { + // Test 1: Empty input + let hash_empty = CheekyHash::new([]); + assert_eq!( + hash_empty.as_bytes()[0], + 0, + "Empty input should have length 0" + ); + assert_eq!( + hash_empty.as_bytes().len(), + 1, + "Empty input should only have length byte" + ); + + // Test 2: Single byte input + let hash_single = CheekyHash::new([42]); + assert_eq!( + hash_single.as_bytes()[0], + 1, + "Single byte should have length 1" + ); + assert_eq!( + hash_single.as_bytes()[1], + 42, + "Single byte value should be preserved" + ); + assert_eq!(hash_single.as_bytes().len(), 2); + + // Test 3: Small input (less than HASH_LEN) + let small_data = b"hello"; + let hash_small = CheekyHash::new(small_data); + assert_eq!(hash_small.as_bytes()[0], 5, "Length should be 5"); + assert_eq!( + &hash_small.as_bytes()[1..6], + small_data, + "Small data should be stored directly" + ); + assert_eq!(hash_small.as_bytes().len(), 6); + + // Test 4: Input exactly at HASH_PAYLOAD boundary + let boundary_data = vec![1u8; HASH_PAYLOAD - 1]; + let hash_boundary = CheekyHash::new(&boundary_data); + assert_eq!( + hash_boundary.as_bytes()[0], + (HASH_PAYLOAD - 1) as u8, + "Length should be HASH_LEN" + ); + assert_eq!( + &hash_boundary.as_bytes()[1..], + &boundary_data[..], + "Boundary data should be stored directly" + ); + + // Test 5: Large input (greater than HASH_LEN) - uses hashing + let large_data = vec![7u8; HASH_SIZE]; + let hash_large = CheekyHash::new(&large_data); + assert_eq!( + hash_large.as_bytes()[0], + HASH_SIZE as u8, + "Large data should have length byte set to HASH_LEN" + ); + assert_eq!( + hash_large.as_bytes().len(), + HASH_SIZE, + "Large data hash should be full length" + ); + // Verify it's actually hashed (not raw data) + assert_ne!( + &hash_large.as_bytes()[1..], + &large_data[..HASH_PAYLOAD], + "Large data should be hashed, not stored directly" + ); + + // Test 6: AsRef<[u8]> trait + let hash = CheekyHash::new(b"test"); + let bytes_ref: &[u8] = hash.as_ref(); + assert_eq!(bytes_ref, hash.as_bytes(), "AsRef should match as_bytes"); + + // Test 7: Copy, Clone, PartialEq traits + let hash1 = CheekyHash::new(b"identical"); + let hash2 = hash1; // Copy + assert_eq!(hash1, hash2, "Copied hashes should be equal"); + + // Test 8: Different inputs produce different hashes + let hash_a = CheekyHash::new(b"abc"); + let hash_b = CheekyHash::new(b"def"); + assert_ne!( + hash_a, hash_b, + "Different inputs should produce different hashes" + ); + + // Test 9: Same input produces same hash (deterministic) + let hash_x1 = CheekyHash::new(b"deterministic"); + let hash_x2 = CheekyHash::new(b"deterministic"); + assert_eq!( + hash_x1, hash_x2, + "Same input should produce identical hashes" + ); + + // Test 10: Large inputs with different content produce different hashes + let large1 = vec![1u8; 100]; + let large2 = vec![2u8; 100]; + let hash_large1 = CheekyHash::new(&large1); + let hash_large2 = CheekyHash::new(&large2); + assert_ne!( + hash_large1, hash_large2, + "Different large inputs should produce different hashes" + ); + + // Test 11: Hash trait (can be used in HashMap/HashSet) + use std::collections::HashMap; + let mut map = HashMap::new(); + let key = CheekyHash::new(b"key"); + map.insert(key, "value"); + assert_eq!( + map.get(&key), + Some(&"value"), + "CheekyHash should work as HashMap key" + ); + + // Test 12: Debug trait + let hash = CheekyHash::new(b"debug"); + let debug_str = format!("{:?}", hash); + assert!( + debug_str.contains("CheekyHash"), + "Debug output should contain type name" + ); + + println!("All CheekyHash tests passed!"); + } +} diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index 4957bee6..ea3cd402 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -6,6 +6,7 @@ pub mod bimap; pub mod cache; +pub mod cheeky_hash; pub mod codec; pub mod config; pub mod glob; diff --git a/tests/src/cluster/stress.rs b/tests/src/cluster/stress.rs index 2272a74a..494050a7 100644 --- a/tests/src/cluster/stress.rs +++ b/tests/src/cluster/stress.rs @@ -230,7 +230,7 @@ async fn email_tests(server: Server, client: Arc) { for email_id in &email_ids_in_mailbox { if let Some(mailbox_tags) = server - .get_archive(TEST_USER_ID, Collection::Email, email_id) + .archive(TEST_USER_ID, Collection::Email, email_id) .await .unwrap() { diff --git a/tests/src/jmap/calendar/identity.rs b/tests/src/jmap/calendar/identity.rs index 47bd8b37..5ff1f93d 100644 --- a/tests/src/jmap/calendar/identity.rs +++ b/tests/src/jmap/calendar/identity.rs @@ -144,7 +144,7 @@ pub async fn test(params: &mut JMAPTest) { batch .with_account_id(account.id().document_id()) .with_collection(Collection::Principal) - .update_document(0) + .with_document(0) .clear(PrincipalField::ParticipantIdentities); params.server.commit_batch(batch).await.unwrap(); params.assert_is_empty().await; diff --git a/tests/src/jmap/mail/changes.rs b/tests/src/jmap/mail/changes.rs index caabb8c7..4f3289a2 100644 --- a/tests/src/jmap/mail/changes.rs +++ b/tests/src/jmap/mail/changes.rs @@ -4,7 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::jmap::{JMAPTest}; +use crate::jmap::JMAPTest; use jmap_proto::types::state::State; use std::str::FromStr; use store::{ahash::AHashSet, write::BatchBuilder}; @@ -142,17 +142,17 @@ pub async fn test(params: &mut JMAPTest) { match change { LogAction::Insert(id) => { batch - .update_document(id as u32) + .with_document(id as u32) .log_item_insert(SyncCollection::Email, None); } LogAction::Update(id) => { batch - .update_document(id as u32) + .with_document(id as u32) .log_item_update(SyncCollection::Email, None); } LogAction::Delete(id) => { batch - .update_document(id as u32) + .with_document(id as u32) .log_item_delete(SyncCollection::Email, None); } LogAction::UpdateChild(id) => { @@ -160,9 +160,9 @@ pub async fn test(params: &mut JMAPTest) { } LogAction::Move(old_id, new_id) => { batch - .update_document(old_id as u32) + .with_document(old_id as u32) .log_item_delete(SyncCollection::Email, None) - .update_document(new_id as u32) + .with_document(new_id as u32) .log_item_insert(SyncCollection::Email, None); } } diff --git a/tests/src/jmap/mail/query_changes.rs b/tests/src/jmap/mail/query_changes.rs index 94f431b4..0d0e9f4b 100644 --- a/tests/src/jmap/mail/query_changes.rs +++ b/tests/src/jmap/mail/query_changes.rs @@ -120,7 +120,7 @@ pub async fn test(params: &mut JMAPTest) { let id = *id_map.get(id).unwrap(); let mut batch = BatchBuilder::new(); batch - .update_document(id.document_id()) + .with_document(id.document_id()) .log_item_update(SyncCollection::Email, id.prefix_id().into()); server.store().write(batch.build_all()).await.unwrap(); updated_ids.insert(id); @@ -137,7 +137,7 @@ pub async fn test(params: &mut JMAPTest) { //let new_thread_id = store::rand::random::(); let old_message_ = server - .get_archive( + .archive( account.id().document_id(), Collection::Email, id.document_id(), @@ -157,7 +157,7 @@ pub async fn test(params: &mut JMAPTest) { BatchBuilder::new() .with_account_id(account.id().document_id()) .with_collection(Collection::Email) - .update_document(id.document_id()) + .with_document(id.document_id()) .custom( ObjectIndexBuilder::new() .with_current(old_message) diff --git a/tests/src/jmap/mod.rs b/tests/src/jmap/mod.rs index 13880af8..d28e0ab8 100644 --- a/tests/src/jmap/mod.rs +++ b/tests/src/jmap/mod.rs @@ -224,19 +224,13 @@ pub async fn wait_for_index(server: &Server) { account_id: 0, collection: 0, document_id: 0, - class: ValueClass::TaskQueue(TaskQueueClass::IndexEmail { - due: 0, - hash: BlobHash::default(), - }), + class: ValueClass::TaskQueue(TaskQueueClass::IndexEmail { due: 0 }), }, ValueKey:: { account_id: u32::MAX, collection: u8::MAX, document_id: u32::MAX, - class: ValueClass::TaskQueue(TaskQueueClass::IndexEmail { - due: u64::MAX, - hash: BlobHash::default(), - }), + class: ValueClass::TaskQueue(TaskQueueClass::IndexEmail { due: u64::MAX }), }, ) .ascending(), @@ -290,6 +284,7 @@ pub async fn assert_is_empty(server: &Server) { } pub async fn emails_purge_tombstoned(server: &Server) { + let todo = "remove"; let mut account_ids = RoaringBitmap::new(); server .core @@ -326,7 +321,7 @@ pub async fn emails_purge_tombstoned(server: &Server) { .access_tokens .insert(account_id, Arc::new(AccessToken::from_id(account_id))); } - server.emails_purge_tombstoned(account_id).await.unwrap(); + //server.emails_purge_tombstoned(account_id).await.unwrap(); if do_add { server.inner.cache.access_tokens.remove(&account_id); } diff --git a/tests/src/store/blob.rs b/tests/src/store/blob.rs index 3b232a10..1a7820b7 100644 --- a/tests/src/store/blob.rs +++ b/tests/src/store/blob.rs @@ -178,7 +178,7 @@ pub async fn blob_tests() { BatchBuilder::new() .with_account_id(if document_id > 0 { 0 } else { 1 }) .with_collection(Collection::Email) - .update_document(document_id as u32) + .with_document(document_id as u32) .set(blob_op, blob_value) .set(BlobOp::Commit { hash: hash.clone() }, vec![]) .build_all(), @@ -291,7 +291,7 @@ pub async fn blob_tests() { BatchBuilder::new() .with_account_id(0) .with_collection(Collection::Email) - .update_document(2) + .with_document(2) .clear(BlobOp::Link { hash: BlobHash::generate(b"789".as_slice()), }) diff --git a/tests/src/store/ops.rs b/tests/src/store/ops.rs index 4695942c..3992e709 100644 --- a/tests/src/store/ops.rs +++ b/tests/src/store/ops.rs @@ -28,7 +28,7 @@ pub async fn test(db: Store) { batch .with_account_id(0) .with_collection(Collection::Email) - .update_document(0); + .with_document(0); for n in 0..900000 { batch.set( ValueClass::Config(format!("key{n:10}").into_bytes()), @@ -41,7 +41,7 @@ pub async fn test(db: Store) { batch .with_account_id(0) .with_collection(Collection::Email) - .update_document(0); + .with_document(0); } } db.write(batch.build_all()).await.unwrap(); @@ -83,7 +83,7 @@ pub async fn test(db: Store) { batch .with_account_id(0) .with_collection(Collection::Email) - .update_document(0); + .with_document(0); for n in 0..900000 { batch.clear(ValueClass::Config(format!("key{n:10}").into_bytes())); @@ -93,7 +93,7 @@ pub async fn test(db: Store) { batch .with_account_id(0) .with_collection(Collection::Email) - .update_document(0); + .with_document(0); } } db.write(batch.build_all()).await.unwrap(); @@ -110,7 +110,7 @@ pub async fn test(db: Store) { builder .with_account_id(0) .with_collection(Collection::Email) - .update_document(0) + .with_document(0) .merge(ValueClass::Property(3), |bytes| { if let Some(bytes) = bytes { Ok((u64::from_be_bytes(bytes.try_into().unwrap()) + 1) @@ -154,7 +154,7 @@ pub async fn test(db: Store) { builder .with_account_id(0) .with_collection(Collection::Email) - .update_document(0) + .with_document(0) .add_and_get(ValueClass::Directory(DirectoryClass::UsedQuota(0)), 1); db.write(builder.build_all()) .await @@ -208,7 +208,7 @@ pub async fn test(db: Store) { builder .with_account_id(0) .with_collection(Collection::Email) - .update_document(document_id) + .with_document(document_id) .set_versioned(ValueClass::Property(5), archived_value, offset) .log_container_insert(SyncCollection::Email); db.write(builder.build_all()) @@ -273,7 +273,7 @@ pub async fn test(db: Store) { BatchBuilder::new() .with_account_id(0) .with_collection(Collection::Email) - .update_document(0) + .with_document(0) .set(ValueClass::Property(1), value.as_slice()) .set(ValueClass::Property(0), "check1".as_bytes()) .set(ValueClass::Property(2), "check2".as_bytes()) @@ -302,7 +302,7 @@ pub async fn test(db: Store) { BatchBuilder::new() .with_account_id(0) .with_collection(Collection::Email) - .update_document(0) + .with_document(0) .clear(ValueClass::Property(1)) .build_all(), ) @@ -346,7 +346,7 @@ pub async fn test(db: Store) { .with_account_id(0) .with_collection(Collection::Email) .with_account_id(0) - .update_document(0) + .with_document(0) .clear(ValueClass::Property(0)) .clear(ValueClass::Property(2)) .clear(ValueClass::Property(3)) @@ -355,7 +355,7 @@ pub async fn test(db: Store) { for document_id in 0..1000 { batch - .update_document(document_id) + .with_document(document_id) .clear(ValueClass::Property(5)); } diff --git a/tests/src/webdav/mod.rs b/tests/src/webdav/mod.rs index 73ccd462..cf0e3f72 100644 --- a/tests/src/webdav/mod.rs +++ b/tests/src/webdav/mod.rs @@ -1050,7 +1050,7 @@ impl WebDavTest { pub async fn fetch_email(&self, account_id: u32, document_id: u32) -> Vec { let metadata_ = self .server - .get_archive_by_property( + .archive_by_property( account_id, Collection::Email, document_id,