From 50dce48a855cf3c08ef00c523b0d043056f868bf Mon Sep 17 00:00:00 2001 From: mdecimus <11444311+mdecimus@users.noreply.github.com> Date: Wed, 3 Dec 2025 19:26:28 +0100 Subject: [PATCH] Spam filter performance and accuracy improvements (part 4) --- Cargo.lock | 23 +- crates/common/src/config/inner.rs | 12 +- crates/common/src/config/network.rs | 6 +- crates/common/src/config/spamfilter.rs | 72 +++-- crates/common/src/core.rs | 106 +++---- crates/common/src/ipc.rs | 1 + crates/common/src/lib.rs | 13 +- crates/common/src/manager/boot.rs | 8 + crates/dav/src/calendar/copy_move.rs | 65 +++- crates/dav/src/calendar/delete.rs | 17 +- crates/dav/src/calendar/freebusy.rs | 11 +- crates/dav/src/calendar/get.rs | 9 +- crates/dav/src/calendar/proppatch.rs | 11 +- crates/dav/src/calendar/scheduling.rs | 14 +- crates/dav/src/calendar/update.rs | 11 +- crates/dav/src/card/copy_move.rs | 65 +++- crates/dav/src/card/delete.rs | 17 +- crates/dav/src/card/get.rs | 11 +- crates/dav/src/card/proppatch.rs | 11 +- crates/dav/src/card/update.rs | 11 +- crates/dav/src/common/acl.rs | 18 +- crates/dav/src/common/lock.rs | 6 +- crates/dav/src/common/propfind.rs | 12 +- crates/dav/src/file/copy_move.rs | 67 +++- crates/dav/src/file/get.rs | 32 +- crates/dav/src/file/proppatch.rs | 11 +- crates/dav/src/file/update.rs | 11 +- crates/email/src/cache/email.rs | 10 +- crates/email/src/cache/mailbox.rs | 11 +- crates/email/src/mailbox/destroy.rs | 11 +- crates/email/src/message/copy.rs | 11 +- crates/email/src/message/ingest.rs | 55 +++- crates/email/src/sieve/delete.rs | 11 +- crates/email/src/sieve/ingest.rs | 7 +- crates/groupware/src/cache/mod.rs | 14 +- crates/groupware/src/calendar/itip.rs | 18 +- crates/groupware/src/calendar/storage.rs | 20 +- crates/groupware/src/contact/storage.rs | 12 +- crates/groupware/src/file/storage.rs | 12 +- crates/http/src/management/crypto.rs | 9 +- crates/http/src/management/stores.rs | 21 +- crates/imap/src/core/mailbox.rs | 11 +- crates/imap/src/op/acl.rs | 12 +- crates/imap/src/op/copy_move.rs | 22 +- crates/imap/src/op/fetch.rs | 18 +- crates/imap/src/op/rename.rs | 12 +- crates/imap/src/op/store.rs | 31 +- crates/imap/src/op/subscribe.rs | 12 +- crates/jmap/src/addressbook/get.rs | 9 +- crates/jmap/src/addressbook/set.rs | 23 +- crates/jmap/src/blob/copy.rs | 5 +- crates/jmap/src/blob/download.rs | 9 +- crates/jmap/src/blob/get.rs | 8 +- crates/jmap/src/calendar/get.rs | 9 +- crates/jmap/src/calendar/set.rs | 23 +- crates/jmap/src/calendar_event/copy.rs | 7 +- crates/jmap/src/calendar_event/get.rs | 10 +- crates/jmap/src/calendar_event/query.rs | 11 +- crates/jmap/src/calendar_event/set.rs | 25 +- .../src/calendar_event_notification/get.rs | 7 +- .../src/calendar_event_notification/set.rs | 7 +- crates/jmap/src/contact/copy.rs | 9 +- crates/jmap/src/contact/get.rs | 9 +- crates/jmap/src/contact/set.rs | 16 +- crates/jmap/src/email/get.rs | 11 +- crates/jmap/src/email/set.rs | 9 +- crates/jmap/src/email/snippet.rs | 13 +- crates/jmap/src/file/get.rs | 9 +- crates/jmap/src/file/set.rs | 10 +- crates/jmap/src/identity/get.rs | 11 +- crates/jmap/src/identity/set.rs | 9 +- crates/jmap/src/mailbox/set.rs | 17 +- crates/jmap/src/participant_identity/get.rs | 24 +- crates/jmap/src/principal/availability.rs | 9 +- crates/jmap/src/push/get.rs | 11 +- crates/jmap/src/push/set.rs | 11 +- crates/jmap/src/sieve/get.rs | 8 +- crates/jmap/src/sieve/set.rs | 11 +- crates/jmap/src/submission/get.rs | 11 +- crates/jmap/src/submission/set.rs | 33 +- crates/jmap/src/vacation/get.rs | 11 +- crates/jmap/src/vacation/set.rs | 11 +- crates/managesieve/src/op/getscript.rs | 11 +- crates/managesieve/src/op/listscripts.rs | 11 +- crates/managesieve/src/op/putscript.rs | 11 +- crates/managesieve/src/op/renamescript.rs | 12 +- crates/migration/src/addressbook_v2.rs | 7 +- crates/migration/src/calendar_v2.rs | 7 +- crates/migration/src/contact_v2.rs | 7 +- crates/migration/src/event_v1.rs | 7 +- crates/migration/src/event_v2.rs | 12 +- crates/migration/src/push_v2.rs | 7 +- crates/migration/src/sieve_v2.rs | 7 +- crates/nlp/Cargo.toml | 2 +- crates/nlp/src/classifier/feature.rs | 16 +- crates/nlp/src/classifier/sgd.rs | 44 +-- crates/pop3/src/op/fetch.rs | 11 +- crates/services/src/broadcast/mod.rs | 5 + crates/services/src/broadcast/subscriber.rs | 9 + crates/services/src/housekeeper/mod.rs | 53 ++++ crates/services/src/state_manager/push.rs | 10 +- crates/services/src/task_manager/alarm.rs | 19 +- crates/services/src/task_manager/index.rs | 34 +- .../src/task_manager/merge_threads.rs | 12 +- crates/smtp/src/queue/spool.rs | 45 +-- crates/spam-filter/Cargo.toml | 1 + crates/spam-filter/src/analysis/domain.rs | 3 + crates/spam-filter/src/modules/classifier.rs | 294 +++++++++++++++--- crates/store/src/write/blob.rs | 252 +++++++++------ crates/store/src/write/key.rs | 31 +- crates/types/src/field.rs | 2 + 111 files changed, 1714 insertions(+), 635 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d0b71419..44cc64ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2141,12 +2141,6 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" -[[package]] -name = "endian-type" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "869b0adbda23651a9c5c0c3d270aac9fcb52e8622a8f2b17e57802d7791962f2" - [[package]] name = "enum-as-inner" version = "0.6.1" @@ -2843,7 +2837,7 @@ dependencies = [ "futures-util", "hickory-proto 0.24.4", "once_cell", - "radix_trie 0.2.1", + "radix_trie", "rand 0.8.5", "rustls 0.21.12", "thiserror 1.0.69", @@ -4626,8 +4620,8 @@ dependencies = [ "nohash", "parking_lot", "psl", - "radix_trie 0.3.0", "rand 0.9.2", + "rkyv", "rust-stemmers", "serde", "tokio", @@ -5803,17 +5797,7 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd" dependencies = [ - "endian-type 0.1.2", - "nibble_vec", -] - -[[package]] -name = "radix_trie" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b4431027dcd37fc2a73ef740b5f233aa805897935b8bce0195e41bbf9a3289a" -dependencies = [ - "endian-type 0.2.0", + "endian-type", "nibble_vec", ] @@ -7450,6 +7434,7 @@ dependencies = [ "nlp", "psl", "reqwest", + "rkyv", "sha1", "sha2 0.10.9", "smtp-proto", diff --git a/crates/common/src/config/inner.rs b/crates/common/src/config/inner.rs index cb642b3a..8da4256c 100644 --- a/crates/common/src/config/inner.rs +++ b/crates/common/src/config/inner.rs @@ -7,12 +7,9 @@ use super::server::tls::{build_self_signed_cert, parse_certificates}; use crate::{ CacheSwap, Caches, Data, DavResource, DavResources, MailboxCache, MessageStoreCache, - MessageUidCache, TlsConnectors, + MessageUidCache, SpamClassifier, TlsConnectors, auth::{AccessToken, roles::RolePermissions}, - config::{ - smtp::resolver::{Policy, Tlsa}, - spamfilter::Reputation, - }, + config::smtp::resolver::{Policy, Tlsa}, listener::blocked::BlockedIps, manager::webadmin::WebAdminManager, }; @@ -20,7 +17,6 @@ use ahash::{AHashMap, AHashSet}; use arc_swap::ArcSwap; use mail_auth::{MX, Parameters, Txt}; use mail_send::smtp::tls::build_tls_connector; -use nlp::classifier::sgd::SGDClassifier; use parking_lot::RwLock; use std::{ net::{IpAddr, Ipv4Addr, Ipv6Addr}, @@ -52,8 +48,7 @@ impl Data { } Data { - spam_classifier: ArcSwap::from_pointee(SGDClassifier::default()), - spam_reputation: ArcSwap::from_pointee(Reputation::default()), + spam_classifier: ArcSwap::from_pointee(SpamClassifier::default()), tls_certificates: ArcSwap::from_pointee(certificates), tls_self_signed_cert: build_self_signed_cert( subject_names.into_iter().collect::>(), @@ -223,7 +218,6 @@ impl Default for Data { fn default() -> Self { Self { spam_classifier: Default::default(), - spam_reputation: Default::default(), tls_certificates: Default::default(), tls_self_signed_cert: Default::default(), blocked_ips: Default::default(), diff --git a/crates/common/src/config/network.rs b/crates/common/src/config/network.rs index 34dfc163..9d251884 100644 --- a/crates/common/src/config/network.rs +++ b/crates/common/src/config/network.rs @@ -42,7 +42,7 @@ pub struct ClusterRoles { pub purge_accounts: ClusterRole, pub push_notifications: ClusterRole, pub fts_indexing: ClusterRole, - pub bayes_training: ClusterRole, + pub spam_training: ClusterRole, pub imip_processing: ClusterRole, pub merge_threads: ClusterRole, pub calendar_alerts: ClusterRole, @@ -246,8 +246,8 @@ impl Network { "cluster.roles.fts-indexing", ), ( - &mut network.roles.bayes_training, - "cluster.roles.bayes-training", + &mut network.roles.spam_training, + "cluster.roles.spam-training", ), ( &mut network.roles.imip_processing, diff --git a/crates/common/src/config/spamfilter.rs b/crates/common/src/config/spamfilter.rs index 43a7478e..e12bdd34 100644 --- a/crates/common/src/config/spamfilter.rs +++ b/crates/common/src/config/spamfilter.rs @@ -5,9 +5,9 @@ */ use super::{Variable, functions::ResolveVariable, if_block::IfBlock, tokenizer::TokenMap}; -use ahash::{AHashMap, AHashSet}; -use compact_str::CompactString; +use ahash::AHashSet; use mail_auth::common::resolver::ToReverseName; +use nlp::classifier::sgd::TextClassifier; use std::{ net::{IpAddr, SocketAddr}, time::Duration, @@ -19,30 +19,20 @@ use utils::{ glob::GlobMap, }; -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub enum ReputationType { - Domain(CompactString), - Asn(u32), - Ip(IpAddr), -} - -#[derive(Debug, Clone, Copy, Default)] -pub struct ReputationCount { - pub ham: u32, - pub spam: u32, -} - -#[derive(Debug, Clone, Default)] -pub struct Reputation { - pub items: AHashMap, - pub total: ReputationCount, - pub last_fetch: u64, +#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default)] +pub struct SpamClassifierModel { + pub classifier: TextClassifier, + pub ham_count: u64, + pub spam_count: u64, + pub last_sample_expiry: u64, + pub last_trained_at: u64, } #[derive(Debug, Clone, Default)] pub struct SpamFilterConfig { pub enabled: bool, pub card_is_ham: bool, + pub trusted_reply: bool, pub grey_list_expiry: Option, pub dnsbl: DnsBlConfig, @@ -88,9 +78,13 @@ pub struct ClassifierConfig { pub epochs: usize, pub feature_hash_size: usize, pub alpha: f32, + pub train_batch_size: usize, + pub min_ham_samples: u64, + pub min_spam_samples: u64, pub auto_learn_reply_ham: bool, pub auto_learn_card_is_ham: bool, pub hold_samples_for: u64, + pub train_frequency: Option, } #[derive(Debug, Clone)] @@ -169,7 +163,10 @@ impl SpamFilterConfig { .property_or_default("spam-filter.enable", "true") .unwrap_or(true), card_is_ham: config - .property_or_default("spam-filter.card-is-ham", "true") + .property_or_default("spam-filter.card-is-ham.enable", "true") + .unwrap_or(true), + trusted_reply: config + .property_or_default("spam-filter.trusted-reply.enable", "true") .unwrap_or(true), dnsbl: DnsBlConfig::parse(config), rules: SpamFilterRules::parse(config), @@ -447,34 +444,49 @@ impl ClassifierConfig { } let feature_hash_size: usize = config - .property_or_default("spam-filter.classifier.feature-hash-size", "1048576") + .property_or_default("spam-filter.classifier.parameters.features", "1048576") .unwrap_or(1048576); if !feature_hash_size.is_power_of_two() { config.new_build_error( - "spam-filter.classifier.feature-hash-size", - "Feature hash size must be a power of two.", + "spam-filter.classifier.parameters.features", + "Feature size must be a power of two.", ); } ClassifierConfig { feature_hash_size, epochs: config - .property_or_default("spam-filter.classifier.epochs", "1000") + .property_or_default("spam-filter.classifier.parameters.epochs", "1000") .unwrap_or(1000), alpha: config - .property_or_default("spam-filter.classifier.alpha", "0.00001") + .property_or_default("spam-filter.classifier.parameters.alpha", "0.00001") .unwrap_or(0.00001), auto_learn_card_is_ham: config - .property_or_default("spam-filter.classifier.auto-learn.card-is-ham", "true") + .property_or_default("spam-filter.card-is-ham.learn", "true") .unwrap_or(true), auto_learn_reply_ham: config - .property_or_default("spam-filter.classifier.auto-learn.trusted-reply", "true") + .property_or_default("spam-filter.trusted-reply.learn", "true") .unwrap_or(true), hold_samples_for: config - .property_or_default::("spam-filter.classifier.hold-samples-for", "365d") - .unwrap_or(Duration::from_secs(365 * 24 * 60 * 60)) + .property_or_default::("spam-filter.classifier.samples.hold-for", "180d") + .unwrap_or(Duration::from_secs(180 * 24 * 60 * 60)) .as_secs(), + min_ham_samples: config + .property_or_default("spam-filter.classifier.samples.min-ham", "10") + .unwrap_or(10), + min_spam_samples: config + .property_or_default("spam-filter.classifier.samples.min-spam", "10") + .unwrap_or(10), + train_batch_size: config + .property_or_default("spam-filter.classifier.training.batch-size", "100") + .unwrap_or(100), + train_frequency: config + .property_or_default::>( + "spam-filter.classifier.training.frequency", + "12h", + ) + .unwrap_or(Some(Duration::from_secs(12 * 60 * 60))), } .into() } diff --git a/crates/common/src/core.rs b/crates/common/src/core.rs index d8746f9f..9d1fc1c1 100644 --- a/crates/common/src/core.rs +++ b/crates/common/src/core.rs @@ -5,14 +5,17 @@ */ use crate::{ - Inner, Server, + Inner, Server, SpamClassifier, auth::{AccessToken, ResourceToken, TenantInfo}, - config::smtp::{ - auth::{ArcSealer, DkimSigner, LazySignature, ResolvedSignature, build_signature}, - queue::{ - ConnectionStrategy, DEFAULT_QUEUE_NAME, MxConfig, QueueExpiry, QueueName, - QueueStrategy, RequireOptional, RoutingStrategy, TlsStrategy, VirtualQueue, + config::{ + smtp::{ + auth::{ArcSealer, DkimSigner, LazySignature, ResolvedSignature, build_signature}, + queue::{ + ConnectionStrategy, DEFAULT_QUEUE_NAME, MxConfig, QueueExpiry, QueueName, + QueueStrategy, RequireOptional, RoutingStrategy, TlsStrategy, VirtualQueue, + }, }, + spamfilter::SpamClassifierModel, }, ipc::{BroadcastEvent, PushEvent, PushNotification}, }; @@ -38,7 +41,7 @@ use types::{ blob::{BlobClass, BlobId}, blob_hash::BlobHash, collection::{Collection, SyncCollection}, - field::Field, + field::{Field, PrincipalField}, type_state::{DataType, StateChange}, }; use utils::{map::bitmap::Bitmap, snowflake::SnowflakeIdGenerator}; @@ -441,57 +444,6 @@ impl Server { }) } - #[inline(always)] - pub async fn archive( - &self, - account_id: u32, - collection: Collection, - document_id: u32, - ) -> trc::Result>> { - self.core - .storage - .data - .get_value(ValueKey { - account_id, - collection: collection.into(), - document_id, - class: ValueClass::Property(Field::ARCHIVE.into()), - }) - .await - .add_context(|err| { - err.caused_by(trc::location!()) - .account_id(account_id) - .collection(collection) - .document_id(document_id) - }) - } - - #[inline(always)] - pub async fn archive_by_property( - &self, - account_id: u32, - collection: Collection, - document_id: u32, - property: Field, - ) -> trc::Result>> { - self.core - .storage - .data - .get_value(ValueKey { - account_id, - collection: collection.into(), - document_id, - class: ValueClass::Property(property.into()), - }) - .await - .add_context(|err| { - err.caused_by(trc::location!()) - .account_id(account_id) - .collection(collection) - .document_id(document_id) - }) - } - pub async fn archives( &self, account_id: u32, @@ -1091,6 +1043,44 @@ impl Server { .caused_by(trc::location!()) } + pub async fn spam_model_reload(&self) -> trc::Result<()> { + if let Some(config) = &self.core.spam.classifier { + if let Some(model) = self + .store() + .get_value::>(ValueKey::property( + u32::MAX, + Collection::Principal, + u32::MAX, + PrincipalField::SpamModel, + )) + .await + .and_then(|archive| match archive { + Some(archive) => archive.deserialize::().map(Some), + None => Ok(None), + }) + .caused_by(trc::location!())? + { + if model.ham_count >= config.min_ham_samples + && model.spam_count >= config.min_spam_samples + { + self.inner + .data + .spam_classifier + .store(Arc::new(SpamClassifier { + model: model.classifier, + last_trained_at: model.last_trained_at, + })); + } else { + let todo = "log insufficient samples, keep existing model"; + } + } else { + let todo = "log missing model, keep existing one"; + } + } + + Ok(()) + } + #[cfg(not(feature = "enterprise"))] pub async fn logo_resource( &self, diff --git a/crates/common/src/ipc.rs b/crates/common/src/ipc.rs index b827d60b..5ba789a9 100644 --- a/crates/common/src/ipc.rs +++ b/crates/common/src/ipc.rs @@ -100,6 +100,7 @@ pub enum BroadcastEvent { ReloadPushServers(u32), ReloadSettings, ReloadBlockedIps, + ReloadSpamFilter, } #[derive(Debug)] diff --git a/crates/common/src/lib.rs b/crates/common/src/lib.rs index 1d3abc28..d02756b6 100644 --- a/crates/common/src/lib.rs +++ b/crates/common/src/lib.rs @@ -28,7 +28,7 @@ use ipc::{BroadcastEvent, HousekeeperEvent, PushEvent, QueueEvent, ReportingEven use listener::{asn::AsnGeoLookupData, blocked::Security, tls::AcmeProviders}; use mail_auth::{MX, Txt}; use manager::webadmin::{Resource, WebAdminManager}; -use nlp::classifier::sgd::SGDClassifier; +use nlp::classifier::sgd::TextClassifier; use parking_lot::{Mutex, RwLock}; use rustls::sign::CertifiedKey; use std::{ @@ -73,8 +73,6 @@ pub mod enterprise; pub use psl; -use crate::config::spamfilter::Reputation; - pub static VERSION_PRIVATE: &str = env!("CARGO_PKG_VERSION"); pub static VERSION_PUBLIC: &str = "1.0.0"; @@ -133,9 +131,14 @@ pub struct Inner { pub ipc: Ipc, } +#[derive(Default)] +pub struct SpamClassifier { + pub model: TextClassifier, + pub last_trained_at: u64, +} + pub struct Data { - pub spam_classifier: ArcSwap, - pub spam_reputation: ArcSwap, + pub spam_classifier: ArcSwap, pub tls_certificates: ArcSwap>>, pub tls_self_signed_cert: Option>, diff --git a/crates/common/src/manager/boot.rs b/crates/common/src/manager/boot.rs index 24be0e4c..ead44f05 100644 --- a/crates/common/src/manager/boot.rs +++ b/crates/common/src/manager/boot.rs @@ -484,6 +484,14 @@ impl BootManager { cache, }); + // Load spam model + if let Err(err) = inner.build_server().spam_model_reload().await { + trc::error!( + err.details("Failed to load spam filter model") + .caused_by(trc::location!()) + ); + } + // Fetch ASN database if has_remote_asn { inner diff --git a/crates/dav/src/calendar/copy_move.rs b/crates/dav/src/calendar/copy_move.rs index f0057b54..0f96ea86 100644 --- a/crates/dav/src/calendar/copy_move.rs +++ b/crates/dav/src/calendar/copy_move.rs @@ -24,6 +24,10 @@ use groupware::{ use http_proto::HttpResponse; use hyper::StatusCode; use store::write::{BatchBuilder, now}; +use store::{ + ValueKey, + write::{AlignedBytes, Archive}, +}; use trc::AddContext; use types::{ acl::Acl, @@ -455,7 +459,12 @@ async fn copy_event( ) -> crate::Result { // Fetch event let event_ = server - .archive(from_account_id, Collection::CalendarEvent, from_document_id) + .store() + .get_value::>(ValueKey::archive( + from_account_id, + Collection::CalendarEvent, + from_document_id, + )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -523,7 +532,12 @@ async fn copy_event( let response = if let Some(to_document_id) = to_document_id { // Overwrite event on destination let event_ = server - .archive(to_account_id, Collection::CalendarEvent, to_document_id) + .store() + .get_value::>(ValueKey::archive( + to_account_id, + Collection::CalendarEvent, + to_document_id, + )) .await .caused_by(trc::location!())?; if let Some(event_) = event_ { @@ -574,7 +588,12 @@ async fn move_event( ) -> crate::Result { // Fetch event let event_ = server - .archive(from_account_id, Collection::CalendarEvent, from_document_id) + .store() + .get_value::>(ValueKey::archive( + from_account_id, + Collection::CalendarEvent, + from_document_id, + )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -683,7 +702,12 @@ async fn move_event( let response = if let Some(to_document_id) = to_document_id { // Overwrite event on destination let event_ = server - .archive(to_account_id, Collection::CalendarEvent, to_document_id) + .store() + .get_value::>(ValueKey::archive( + to_account_id, + Collection::CalendarEvent, + to_document_id, + )) .await .caused_by(trc::location!())?; if let Some(event_) = event_ { @@ -730,7 +754,12 @@ async fn rename_event( ) -> crate::Result { // Fetch event let event_ = server - .archive(account_id, Collection::CalendarEvent, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::CalendarEvent, + document_id, + )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -779,7 +808,12 @@ async fn copy_container( ) -> crate::Result { // Fetch calendar let calendar_ = server - .archive(from_account_id, Collection::Calendar, from_document_id) + .store() + .get_value::>(ValueKey::archive( + from_account_id, + Collection::Calendar, + from_document_id, + )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -823,7 +857,12 @@ async fn copy_container( let to_document_id = if let Some(to_document_id) = to_document_id { // Overwrite destination let calendar_ = server - .archive(to_account_id, Collection::Calendar, to_document_id) + .store() + .get_value::>(ValueKey::archive( + to_account_id, + Collection::Calendar, + to_document_id, + )) .await .caused_by(trc::location!())?; if let Some(calendar_) = calendar_ { @@ -862,11 +901,12 @@ async fn copy_container( let mut required_space = 0; for from_child_document_id in from_children_ids { if let Some(event_) = server - .archive( + .store() + .get_value::>(ValueKey::archive( from_account_id, Collection::CalendarEvent, from_child_document_id, - ) + )) .await? { let event = event_ @@ -983,7 +1023,12 @@ async fn rename_container( ) -> crate::Result { // Fetch calendar let calendar_ = server - .archive(account_id, Collection::Calendar, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::Calendar, + document_id, + )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; diff --git a/crates/dav/src/calendar/delete.rs b/crates/dav/src/calendar/delete.rs index c4f0987c..26e18f21 100644 --- a/crates/dav/src/calendar/delete.rs +++ b/crates/dav/src/calendar/delete.rs @@ -22,9 +22,10 @@ use groupware::{ }; use http_proto::HttpResponse; use hyper::StatusCode; +use store::write::{BatchBuilder, ValueClass}; use store::{ ValueKey, - write::{BatchBuilder, ValueClass}, + write::{AlignedBytes, Archive}, }; use trc::AddContext; use types::{ @@ -91,7 +92,12 @@ impl CalendarDeleteRequestHandler for Server { } let calendar_ = self - .archive(account_id, Collection::Calendar, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::Calendar, + document_id, + )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -175,7 +181,12 @@ impl CalendarDeleteRequestHandler for Server { } let event_ = self - .archive(account_id, Collection::CalendarEvent, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::CalendarEvent, + document_id, + )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; diff --git a/crates/dav/src/calendar/freebusy.rs b/crates/dav/src/calendar/freebusy.rs index 03e000ca..141c2b61 100644 --- a/crates/dav/src/calendar/freebusy.rs +++ b/crates/dav/src/calendar/freebusy.rs @@ -22,6 +22,10 @@ use groupware::{cache::GroupwareCache, calendar::CalendarEvent}; use http_proto::HttpResponse; use hyper::StatusCode; use std::str::FromStr; +use store::{ + ValueKey, + write::{AlignedBytes, Archive}, +}; use store::{ ahash::AHashMap, write::{now, serialize::rkyv_deserialize}, @@ -155,7 +159,12 @@ impl CalendarFreebusyRequestHandler for Server { for document_id in document_ids { let Some(archive) = self - .archive(account_id, Collection::CalendarEvent, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::CalendarEvent, + document_id, + )) .await .caused_by(trc::location!())? else { diff --git a/crates/dav/src/calendar/get.rs b/crates/dav/src/calendar/get.rs index 9ec08f46..d10826a6 100644 --- a/crates/dav/src/calendar/get.rs +++ b/crates/dav/src/calendar/get.rs @@ -17,6 +17,10 @@ use dav_proto::{RequestHeaders, schema::property::Rfc1123DateTime}; use groupware::{cache::GroupwareCache, calendar::CalendarEvent}; use http_proto::HttpResponse; use hyper::StatusCode; +use store::{ + ValueKey, + write::{AlignedBytes, Archive}, +}; use trc::AddContext; use types::{ acl::Acl, @@ -73,11 +77,12 @@ impl CalendarGetRequestHandler for Server { // Fetch event let event_ = self - .archive( + .store() + .get_value::>(ValueKey::archive( account_id, Collection::CalendarEvent, resource.document_id(), - ) + )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; diff --git a/crates/dav/src/calendar/proppatch.rs b/crates/dav/src/calendar/proppatch.rs index 3050bd69..1304663c 100644 --- a/crates/dav/src/calendar/proppatch.rs +++ b/crates/dav/src/calendar/proppatch.rs @@ -31,6 +31,10 @@ use http_proto::HttpResponse; use hyper::StatusCode; use std::str::FromStr; use store::write::BatchBuilder; +use store::{ + ValueKey, + write::{AlignedBytes, Archive}, +}; use trc::AddContext; use types::{ acl::Acl, @@ -112,7 +116,12 @@ impl CalendarPropPatchRequestHandler for Server { // Fetch archive let archive = self - .archive(account_id, collection, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + collection, + document_id, + )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; diff --git a/crates/dav/src/calendar/scheduling.rs b/crates/dav/src/calendar/scheduling.rs index 97a0c736..7162671f 100644 --- a/crates/dav/src/calendar/scheduling.rs +++ b/crates/dav/src/calendar/scheduling.rs @@ -32,6 +32,10 @@ use dav_proto::{ use groupware::{DestroyArchive, cache::GroupwareCache, calendar::CalendarEventNotification}; use http_proto::HttpResponse; use hyper::StatusCode; +use store::{ + ValueKey, + write::{AlignedBytes, Archive}, +}; use store::{ahash::AHashMap, write::BatchBuilder}; use trc::AddContext; use types::collection::{Collection, SyncCollection}; @@ -98,11 +102,12 @@ impl CalendarEventNotificationHandler for Server { // Fetch event let event_ = self - .archive( + .store() + .get_value::>(ValueKey::archive( account_id, Collection::CalendarEventNotification, resource.document_id(), - ) + )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -181,11 +186,12 @@ impl CalendarEventNotificationHandler for Server { let document_id = resource.document_id(); let event_ = self - .archive( + .store() + .get_value::>(ValueKey::archive( account_id, Collection::CalendarEventNotification, document_id, - ) + )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; diff --git a/crates/dav/src/calendar/update.rs b/crates/dav/src/calendar/update.rs index 194dd593..d454bf8e 100644 --- a/crates/dav/src/calendar/update.rs +++ b/crates/dav/src/calendar/update.rs @@ -36,6 +36,10 @@ use http_proto::HttpResponse; use hyper::StatusCode; use std::collections::HashSet; use store::write::{BatchBuilder, now}; +use store::{ + ValueKey, + write::{AlignedBytes, Archive}, +}; use trc::AddContext; use types::{ acl::Acl, @@ -121,7 +125,12 @@ impl CalendarUpdateRequestHandler for Server { // Update let event_ = self - .archive(account_id, Collection::CalendarEvent, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::CalendarEvent, + document_id, + )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; diff --git a/crates/dav/src/card/copy_move.rs b/crates/dav/src/card/copy_move.rs index ad0a9a13..dcd54114 100644 --- a/crates/dav/src/card/copy_move.rs +++ b/crates/dav/src/card/copy_move.rs @@ -23,6 +23,10 @@ use groupware::{ use http_proto::HttpResponse; use hyper::StatusCode; use store::write::BatchBuilder; +use store::{ + ValueKey, + write::{AlignedBytes, Archive}, +}; use trc::AddContext; use types::{ acl::Acl, @@ -437,7 +441,12 @@ async fn copy_card( ) -> crate::Result { // Fetch card let card_ = server - .archive(from_account_id, Collection::ContactCard, from_document_id) + .store() + .get_value::>(ValueKey::archive( + from_account_id, + Collection::ContactCard, + from_document_id, + )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -498,7 +507,12 @@ async fn copy_card( let response = if let Some(to_document_id) = to_document_id { // Overwrite card on destination let card_ = server - .archive(to_account_id, Collection::ContactCard, to_document_id) + .store() + .get_value::>(ValueKey::archive( + to_account_id, + Collection::ContactCard, + to_document_id, + )) .await .caused_by(trc::location!())?; if let Some(card_) = card_ { @@ -547,7 +561,12 @@ async fn move_card( ) -> crate::Result { // Fetch card let card_ = server - .archive(from_account_id, Collection::ContactCard, from_document_id) + .store() + .get_value::>(ValueKey::archive( + from_account_id, + Collection::ContactCard, + from_document_id, + )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -641,7 +660,12 @@ async fn move_card( let response = if let Some(to_document_id) = to_document_id { // Overwrite card on destination let card_ = server - .archive(to_account_id, Collection::ContactCard, to_document_id) + .store() + .get_value::>(ValueKey::archive( + to_account_id, + Collection::ContactCard, + to_document_id, + )) .await .caused_by(trc::location!())?; if let Some(card_) = card_ { @@ -687,7 +711,12 @@ async fn rename_card( ) -> crate::Result { // Fetch card let card_ = server - .archive(account_id, Collection::ContactCard, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::ContactCard, + document_id, + )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -736,7 +765,12 @@ async fn copy_container( ) -> crate::Result { // Fetch book let book_ = server - .archive(from_account_id, Collection::AddressBook, from_document_id) + .store() + .get_value::>(ValueKey::archive( + from_account_id, + Collection::AddressBook, + from_document_id, + )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -777,7 +811,12 @@ async fn copy_container( let to_document_id = if let Some(to_document_id) = to_document_id { // Overwrite destination let book_ = server - .archive(to_account_id, Collection::AddressBook, to_document_id) + .store() + .get_value::>(ValueKey::archive( + to_account_id, + Collection::AddressBook, + to_document_id, + )) .await .caused_by(trc::location!())?; if let Some(book_) = book_ { @@ -814,11 +853,12 @@ async fn copy_container( let mut required_space = 0; for from_child_document_id in from_children_ids { if let Some(card_) = server - .archive( + .store() + .get_value::>(ValueKey::archive( from_account_id, Collection::ContactCard, from_child_document_id, - ) + )) .await? { let card = card_ @@ -928,7 +968,12 @@ async fn rename_container( ) -> crate::Result { // Fetch book let book_ = server - .archive(account_id, Collection::AddressBook, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::AddressBook, + document_id, + )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; diff --git a/crates/dav/src/card/delete.rs b/crates/dav/src/card/delete.rs index f5660b1c..d07f040f 100644 --- a/crates/dav/src/card/delete.rs +++ b/crates/dav/src/card/delete.rs @@ -21,9 +21,10 @@ use groupware::{ }; use http_proto::HttpResponse; use hyper::StatusCode; +use store::write::{BatchBuilder, ValueClass}; use store::{ ValueKey, - write::{BatchBuilder, ValueClass}, + write::{AlignedBytes, Archive}, }; use trc::AddContext; use types::{ @@ -71,7 +72,12 @@ impl CardDeleteRequestHandler for Server { let mut batch = BatchBuilder::new(); if delete_resource.is_container() { let book_ = self - .archive(account_id, Collection::AddressBook, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::AddressBook, + document_id, + )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -158,7 +164,12 @@ impl CardDeleteRequestHandler for Server { } let card_ = self - .archive(account_id, Collection::ContactCard, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::ContactCard, + document_id, + )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; diff --git a/crates/dav/src/card/get.rs b/crates/dav/src/card/get.rs index 724ec3b0..1c755efc 100644 --- a/crates/dav/src/card/get.rs +++ b/crates/dav/src/card/get.rs @@ -17,6 +17,10 @@ use dav_proto::{RequestHeaders, schema::property::Rfc1123DateTime}; use groupware::{cache::GroupwareCache, contact::ContactCard}; use http_proto::HttpResponse; use hyper::StatusCode; +use store::{ + ValueKey, + write::{AlignedBytes, Archive}, +}; use trc::AddContext; use types::{ acl::Acl, @@ -73,7 +77,12 @@ impl CardGetRequestHandler for Server { // Fetch card let card_ = self - .archive(account_id, Collection::ContactCard, resource.document_id()) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::ContactCard, + resource.document_id(), + )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; diff --git a/crates/dav/src/card/proppatch.rs b/crates/dav/src/card/proppatch.rs index faa9aefe..a79ed2fe 100644 --- a/crates/dav/src/card/proppatch.rs +++ b/crates/dav/src/card/proppatch.rs @@ -29,6 +29,10 @@ use groupware::{ use http_proto::HttpResponse; use hyper::StatusCode; use store::write::BatchBuilder; +use store::{ + ValueKey, + write::{AlignedBytes, Archive}, +}; use trc::AddContext; use types::{ acl::Acl, @@ -109,7 +113,12 @@ impl CardPropPatchRequestHandler for Server { // Fetch archive let archive = self - .archive(account_id, collection, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + collection, + document_id, + )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; diff --git a/crates/dav/src/card/update.rs b/crates/dav/src/card/update.rs index eb0d156a..01b33b94 100644 --- a/crates/dav/src/card/update.rs +++ b/crates/dav/src/card/update.rs @@ -25,6 +25,10 @@ use groupware::{cache::GroupwareCache, contact::ContactCard}; use http_proto::HttpResponse; use hyper::StatusCode; use store::write::BatchBuilder; +use store::{ + ValueKey, + write::{AlignedBytes, Archive}, +}; use trc::AddContext; use types::{ acl::Acl, @@ -110,7 +114,12 @@ impl CardUpdateRequestHandler for Server { // Update let card_ = self - .archive(account_id, Collection::ContactCard, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::ContactCard, + document_id, + )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; diff --git a/crates/dav/src/common/acl.rs b/crates/dav/src/common/acl.rs index 9e6460c7..1c992020 100644 --- a/crates/dav/src/common/acl.rs +++ b/crates/dav/src/common/acl.rs @@ -24,6 +24,10 @@ use groupware::{cache::GroupwareCache, calendar::Calendar, contact::AddressBook, use http_proto::HttpResponse; use hyper::StatusCode; use rkyv::vec::ArchivedVec; +use store::{ + ValueKey, + write::{AlignedBytes, Archive}, +}; use store::{ahash::AHashSet, roaring::RoaringBitmap, write::BatchBuilder}; use trc::AddContext; use types::{ @@ -108,7 +112,12 @@ impl DavAclHandler for Server { // Fetch node let archive = self - .archive(account_id, collection, resource.document_id()) + .store() + .get_value::>(ValueKey::archive( + account_id, + collection, + resource.document_id(), + )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -212,7 +221,12 @@ impl DavAclHandler for Server { } let archive = self - .archive(uri.account_id, uri.collection, uri.resource) + .store() + .get_value::>(ValueKey::archive( + uri.account_id, + uri.collection, + uri.resource, + )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; diff --git a/crates/dav/src/common/lock.rs b/crates/dav/src/common/lock.rs index e04f689f..75b1daef 100644 --- a/crates/dav/src/common/lock.rs +++ b/crates/dav/src/common/lock.rs @@ -18,6 +18,7 @@ use groupware::cache::GroupwareCache; use http_proto::HttpResponse; use hyper::StatusCode; use std::collections::HashMap; +use store::ValueKey; use store::dispatch::lookup::KeyValue; use store::write::serialize::rkyv_deserialize; use store::write::{AlignedBytes, Archive, Archiver, now}; @@ -526,11 +527,12 @@ impl LockRequestHandler for Server { if let Some(document_id) = resource_state.document_id.filter(|&id| id != u32::MAX) && let Some(archive) = self - .archive( + .store() + .get_value::>(ValueKey::archive( resource_state.account_id, resource_state.collection, document_id, - ) + )) .await .caused_by(trc::location!())? { diff --git a/crates/dav/src/common/propfind.rs b/crates/dav/src/common/propfind.rs index 44b61cb9..8d886c09 100644 --- a/crates/dav/src/common/propfind.rs +++ b/crates/dav/src/common/propfind.rs @@ -55,11 +55,14 @@ use groupware::{ use http_proto::HttpResponse; use hyper::StatusCode; use std::sync::Arc; +use store::{ + ValueKey, + write::{AlignedBytes, Archive}, +}; use store::{ ahash::AHashMap, query::log::{Change, Query}, roaring::RoaringBitmap, - write::{AlignedBytes, Archive}, }; use trc::AddContext; use types::{ @@ -431,7 +434,12 @@ impl PropFindRequestHandler for Server { item.document_id == SCHEDULE_INBOX_ID, ) } else if let Some(archive) = self - .archive(account_id, collection, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + collection, + document_id, + )) .await .caused_by(trc::location!())? { diff --git a/crates/dav/src/file/copy_move.rs b/crates/dav/src/file/copy_move.rs index 93c6f9ca..90547468 100644 --- a/crates/dav/src/file/copy_move.rs +++ b/crates/dav/src/file/copy_move.rs @@ -22,6 +22,10 @@ use groupware::{DestroyArchive, cache::GroupwareCache, file::FileNode}; use http_proto::HttpResponse; use hyper::StatusCode; use std::sync::Arc; +use store::{ + ValueKey, + write::{AlignedBytes, Archive}, +}; use store::{ ahash::AHashMap, write::{BatchBuilder, now}, @@ -358,7 +362,12 @@ async fn move_container( if from_account_id == to_account_id { let node_ = server - .archive(from_account_id, Collection::FileNode, from_document_id) + .store() + .get_value::>(ValueKey::archive( + from_account_id, + Collection::FileNode, + from_document_id, + )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -459,7 +468,12 @@ async fn copy_container( .caused_by(trc::location!())?; for (document_id, _) in copy_files.into_iter() { let node_ = server - .archive(from_account_id, Collection::FileNode, document_id) + .store() + .get_value::>(ValueKey::archive( + from_account_id, + Collection::FileNode, + document_id, + )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))? @@ -550,7 +564,12 @@ async fn overwrite_and_delete_item( // dest_node is the current file at the destination let dest_node_ = server - .archive(to_account_id, Collection::FileNode, to_document_id) + .store() + .get_value::>(ValueKey::archive( + to_account_id, + Collection::FileNode, + to_document_id, + )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -561,7 +580,12 @@ async fn overwrite_and_delete_item( // source_node is the file to be copied let source_node__ = server - .archive(from_account_id, Collection::FileNode, from_document_id) + .store() + .get_value::>(ValueKey::archive( + from_account_id, + Collection::FileNode, + from_document_id, + )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -620,7 +644,12 @@ async fn overwrite_item( // dest_node is the current file at the destination let dest_node_ = server - .archive(to_account_id, Collection::FileNode, to_document_id) + .store() + .get_value::>(ValueKey::archive( + to_account_id, + Collection::FileNode, + to_document_id, + )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -631,7 +660,12 @@ async fn overwrite_item( // source_node is the file to be copied let mut source_node = server - .archive(from_account_id, Collection::FileNode, from_document_id) + .store() + .get_value::>(ValueKey::archive( + from_account_id, + Collection::FileNode, + from_document_id, + )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))? @@ -676,7 +710,12 @@ async fn move_item( let parent_id = destination.document_id.map(|id| id + 1).unwrap_or(0); let node_ = server - .archive(from_account_id, Collection::FileNode, from_document_id) + .store() + .get_value::>(ValueKey::archive( + from_account_id, + Collection::FileNode, + from_document_id, + )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; @@ -746,7 +785,12 @@ async fn copy_item( let parent_id = destination.document_id.map(|id| id + 1).unwrap_or(0); let mut node = server - .archive(from_account_id, Collection::FileNode, from_document_id) + .store() + .get_value::>(ValueKey::archive( + from_account_id, + Collection::FileNode, + from_document_id, + )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))? @@ -786,7 +830,12 @@ async fn rename_item( let from_document_id = from_resource.resource.document_id; let node_ = server - .archive(from_account_id, Collection::FileNode, from_document_id) + .store() + .get_value::>(ValueKey::archive( + from_account_id, + Collection::FileNode, + from_document_id, + )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; diff --git a/crates/dav/src/file/get.rs b/crates/dav/src/file/get.rs index edcf04e8..a49ed4e5 100644 --- a/crates/dav/src/file/get.rs +++ b/crates/dav/src/file/get.rs @@ -4,17 +4,6 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use common::{Server, auth::AccessToken, sharing::EffectiveAcl}; -use dav_proto::{RequestHeaders, schema::property::Rfc1123DateTime}; -use groupware::{cache::GroupwareCache, file::FileNode}; -use http_proto::HttpResponse; -use hyper::StatusCode; -use trc::AddContext; -use types::{ - acl::Acl, - collection::{Collection, SyncCollection}, -}; - use crate::{ DavError, DavMethod, common::{ @@ -24,6 +13,20 @@ use crate::{ }, file::DavFileResource, }; +use common::{Server, auth::AccessToken, sharing::EffectiveAcl}; +use dav_proto::{RequestHeaders, schema::property::Rfc1123DateTime}; +use groupware::{cache::GroupwareCache, file::FileNode}; +use http_proto::HttpResponse; +use hyper::StatusCode; +use store::{ + ValueKey, + write::{AlignedBytes, Archive}, +}; +use trc::AddContext; +use types::{ + acl::Acl, + collection::{Collection, SyncCollection}, +}; pub(crate) trait FileGetRequestHandler: Sync + Send { fn handle_file_get_request( @@ -55,7 +58,12 @@ impl FileGetRequestHandler for Server { // Fetch node let node_ = self - .archive(account_id, Collection::FileNode, resource.resource) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::FileNode, + resource.resource, + )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; diff --git a/crates/dav/src/file/proppatch.rs b/crates/dav/src/file/proppatch.rs index 936254c9..1fa00e8c 100644 --- a/crates/dav/src/file/proppatch.rs +++ b/crates/dav/src/file/proppatch.rs @@ -26,6 +26,10 @@ use groupware::{cache::GroupwareCache, file::FileNode}; use http_proto::HttpResponse; use hyper::StatusCode; use store::write::BatchBuilder; +use store::{ + ValueKey, + write::{AlignedBytes, Archive}, +}; use trc::AddContext; use types::{ acl::Acl, @@ -75,7 +79,12 @@ impl FilePropPatchRequestHandler for Server { // Fetch node let node_ = self - .archive(account_id, Collection::FileNode, resource.resource) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::FileNode, + resource.resource, + )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; diff --git a/crates/dav/src/file/update.rs b/crates/dav/src/file/update.rs index 93420c5e..9a93e2da 100644 --- a/crates/dav/src/file/update.rs +++ b/crates/dav/src/file/update.rs @@ -25,6 +25,10 @@ use groupware::{ use http_proto::HttpResponse; use hyper::StatusCode; use store::write::{BatchBuilder, now}; +use store::{ + ValueKey, + write::{AlignedBytes, Archive}, +}; use trc::AddContext; use types::{ acl::Acl, @@ -74,7 +78,12 @@ impl FileUpdateRequestHandler for Server { { // Update let node_ = self - .archive(account_id, Collection::FileNode, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::FileNode, + document_id, + )) .await .caused_by(trc::location!())? .ok_or(DavError::Code(StatusCode::NOT_FOUND))?; diff --git a/crates/email/src/cache/email.rs b/crates/email/src/cache/email.rs index edfe07af..fd7f539e 100644 --- a/crates/email/src/cache/email.rs +++ b/crates/email/src/cache/email.rs @@ -9,7 +9,8 @@ use common::{ MessageCache, MessageStoreCache, MessageUidCache, MessagesCache, Server, auth::AccessToken, sharing::EffectiveAcl, }; -use store::{ahash::AHashMap, roaring::RoaringBitmap, write::Archive}; +use store::write::{AlignedBytes, Archive}; +use store::{ValueKey, ahash::AHashMap, roaring::RoaringBitmap}; use trc::AddContext; use types::{ acl::Acl, @@ -43,7 +44,12 @@ pub(crate) async fn update_email_cache( for (document_id, is_update) in changed_ids { if *is_update && let Some(archive) = server - .archive(account_id, Collection::Email, *document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::Email, + *document_id, + )) .await .caused_by(trc::location!())? { diff --git a/crates/email/src/cache/mailbox.rs b/crates/email/src/cache/mailbox.rs index 99387283..a315a6db 100644 --- a/crates/email/src/cache/mailbox.rs +++ b/crates/email/src/cache/mailbox.rs @@ -9,6 +9,10 @@ use common::{ MailboxCache, MailboxesCache, MessageStoreCache, Server, auth::AccessToken, sharing::EffectiveAcl, }; +use store::{ + ValueKey, + write::{AlignedBytes, Archive}, +}; use store::{ahash::AHashMap, roaring::RoaringBitmap}; use trc::AddContext; use types::{ @@ -41,7 +45,12 @@ pub(crate) async fn update_mailbox_cache( for (document_id, is_update) in changed_ids { if *is_update && let Some(archive) = server - .archive(account_id, Collection::Mailbox, *document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::Mailbox, + *document_id, + )) .await .caused_by(trc::location!())? { diff --git a/crates/email/src/mailbox/destroy.rs b/crates/email/src/mailbox/destroy.rs index 4ea74ba1..f609624a 100644 --- a/crates/email/src/mailbox/destroy.rs +++ b/crates/email/src/mailbox/destroy.rs @@ -17,6 +17,10 @@ use store::{ roaring::RoaringBitmap, write::{BatchBuilder, SearchIndex, TaskEpoch, TaskQueueClass, ValueClass}, }; +use store::{ + ValueKey, + write::{AlignedBytes, Archive}, +}; use trc::AddContext; use types::{ acl::Acl, @@ -173,7 +177,12 @@ impl MailboxDestroy for Server { // Obtain mailbox if let Some(mailbox_) = self - .archive(account_id, Collection::Mailbox, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::Mailbox, + document_id, + )) .await .caused_by(trc::location!())? { diff --git a/crates/email/src/message/copy.rs b/crates/email/src/message/copy.rs index d8bc52d6..09dd734a 100644 --- a/crates/email/src/message/copy.rs +++ b/crates/email/src/message/copy.rs @@ -23,6 +23,10 @@ use mail_parser::parsers::fields::thread::thread_name; use store::write::{ BatchBuilder, IndexPropertyClass, SearchIndex, TaskEpoch, TaskQueueClass, ValueClass, }; +use store::{ + ValueKey, + write::{AlignedBytes, Archive}, +}; use trc::AddContext; use types::{ blob::{BlobClass, BlobId}, @@ -66,12 +70,13 @@ impl EmailCopy for Server { // Obtain metadata let account_id = resource_token.account_id; let mut metadata = if let Some(metadata) = self - .archive_by_property( + .store() + .get_value::>(ValueKey::property( from_account_id, Collection::Email, from_message_id, - EmailField::Metadata.into(), - ) + EmailField::Metadata, + )) .await? { metadata diff --git a/crates/email/src/message/ingest.rs b/crates/email/src/message/ingest.rs index e9b61d38..7764b241 100644 --- a/crates/email/src/message/ingest.rs +++ b/crates/email/src/message/ingest.rs @@ -26,6 +26,7 @@ use mail_parser::{ }; use std::{borrow::Cow, cmp::Ordering, fmt::Write, time::Instant}; use std::{future::Future, hash::Hasher}; +use store::write::{AlignedBytes, Archive}; use store::{ IndexKeyPrefix, IterateParams, U32_LEN, ValueKey, ahash::{AHashMap, AHashSet}, @@ -111,8 +112,8 @@ pub trait EmailIngest: Sync + Send { &self, batch: &mut BatchBuilder, hash: BlobHash, - hold_override: Option, is_spam: bool, + hold_sample: bool, ); } @@ -268,12 +269,23 @@ impl EmailIngest for Server { .caused_by(trc::location!())? { is_spam = false; - train_spam = Some(false); + if self + .core + .spam + .classifier + .as_ref() + .is_some_and(|c| c.auto_learn_card_is_ham) + { + train_spam = Some(false); + } overridden = Some("card-exists"); } // Check if the message is a trusted reply to a previous message - if is_spam && let Some(thread_id) = thread_result.thread_id { + if is_spam + && self.core.spam.trusted_reply + && let Some(thread_id) = thread_result.thread_id + { let cache = self .get_cached_messages(account_id) .await @@ -288,7 +300,15 @@ impl EmailIngest for Server { .any(|m| m.mailboxes.iter().any(|mb| mb.mailbox_id == sent_folder_id)) { is_spam = false; - train_spam = Some(false); + if self + .core + .spam + .classifier + .as_ref() + .is_some_and(|c| c.auto_learn_reply_ham) + { + train_spam = Some(false); + } overridden = Some("trusted-reply"); } } @@ -446,12 +466,13 @@ impl EmailIngest for Server { let is_encrypted = if do_encrypt && !message.is_encrypted() && let Some(encrypt_params_) = self - .archive_by_property( + .store() + .get_value::>(ValueKey::property( account_id, Collection::Principal, 0, - PrincipalField::EncryptionKeys.into(), - ) + PrincipalField::EncryptionKeys, + )) .await .caused_by(trc::location!())? { @@ -607,8 +628,8 @@ impl EmailIngest for Server { self.add_spam_sample( &mut batch, params.blob_hash.unwrap_or(&blob_hash).clone(), - is_encrypted.then_some(2 * 86400), learn_spam, + !is_encrypted, ); } @@ -816,19 +837,20 @@ impl EmailIngest for Server { is_spam: bool, ) -> trc::Result<()> { if let Some(archive) = self - .archive_by_property( + .store() + .get_value::>(ValueKey::property( account_id, Collection::Email, document_id, - EmailField::Metadata.into(), - ) + EmailField::Metadata, + )) .await .caused_by(trc::location!())? { let metadata = archive .to_unarchived::() .caused_by(trc::location!())?; - self.add_spam_sample(batch, (&metadata.inner.blob_hash).into(), None, is_spam); + self.add_spam_sample(batch, (&metadata.inner.blob_hash).into(), is_spam, true); } Ok(()) @@ -838,15 +860,15 @@ impl EmailIngest for Server { &self, batch: &mut BatchBuilder, hash: BlobHash, - hold_override: Option, is_spam: bool, + hold_sample: bool, ) { if let Some(config) = &self.core.spam.classifier { let mut dt = DateTime::from_timestamp(now() as i64); dt.hour = 0; dt.minute = 0; dt.second = 0; - let until = dt.to_timestamp() as u64 + hold_override.unwrap_or(config.hold_samples_for); + let until = dt.to_timestamp() as u64 + config.hold_samples_for; batch .set( @@ -856,7 +878,10 @@ impl EmailIngest for Server { }, vec![BlobLink::SPAM_SAMPLE_LINK], ) - .set(BlobOp::SpamSample { hash, until }, vec![u8::from(is_spam)]); + .set( + BlobOp::SpamSample { hash, until }, + vec![u8::from(is_spam), u8::from(hold_sample)], + ); } } } diff --git a/crates/email/src/sieve/delete.rs b/crates/email/src/sieve/delete.rs index 99d8b00a..49fbe75d 100644 --- a/crates/email/src/sieve/delete.rs +++ b/crates/email/src/sieve/delete.rs @@ -7,6 +7,10 @@ use super::SieveScript; use common::{Server, auth::AccessToken, storage::index::ObjectIndexBuilder}; use store::write::BatchBuilder; +use store::{ + ValueKey, + write::{AlignedBytes, Archive}, +}; use trc::AddContext; use types::{collection::Collection, field::SieveField}; @@ -30,7 +34,12 @@ impl SieveScriptDelete for Server { ) -> trc::Result { // Fetch record if let Some(obj_) = self - .archive(account_id, Collection::SieveScript, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::SieveScript, + document_id, + )) .await? { // Delete record diff --git a/crates/email/src/sieve/ingest.rs b/crates/email/src/sieve/ingest.rs index f4c79e95..ec95d94b 100644 --- a/crates/email/src/sieve/ingest.rs +++ b/crates/email/src/sieve/ingest.rs @@ -636,7 +636,12 @@ impl SieveScriptIngest for Server { ) -> trc::Result> { // Obtain script object let Some(script_object) = self - .archive(account_id, Collection::SieveScript, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::SieveScript, + document_id, + )) .await? else { return Ok(None); diff --git a/crates/groupware/src/cache/mod.rs b/crates/groupware/src/cache/mod.rs index a5a066b6..31f6bf46 100644 --- a/crates/groupware/src/cache/mod.rs +++ b/crates/groupware/src/cache/mod.rs @@ -450,7 +450,12 @@ async fn process_changes( Change::InsertItem(id) | Change::UpdateItem(id) => { let document_id = id as u32; if let Some(archive) = server - .archive(account_id, collection.collection(false), document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + collection.collection(false), + document_id, + )) .await .caused_by(trc::location!())? { @@ -473,7 +478,12 @@ async fn process_changes( Change::InsertContainer(id) | Change::UpdateContainer(id) => { let document_id = id as u32; if let Some(archive) = server - .archive(account_id, collection.collection(true), document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + collection.collection(true), + document_id, + )) .await .caused_by(trc::location!())? { diff --git a/crates/groupware/src/calendar/itip.rs b/crates/groupware/src/calendar/itip.rs index 4448f770..10ec2373 100644 --- a/crates/groupware/src/calendar/itip.rs +++ b/crates/groupware/src/calendar/itip.rs @@ -34,8 +34,8 @@ use common::{ i18n, }; use store::{ - rand, - write::{BatchBuilder, now}, + ValueKey, rand, + write::{AlignedBytes, Archive, BatchBuilder, now}, }; use trc::AddContext; use types::{ @@ -164,7 +164,12 @@ impl ItipIngest for Server { if let Some(document_id) = document_id { if let Some(archive) = self - .archive(account_id, Collection::CalendarEvent, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::CalendarEvent, + document_id, + )) .await .caused_by(trc::location!())? { @@ -403,7 +408,12 @@ impl ItipIngest for Server { async fn http_rsvp_handle(&self, query: &str, language: &str) -> trc::Result { let response = if let Some(rsvp) = decode_rsvp_response(self, query).await { if let Some(archive) = self - .archive(rsvp.account_id, Collection::CalendarEvent, rsvp.document_id) + .store() + .get_value::>(ValueKey::archive( + rsvp.account_id, + Collection::CalendarEvent, + rsvp.document_id, + )) .await .caused_by(trc::location!())? { diff --git a/crates/groupware/src/calendar/storage.rs b/crates/groupware/src/calendar/storage.rs index ef20c0d4..962d20a6 100644 --- a/crates/groupware/src/calendar/storage.rs +++ b/crates/groupware/src/calendar/storage.rs @@ -21,7 +21,8 @@ use store::{ IterateParams, U16_LEN, U32_LEN, U64_LEN, ValueKey, roaring::RoaringBitmap, write::{ - Archive, BatchBuilder, IndexPropertyClass, TaskEpoch, TaskQueueClass, ValueClass, + AlignedBytes, Archive, BatchBuilder, IndexPropertyClass, TaskEpoch, TaskQueueClass, + ValueClass, key::{DeserializeBigEndian, KeySerializer}, now, }, @@ -136,11 +137,12 @@ impl ItipAutoExpunge for Server { for document_id in destroy_ids { // Fetch event if let Some(event_) = self - .archive( + .store() + .get_value::>(ValueKey::archive( account_id, Collection::CalendarEventNotification, document_id, - ) + )) .await .caused_by(trc::location!())? { @@ -328,7 +330,12 @@ impl DestroyArchive> { let calendar_id = document_id; for document_id in children_ids { if let Some(event_) = server - .archive(account_id, Collection::CalendarEvent, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::CalendarEvent, + document_id, + )) .await? { DestroyArchive( @@ -567,11 +574,12 @@ impl ArchivedCalendarEvent { ) -> trc::Result { for event_name in self.names.iter() { if let Some(calendar_) = server - .archive( + .store() + .get_value::>(ValueKey::archive( access_token.primary_id, Collection::Calendar, event_name.parent_id.to_native(), - ) + )) .await .caused_by(trc::location!())? { diff --git a/crates/groupware/src/contact/storage.rs b/crates/groupware/src/contact/storage.rs index 9a2c821f..0cd692da 100644 --- a/crates/groupware/src/contact/storage.rs +++ b/crates/groupware/src/contact/storage.rs @@ -7,7 +7,10 @@ use super::{AddressBook, ArchivedAddressBook, ArchivedContactCard, ContactCard}; use crate::DestroyArchive; use common::{Server, auth::AccessToken, storage::index::ObjectIndexBuilder}; -use store::write::{Archive, BatchBuilder, now}; +use store::{ + ValueKey, + write::{AlignedBytes, Archive, BatchBuilder, now}, +}; use trc::AddContext; use types::collection::{Collection, VanishedCollection}; @@ -136,7 +139,12 @@ impl DestroyArchive> { let addressbook_id = document_id; for document_id in children_ids { if let Some(card_) = server - .archive(account_id, Collection::ContactCard, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::ContactCard, + document_id, + )) .await? { DestroyArchive( diff --git a/crates/groupware/src/file/storage.rs b/crates/groupware/src/file/storage.rs index b18294c1..4746cefc 100644 --- a/crates/groupware/src/file/storage.rs +++ b/crates/groupware/src/file/storage.rs @@ -7,7 +7,10 @@ use super::{ArchivedFileNode, FileNode}; use crate::DestroyArchive; use common::{Server, auth::AccessToken, storage::index::ObjectIndexBuilder}; -use store::write::{Archive, BatchBuilder, now}; +use store::{ + ValueKey, + write::{AlignedBytes, Archive, BatchBuilder, now}, +}; use trc::AddContext; use types::collection::{Collection, VanishedCollection}; @@ -124,7 +127,12 @@ impl DestroyArchive> { .with_collection(Collection::FileNode); for document_id in self.0 { if let Some(node) = server - .archive(account_id, Collection::FileNode, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::FileNode, + document_id, + )) .await? { // Delete record diff --git a/crates/http/src/management/crypto.rs b/crates/http/src/management/crypto.rs index ffd17f74..101289a1 100644 --- a/crates/http/src/management/crypto.rs +++ b/crates/http/src/management/crypto.rs @@ -16,7 +16,7 @@ use mail_parser::MessageParser; use serde_json::json; use std::{future::Future, sync::Arc}; use store::{ - Deserialize, Serialize, + Deserialize, Serialize, ValueKey, write::{AlignedBytes, Archive, Archiver, BatchBuilder}, }; use trc::AddContext; @@ -38,12 +38,13 @@ pub trait CryptoHandler: Sync + Send { impl CryptoHandler for Server { async fn handle_crypto_get(&self, access_token: Arc) -> trc::Result { let ec = if let Some(params_) = self - .archive_by_property( + .store() + .get_value::>(ValueKey::property( access_token.primary_id(), Collection::Principal, 0, - PrincipalField::EncryptionKeys.into(), - ) + PrincipalField::EncryptionKeys, + )) .await? { let params = params_ diff --git a/crates/http/src/management/stores.rs b/crates/http/src/management/stores.rs index bfd4a589..8b87be47 100644 --- a/crates/http/src/management/stores.rs +++ b/crates/http/src/management/stores.rs @@ -35,9 +35,12 @@ use serde_json::json; use services::task_manager::index::ReindexIndexTask; use std::future::Future; use store::{ - Serialize, rand, + Serialize, ValueKey, rand, search::SearchQuery, - write::{Archiver, BatchBuilder, BlobLink, BlobOp, DirectoryClass, SearchIndex, ValueClass}, + write::{ + AlignedBytes, Archive, Archiver, BatchBuilder, BlobLink, BlobOp, DirectoryClass, + SearchIndex, ValueClass, + }, }; use trc::AddContext; use types::{ @@ -506,7 +509,12 @@ pub async fn reset_imap_uids(server: &Server, account_id: u32) -> trc::Result<(u for &mailbox_id in cache.mailboxes.index.keys() { let mailbox = server - .archive(account_id, Collection::Mailbox, mailbox_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::Mailbox, + mailbox_id, + )) .await .caused_by(trc::location!())? .ok_or_else(|| trc::ImapEvent::Error.into_err().caused_by(trc::location!()))? @@ -537,7 +545,12 @@ pub async fn reset_imap_uids(server: &Server, account_id: u32) -> trc::Result<(u // Reset all UIDs for message_id in cache.emails.items.iter().map(|i| i.document_id) { let data = server - .archive(account_id, Collection::Email, message_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::Email, + message_id, + )) .await .caused_by(trc::location!())?; let data_ = if let Some(data) = data { diff --git a/crates/imap/src/core/mailbox.rs b/crates/imap/src/core/mailbox.rs index 002414a1..66b0d9dc 100644 --- a/crates/imap/src/core/mailbox.rs +++ b/crates/imap/src/core/mailbox.rs @@ -23,6 +23,10 @@ use std::{ collections::BTreeMap, sync::{Arc, atomic::Ordering}, }; +use store::{ + ValueKey, + write::{AlignedBytes, Archive}, +}; use trc::AddContext; use types::{acl::Acl, collection::Collection, id::Id, keyword::Keyword, special_use::SpecialUse}; @@ -396,7 +400,12 @@ impl SessionData { Ok(access_token.is_member(account_id) || self .server - .archive(account_id, Collection::Mailbox, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::Mailbox, + document_id, + )) .await .and_then(|mailbox| { if let Some(mailbox) = mailbox { diff --git a/crates/imap/src/op/acl.rs b/crates/imap/src/op/acl.rs index f35a5c1f..7f083444 100644 --- a/crates/imap/src/op/acl.rs +++ b/crates/imap/src/op/acl.rs @@ -29,7 +29,10 @@ use imap_proto::{ receiver::Request, }; use std::{sync::Arc, time::Instant}; -use store::write::{AlignedBytes, Archive, BatchBuilder}; +use store::{ + ValueKey, + write::{AlignedBytes, Archive, BatchBuilder}, +}; use trc::AddContext; use types::{ acl::{Acl, AclGrant}, @@ -460,7 +463,12 @@ impl SessionData { if let Some(mailbox) = self.get_mailbox_by_name(&arguments.mailbox_name) { if let Some(values) = self .server - .archive(mailbox.account_id, Collection::Mailbox, mailbox.mailbox_id) + .store() + .get_value::>(ValueKey::archive( + mailbox.account_id, + Collection::Mailbox, + mailbox.mailbox_id, + )) .await .caused_by(trc::location!())? { diff --git a/crates/imap/src/op/copy_move.rs b/crates/imap/src/op/copy_move.rs index d8f12e76..67aabd1a 100644 --- a/crates/imap/src/op/copy_move.rs +++ b/crates/imap/src/op/copy_move.rs @@ -26,6 +26,7 @@ use imap_proto::{ }; use std::{sync::Arc, time::Instant}; use store::{ + ValueKey, roaring::RoaringBitmap, write::{AlignedBytes, Archive, BatchBuilder}, }; @@ -319,18 +320,16 @@ impl SessionData { } // Add message to training queue - let learn_spam = if dest_mailbox_id.mailbox_id == JUNK_ID { - Some(true) + if dest_mailbox_id.mailbox_id == JUNK_ID { + self.server + .add_account_spam_sample(&mut batch, account_id, id, true) + .await + .imap_ctx(&arguments.tag, trc::location!())?; } else if src_mailbox.id.mailbox_id == JUNK_ID && dest_mailbox_id.mailbox_id != TRASH_ID { - Some(false) - } else { - None - }; - if let Some(learn_spam) = learn_spam { self.server - .add_account_spam_sample(&mut batch, account_id, id, learn_spam) + .add_account_spam_sample(&mut batch, account_id, id, false) .await .imap_ctx(&arguments.tag, trc::location!())?; } @@ -551,7 +550,12 @@ impl SessionData { ) -> trc::Result>> { if let Some(data) = self .server - .archive(account_id, Collection::Email, id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::Email, + id, + )) .await? { Ok(Some(data)) diff --git a/crates/imap/src/op/fetch.rs b/crates/imap/src/op/fetch.rs index 13dacbe7..19f1f743 100644 --- a/crates/imap/src/op/fetch.rs +++ b/crates/imap/src/op/fetch.rs @@ -34,6 +34,10 @@ use imap_proto::{ receiver::Request, }; use std::{borrow::Cow, sync::Arc, time::Instant}; +use store::{ + ValueKey, + write::{AlignedBytes, Archive}, +}; use store::{ query::log::{Change, Query}, rkyv::rend::u16_le, @@ -323,12 +327,13 @@ impl SessionData { // Obtain attributes and keywords let (metadata_, data) = if let (Some(email), Some(data)) = ( self.server - .archive_by_property( + .store() + .get_value::>(ValueKey::property( account_id, Collection::Email, id, - EmailField::Metadata.into(), - ) + EmailField::Metadata, + )) .await .imap_ctx(&arguments.tag, trc::location!())?, message_cache.email_by_id(&id), @@ -545,7 +550,12 @@ impl SessionData { if set_seen_flag && let Some(data_) = self .server - .archive(account_id, Collection::Email, id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::Email, + id, + )) .await .imap_ctx(&arguments.tag, trc::location!())? { diff --git a/crates/imap/src/op/rename.rs b/crates/imap/src/op/rename.rs index daa4550d..2b6b0bdf 100644 --- a/crates/imap/src/op/rename.rs +++ b/crates/imap/src/op/rename.rs @@ -14,7 +14,10 @@ use imap_proto::{ Command, ResponseCode, StatusResponse, protocol::rename::Arguments, receiver::Request, }; use std::time::Instant; -use store::write::BatchBuilder; +use store::{ + ValueKey, + write::{AlignedBytes, Archive, BatchBuilder}, +}; use trc::AddContext; use types::{acl::Acl, collection::Collection}; @@ -85,7 +88,12 @@ impl SessionData { // Obtain mailbox let mailbox_ = self .server - .archive(params.account_id, Collection::Mailbox, mailbox_id) + .store() + .get_value::>(ValueKey::archive( + params.account_id, + Collection::Mailbox, + mailbox_id, + )) .await .imap_ctx(&arguments.tag, trc::location!())? .ok_or_else(|| { diff --git a/crates/imap/src/op/store.rs b/crates/imap/src/op/store.rs index ceee118e..96f89e07 100644 --- a/crates/imap/src/op/store.rs +++ b/crates/imap/src/op/store.rs @@ -12,7 +12,7 @@ use crate::{ use ahash::AHashSet; use common::{listener::SessionStream, storage::index::ObjectIndexBuilder}; use directory::Permission; -use email::message::metadata::MessageData; +use email::message::{ingest::EmailIngest, metadata::MessageData}; use imap_proto::{ Command, ResponseCode, ResponseType, StatusResponse, protocol::{ @@ -24,8 +24,9 @@ use imap_proto::{ }; use std::{sync::Arc, time::Instant}; use store::{ + ValueKey, query::log::{Change, Query}, - write::{BatchBuilder, TaskEpoch, TaskQueueClass, ValueClass}, + write::{AlignedBytes, Archive, BatchBuilder}, }; use trc::AddContext; use types::{ @@ -189,14 +190,18 @@ impl SessionData { .map(|k| Keyword::from(k.clone())) .collect::>(); let mut changed_mailboxes = AHashSet::new(); - let mut has_spam_train_tasks = false; let mut batch = BatchBuilder::new(); for (id, imap_id) in &ids { // Obtain message data let data_ = if let Some(data) = self .server - .archive(account_id, Collection::Email, *id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::Email, + *id, + )) .await .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())? { @@ -292,15 +297,10 @@ impl SessionData { // Add spam train task if let Some(learn_spam) = train_spam { - batch.set( - ValueClass::TaskQueue(TaskQueueClass::SpamTrain { - due: TaskEpoch::now(), - blob_hash: Default::default(), - learn_spam, - }), - vec![], - ); - has_spam_train_tasks = true; + self.server + .add_account_spam_sample(&mut batch, account_id, *id, learn_spam) + .await + .imap_ctx(response.tag.as_ref().unwrap(), trc::location!())?; } // Set commit point @@ -335,11 +335,6 @@ impl SessionData { } } - // Trigger Bayes training - if has_spam_train_tasks { - self.server.notify_task_queue(); - } - // Write changes if !batch.is_empty() { match self diff --git a/crates/imap/src/op/subscribe.rs b/crates/imap/src/op/subscribe.rs index 3d99e364..18420683 100644 --- a/crates/imap/src/op/subscribe.rs +++ b/crates/imap/src/op/subscribe.rs @@ -13,7 +13,10 @@ use common::{listener::SessionStream, storage::index::ObjectIndexBuilder}; use directory::Permission; use imap_proto::{Command, ResponseCode, StatusResponse, receiver::Request}; use std::time::Instant; -use store::write::BatchBuilder; +use store::{ + ValueKey, + write::{AlignedBytes, Archive, BatchBuilder}, +}; use types::collection::Collection; impl Session { @@ -92,7 +95,12 @@ impl SessionData { // Obtain mailbox let mailbox_ = self .server - .archive(account_id, Collection::Mailbox, mailbox_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::Mailbox, + mailbox_id, + )) .await .imap_ctx(&tag, trc::location!())? .ok_or_else(|| { diff --git a/crates/jmap/src/addressbook/get.rs b/crates/jmap/src/addressbook/get.rs index 6196d4ac..8dbad3e0 100644 --- a/crates/jmap/src/addressbook/get.rs +++ b/crates/jmap/src/addressbook/get.rs @@ -12,7 +12,7 @@ use jmap_proto::{ object::addressbook::{self, AddressBookProperty, AddressBookValue}, }; use jmap_tools::{Map, Value}; -use store::{ValueKey, roaring::RoaringBitmap, write::ValueClass}; +use store::{ValueKey, roaring::RoaringBitmap, write::{AlignedBytes, Archive, ValueClass}}; use trc::AddContext; use types::{ acl::{Acl, AclGrant}, @@ -95,7 +95,12 @@ impl AddressBookGet for Server { continue; } let _address_book = if let Some(address_book) = self - .archive(account_id, Collection::AddressBook, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::AddressBook, + document_id, + )) .await? { address_book diff --git a/crates/jmap/src/addressbook/set.rs b/crates/jmap/src/addressbook/set.rs index 292c8a13..cf868521 100644 --- a/crates/jmap/src/addressbook/set.rs +++ b/crates/jmap/src/addressbook/set.rs @@ -24,7 +24,7 @@ use rand::{Rng, distr::Alphanumeric}; use store::{ SerializeInfallible, ValueKey, ahash::AHashSet, - write::{BatchBuilder, ValueClass}, + write::{AlignedBytes, Archive, BatchBuilder, ValueClass}, }; use trc::AddContext; use types::{ @@ -131,7 +131,12 @@ impl AddressBookSet for Server { // Obtain address book let document_id = id.document_id(); let address_book_ = if let Some(address_book_) = self - .archive(account_id, Collection::AddressBook, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::AddressBook, + document_id, + )) .await? { address_book_ @@ -223,7 +228,12 @@ impl AddressBookSet for Server { }; let Some(address_book_) = self - .archive(account_id, Collection::AddressBook, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::AddressBook, + document_id, + )) .await .caused_by(trc::location!())? else { @@ -278,7 +288,12 @@ impl AddressBookSet for Server { if !destroy_children.is_empty() { for document_id in destroy_children { if let Some(card_) = self - .archive(account_id, Collection::ContactCard, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::ContactCard, + document_id, + )) .await? { let card = card_ diff --git a/crates/jmap/src/blob/copy.rs b/crates/jmap/src/blob/copy.rs index 05eccf05..47ffbece 100644 --- a/crates/jmap/src/blob/copy.rs +++ b/crates/jmap/src/blob/copy.rs @@ -13,10 +13,7 @@ use jmap_proto::{ request::IntoValid, }; use std::future::Future; -use store::{ - SerializeInfallible, - write::{BatchBuilder, BlobLink, BlobOp, now}, -}; +use store::write::{BatchBuilder, BlobLink, BlobOp, now}; use trc::AddContext; use types::blob::{BlobClass, BlobId}; use utils::map::vec_map::VecMap; diff --git a/crates/jmap/src/blob/download.rs b/crates/jmap/src/blob/download.rs index 84c6630c..086d40b4 100644 --- a/crates/jmap/src/blob/download.rs +++ b/crates/jmap/src/blob/download.rs @@ -9,6 +9,8 @@ use email::cache::MessageCacheFetch; use email::cache::email::MessageCacheAccess; use email::message::metadata::MessageMetadata; use groupware::cache::GroupwareCache; +use store::ValueKey; +use store::write::{AlignedBytes, Archive}; use std::future::Future; use trc::AddContext; use types::acl::Acl; @@ -59,12 +61,13 @@ impl BlobDownload for Server { Ok(Some(data)), ) if *collection == Collection::Email as u8 => { let Some(archive) = self - .archive_by_property( + .store() + .get_value::>(ValueKey::property( *account_id, Collection::Email, *document_id, - EmailField::Metadata.into(), - ) + EmailField::Metadata, + )) .await .caused_by(trc::location!())? else { diff --git a/crates/jmap/src/blob/get.rs b/crates/jmap/src/blob/get.rs index 9facf38a..25ed5a76 100644 --- a/crates/jmap/src/blob/get.rs +++ b/crates/jmap/src/blob/get.rs @@ -19,6 +19,7 @@ use jmap_tools::{Map, Value}; use mail_builder::encoders::base64::base64_encode; use sha1::{Digest, Sha1}; use sha2::{Sha256, Sha512}; +use store::{ValueKey, write::{AlignedBytes, Archive}}; use std::future::Future; use trc::AddContext; use types::{blob::BlobClass, collection::Collection, id::Id, type_state::DataType}; @@ -203,7 +204,12 @@ impl BlobOperations for Server { let collection = Collection::from(*collection); if collection == Collection::Email { if let Some(data_) = self - .archive(req_account_id, Collection::Email, *document_id) + .store() + .get_value::>(ValueKey::archive( + req_account_id, + Collection::Email, + *document_id, + )) .await? { let data = data_ diff --git a/crates/jmap/src/calendar/get.rs b/crates/jmap/src/calendar/get.rs index ceb4daf2..40cf5872 100644 --- a/crates/jmap/src/calendar/get.rs +++ b/crates/jmap/src/calendar/get.rs @@ -19,7 +19,7 @@ use jmap_proto::{ object::calendar::{self, CalendarProperty, CalendarValue, IncludeInAvailability}, }; use jmap_tools::{Key, Map, Value}; -use store::{ValueKey, roaring::RoaringBitmap, write::ValueClass}; +use store::{ValueKey, roaring::RoaringBitmap, write::{AlignedBytes, Archive, ValueClass}}; use trc::AddContext; use types::{ acl::{Acl, AclGrant}, @@ -105,7 +105,12 @@ impl CalendarGet for Server { continue; } let _calendar = if let Some(calendar) = self - .archive(account_id, Collection::Calendar, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::Calendar, + document_id, + )) .await? { calendar diff --git a/crates/jmap/src/calendar/set.rs b/crates/jmap/src/calendar/set.rs index e4aa3c90..08b3042c 100644 --- a/crates/jmap/src/calendar/set.rs +++ b/crates/jmap/src/calendar/set.rs @@ -29,7 +29,7 @@ use rand::{Rng, distr::Alphanumeric}; use store::{ SerializeInfallible, ValueKey, ahash::AHashSet, - write::{BatchBuilder, ValueClass}, + write::{AlignedBytes, Archive, BatchBuilder, ValueClass}, }; use trc::AddContext; use types::{ @@ -136,7 +136,12 @@ impl CalendarSet for Server { // Obtain calendar let document_id = id.document_id(); let calendar_ = if let Some(calendar_) = self - .archive(account_id, Collection::Calendar, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::Calendar, + document_id, + )) .await? { calendar_ @@ -214,7 +219,12 @@ impl CalendarSet for Server { }; let Some(calendar_) = self - .archive(account_id, Collection::Calendar, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::Calendar, + document_id, + )) .await .caused_by(trc::location!())? else { @@ -269,7 +279,12 @@ impl CalendarSet for Server { if !destroy_children.is_empty() { for document_id in destroy_children { if let Some(event_) = self - .archive(account_id, Collection::CalendarEvent, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::CalendarEvent, + document_id, + )) .await? { let event = event_ diff --git a/crates/jmap/src/calendar_event/copy.rs b/crates/jmap/src/calendar_event/copy.rs index d001100c..e6e3c823 100644 --- a/crates/jmap/src/calendar_event/copy.rs +++ b/crates/jmap/src/calendar_event/copy.rs @@ -26,7 +26,7 @@ use jmap_proto::{ }, types::state::State, }; -use store::{roaring::RoaringBitmap, write::BatchBuilder}; +use store::{ValueKey, roaring::RoaringBitmap, write::{AlignedBytes, Archive, BatchBuilder}}; use trc::AddContext; use types::{ acl::Acl, @@ -123,11 +123,12 @@ impl JmapCalendarEventCopy for Server { } let Some(_calendar_event) = self - .archive( + .store() + .get_value::>(ValueKey::archive( from_account_id, Collection::CalendarEvent, from_calendar_event_id, - ) + )) .await? else { response.not_created.append( diff --git a/crates/jmap/src/calendar_event/get.rs b/crates/jmap/src/calendar_event/get.rs index 8fad0292..cc26d701 100644 --- a/crates/jmap/src/calendar_event/get.rs +++ b/crates/jmap/src/calendar_event/get.rs @@ -33,8 +33,7 @@ use jmap_proto::{ use jmap_tools::{Key, Map, Value}; use std::{str::FromStr, sync::Arc}; use store::{ - ahash::{AHashMap, AHashSet}, - roaring::RoaringBitmap, + ValueKey, ahash::{AHashMap, AHashSet}, roaring::RoaringBitmap, write::{AlignedBytes, Archive} }; use trc::AddContext; use types::{ @@ -203,7 +202,12 @@ impl CalendarEventGet for Server { } let Some(_calendar_event) = self - .archive(account_id, Collection::CalendarEvent, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::CalendarEvent, + document_id, + )) .await? else { response.not_found.push(id); diff --git a/crates/jmap/src/calendar_event/query.rs b/crates/jmap/src/calendar_event/query.rs index 658d2c6f..e1bc79d9 100644 --- a/crates/jmap/src/calendar_event/query.rs +++ b/crates/jmap/src/calendar_event/query.rs @@ -17,9 +17,7 @@ use jmap_proto::{ use nlp::language::Language; use std::{cmp::Ordering, sync::Arc}; use store::{ - roaring::RoaringBitmap, - search::{CalendarSearchField, SearchComparator, SearchFilter, SearchQuery}, - write::SearchIndex, + ValueKey, roaring::RoaringBitmap, search::{CalendarSearchField, SearchComparator, SearchFilter, SearchQuery}, write::{AlignedBytes, Archive, SearchIndex} }; use trc::AddContext; use types::{ @@ -260,7 +258,12 @@ impl CalendarEventQuery for Server { for document_id in results { let Some(_calendar_event) = self - .archive(account_id, Collection::CalendarEvent, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::CalendarEvent, + document_id, + )) .await? else { continue; diff --git a/crates/jmap/src/calendar_event/set.rs b/crates/jmap/src/calendar_event/set.rs index 427efd21..4e18888c 100644 --- a/crates/jmap/src/calendar_event/set.rs +++ b/crates/jmap/src/calendar_event/set.rs @@ -38,9 +38,7 @@ use jmap_proto::{ use jmap_tools::{JsonPointerHandler, JsonPointerItem, Key, Map, Value}; use std::{borrow::Cow, str::FromStr}; use store::{ - ahash::AHashSet, - roaring::RoaringBitmap, - write::{BatchBuilder, now, serialize::rkyv_deserialize}, + ValueKey, ahash::AHashSet, roaring::RoaringBitmap, write::{AlignedBytes, Archive, BatchBuilder, now, serialize::rkyv_deserialize} }; use trc::AddContext; use types::{ @@ -150,7 +148,12 @@ impl CalendarEventSet for Server { // Obtain calendar_event card let document_id = id.document_id(); let calendar_event_ = if let Some(calendar_event_) = self - .archive(account_id, Collection::CalendarEvent, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::CalendarEvent, + document_id, + )) .await? { calendar_event_ @@ -436,7 +439,12 @@ impl CalendarEventSet for Server { } let Some(calendar_event_) = self - .archive(account_id, Collection::CalendarEvent, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::CalendarEvent, + document_id, + )) .await .caused_by(trc::location!())? else { @@ -546,7 +554,12 @@ impl CalendarEventSet for Server { )))); } else if let Some(show_without_time) = use_default_alerts && let Some(_calendar) = self - .archive(account_id, Collection::Calendar, name.parent_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::Calendar, + name.parent_id, + )) .await? { ical.components.extend( diff --git a/crates/jmap/src/calendar_event_notification/get.rs b/crates/jmap/src/calendar_event_notification/get.rs index 5453a4e7..d2dc6127 100644 --- a/crates/jmap/src/calendar_event_notification/get.rs +++ b/crates/jmap/src/calendar_event_notification/get.rs @@ -25,7 +25,7 @@ use jmap_proto::{ }, types::date::UTCDate, }; -use store::write::serialize::rkyv_deserialize; +use store::{ValueKey, write::{AlignedBytes, Archive, serialize::rkyv_deserialize}}; use trc::AddContext; use types::{ blob::BlobId, @@ -84,11 +84,12 @@ impl CalendarEventNotificationGet for Server { // Obtain the event object let document_id = id.document_id(); let _event = if let Some(event) = self - .archive( + .store() + .get_value::>(ValueKey::archive( account_id, Collection::CalendarEventNotification, document_id, - ) + )) .await? { event diff --git a/crates/jmap/src/calendar_event_notification/set.rs b/crates/jmap/src/calendar_event_notification/set.rs index bb577404..9330021d 100644 --- a/crates/jmap/src/calendar_event_notification/set.rs +++ b/crates/jmap/src/calendar_event_notification/set.rs @@ -14,7 +14,7 @@ use jmap_proto::{ request::IntoValid, types::state::State, }; -use store::write::BatchBuilder; +use store::{ValueKey, write::{AlignedBytes, Archive, BatchBuilder}}; use trc::AddContext; use types::collection::{Collection, SyncCollection}; @@ -72,11 +72,12 @@ impl CalendarEventNotificationSet for Server { }; let _event = if let Some(event) = self - .archive( + .store() + .get_value::>(ValueKey::archive( account_id, Collection::CalendarEventNotification, document_id, - ) + )) .await? { event diff --git a/crates/jmap/src/contact/copy.rs b/crates/jmap/src/contact/copy.rs index 5d0d123a..0ef99856 100644 --- a/crates/jmap/src/contact/copy.rs +++ b/crates/jmap/src/contact/copy.rs @@ -22,7 +22,7 @@ use jmap_proto::{ }, types::state::State, }; -use store::{roaring::RoaringBitmap, write::BatchBuilder}; +use store::{ValueKey, roaring::RoaringBitmap, write::{AlignedBytes, Archive, BatchBuilder}}; use trc::AddContext; use types::{ acl::Acl, @@ -107,7 +107,12 @@ impl JmapContactCardCopy for Server { } let Some(_contact) = self - .archive(from_account_id, Collection::ContactCard, from_contact_id) + .store() + .get_value::>(ValueKey::archive( + from_account_id, + Collection::ContactCard, + from_contact_id, + )) .await? else { response.not_created.append( diff --git a/crates/jmap/src/contact/get.rs b/crates/jmap/src/contact/get.rs index ed4d1bc7..e5208ddc 100644 --- a/crates/jmap/src/contact/get.rs +++ b/crates/jmap/src/contact/get.rs @@ -14,7 +14,7 @@ use jmap_proto::{ request::reference::MaybeResultReference, }; use jmap_tools::{Map, Value}; -use store::roaring::RoaringBitmap; +use store::{ValueKey, roaring::RoaringBitmap, write::{AlignedBytes, Archive}}; use trc::AddContext; use types::{ acl::Acl, @@ -98,7 +98,12 @@ impl ContactCardGet for Server { } let _contact = if let Some(contact) = self - .archive(account_id, Collection::ContactCard, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::ContactCard, + document_id, + )) .await? { contact diff --git a/crates/jmap/src/contact/set.rs b/crates/jmap/src/contact/set.rs index c6752bae..110f022e 100644 --- a/crates/jmap/src/contact/set.rs +++ b/crates/jmap/src/contact/set.rs @@ -17,7 +17,7 @@ use jmap_proto::{ types::state::State, }; use jmap_tools::{JsonPointerHandler, JsonPointerItem, Key, Value}; -use store::{ahash::AHashSet, roaring::RoaringBitmap, write::BatchBuilder}; +use store::{ValueKey, ahash::AHashSet, roaring::RoaringBitmap, write::{AlignedBytes, Archive, BatchBuilder}}; use trc::AddContext; use types::{ acl::Acl, @@ -115,7 +115,12 @@ impl ContactCardSet for Server { // Obtain contact card let document_id = id.document_id(); let contact_card_ = if let Some(contact_card_) = self - .archive(account_id, Collection::ContactCard, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::ContactCard, + document_id, + )) .await? { contact_card_ @@ -281,7 +286,12 @@ impl ContactCardSet for Server { }; let Some(contact_card_) = self - .archive(account_id, Collection::ContactCard, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::ContactCard, + document_id, + )) .await .caused_by(trc::location!())? else { diff --git a/crates/jmap/src/email/get.rs b/crates/jmap/src/email/get.rs index f1ea9cd8..b08aabd6 100644 --- a/crates/jmap/src/email/get.rs +++ b/crates/jmap/src/email/get.rs @@ -26,6 +26,10 @@ use jmap_proto::{ use jmap_tools::{Key, Map, Value}; use mail_parser::HeaderValue; use std::future::Future; +use store::{ + ValueKey, + write::{AlignedBytes, Archive}, +}; use trc::{AddContext, StoreEvent}; use types::{ acl::Acl, @@ -153,12 +157,13 @@ impl EmailGet for Server { continue; } let metadata_ = match self - .archive_by_property( + .store() + .get_value::>(ValueKey::property( account_id, Collection::Email, id.document_id(), - EmailField::Metadata.into(), - ) + EmailField::Metadata, + )) .await? { Some(metadata) => metadata, diff --git a/crates/jmap/src/email/set.rs b/crates/jmap/src/email/set.rs index c3f37b6e..5c4cbfe0 100644 --- a/crates/jmap/src/email/set.rs +++ b/crates/jmap/src/email/set.rs @@ -43,7 +43,7 @@ use mail_builder::{ use mail_parser::MessageParser; use std::future::Future; use std::{borrow::Cow, collections::HashMap}; -use store::{ahash::AHashMap, roaring::RoaringBitmap, write::BatchBuilder}; +use store::{ValueKey, ahash::AHashMap, roaring::RoaringBitmap, write::{AlignedBytes, Archive, BatchBuilder}}; use trc::AddContext; use types::{ acl::Acl, @@ -795,7 +795,12 @@ impl EmailSet for Server { // Obtain message data let document_id = id.document_id(); let data_ = match self - .archive(account_id, Collection::Email, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::Email, + document_id, + )) .await? { Some(data) => data, diff --git a/crates/jmap/src/email/snippet.rs b/crates/jmap/src/email/snippet.rs index 6391170a..0d564183 100644 --- a/crates/jmap/src/email/snippet.rs +++ b/crates/jmap/src/email/snippet.rs @@ -22,7 +22,11 @@ use jmap_proto::{ use mail_parser::decoders::html::html_to_text; use nlp::language::{Language, search_snippet::generate_snippet, stemmer::Stemmer}; use std::future::Future; -use store::backend::MAX_TOKEN_LENGTH; +use store::{ + ValueKey, + backend::MAX_TOKEN_LENGTH, + write::{AlignedBytes, Archive}, +}; use trc::AddContext; use types::{acl::Acl, collection::Collection, field::EmailField}; use utils::chained_bytes::ChainedBytes; @@ -126,12 +130,13 @@ impl EmailSearchSnippet for Server { continue; } let metadata_ = match self - .archive_by_property( + .store() + .get_value::>(ValueKey::property( account_id, Collection::Email, document_id, - EmailField::Metadata.into(), - ) + EmailField::Metadata, + )) .await? { Some(metadata) => metadata, diff --git a/crates/jmap/src/file/get.rs b/crates/jmap/src/file/get.rs index 92db2c9d..badafde0 100644 --- a/crates/jmap/src/file/get.rs +++ b/crates/jmap/src/file/get.rs @@ -13,7 +13,7 @@ use jmap_proto::{ types::date::UTCDate, }; use jmap_tools::{Map, Value}; -use store::{roaring::RoaringBitmap, write::now}; +use store::{ValueKey, roaring::RoaringBitmap, write::{AlignedBytes, Archive, now}}; use trc::AddContext; use types::{ acl::{Acl, AclGrant}, @@ -81,7 +81,12 @@ impl FileNodeGet for Server { continue; } let _file_node = if let Some(file_node) = self - .archive(account_id, Collection::FileNode, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::FileNode, + document_id, + )) .await? { file_node diff --git a/crates/jmap/src/file/set.rs b/crates/jmap/src/file/set.rs index dcb12eed..fde7c741 100644 --- a/crates/jmap/src/file/set.rs +++ b/crates/jmap/src/file/set.rs @@ -21,8 +21,7 @@ use jmap_proto::{ }; use jmap_tools::{JsonPointerItem, Key, Value}; use store::{ - ahash::{AHashMap, AHashSet}, - write::BatchBuilder, + ValueKey, ahash::{AHashMap, AHashSet}, write::{AlignedBytes, Archive, BatchBuilder} }; use trc::AddContext; use types::{ @@ -187,7 +186,12 @@ impl FileNodeSet for Server { // Obtain file node let document_id = id.document_id(); let file_node_ = if let Some(file_node_) = self - .archive(account_id, Collection::FileNode, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::FileNode, + document_id, + )) .await? { file_node_ diff --git a/crates/jmap/src/identity/get.rs b/crates/jmap/src/identity/get.rs index ab893f21..2b3c0e0a 100644 --- a/crates/jmap/src/identity/get.rs +++ b/crates/jmap/src/identity/get.rs @@ -15,9 +15,7 @@ use jmap_proto::{ use jmap_tools::{Map, Value}; use std::future::Future; use store::{ - rkyv::{option::ArchivedOption, vec::ArchivedVec}, - roaring::RoaringBitmap, - write::BatchBuilder, + ValueKey, rkyv::{option::ArchivedOption, vec::ArchivedVec}, roaring::RoaringBitmap, write::{AlignedBytes, Archive, BatchBuilder} }; use trc::AddContext; use types::{ @@ -83,7 +81,12 @@ impl IdentityGet for Server { continue; } let _identity = if let Some(identity) = self - .archive(account_id, Collection::Identity, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::Identity, + document_id, + )) .await? { identity diff --git a/crates/jmap/src/identity/set.rs b/crates/jmap/src/identity/set.rs index 19eaecb0..17e191bf 100644 --- a/crates/jmap/src/identity/set.rs +++ b/crates/jmap/src/identity/set.rs @@ -17,7 +17,7 @@ use jmap_proto::{ }; use jmap_tools::{Key, Value}; use std::future::Future; -use store::write::BatchBuilder; +use store::{ValueKey, write::{AlignedBytes, Archive, BatchBuilder}}; use trc::AddContext; use types::{ collection::{Collection, SyncCollection}, @@ -129,7 +129,12 @@ impl IdentitySet for Server { // Obtain identity let document_id = id.document_id(); let identity_ = if let Some(identity_) = self - .archive(account_id, Collection::Identity, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::Identity, + document_id, + )) .await? { identity_ diff --git a/crates/jmap/src/mailbox/set.rs b/crates/jmap/src/mailbox/set.rs index f25febda..87834afa 100644 --- a/crates/jmap/src/mailbox/set.rs +++ b/crates/jmap/src/mailbox/set.rs @@ -31,8 +31,7 @@ use jmap_proto::{ use jmap_tools::{JsonPointerItem, Key, Map, Value}; use std::future::Future; use store::{ - roaring::RoaringBitmap, - write::{Archive, BatchBuilder, assert::AssertValue}, + ValueKey, roaring::RoaringBitmap, write::{AlignedBytes, Archive, BatchBuilder, assert::AssertValue} }; use trc::AddContext; use types::{ @@ -170,7 +169,12 @@ impl MailboxSet for Server { // Obtain mailbox let document_id = id.document_id(); if let Some(mailbox) = self - .archive(account_id, Collection::Mailbox, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::Mailbox, + document_id, + )) .await? { // Validate ACL @@ -458,7 +462,12 @@ impl MailboxSet for Server { let parent_document_id = mailbox_parent_id - 1; if let Some(mailbox_) = self - .archive(ctx.account_id, Collection::Mailbox, parent_document_id) + .store() + .get_value::>(ValueKey::archive( + ctx.account_id, + Collection::Mailbox, + parent_document_id, + )) .await? { let mailbox = mailbox_ diff --git a/crates/jmap/src/participant_identity/get.rs b/crates/jmap/src/participant_identity/get.rs index edf6c8ab..faba4b10 100644 --- a/crates/jmap/src/participant_identity/get.rs +++ b/crates/jmap/src/participant_identity/get.rs @@ -13,7 +13,7 @@ use jmap_proto::{ }; use jmap_tools::{Map, Value}; use store::{ - Serialize, + Serialize, ValueKey, write::{AlignedBytes, Archive, Archiver, BatchBuilder}, }; use trc::AddContext; @@ -116,12 +116,13 @@ impl ParticipantIdentityGet for Server { account_id: u32, ) -> trc::Result>> { if let Some(identities) = self - .archive_by_property( + .store() + .get_value::>(ValueKey::property( account_id, Collection::Principal, 0, - PrincipalField::ParticipantIdentities.into(), - ) + PrincipalField::ParticipantIdentities, + )) .await? { return Ok(Some(identities)); @@ -183,12 +184,13 @@ impl ParticipantIdentityGet for Server { self.commit_batch(batch).await.caused_by(trc::location!())?; - self.archive_by_property( - account_id, - Collection::Principal, - 0, - PrincipalField::ParticipantIdentities.into(), - ) - .await + self.store() + .get_value::>(ValueKey::property( + account_id, + Collection::Principal, + 0, + PrincipalField::ParticipantIdentities, + )) + .await } } diff --git a/crates/jmap/src/principal/availability.rs b/crates/jmap/src/principal/availability.rs index 3cd467c2..1cbc241f 100644 --- a/crates/jmap/src/principal/availability.rs +++ b/crates/jmap/src/principal/availability.rs @@ -30,7 +30,7 @@ use jmap_proto::{ }; use jmap_tools::{Key, Map, Value}; use std::{collections::hash_map::Entry, future::Future}; -use store::ahash::AHashMap; +use store::{ValueKey, ahash::AHashMap, write::{AlignedBytes, Archive}}; use trc::AddContext; use types::{ TimeRange, @@ -178,7 +178,12 @@ impl PrincipalGetAvailability for Server { // Fetch event let document_id = resource.document_id; let Some(archive) = self - .archive(account_id, Collection::CalendarEvent, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::CalendarEvent, + document_id, + )) .await .caused_by(trc::location!())? else { diff --git a/crates/jmap/src/push/get.rs b/crates/jmap/src/push/get.rs index 8692b84f..2714ea81 100644 --- a/crates/jmap/src/push/get.rs +++ b/crates/jmap/src/push/get.rs @@ -14,8 +14,8 @@ use jmap_proto::{ use jmap_tools::{Map, Value}; use std::future::Future; use store::{ - Serialize, - write::{Archiver, BatchBuilder, now}, + Serialize, ValueKey, + write::{AlignedBytes, Archive, Archiver, BatchBuilder, now}, }; use trc::{AddContext, ServerEvent}; use types::{collection::Collection, field::PrincipalField, id::Id}; @@ -54,12 +54,13 @@ impl PushSubscriptionFetch for Server { }; let Some(subscriptions_) = self - .archive_by_property( + .store() + .get_value::>(ValueKey::property( account_id, Collection::Principal, 0, - PrincipalField::PushSubscriptions.into(), - ) + PrincipalField::PushSubscriptions, + )) .await? else { for id in ids.unwrap_or_default() { diff --git a/crates/jmap/src/push/set.rs b/crates/jmap/src/push/set.rs index 5970f7f9..8b5578bf 100644 --- a/crates/jmap/src/push/set.rs +++ b/crates/jmap/src/push/set.rs @@ -19,9 +19,9 @@ use jmap_tools::{Key, Map, Value}; use rand::distr::Alphanumeric; use std::future::Future; use store::{ - Serialize, + Serialize, ValueKey, rand::{Rng, rng}, - write::{Archiver, BatchBuilder, now}, + write::{AlignedBytes, Archive, Archiver, BatchBuilder, now}, }; use trc::{AddContext, ServerEvent}; use types::{collection::Collection, field::PrincipalField}; @@ -47,12 +47,13 @@ impl PushSubscriptionSet for Server { // Load existing push subscriptions let account_id = access_token.primary_id(); let subscriptions_archive = self - .archive_by_property( + .store() + .get_value::>(ValueKey::property( account_id, Collection::Principal, 0, - PrincipalField::PushSubscriptions.into(), - ) + PrincipalField::PushSubscriptions, + )) .await?; let mut subscriptions = if let Some(subscriptions) = &subscriptions_archive { subscriptions diff --git a/crates/jmap/src/sieve/get.rs b/crates/jmap/src/sieve/get.rs index 884fc086..d07664cb 100644 --- a/crates/jmap/src/sieve/get.rs +++ b/crates/jmap/src/sieve/get.rs @@ -12,6 +12,7 @@ use jmap_proto::{ object::sieve::{Sieve, SieveProperty, SieveValue}, }; use jmap_tools::{Map, Value}; +use store::{ValueKey, write::{AlignedBytes, Archive}}; use std::future::Future; use trc::AddContext; use types::{ @@ -71,7 +72,12 @@ impl SieveScriptGet for Server { continue; } let sieve_ = if let Some(sieve) = self - .archive(account_id, Collection::SieveScript, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::SieveScript, + document_id, + )) .await? { sieve diff --git a/crates/jmap/src/sieve/set.rs b/crates/jmap/src/sieve/set.rs index 5d787455..bc8a928f 100644 --- a/crates/jmap/src/sieve/set.rs +++ b/crates/jmap/src/sieve/set.rs @@ -27,9 +27,7 @@ use rand::distr::Alphanumeric; use sieve::compiler::ErrorType; use std::future::Future; use store::{ - Serialize, SerializeInfallible, - rand::{Rng, rng}, - write::{Archive, Archiver, BatchBuilder}, + Serialize, SerializeInfallible, ValueKey, rand::{Rng, rng}, write::{AlignedBytes, Archive, Archiver, BatchBuilder} }; use trc::AddContext; use types::{ @@ -199,7 +197,12 @@ impl SieveScriptSet for Server { // Obtain sieve script let document_id = id.document_id(); if let Some(sieve_) = self - .archive(account_id, Collection::SieveScript, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::SieveScript, + document_id, + )) .await? { let sieve = sieve_ diff --git a/crates/jmap/src/submission/get.rs b/crates/jmap/src/submission/get.rs index 25379e74..5153783c 100644 --- a/crates/jmap/src/submission/get.rs +++ b/crates/jmap/src/submission/get.rs @@ -22,7 +22,9 @@ use std::future::Future; use store::{ IterateParams, U32_LEN, ValueKey, rkyv::option::ArchivedOption, - write::{IndexPropertyClass, ValueClass, key::DeserializeBigEndian, now}, + write::{ + AlignedBytes, Archive, IndexPropertyClass, ValueClass, key::DeserializeBigEndian, now, + }, }; use trc::AddContext; use types::{ @@ -112,7 +114,12 @@ impl EmailSubmissionGet for Server { // Obtain the email_submission object let document_id = id.document_id(); let submission_ = if let Some(submission) = self - .archive(account_id, Collection::EmailSubmission, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::EmailSubmission, + document_id, + )) .await? { submission diff --git a/crates/jmap/src/submission/set.rs b/crates/jmap/src/submission/set.rs index fab539f7..e28a1ef7 100644 --- a/crates/jmap/src/submission/set.rs +++ b/crates/jmap/src/submission/set.rs @@ -35,7 +35,10 @@ use smtp::{ use smtp_proto::{MailFrom, RcptTo, request::parser::Rfc5321Parser}; use std::{borrow::Cow, future::Future}; use std::{collections::HashMap, sync::Arc, time::Duration}; -use store::write::{BatchBuilder, now}; +use store::{ + ValueKey, + write::{AlignedBytes, Archive, BatchBuilder, now}, +}; use trc::AddContext; use types::{collection::Collection, field::EmailField, id::Id}; use utils::{map::vec_map::VecMap, sanitize_email}; @@ -117,7 +120,12 @@ impl EmailSubmissionSet for Server { // Obtain submission let document_id = id.document_id(); let submission = if let Some(submission) = self - .archive(account_id, Collection::EmailSubmission, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::EmailSubmission, + document_id, + )) .await? { submission @@ -210,7 +218,12 @@ impl EmailSubmissionSet for Server { for id in will_destroy { let document_id = id.document_id(); if let Some(submission) = self - .archive(account_id, Collection::EmailSubmission, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::EmailSubmission, + document_id, + )) .await? { // Update record @@ -455,7 +468,12 @@ impl EmailSubmissionSet for Server { // Fetch identity's mailFrom let identity_mail_from = if let Some(identity) = self - .archive(account_id, Collection::Identity, submission.identity_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::Identity, + submission.identity_id, + )) .await? { identity @@ -491,12 +509,13 @@ impl EmailSubmissionSet for Server { // Obtain message metadata let metadata_ = if let Some(metadata) = self - .archive_by_property( + .store() + .get_value::>(ValueKey::property( account_id, Collection::Email, submission.email_id, - EmailField::Metadata.into(), - ) + EmailField::Metadata, + )) .await? { metadata diff --git a/crates/jmap/src/vacation/get.rs b/crates/jmap/src/vacation/get.rs index 12618b71..9f25526c 100644 --- a/crates/jmap/src/vacation/get.rs +++ b/crates/jmap/src/vacation/get.rs @@ -17,6 +17,10 @@ use jmap_proto::{ }; use jmap_tools::{Map, Value}; use std::future::Future; +use store::{ + ValueKey, + write::{AlignedBytes, Archive}, +}; use trc::AddContext; use types::{ collection::{Collection, SyncCollection}, @@ -81,7 +85,12 @@ impl VacationResponseGet for Server { if do_get { if let Some(document_id) = self.get_vacation_sieve_script_id(account_id).await? { if let Some(sieve_) = self - .archive(account_id, Collection::SieveScript, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::SieveScript, + document_id, + )) .await? { let active_script_id = self.sieve_script_get_active_id(account_id).await?; diff --git a/crates/jmap/src/vacation/set.rs b/crates/jmap/src/vacation/set.rs index 1dccd288..9080c678 100644 --- a/crates/jmap/src/vacation/set.rs +++ b/crates/jmap/src/vacation/set.rs @@ -24,8 +24,8 @@ use mail_parser::decoders::html::html_to_text; use std::borrow::Cow; use std::future::Future; use store::{ - Serialize, SerializeInfallible, - write::{Archiver, BatchBuilder}, + Serialize, SerializeInfallible, ValueKey, + write::{AlignedBytes, Archive, Archiver, BatchBuilder}, }; use trc::AddContext; use types::{ @@ -132,7 +132,12 @@ impl VacationResponseSet for Server { let (mut sieve, prev_sieve) = if let Some(document_id) = document_id { let prev_sieve = self - .archive(account_id, Collection::SieveScript, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::SieveScript, + document_id, + )) .await? .ok_or_else(|| { trc::StoreEvent::NotFound diff --git a/crates/managesieve/src/op/getscript.rs b/crates/managesieve/src/op/getscript.rs index 26d0bd20..de37a18b 100644 --- a/crates/managesieve/src/op/getscript.rs +++ b/crates/managesieve/src/op/getscript.rs @@ -10,6 +10,10 @@ use directory::Permission; use email::sieve::SieveScript; use imap_proto::receiver::Request; use std::time::Instant; +use store::{ + ValueKey, + write::{AlignedBytes, Archive}, +}; use trc::AddContext; use types::{blob::BlobSection, blob_hash::BlobHash, collection::Collection}; @@ -33,7 +37,12 @@ impl Session { let document_id = self.get_script_id(account_id, &name).await?; let sieve_ = self .server - .archive(account_id, Collection::SieveScript, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::SieveScript, + document_id, + )) .await .caused_by(trc::location!())? .ok_or_else(|| { diff --git a/crates/managesieve/src/op/listscripts.rs b/crates/managesieve/src/op/listscripts.rs index f01692de..6b9cede9 100644 --- a/crates/managesieve/src/op/listscripts.rs +++ b/crates/managesieve/src/op/listscripts.rs @@ -9,6 +9,10 @@ use common::listener::SessionStream; use directory::Permission; use email::sieve::{SieveScript, ingest::SieveScriptIngest}; use std::time::Instant; +use store::{ + ValueKey, + write::{AlignedBytes, Archive}, +}; use trc::AddContext; use types::{collection::Collection, field::SieveField}; @@ -36,7 +40,12 @@ impl Session { for document_id in document_ids { if let Some(script_) = self .server - .archive(account_id, Collection::SieveScript, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::SieveScript, + document_id, + )) .await .caused_by(trc::location!())? { diff --git a/crates/managesieve/src/op/putscript.rs b/crates/managesieve/src/op/putscript.rs index e52abf7d..8cd91591 100644 --- a/crates/managesieve/src/op/putscript.rs +++ b/crates/managesieve/src/op/putscript.rs @@ -12,8 +12,8 @@ use imap_proto::receiver::Request; use sieve::compiler::ErrorType; use std::time::Instant; use store::{ - Serialize, - write::{Archiver, BatchBuilder}, + Serialize, ValueKey, + write::{AlignedBytes, Archive, Archiver, BatchBuilder}, }; use trc::AddContext; use types::{collection::Collection, field::SieveField}; @@ -102,7 +102,12 @@ impl Session { // Obtain script values let script_ = self .server - .archive(account_id, Collection::SieveScript, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::SieveScript, + document_id, + )) .await .caused_by(trc::location!())? .ok_or_else(|| { diff --git a/crates/managesieve/src/op/renamescript.rs b/crates/managesieve/src/op/renamescript.rs index cd4bf293..1602532b 100644 --- a/crates/managesieve/src/op/renamescript.rs +++ b/crates/managesieve/src/op/renamescript.rs @@ -10,7 +10,10 @@ use directory::Permission; use email::sieve::SieveScript; use imap_proto::receiver::Request; use std::time::Instant; -use store::write::BatchBuilder; +use store::{ + ValueKey, + write::{AlignedBytes, Archive, BatchBuilder}, +}; use trc::AddContext; use types::collection::Collection; @@ -58,7 +61,12 @@ impl Session { // Obtain script values let script = self .server - .archive(account_id, Collection::SieveScript, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::SieveScript, + document_id, + )) .await .caused_by(trc::location!())? .ok_or_else(|| { diff --git a/crates/migration/src/addressbook_v2.rs b/crates/migration/src/addressbook_v2.rs index e94b300f..fdf675e2 100644 --- a/crates/migration/src/addressbook_v2.rs +++ b/crates/migration/src/addressbook_v2.rs @@ -43,7 +43,12 @@ pub(crate) async fn migrate_addressbook_v013(server: &Server, account_id: u32) - for document_id in document_ids.iter() { let Some(archive) = server - .archive(account_id, Collection::AddressBook, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::AddressBook, + document_id, + )) .await .caused_by(trc::location!())? else { diff --git a/crates/migration/src/calendar_v2.rs b/crates/migration/src/calendar_v2.rs index 129a69a0..b567bda8 100644 --- a/crates/migration/src/calendar_v2.rs +++ b/crates/migration/src/calendar_v2.rs @@ -74,7 +74,12 @@ pub(crate) async fn migrate_calendar_v013(server: &Server, account_id: u32) -> t for document_id in document_ids.iter() { let Some(archive) = server - .archive(account_id, Collection::Calendar, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::Calendar, + document_id, + )) .await .caused_by(trc::location!())? else { diff --git a/crates/migration/src/contact_v2.rs b/crates/migration/src/contact_v2.rs index 5eaa3f45..ef960693 100644 --- a/crates/migration/src/contact_v2.rs +++ b/crates/migration/src/contact_v2.rs @@ -37,7 +37,12 @@ pub(crate) async fn migrate_contacts_v013(server: &Server, account_id: u32) -> t for document_id in document_ids.iter() { let Some(archive) = server - .archive(account_id, Collection::ContactCard, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::ContactCard, + document_id, + )) .await .caused_by(trc::location!())? else { diff --git a/crates/migration/src/event_v1.rs b/crates/migration/src/event_v1.rs index 7e94ce79..8bca8264 100644 --- a/crates/migration/src/event_v1.rs +++ b/crates/migration/src/event_v1.rs @@ -89,7 +89,12 @@ pub(crate) async fn migrate_calendar_events_v012(server: &Server) -> trc::Result for document_id in document_ids.iter() { let Some(archive) = server - .archive(account_id, Collection::CalendarEvent, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::CalendarEvent, + document_id, + )) .await .caused_by(trc::location!())? else { diff --git a/crates/migration/src/event_v2.rs b/crates/migration/src/event_v2.rs index 5ce20f40..4beab376 100644 --- a/crates/migration/src/event_v2.rs +++ b/crates/migration/src/event_v2.rs @@ -77,7 +77,12 @@ pub(crate) async fn migrate_calendar_events_v013( for document_id in document_ids.iter() { let Some(archive) = server - .archive(account_id, Collection::CalendarEvent, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::CalendarEvent, + document_id, + )) .await .caused_by(trc::location!())? else { @@ -151,11 +156,12 @@ pub(crate) async fn migrate_calendar_scheduling_v013( for document_id in document_ids.iter() { let Some(archive) = server - .archive( + .store() + .get_value::>(ValueKey::archive( account_id, Collection::CalendarEventNotification, document_id, - ) + )) .await .caused_by(trc::location!())? else { diff --git a/crates/migration/src/push_v2.rs b/crates/migration/src/push_v2.rs index 284e46ad..ea6e842d 100644 --- a/crates/migration/src/push_v2.rs +++ b/crates/migration/src/push_v2.rs @@ -36,7 +36,12 @@ pub(crate) async fn migrate_push_subscriptions_v013( for push_id in &push_ids { match server - .archive(account_id, Collection::PushSubscription, push_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::PushSubscription, + push_id, + )) .await { Ok(Some(legacy)) => match legacy.deserialize_untrusted::() { diff --git a/crates/migration/src/sieve_v2.rs b/crates/migration/src/sieve_v2.rs index 2036176d..b47640f2 100644 --- a/crates/migration/src/sieve_v2.rs +++ b/crates/migration/src/sieve_v2.rs @@ -32,7 +32,12 @@ pub(crate) async fn migrate_sieve_v013(server: &Server, account_id: u32) -> trc: for script_id in &script_ids { match server - .archive(account_id, Collection::SieveScript, script_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::SieveScript, + script_id, + )) .await { Ok(Some(legacy)) => match legacy.deserialize_untrusted::() { diff --git a/crates/nlp/Cargo.toml b/crates/nlp/Cargo.toml index db732c5a..a3535196 100644 --- a/crates/nlp/Cargo.toml +++ b/crates/nlp/Cargo.toml @@ -15,10 +15,10 @@ jieba-rs = "0.8" # Chinese stemmer lru-cache = "0.1.2" parking_lot = "0.12.1" psl = "2" -radix_trie = "0.3" maplit = "1.0.2" hashify = "0.2.1" rand = "0.9.2" +rkyv = { version = "0.8.10", features = ["little_endian"] } [features] test_mode = [] diff --git a/crates/nlp/src/classifier/feature.rs b/crates/nlp/src/classifier/feature.rs index 39369df6..345a634d 100644 --- a/crates/nlp/src/classifier/feature.rs +++ b/crates/nlp/src/classifier/feature.rs @@ -22,8 +22,6 @@ pub struct FeatureBuilder { pub trait Feature { fn prefix(&self) -> u16; fn value(&self) -> &[u8]; - fn is_global_feature(&self) -> bool; - fn is_local_feature(&self) -> bool; } impl FeatureBuilder { @@ -47,17 +45,13 @@ impl FeatureBuilder { buf.extend_from_slice(&feature.prefix().to_be_bytes()); buf.extend_from_slice(feature.value()); - if feature.is_global_feature() { - let big_hash = xxh3_64_with_seed(&buf, 0); - let hash = big_hash as u32 & self.features_mask; - let sign = if big_hash & (1 << 63) == 0 { 1.0 } else { -1.0 }; + let big_hash = xxh3_64_with_seed(&buf, 0); + let hash = big_hash as u32 & self.features_mask; + let sign = if big_hash & (1 << 63) == 0 { 1.0 } else { -1.0 }; - *features_map.entry(hash).or_default() += sign * count; - } + *features_map.entry(hash).or_default() += sign * count; - if feature.is_local_feature() - && let Some(account_id) = account_id - { + if let Some(account_id) = account_id { buf.extend_from_slice(&account_id.to_be_bytes()); let big_hash = xxh3_64_with_seed(&buf, 0); let hash = big_hash as u32 & self.features_mask; diff --git a/crates/nlp/src/classifier/sgd.rs b/crates/nlp/src/classifier/sgd.rs index e218d374..ed964b51 100644 --- a/crates/nlp/src/classifier/sgd.rs +++ b/crates/nlp/src/classifier/sgd.rs @@ -7,39 +7,33 @@ use crate::classifier::feature::{FeatureBuilder, Features, Sample}; use rand::{SeedableRng, rngs::StdRng, seq::SliceRandom}; -#[derive(Default)] -pub struct SGDClassifier { +#[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default)] +pub struct TextClassifier { weights: Vec, intercept: f32, - n_epochs: usize, - alpha: f32, - random_state: u64, } const MAX_DLOSS: f32 = 1e4; -impl SGDClassifier { - pub fn new(n_features: usize, n_epochs: usize, alpha: f32, random_state: u64) -> Self { - SGDClassifier { +impl TextClassifier { + pub fn new(n_features: usize) -> Self { + TextClassifier { weights: vec![0.0; n_features], - n_epochs, - random_state, - alpha, intercept: 0.0, } } - pub fn fit(&mut self, samples: &mut [impl AsRef]) { - let mut rng = StdRng::seed_from_u64(self.random_state); + pub fn fit(&mut self, samples: &mut [impl AsRef], n_epochs: usize, alpha: f32) { + let mut rng = StdRng::seed_from_u64(42); let mut t = 1; let mut w_scale = 1.0; // Heuristic to initialize 'optimal' learning rate - let typw = (1.0 / self.alpha.sqrt()).sqrt(); + let typw = (1.0 / alpha.sqrt()).sqrt(); let initial_eta0 = typw / 1.0_f32.max(gradient(1.0, -typw)); - let optimal_init = 1.0 / (initial_eta0 * self.alpha); + let optimal_init = 1.0 / (initial_eta0 * alpha); - for _ in 0..self.n_epochs { + for _ in 0..n_epochs { samples.shuffle(&mut rng); for sample in samples.iter() { @@ -50,13 +44,13 @@ impl SGDClassifier { dot += self.weights[*idx as usize] * *feature; } let p = (dot * w_scale) + self.intercept; - let eta = 1.0 / (self.alpha * (optimal_init + (t as f32) - 1.0)); + let eta = 1.0 / (alpha * (optimal_init + (t as f32) - 1.0)); // Compute Loss & Gradient let dloss = gradient(sample.class, p).clamp(-MAX_DLOSS, MAX_DLOSS); // Lazy weight decay - w_scale *= 1.0 - (eta * self.alpha); + w_scale *= 1.0 - (eta * alpha); // Update weights let update = -eta * dloss; @@ -165,7 +159,7 @@ fn log1pexp(x: f32) -> f32 { pub mod tests { use crate::classifier::{ feature::{Feature, Sample}, - sgd::SGDClassifier, + sgd::TextClassifier, }; use rand::{SeedableRng, rngs::StdRng, seq::SliceRandom}; use std::{ @@ -274,14 +268,6 @@ pub mod tests { fn value(&self) -> &[u8] { self.as_bytes() } - - fn is_global_feature(&self) -> bool { - true - } - - fn is_local_feature(&self) -> bool { - false - } } #[test] @@ -292,7 +278,7 @@ pub mod tests { ); let mut samples = Vec::with_capacity(1024); - let mut model = SGDClassifier::new(1 << 20, 1000, 0.0001, 42); + let mut model = TextClassifier::new(1 << 20); let builder = model.feature_builder(); let time = Instant::now(); @@ -329,7 +315,7 @@ pub mod tests { println!("Training SGD Classifier..."); let time = Instant::now(); - model.fit(&mut train_samples); + model.fit(&mut train_samples, 1000, 0.0001); println!("SGD Classifier trained in {:?}", time.elapsed()); let y_pred = model.predict_batch(test_samples.iter().map(|s| &s.features)); diff --git a/crates/pop3/src/op/fetch.rs b/crates/pop3/src/op/fetch.rs index bb292ab6..4e396075 100644 --- a/crates/pop3/src/op/fetch.rs +++ b/crates/pop3/src/op/fetch.rs @@ -9,6 +9,10 @@ use common::listener::SessionStream; use directory::Permission; use email::message::metadata::MessageMetadata; use std::time::Instant; +use store::{ + ValueKey, + write::{AlignedBytes, Archive}, +}; use trc::AddContext; use types::{collection::Collection, field::EmailField}; use utils::chained_bytes::ChainedBytes; @@ -25,12 +29,13 @@ impl Session { if let Some(message) = mailbox.messages.get(msg.saturating_sub(1) as usize) { if let Some(metadata_) = self .server - .archive_by_property( + .store() + .get_value::>(ValueKey::property( mailbox.account_id, Collection::Email, message.id, - EmailField::Metadata.into(), - ) + EmailField::Metadata, + )) .await .caused_by(trc::location!())? { diff --git a/crates/services/src/broadcast/mod.rs b/crates/services/src/broadcast/mod.rs index c41f5a68..678dbdab 100644 --- a/crates/services/src/broadcast/mod.rs +++ b/crates/services/src/broadcast/mod.rs @@ -90,6 +90,9 @@ impl BroadcastBatch> { serialized.push(7u8); let _ = serialized.write_leb128(*account_id); } + BroadcastEvent::ReloadSpamFilter => { + serialized.push(8u8); + } } } serialized @@ -186,6 +189,8 @@ where Ok(Some(BroadcastEvent::ReloadPushServers(account_id))) } + 8 => Ok(Some(BroadcastEvent::ReloadSpamFilter)), + _ => Err(()), } } else { diff --git a/crates/services/src/broadcast/subscriber.rs b/crates/services/src/broadcast/subscriber.rs index ae958fba..6dce0a7a 100644 --- a/crates/services/src/broadcast/subscriber.rs +++ b/crates/services/src/broadcast/subscriber.rs @@ -189,6 +189,14 @@ pub fn spawn_broadcast_subscriber(inner: Arc, mut shutdown_rx: watch::Rec ); } } + BroadcastEvent::ReloadSpamFilter => { + if let Err(err) = inner.build_server().spam_model_reload().await { + trc::error!( + err.details("Failed to reload spam filter model") + .caused_by(trc::location!()) + ); + } + } } } Ok(None) => break, @@ -264,5 +272,6 @@ fn log_event(event: &BroadcastEvent) -> trc::Value { BroadcastEvent::ReloadPushServers(account_id) => { trc::Value::Array(vec!["ReloadPushServers".into(), (*account_id).into()]) } + BroadcastEvent::ReloadSpamFilter => CompactString::const_new("ReloadSpamFilter").into(), } } diff --git a/crates/services/src/housekeeper/mod.rs b/crates/services/src/housekeeper/mod.rs index 1b071890..2daf12b5 100644 --- a/crates/services/src/housekeeper/mod.rs +++ b/crates/services/src/housekeeper/mod.rs @@ -12,6 +12,7 @@ use common::{ }; use email::message::delete::EmailDeletion; use smtp::reporting::SmtpReporting; +use spam_filter::modules::classifier::SpamClassifier; use std::{ collections::BinaryHeap, future::Future, @@ -55,6 +56,7 @@ enum ActionClass { #[cfg(feature = "enterprise")] RenewLicense, // SPDX-SnippetEnd + TrainSpamClassifier, } #[derive(Default)] @@ -98,6 +100,22 @@ pub fn spawn_housekeeper(inner: Arc, mut rx: mpsc::Receiver, mut rx: mpsc::Receiver { + if server + .core + .network + .roles + .spam_training + .is_enabled_or_sharded() + && let Some(train_frequency) = server + .core + .spam + .classifier + .as_ref() + .and_then(|c| c.train_frequency) + { + trc::event!( + Housekeeper(trc::HousekeeperEvent::Run), + Type = "spam_classifier_train" + ); + + // Schedule next training + queue.schedule( + Instant::now() + train_frequency, + ActionClass::TrainSpamClassifier, + ); + + let server = server.clone(); + tokio::spawn(async move { + if let Err(err) = server.spam_train(false).await { + trc::error!( + err.details("Failed to train spam classifier") + ); + } + }); + } + } // SPDX-SnippetBegin // SPDX-FileCopyrightText: 2020 Stalwart Labs LLC diff --git a/crates/services/src/state_manager/push.rs b/crates/services/src/state_manager/push.rs index 17015d94..bdebedcd 100644 --- a/crates/services/src/state_manager/push.rs +++ b/crates/services/src/state_manager/push.rs @@ -18,8 +18,9 @@ use std::{ time::{Duration, Instant}, }; use store::{ + ValueKey, ahash::{AHashMap, AHashSet}, - write::now, + write::{AlignedBytes, Archive, now}, }; use tokio::sync::mpsc; use trc::{AddContext, PushSubscriptionEvent, ServerEvent}; @@ -488,12 +489,13 @@ async fn load_push_subscriptions( .collect::>(); if let Some(push_subscriptions) = server - .archive_by_property( + .store() + .get_value::>(ValueKey::property( account_id, Collection::Principal, 0, - PrincipalField::PushSubscriptions.into(), - ) + PrincipalField::PushSubscriptions, + )) .await? { push_subscriptions diff --git a/crates/services/src/task_manager/alarm.rs b/crates/services/src/task_manager/alarm.rs index 42ce0f5d..0954eb86 100644 --- a/crates/services/src/task_manager/alarm.rs +++ b/crates/services/src/task_manager/alarm.rs @@ -31,7 +31,10 @@ use mail_parser::decoders::html::html_to_text; use smtp::core::{Session, SessionData}; use smtp_proto::{MailFrom, RcptTo}; use std::{str::FromStr, sync::Arc, time::Duration}; -use store::write::{BatchBuilder, now}; +use store::{ + ValueKey, + write::{AlignedBytes, Archive, BatchBuilder, now}, +}; use trc::{AddContext, TaskQueueEvent}; use types::collection::Collection; use utils::{sanitize_email, template::Variables}; @@ -121,7 +124,12 @@ async fn send_email_alarm( // Fetch event let Some(event_) = server - .archive(account_id, Collection::CalendarEvent, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::CalendarEvent, + document_id, + )) .await .caused_by(trc::location!())? else { @@ -310,7 +318,12 @@ async fn send_display_alarm( ) -> trc::Result { // Fetch event let Some(event_) = server - .archive(account_id, Collection::CalendarEvent, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::CalendarEvent, + document_id, + )) .await .caused_by(trc::location!())? else { diff --git a/crates/services/src/task_manager/index.rs b/crates/services/src/task_manager/index.rs index 6a2f1f10..96df283b 100644 --- a/crates/services/src/task_manager/index.rs +++ b/crates/services/src/task_manager/index.rs @@ -17,11 +17,13 @@ use email::{cache::MessageCacheFetch, message::metadata::MessageMetadata}; use groupware::{cache::GroupwareCache, calendar::CalendarEvent, contact::ContactCard}; use std::cmp::Ordering; use store::{ - SerializeInfallible, + SerializeInfallible, ValueKey, ahash::AHashMap, roaring::RoaringBitmap, search::{IndexDocument, SearchField, SearchFilter, SearchQuery}, - write::{BatchBuilder, SearchIndex, TaskEpoch, TaskQueueClass, ValueClass}, + write::{ + AlignedBytes, Archive, BatchBuilder, SearchIndex, TaskEpoch, TaskQueueClass, ValueClass, + }, }; use trc::{AddContext, TaskQueueEvent}; use types::{ @@ -402,12 +404,13 @@ async fn build_email_document( }; match server - .archive_by_property( + .store() + .get_value::>(ValueKey::property( account_id, Collection::Email, document_id, - EmailField::Metadata.into(), - ) + EmailField::Metadata, + )) .await? { Some(metadata_) => { @@ -448,7 +451,12 @@ async fn build_calendar_document( }; match server - .archive(account_id, Collection::CalendarEvent, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::CalendarEvent, + document_id, + )) .await? { Some(metadata_) => Ok(Some( @@ -476,7 +484,12 @@ async fn build_contact_document( }; match server - .archive(account_id, Collection::ContactCard, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::ContactCard, + document_id, + )) .await? { Some(metadata_) => Ok(Some( @@ -520,12 +533,13 @@ async fn delete_email_metadata( document_id: u32, ) -> trc::Result<()> { match server - .archive_by_property( + .store() + .get_value::>(ValueKey::property( account_id, Collection::Email, document_id, - EmailField::Metadata.into(), - ) + EmailField::Metadata, + )) .await? { Some(metadata_) => { diff --git a/crates/services/src/task_manager/merge_threads.rs b/crates/services/src/task_manager/merge_threads.rs index 7f8f6b35..4ff1a9eb 100644 --- a/crates/services/src/task_manager/merge_threads.rs +++ b/crates/services/src/task_manager/merge_threads.rs @@ -14,7 +14,10 @@ use store::{ IndexKeyPrefix, IterateParams, U32_LEN, ValueKey, ahash::{AHashMap, AHashSet}, rand::Rng, - write::{BatchBuilder, IndexPropertyClass, ValueClass, key::DeserializeBigEndian}, + write::{ + AlignedBytes, Archive, BatchBuilder, IndexPropertyClass, ValueClass, + key::DeserializeBigEndian, + }, }; use trc::AddContext; use types::{ @@ -132,7 +135,12 @@ async fn merge_threads( if thread_id != group_thread_id { for &document_id in document_ids { if let Some(data_) = server - .archive(account_id, Collection::Email, document_id) + .store() + .get_value::>(ValueKey::archive( + account_id, + Collection::Email, + document_id, + )) .await .caused_by(trc::location!())? { diff --git a/crates/smtp/src/queue/spool.rs b/crates/smtp/src/queue/spool.rs index c65fc2ef..5af99828 100644 --- a/crates/smtp/src/queue/spool.rs +++ b/crates/smtp/src/queue/spool.rs @@ -25,9 +25,9 @@ use store::write::key::DeserializeBigEndian; use store::write::serialize::rkyv_deserialize; use store::write::{ AlignedBytes, Archive, Archiver, BatchBuilder, BlobLink, BlobOp, MergeResult, Params, - QueueClass, TaskEpoch, TaskQueueClass, ValueClass, now, + QueueClass, ValueClass, now, }; -use store::{Deserialize, IterateParams, Serialize, SerializeInfallible, U64_LEN, ValueKey}; +use store::{Deserialize, IterateParams, Serialize, U64_LEN, ValueKey}; use trc::{AddContext, ServerEvent}; use types::blob_hash::BlobHash; use utils::DomainPart; @@ -352,7 +352,8 @@ impl MessageWrapper { // Reserve and write blob let mut batch = BatchBuilder::new(); - let reserve_until = now() + if !train_spam { 120 } else { 86400 }; + let now = now(); + let reserve_until = now + 120; batch.set( BlobOp::Link { hash: self.message.blob_hash.clone(), @@ -438,25 +439,33 @@ impl MessageWrapper { ); } - if !train_spam { - batch.clear(BlobOp::Link { + if train_spam && let Some(config) = &server.core.spam.classifier { + let hold_period = now + config.hold_samples_for; + + batch + .set( + BlobOp::Link { + hash: self.message.blob_hash.clone(), + to: BlobLink::Temporary { until: hold_period }, + }, + vec![BlobLink::SPAM_SAMPLE_LINK], + ) + .set( + BlobOp::SpamSample { + hash: self.message.blob_hash.clone(), + until: hold_period, + }, + vec![1, 1], + ); + } + + batch + .clear(BlobOp::Link { hash: self.message.blob_hash.clone(), to: BlobLink::Temporary { until: reserve_until, }, - }); - } else { - batch.set( - ValueClass::TaskQueue(TaskQueueClass::SpamTrain { - due: TaskEpoch::now(), - blob_hash: self.message.blob_hash.clone(), - learn_spam: true, - }), - reserve_until.serialize(), - ); - } - - batch + }) .set( BlobOp::Link { hash: self.message.blob_hash.clone(), diff --git a/crates/spam-filter/Cargo.toml b/crates/spam-filter/Cargo.toml index 5a9559c8..652fe983 100644 --- a/crates/spam-filter/Cargo.toml +++ b/crates/spam-filter/Cargo.toml @@ -26,6 +26,7 @@ infer = "0.19" sha1 = "0.10" sha2 = "0.10.6" compact_str = "0.9.0" +rkyv = { version = "0.8.10", features = ["little_endian"] } [features] test_mode = [] diff --git a/crates/spam-filter/src/analysis/domain.rs b/crates/spam-filter/src/analysis/domain.rs index c783d70b..99c87694 100644 --- a/crates/spam-filter/src/analysis/domain.rs +++ b/crates/spam-filter/src/analysis/domain.rs @@ -57,6 +57,7 @@ impl SpamFilterAnalyzeDomain for Server { { if let Host::Name(name) = host && let Some(name) = Hostname::new(name.as_ref()).sld + && !is_trusted_domain(self, &name, ctx.input.span_id).await { domains.insert(ElementLocation::new(name, Location::HeaderReceived)); } @@ -70,6 +71,8 @@ impl SpamFilterAnalyzeDomain for Server { let host = Hostname::new(d); if host.sld.is_some() { Some(host) } else { None } }) + && !is_trusted_domain(self, mid_domain.sld_or_default(), ctx.input.span_id) + .await { domains.insert(ElementLocation::new(mid_domain.fqdn, Location::HeaderMid)); } diff --git a/crates/spam-filter/src/modules/classifier.rs b/crates/spam-filter/src/modules/classifier.rs index 456ea2bb..8bbb6a6d 100644 --- a/crates/spam-filter/src/modules/classifier.rs +++ b/crates/spam-filter/src/modules/classifier.rs @@ -5,12 +5,12 @@ */ use crate::{Email, IpParts, SpamFilterContext, TextPart, analysis::url::UrlParts}; -use common::{Server, config::spamfilter::Location}; -use core::hash; +use common::config::spamfilter::SpamClassifierModel; +use common::{Server, config::spamfilter::Location, ipc::BroadcastEvent}; use mail_auth::DmarcResult; -use mail_parser::{DateTime, MimeHeaders}; +use mail_parser::MimeHeaders; use nlp::{ - classifier::feature::Feature, + classifier::{feature::Feature, sgd::TextClassifier}, tokenizers::{ stream::{WordStemTokenizer, symbols}, types::TokenType, @@ -20,34 +20,226 @@ use std::{ borrow::Cow, collections::{HashMap, hash_map::Entry}, hash::{Hash, RandomState}, + sync::Arc, +}; +use store::write::{BlobLink, now}; +use store::{ + IterateParams, Serialize, U32_LEN, U64_LEN, ValueKey, + write::{ + AlignedBytes, Archive, Archiver, BatchBuilder, BlobOp, ValueClass, + key::DeserializeBigEndian, + }, }; -use store::write::{BatchBuilder, BlobLink, BlobOp, now}; use trc::AddContext; -use types::blob_hash::BlobHash; +use types::{blob_hash::BlobHash, collection::Collection, field::PrincipalField}; pub trait SpamClassifier { + fn spam_train(&self, retrain: bool) -> impl Future> + Send; + fn spam_classify( &self, ctx: &mut SpamFilterContext<'_>, ) -> impl Future> + Send; +} - fn spam_train(&self) -> impl Future> + Send; +struct TrainingSample { + hash: BlobHash, + account_id: u32, + is_spam: bool, + remove: Option, } impl SpamClassifier for Server { - async fn spam_train(&self) -> trc::Result<()> { + async fn spam_train(&self, retrain: bool) -> trc::Result<()> { let todo = "parse ASN and other stuff, build context properly"; + let Some(config) = &self.core.spam.classifier else { + return Ok(()); + }; - todo!() + // Fetch model + let mut model = if !retrain + && let Some(model) = self + .store() + .get_value::>(ValueKey::property( + u32::MAX, + Collection::Principal, + u32::MAX, + PrincipalField::SpamModel, + )) + .await + .and_then(|archive| match archive { + Some(archive) => archive.deserialize::().map(Some), + None => Ok(None), + }) + .caused_by(trc::location!())? + { + model + } else { + SpamClassifierModel { + classifier: TextClassifier::new(config.feature_hash_size), + ham_count: 0, + spam_count: 0, + last_sample_expiry: 0, + last_trained_at: 0, + } + }; + + // Fetch blob hashes for samples + let mut samples = Vec::new(); + let mut remove_entries = false; + let from_key = ValueKey { + account_id: 0, + collection: 0, + document_id: 0, + class: ValueClass::Blob(BlobOp::SpamSample { + hash: BlobHash::default(), + until: model.last_sample_expiry + 1, + }), + }; + let to_key = ValueKey { + account_id: u32::MAX, + collection: u8::MAX, + document_id: u32::MAX, + class: ValueClass::Blob(BlobOp::SpamSample { + hash: BlobHash::new_max(), + until: u64::MAX, + }), + }; + self.store() + .iterate( + IterateParams::new(from_key, to_key).ascending(), + |key, value| { + let until = key.deserialize_be_u64(1)?; + let account_id = key.deserialize_be_u32(U64_LEN + 1)?; + let hash = BlobHash::try_from_hash_slice( + key.get(U64_LEN + U32_LEN + 1..).ok_or_else(|| { + trc::Error::corrupted_key(key, value.into(), trc::location!()) + })?, + ) + .unwrap(); + let (Some(is_spam), Some(hold)) = (value.first(), value.get(1)) else { + return Err(trc::Error::corrupted_key( + key, + value.into(), + trc::location!(), + )); + }; + + let do_remove = *hold == 0; + samples.push(TrainingSample { + hash, + account_id, + is_spam: *is_spam == 1, + remove: do_remove.then_some(until), + }); + + remove_entries |= do_remove; + model.last_sample_expiry = until; + + Ok(true) + }, + ) + .await + .caused_by(trc::location!())?; + + if !samples.is_empty() { + let todo = "log no new samples"; + return Ok(()); + } + + // Train model + for chunk in samples.chunks(config.train_batch_size.max(10)) { + let todo = "do magic here"; + + let mut samples = Vec::with_capacity(chunk.len()); + for sample in chunk { + if sample.is_spam { + model.spam_count += 1; + } else { + model.ham_count += 1; + } + samples.push(sample); + } + + let todo = "use blocking"; + } + + // Store updated model + model.last_trained_at = now(); + let archiver = Archiver::new(model); + let mut batch = BatchBuilder::new(); + batch + .with_account_id(u32::MAX) + .with_collection(Collection::Principal) + .with_document(u32::MAX) + .set( + ValueClass::Property(PrincipalField::SpamModel.into()), + archiver.serialize().caused_by(trc::location!())?, + ); + self.store() + .write(batch.build_all()) + .await + .caused_by(trc::location!())?; + + // Reload model + let model = archiver.inner; + if model.ham_count >= config.min_ham_samples && model.spam_count >= config.min_spam_samples + { + self.inner + .data + .spam_classifier + .store(Arc::new(common::SpamClassifier { + model: model.classifier, + last_trained_at: model.last_trained_at, + })); + self.cluster_broadcast(BroadcastEvent::ReloadSpamFilter) + .await; + } + + // Remove samples marked for deletion + if remove_entries { + let mut batch = BatchBuilder::new(); + for sample in samples { + if let Some(until) = sample.remove { + batch + .with_account_id(sample.account_id) + .clear(BlobOp::Link { + hash: sample.hash.clone(), + to: BlobLink::Temporary { until }, + }) + .clear(BlobOp::SpamSample { + hash: sample.hash, + until, + }); + if batch.is_large_batch() { + self.store() + .write(batch.build_all()) + .await + .caused_by(trc::location!())?; + batch = BatchBuilder::new(); + } + } + } + if !batch.is_empty() { + self.store() + .write(batch.build_all()) + .await + .caused_by(trc::location!())?; + } + } + + Ok(()) } async fn spam_classify(&self, ctx: &mut SpamFilterContext<'_>) -> trc::Result<()> { let classifier = self.inner.data.spam_classifier.load_full(); + let model = &classifier.model; - if classifier.is_active() { + if model.is_active() { let mut classifier_confidence = Vec::with_capacity(ctx.input.env_rcpt_to.len()); + let mut has_prediction = false; let mut tokens = ctx.classifier_tokens().0; - let feature_builder = classifier.feature_builder(); + let feature_builder = model.feature_builder(); feature_builder.scale(&mut tokens); for rcpt in &ctx.input.env_rcpt_to { @@ -57,7 +249,8 @@ impl SpamClassifier for Server { .await .caused_by(trc::location!())? { - classifier + has_prediction = true; + model .predict(&feature_builder.build(&tokens, account_id.into())) .into() } else { @@ -66,7 +259,14 @@ impl SpamClassifier for Server { classifier_confidence.push(prediction); } - ctx.result.classifier_confidence = classifier_confidence; + if has_prediction { + ctx.result.classifier_confidence = classifier_confidence; + } else { + // None of the recipients are local, default to global model prediction + let prediction = model.predict(&feature_builder.build(&tokens, None)); + ctx.result.classifier_confidence = + vec![prediction.into(); ctx.input.env_rcpt_to.len()]; + } } Ok(()) @@ -90,6 +290,7 @@ pub enum Token<'x> { Email { value: Cow<'x, str> }, Hostname { value: &'x str }, Attachment { value: Cow<'x, str> }, + MimeType { value: String }, } #[derive(Debug)] @@ -168,35 +369,40 @@ impl<'x> SpamFilterContext<'x> { }); } - // Add attachment indicators + // Add MIME and attachment indicators for part in &self.input.message.parts { - if let Some(name) = part.attachment_name() { - if let Some((name, ext)) = name.rsplit_once('.') { - if !ext.is_empty() { + if let Some(name) = part.attachment_name() + && let Some((name, ext)) = name.rsplit_once('.') + { + if !ext.is_empty() { + tokens.insert(Token::Attachment { + value: lower_prefix("!", truncate_word(ext, MAX_TOKEN_LENGTH)).into(), + }); + } + for token in name.split(|c: char| !c.is_alphanumeric()) { + if token.len() > 1 { tokens.insert(Token::Attachment { - value: lower_prefix("!", truncate_word(ext, MAX_TOKEN_LENGTH)).into(), + value: lower_prefix("_", truncate_word(token, MAX_TOKEN_LENGTH)).into(), }); } - for token in name.split(|c: char| !c.is_alphanumeric()) { - if token.len() > 1 { - tokens.insert(Token::Attachment { - value: lower_prefix("_", truncate_word(token, MAX_TOKEN_LENGTH)) - .into(), - }); - } + } + } + + if let Some(ct) = part.content_type() { + let mut ct_lower = String::with_capacity( + ct.c_type.len() + ct.c_subtype.as_ref().map_or(0, |s| s.len()), + ); + for ch in ct.c_type.chars() { + ct_lower.push(ch.to_ascii_lowercase()); + } + if let Some(st) = &ct.c_subtype { + ct_lower.push('/'); + for ch in st.chars() { + ct_lower.push(ch.to_ascii_lowercase()); } } - if let Some(ct) = part.content_type() { - tokens.insert(Token::Attachment { - value: ct.c_type.as_ref().into(), - }); - if let Some(st) = &ct.c_subtype { - tokens.insert(Token::Attachment { - value: st.as_ref().into(), - }); - } - } + tokens.insert(Token::MimeType { value: ct_lower }); } } @@ -506,6 +712,7 @@ impl Feature for Token<'_> { Token::Email { .. } => 7, Token::Hostname { .. } => 8, Token::Attachment { .. } => 9, + Token::MimeType { .. } => 10, } } @@ -521,24 +728,9 @@ impl Feature for Token<'_> { Token::Email { value } => value.as_bytes(), Token::Hostname { value } => value.as_bytes(), Token::Attachment { value } => value.as_bytes(), + Token::MimeType { value } => value.as_bytes(), } } - - fn is_global_feature(&self) -> bool { - matches!( - self, - Token::Sender { .. } - | Token::Asn { .. } - | Token::Url { .. } - | Token::Email { .. } - | Token::Hostname { .. } - | Token::Attachment { .. } - ) - } - - fn is_local_feature(&self) -> bool { - true - } } #[derive(Debug, PartialEq, Eq, Clone, Copy)] diff --git a/crates/store/src/write/blob.rs b/crates/store/src/write/blob.rs index cff5cb18..e0fafa38 100644 --- a/crates/store/src/write/blob.rs +++ b/crates/store/src/write/blob.rs @@ -135,94 +135,19 @@ impl Store { to: BlobLink::Document, }), }; - const TEMP_LINK: usize = BLOB_HASH_LEN + U32_LEN + U64_LEN; - const DOC_LINK: usize = BLOB_HASH_LEN + U64_LEN + 1; - let mut last_hash = BlobHash::default(); - let mut last_hash_is_linked = true; // Avoid deleting non-existing last_hash on first iteration - let mut delete_keys = Vec::new(); - let now = now(); + let mut state = BlobPurgeState::new(); self.iterate( IterateParams::new(from_key, to_key).ascending(), |key, value| { - let hash = BlobHash::try_from_hash_slice( - key.get(0..BLOB_HASH_LEN) - .ok_or_else(|| trc::Error::corrupted_key(key, None, trc::location!()))?, - ) - .unwrap(); + let hash = + BlobHash::try_from_hash_slice(key.get(0..BLOB_HASH_LEN).ok_or_else(|| { + trc::Error::corrupted_key(key, value.into(), trc::location!()) + })?) + .unwrap(); - if last_hash != hash { - if !last_hash_is_linked { - delete_keys.push(( - None, - BlobOp::Commit { - hash: std::mem::replace(&mut last_hash, hash), - }, - )); - } else { - last_hash = hash; - } - last_hash_is_linked = false; - } - - match key.len() { - BLOB_HASH_LEN => { - // Main blob entry - } - TEMP_LINK => { - // Temporary link - let until = key.deserialize_be_u64(BLOB_HASH_LEN + U32_LEN)?; - if until <= now { - let account_id = key.deserialize_be_u32(BLOB_HASH_LEN)?; - delete_keys.push(( - Some(account_id), - BlobOp::Link { - hash: last_hash.clone(), - to: BlobLink::Temporary { until }, - }, - )); - match value.first().copied() { - Some(BlobLink::QUOTA_LINK) => { - delete_keys.push(( - Some(account_id), - BlobOp::Quota { - hash: last_hash.clone(), - until, - }, - )); - } - Some(BlobLink::UNDELETE_LINK) => { - delete_keys.push(( - Some(account_id), - BlobOp::Undelete { - hash: last_hash.clone(), - until, - }, - )); - } - Some(BlobLink::SPAM_SAMPLE_LINK) => { - delete_keys.push(( - Some(account_id), - BlobOp::SpamSample { - hash: last_hash.clone(), - until, - }, - )); - } - _ => {} - } - } else { - last_hash_is_linked = true; - } - } - DOC_LINK => { - // Document link - last_hash_is_linked = true; - } - _ => { - return Err(trc::Error::corrupted_key(key, None, trc::location!())); - } - } + state.update_hash(hash); + state.process_key(key, value)?; Ok(true) }, @@ -230,12 +155,10 @@ impl Store { .await .caused_by(trc::location!())?; - if !last_hash_is_linked { - delete_keys.push((None, BlobOp::Commit { hash: last_hash })); - } + state.finalize(BlobHash::default()); // Delete expired or unlinked blobs - for (_, op) in &delete_keys { + for (_, op) in &state.delete_keys { if let BlobOp::Commit { hash } = op { blob_store .delete_blob(hash.as_ref()) @@ -246,7 +169,7 @@ impl Store { // Delete hashes let mut batch = BatchBuilder::new(); - for (account_id, op) in delete_keys { + for (account_id, op) in state.delete_keys { if batch.is_large_batch() { self.write(batch.build_all()) .await @@ -272,3 +195,156 @@ impl Store { Ok(()) } } + +struct BlobPurgeState { + last_hash: BlobHash, + last_hash_is_linked: bool, + delete_keys: Vec<(Option, BlobOp)>, + spam_train_samples: Vec<(u32, u64)>, + now: u64, +} + +impl BlobPurgeState { + fn new() -> Self { + Self { + last_hash: BlobHash::default(), + last_hash_is_linked: true, // Avoid deleting non-existing last_hash on first iteration + delete_keys: Vec::new(), + spam_train_samples: Vec::new(), + now: now(), + } + } + + pub fn update_hash(&mut self, hash: BlobHash) { + if self.last_hash != hash { + self.finalize(hash); + self.last_hash_is_linked = false; + } + } + + pub fn finalize(&mut self, new_hash: BlobHash) { + if !self.last_hash_is_linked { + self.delete_keys.push(( + None, + BlobOp::Commit { + hash: std::mem::replace(&mut self.last_hash, new_hash), + }, + )); + } else { + if !self.spam_train_samples.is_empty() { + if self.spam_train_samples.len() > 1 { + // Sort by account_id ascending, then until descending + self.spam_train_samples + .sort_unstable_by(|(a_id, a_until), (b_id, b_until)| { + a_id.cmp(b_id).then_with(|| b_until.cmp(a_until)) + }); + let mut samples = self.spam_train_samples.iter().peekable(); + while let Some((account_id, until)) = samples.next() { + // Keep only the latest sample per account + let mut last_until = *until; + while let Some((next_account_id, next_until)) = samples.peek() { + if next_account_id == account_id { + self.delete_keys.push(( + Some(*account_id), + BlobOp::SpamSample { + hash: self.last_hash.clone(), + until: last_until, + }, + )); + self.delete_keys.push(( + Some(*account_id), + BlobOp::Link { + hash: self.last_hash.clone(), + to: BlobLink::Temporary { until: last_until }, + }, + )); + samples.next(); + last_until = *next_until; + } else { + break; + } + } + } + } + + self.spam_train_samples.clear(); + } + self.last_hash = new_hash; + } + } + + pub fn process_key(&mut self, key: &[u8], value: &[u8]) -> trc::Result<()> { + const TEMP_LINK: usize = BLOB_HASH_LEN + U32_LEN + U64_LEN; + const DOC_LINK: usize = BLOB_HASH_LEN + U64_LEN + 1; + + match key.len() { + BLOB_HASH_LEN => { + // Main blob entry + Ok(()) + } + TEMP_LINK => { + // Temporary link + let until = key.deserialize_be_u64(BLOB_HASH_LEN + U32_LEN)?; + if until <= self.now { + let account_id = key.deserialize_be_u32(BLOB_HASH_LEN)?; + self.delete_keys.push(( + Some(account_id), + BlobOp::Link { + hash: self.last_hash.clone(), + to: BlobLink::Temporary { until }, + }, + )); + match value.first().copied() { + Some(BlobLink::QUOTA_LINK) => { + self.delete_keys.push(( + Some(account_id), + BlobOp::Quota { + hash: self.last_hash.clone(), + until, + }, + )); + } + Some(BlobLink::UNDELETE_LINK) => { + self.delete_keys.push(( + Some(account_id), + BlobOp::Undelete { + hash: self.last_hash.clone(), + until, + }, + )); + } + Some(BlobLink::SPAM_SAMPLE_LINK) => { + self.delete_keys.push(( + Some(account_id), + BlobOp::SpamSample { + hash: self.last_hash.clone(), + until, + }, + )); + } + _ => {} + } + } else { + // Delete attempts to train the same message multiple times + if matches!(value.first(), Some(&BlobLink::SPAM_SAMPLE_LINK)) { + let account_id = key.deserialize_be_u32(BLOB_HASH_LEN)?; + self.spam_train_samples.push((account_id, until)); + } + + self.last_hash_is_linked = true; + } + Ok(()) + } + DOC_LINK => { + // Document link + self.last_hash_is_linked = true; + Ok(()) + } + _ => Err(trc::Error::corrupted_key( + key, + value.into(), + trc::location!(), + )), + } + } +} diff --git a/crates/store/src/write/key.rs b/crates/store/src/write/key.rs index 52d56f84..de53fd6a 100644 --- a/crates/store/src/write/key.rs +++ b/crates/store/src/write/key.rs @@ -19,7 +19,7 @@ use crate::{ write::{BlobLink, IndexPropertyClass, SearchIndex, SearchIndexId, SearchIndexType}, }; use std::convert::TryInto; -use types::{blob_hash::BLOB_HASH_LEN, collection::SyncCollection}; +use types::{blob_hash::BLOB_HASH_LEN, collection::SyncCollection, field::Field}; use utils::codec::leb128::Leb128_; pub struct KeySerializer { @@ -154,6 +154,19 @@ impl DeserializeBigEndian for &[u8] { } impl> ValueKey { + pub fn with_document_id(self, document_id: u32) -> Self { + Self { + document_id, + ..self + } + } + + pub fn is_counter(&self) -> bool { + self.class.as_ref().is_counter(self.collection) + } +} + +impl ValueKey { pub fn property( account_id: u32, collection: impl Into, @@ -168,16 +181,18 @@ impl> ValueKey { } } - pub fn with_document_id(self, document_id: u32) -> Self { - Self { + pub fn archive( + account_id: u32, + collection: impl Into, + document_id: u32, + ) -> ValueKey { + ValueKey { + account_id, + collection: collection.into(), document_id, - ..self + class: ValueClass::Property(Field::ARCHIVE.into()), } } - - pub fn is_counter(&self) -> bool { - self.class.as_ref().is_counter(self.collection) - } } impl Key for IndexKeyPrefix { diff --git a/crates/types/src/field.rs b/crates/types/src/field.rs index 3d22eb11..36031683 100644 --- a/crates/types/src/field.rs +++ b/crates/types/src/field.rs @@ -83,6 +83,7 @@ pub enum PrincipalField { DefaultAddressBookId, ActiveScriptId, PushSubscriptions, + SpamModel, } impl From for u8 { @@ -162,6 +163,7 @@ impl From for u8 { PrincipalField::DefaultAddressBookId => 48, PrincipalField::ActiveScriptId => 49, PrincipalField::PushSubscriptions => 44, + PrincipalField::SpamModel => 52, PrincipalField::Archive => ARCHIVE_FIELD, } }