From 65fe4b1b492acef327cc8542412821aece5e6195 Mon Sep 17 00:00:00 2001 From: mdecimus Date: Sat, 17 May 2025 21:48:30 +0200 Subject: [PATCH] Sequential change ids --- crates/common/src/config/inner.rs | 3 +- crates/common/src/config/jmap/settings.rs | 4 +- crates/common/src/core.rs | 108 +++- crates/common/src/listener/acme/directory.rs | 5 +- crates/common/src/listener/acme/resolver.rs | 4 +- crates/common/src/manager/restore.rs | 40 +- crates/common/src/storage/index.rs | 24 +- crates/dav/src/common/lock.rs | 9 +- crates/dav/src/common/mod.rs | 19 +- crates/dav/src/common/propfind.rs | 8 +- crates/directory/src/lib.rs | 8 +- crates/email/src/cache/email.rs | 19 +- crates/email/src/cache/mod.rs | 23 +- crates/email/src/identity/index.rs | 6 +- crates/email/src/identity/mod.rs | 8 - crates/email/src/mailbox/destroy.rs | 9 +- crates/email/src/mailbox/index.rs | 6 +- crates/email/src/mailbox/mod.rs | 8 - crates/email/src/message/copy.rs | 10 +- crates/email/src/message/crypto.rs | 8 +- crates/email/src/message/delete.rs | 60 ++- crates/email/src/message/ingest.rs | 13 +- crates/email/src/message/metadata.rs | 16 +- crates/email/src/push/mod.rs | 7 - crates/email/src/sieve/activate.rs | 9 +- crates/email/src/sieve/index.rs | 6 +- crates/email/src/sieve/ingest.rs | 24 +- crates/email/src/sieve/mod.rs | 10 +- crates/email/src/submission/index.rs | 6 +- crates/email/src/submission/mod.rs | 7 - crates/groupware/src/cache/mod.rs | 37 +- crates/groupware/src/calendar/index.rs | 12 +- crates/groupware/src/calendar/mod.rs | 13 - crates/groupware/src/contact/index.rs | 12 +- crates/groupware/src/contact/mod.rs | 13 - crates/groupware/src/file/index.rs | 9 +- crates/groupware/src/file/mod.rs | 7 - crates/http/src/auth/oauth/auth.rs | 21 +- crates/http/src/auth/oauth/token.rs | 8 +- crates/http/src/management/report.rs | 8 +- crates/imap/src/op/copy_move.rs | 15 +- crates/imap/src/op/expunge.rs | 1 - crates/imap/src/op/fetch.rs | 5 +- crates/imap/src/op/store.rs | 24 +- crates/jmap/src/email/set.rs | 22 +- crates/jmap/src/identity/set.rs | 7 +- crates/jmap/src/mailbox/set.rs | 17 +- crates/jmap/src/push/set.rs | 7 +- crates/jmap/src/sieve/set.rs | 12 +- crates/jmap/src/submission/set.rs | 7 +- crates/jmap/src/vacation/set.rs | 26 +- crates/managesieve/src/op/putscript.rs | 5 +- crates/migration/src/report.rs | 15 +- crates/services/src/broadcast/subscriber.rs | 24 +- crates/smtp/src/queue/mod.rs | 8 +- crates/smtp/src/reporting/analysis.rs | 8 +- crates/smtp/src/reporting/dmarc.rs | 29 +- crates/smtp/src/reporting/tls.rs | 14 +- .../store/src/backend/foundationdb/write.rs | 93 ++-- crates/store/src/backend/mysql/write.rs | 71 ++- crates/store/src/backend/postgres/write.rs | 65 ++- crates/store/src/backend/rocksdb/write.rs | 66 ++- crates/store/src/backend/sqlite/write.rs | 49 +- crates/store/src/dispatch/lookup.rs | 14 +- crates/store/src/dispatch/store.rs | 82 +-- crates/store/src/fts/index.rs | 5 +- crates/store/src/lib.rs | 28 - crates/store/src/query/log.rs | 6 + crates/store/src/write/assert.rs | 48 +- crates/store/src/write/batch.rs | 170 +++--- crates/store/src/write/key.rs | 7 +- crates/store/src/write/mod.rs | 123 +++-- crates/store/src/write/serialize.rs | 510 +++++++----------- crates/trc/src/event/description.rs | 2 - crates/trc/src/event/level.rs | 4 +- crates/trc/src/lib.rs | 1 - crates/trc/src/serializers/binary.rs | 4 +- crates/utils/src/snowflake.rs | 58 +- tests/src/imap/condstore.rs | 4 +- tests/src/imap/mod.rs | 4 +- tests/src/jmap/mod.rs | 4 +- tests/src/store/mod.rs | 2 +- tests/src/store/ops.rs | 97 +++- 83 files changed, 1193 insertions(+), 1197 deletions(-) diff --git a/crates/common/src/config/inner.rs b/crates/common/src/config/inner.rs index 66e55290..6efd607b 100644 --- a/crates/common/src/config/inner.rs +++ b/crates/common/src/config/inner.rs @@ -26,7 +26,7 @@ use std::{ use utils::{ cache::{Cache, CacheWithTtl}, config::Config, - snowflake::{HlcTimestamp, SnowflakeIdGenerator}, + snowflake::SnowflakeIdGenerator, }; impl Data { @@ -44,7 +44,6 @@ impl Data { .property::("cluster.node-id") .unwrap_or_else(store::rand::random); let id_generator = SnowflakeIdGenerator::with_node_id(node_id); - HlcTimestamp::init(node_id as u16); if !id_generator.is_valid() { panic!("Invalid system time, panicking to avoid data corruption"); } diff --git a/crates/common/src/config/jmap/settings.rs b/crates/common/src/config/jmap/settings.rs index 894dc8af..10254902 100644 --- a/crates/common/src/config/jmap/settings.rs +++ b/crates/common/src/config/jmap/settings.rs @@ -17,7 +17,7 @@ pub struct JmapConfig { pub snippet_max_results: usize, pub changes_max_results: Option, - pub changes_max_history: Option, + pub changes_max_history: Option, pub request_max_size: usize, pub request_max_calls: usize, @@ -245,7 +245,7 @@ impl JmapConfig { .property_or_default::>("jmap.protocol.changes.max-results", "5000") .unwrap_or_default(), changes_max_history: config - .property_or_default::>("jmap.protocol.changes.max-history", "30d") + .property_or_default::>("changes.max-history", "100000") .unwrap_or_default(), snippet_max_results: config .property("jmap.protocol.search-snippet.max-results") diff --git a/crates/common/src/core.rs b/crates/common/src/core.rs index 47d58665..7f54485d 100644 --- a/crates/common/src/core.rs +++ b/crates/common/src/core.rs @@ -22,15 +22,15 @@ use jmap_proto::types::{ type_state::DataType, }; use sieve::Sieve; -use std::{sync::Arc, time::Duration}; +use std::sync::Arc; use store::{ BitmapKey, BlobClass, BlobStore, Deserialize, FtsStore, InMemoryStore, IndexKey, IterateParams, - LogKey, SerializeInfallible, Store, U32_LEN, ValueKey, + Key, LogKey, SUBSPACE_LOGS, SerializeInfallible, Store, U32_LEN, U64_LEN, ValueKey, dispatch::DocumentSet, roaring::RoaringBitmap, write::{ - AlignedBytes, Archive, AssignedIds, BatchBuilder, BlobOp, DirectoryClass, QueueClass, - ValueClass, key::DeserializeBigEndian, now, + AlignedBytes, AnyClass, Archive, AssignedIds, BatchBuilder, BlobOp, DirectoryClass, + QueueClass, ValueClass, key::DeserializeBigEndian, now, }, }; use trc::AddContext; @@ -501,14 +501,19 @@ impl Server { pub async fn commit_batch(&self, mut builder: BatchBuilder) -> trc::Result { let mut assigned_ids = AssignedIds::default(); + let mut commit_points = builder.commit_points(); - for batch in builder.build() { - assigned_ids = self.store().write(batch).await?; + for commit_point in commit_points.iter() { + let batch = builder.build_one(commit_point); + assigned_ids + .ids + .extend(self.store().write(batch).await?.ids); } if let Some(changes) = builder.changes() { for (account_id, changed_collections) in changes { - let mut state_change = StateChange::new(account_id, changed_collections.change_id); + let mut state_change = + StateChange::new(account_id, assigned_ids.last_change_id(account_id)?); for changed_collection in changed_collections.changed_containers { if let Some(data_type) = DataType::try_from_id(changed_collection, true) { state_change.set_change(data_type); @@ -522,20 +527,13 @@ impl Server { if state_change.has_changes() { self.broadcast_state_change(state_change).await; } - assigned_ids.change_id = changed_collections.change_id.into(); } } Ok(assigned_ids) } - pub async fn delete_changes(&self, account_id: u32, before: Duration) -> trc::Result<()> { - let reference_cid = self.inner.data.jmap_id_gen.past_id(before).ok_or_else(|| { - trc::StoreEvent::UnexpectedError - .caused_by(trc::location!()) - .ctx(trc::Key::Reason, "Failed to generate reference change id.") - })?; - + pub async fn delete_changes(&self, account_id: u32, max_entries: usize) -> trc::Result<()> { for collection in [ SyncCollection::Email.into(), SyncCollection::Thread.into(), @@ -546,22 +544,74 @@ impl Server { SyncCollection::AddressBook.into(), SyncCollection::Calendar.into(), ] { - self.core - .storage - .data - .delete_range( - LogKey { - account_id, - collection, - change_id: 0, - }, - LogKey { - account_id, - collection, - change_id: reference_cid, + let from_key = LogKey { + account_id, + collection, + change_id: 0, + }; + let to_key = LogKey { + account_id, + collection, + change_id: u64::MAX, + }; + + let mut first_change_id = 0; + let mut num_changes = 0; + + self.store() + .iterate( + IterateParams::new(from_key, to_key) + .descending() + .no_values(), + |key, _| { + first_change_id = key.deserialize_be_u64(key.len() - U64_LEN)?; + num_changes += 1; + + Ok(num_changes <= max_entries) }, ) - .await?; + .await + .caused_by(trc::location!())?; + + if num_changes > max_entries { + self.store() + .delete_range( + LogKey { + account_id, + collection, + change_id: 0, + }, + LogKey { + account_id, + collection, + change_id: first_change_id, + }, + ) + .await + .caused_by(trc::location!())?; + + // Write truncation entry for cache + let mut batch = BatchBuilder::new(); + batch + .with_account_id(account_id) + .with_collection(collection) + .set( + ValueClass::Any(AnyClass { + subspace: SUBSPACE_LOGS, + key: LogKey { + account_id, + collection, + change_id: first_change_id, + } + .serialize(0), + }), + Vec::new(), + ); + self.store() + .write(batch.build_all()) + .await + .caused_by(trc::location!())?; + } } Ok(()) diff --git a/crates/common/src/listener/acme/directory.rs b/crates/common/src/listener/acme/directory.rs index 3dda1677..f095e196 100644 --- a/crates/common/src/listener/acme/directory.rs +++ b/crates/common/src/listener/acme/directory.rs @@ -16,7 +16,7 @@ use ring::signature::{ECDSA_P256_SHA256_FIXED_SIGNING, EcdsaKeyPair, EcdsaSignin use serde::Deserialize; use std::time::Duration; use store::Serialize; -use store::write::UnversionedArchiver; +use store::write::Archiver; use trc::AddContext; use trc::event::conv::AssertSuccess; @@ -188,7 +188,7 @@ impl Account { .reason(err) })?; - UnversionedArchiver::new(SerializedCert { + Archiver::new(SerializedCert { certificate: cert.serialize_der().map_err(|err| { trc::EventType::Acme(trc::AcmeEvent::Error) .caused_by(trc::location!()) @@ -196,6 +196,7 @@ impl Account { })?, private_key: cert.serialize_private_key_der(), }) + .untrusted() .serialize() } } diff --git a/crates/common/src/listener/acme/resolver.rs b/crates/common/src/listener/acme/resolver.rs index 876594b1..830671fc 100644 --- a/crates/common/src/listener/acme/resolver.rs +++ b/crates/common/src/listener/acme/resolver.rs @@ -19,7 +19,7 @@ use rustls_pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer}; use std::sync::Arc; use store::{ dispatch::lookup::KeyValue, - write::{AlignedBytes, UnversionedArchive}, + write::{AlignedBytes, Archive}, }; use trc::AcmeEvent; @@ -48,7 +48,7 @@ impl Server { pub(crate) async fn build_acme_certificate(&self, domain: &str) -> Option> { match self .in_memory_store() - .key_get::>(KeyValue::<()>::build_key(KV_ACME, domain)) + .key_get::>(KeyValue::<()>::build_key(KV_ACME, domain)) .await { Ok(Some(cert_)) => match cert_.unarchive::() { diff --git a/crates/common/src/manager/restore.rs b/crates/common/src/manager/restore.rs index b1a0530d..93989559 100644 --- a/crates/common/src/manager/restore.rs +++ b/crates/common/src/manager/restore.rs @@ -10,6 +10,7 @@ use std::{ }; use crate::Core; +use ahash::AHashMap; use jmap_proto::types::{collection::Collection, property::Property}; use store::{ BlobStore, Key, LogKey, SUBSPACE_LOGS, SerializeInfallible, Store, U32_LEN, @@ -72,6 +73,8 @@ async fn restore_file(store: Store, blob_store: BlobStore, path: &Path) { let mut batch_size = 0; let mut batch = BatchBuilder::new(); + let mut change_ids: AHashMap = AHashMap::new(); + while let Some(op) = reader.next().await { match op { Op::Family(f) => family = f, @@ -194,11 +197,6 @@ async fn restore_file(store: Store, blob_store: BlobStore, path: &Path) { .deserialize_leb128::() .expect("Failed to deserialize principal id"), ), - /*3 => DirectoryClass::Domain( - key.get(1..) - .expect("Failed to read directory string") - .to_vec(), - ),*/ 4 => { batch.add( ValueClass::Directory(DirectoryClass::UsedQuota( @@ -343,20 +341,27 @@ async fn restore_file(store: Store, blob_store: BlobStore, path: &Path) { } } Family::Log => { + let change_id = key + .as_slice() + .deserialize_be_u64(0) + .expect("Failed to deserialize change id"); + let change_ids = change_ids.entry(account_id).or_default(); + *change_ids = std::cmp::max(*change_ids, change_id); + batch.any_op(Operation::Value { class: ValueClass::Any(AnyClass { subspace: SUBSPACE_LOGS, key: LogKey { account_id, collection, - change_id: key - .as_slice() - .deserialize_be_u64(0) - .expect("Failed to deserialize change id"), + change_id, } .serialize(0), }), - op: ValueOp::Set(value), + op: ValueOp::Set { + value, + version_offset: None, + }, }); } Family::None => failed("No family specified in file"), @@ -384,6 +389,21 @@ async fn restore_file(store: Store, blob_store: BlobStore, path: &Path) { .await .failed("Failed to write batch"); } + + if !change_ids.is_empty() { + let mut batch = BatchBuilder::new(); + + for (account_id, change_id) in change_ids { + batch + .with_account_id(account_id) + .add(ValueClass::ChangeId, change_id as i64); + } + + store + .write(batch.build_all()) + .await + .failed("Failed to write batch"); + } } struct OpReader { diff --git a/crates/common/src/storage/index.rs b/crates/common/src/storage/index.rs index 9bae072e..c01225a7 100644 --- a/crates/common/src/storage/index.rs +++ b/crates/common/src/storage/index.rs @@ -13,7 +13,7 @@ use rkyv::{ }; use std::{borrow::Cow, fmt::Debug}; use store::{ - Serialize, SerializeInfallible, SerializedVersion, + Serialize, SerializeInfallible, write::{Archive, Archiver, BatchBuilder, BlobOp, DirectoryClass, IntoOperations, TagValue}, }; use utils::BlobHash; @@ -218,7 +218,6 @@ pub trait IndexableObject: Sync + Send { pub trait IndexableAndSerializableObject: IndexableObject - + SerializedVersion + rkyv::Archive + for<'a> rkyv::Serialize< rkyv::api::high::HighSerializer< @@ -228,6 +227,7 @@ pub trait IndexableAndSerializableObject: >, > { + fn is_versioned() -> bool; } #[derive(Debug)] @@ -295,7 +295,12 @@ impl IntoOperations for item in changes.index_values() { build_index(batch, item, self.tenant_id, true); } - batch.set(Property::Value, Archiver::new(changes).serialize()?); + if N::is_versioned() { + let (offset, bytes) = Archiver::new(changes).serialize_versioned()?; + batch.set_versioned(Property::Value, bytes, offset); + } else { + batch.set(Property::Value, Archiver::new(changes).serialize()?); + } } (Some(current), Some(changes)) => { // Update @@ -318,7 +323,12 @@ impl IntoOperations } } } - batch.set(Property::Value, Archiver::new(changes).serialize()?); + if N::is_versioned() { + let (offset, bytes) = Archiver::new(changes).serialize_versioned()?; + batch.set_versioned(Property::Value, bytes, offset); + } else { + batch.set(Property::Value, Archiver::new(changes).serialize()?); + } } (Some(current), None) => { // Deletion @@ -586,4 +596,8 @@ impl IndexableObject for () { } } -impl IndexableAndSerializableObject for () {} +impl IndexableAndSerializableObject for () { + fn is_versioned() -> bool { + false + } +} diff --git a/crates/dav/src/common/lock.rs b/crates/dav/src/common/lock.rs index 159a4cc6..d0c3a4d2 100644 --- a/crates/dav/src/common/lock.rs +++ b/crates/dav/src/common/lock.rs @@ -20,7 +20,7 @@ use std::collections::HashMap; use store::dispatch::lookup::KeyValue; use store::write::serialize::rkyv_deserialize; use store::write::{AlignedBytes, Archive, Archiver, now}; -use store::{SERIALIZE_DAV_LOCKS_V1, Serialize, SerializedVersion, U32_LEN}; +use store::{Serialize, U32_LEN}; use trc::AddContext; use super::ETag; @@ -318,6 +318,7 @@ impl LockRequestHandler for Server { KeyValue::new( resource_hash, Archiver::new(lock_data) + .untrusted() .serialize() .caused_by(trc::location!())?, ) @@ -753,12 +754,6 @@ impl<'x> LockCaches<'x> { } } -impl SerializedVersion for LockData { - fn serialize_version() -> u8 { - SERIALIZE_DAV_LOCKS_V1 - } -} - impl LockItem { pub fn to_active_lock(&self, href: String) -> ActiveLock { ActiveLock::new( diff --git a/crates/dav/src/common/mod.rs b/crates/dav/src/common/mod.rs index 9759d60d..4717cad0 100644 --- a/crates/dav/src/common/mod.rs +++ b/crates/dav/src/common/mod.rs @@ -27,10 +27,7 @@ use groupware::{ use jmap_proto::types::{collection::Collection, property::Property, value::ArchivedAclGrant}; use propfind::PropFindItem; use rkyv::vec::ArchivedVec; -use store::{ - U32_LEN, - write::{AlignedBytes, Archive, BatchBuilder, Operation, ValueClass, ValueOp}, -}; +use store::write::{AlignedBytes, Archive, BatchBuilder, Operation, ValueClass, ValueOp}; use uri::{OwnedUri, Urn}; pub mod acl; @@ -94,6 +91,7 @@ pub(crate) enum DavQueryFilter { pub(crate) trait ETag { fn etag(&self) -> String; + fn ctag(&self) -> String; } pub(crate) trait ExtractETag { @@ -102,7 +100,11 @@ pub(crate) trait ExtractETag { impl ETag for Archive { fn etag(&self) -> String { - format!("\"{}\"", self.hash) + format!("\"{}\"", self.version.hash().unwrap_or_default()) + } + + fn ctag(&self) -> String { + format!("\"{}\"", self.version.change_id().unwrap_or_default()) } } @@ -113,11 +115,10 @@ impl ExtractETag for BatchBuilder { match op { Operation::Value { class: ValueClass::Property(p_id), - op: ValueOp::Set(value), + op: ValueOp::Set { value, .. }, } if *p_id == p_value => { - return value - .get(value.len() - U32_LEN..) - .map(|v| format!("\"{}\"", u32::from_be_bytes(v.try_into().unwrap()))); + return Archive::::extract_hash(value) + .map(|hash| format!("\"{}\"", hash)); } _ => {} } diff --git a/crates/dav/src/common/propfind.rs b/crates/dav/src/common/propfind.rs index a07e0600..170f8969 100644 --- a/crates/dav/src/common/propfind.rs +++ b/crates/dav/src/common/propfind.rs @@ -875,15 +875,9 @@ impl PropFindRequestHandler for Server { } WebDavProperty::GetCTag => { if item.is_container { - let ctag = data - .resources(self, access_token, account_id, sync_collection) - .await - .caused_by(trc::location!())? - .highest_change_id; - fields.push(DavPropertyValue::new( property.clone(), - DavValue::String(format!("\"{ctag}\"")), + DavValue::String(archive_.ctag()), )); } else { fields_not_found.push(DavPropertyValue::empty(property.clone())); diff --git a/crates/directory/src/lib.rs b/crates/directory/src/lib.rs index 01ab360f..f0a10858 100644 --- a/crates/directory/src/lib.rs +++ b/crates/directory/src/lib.rs @@ -22,7 +22,7 @@ use deadpool::managed::PoolError; use ldap3::LdapError; use mail_send::Credentials; use proc_macros::EnumMethods; -use store::{SERIALIZE_PRINCIPAL_V1, SerializedVersion, Store}; +use store::Store; use trc::ipc::bitset::Bitset; pub mod backend; @@ -46,12 +46,6 @@ pub struct Principal { pub data: Vec, } -impl SerializedVersion for Principal { - fn serialize_version() -> u8 { - SERIALIZE_PRINCIPAL_V1 - } -} - #[derive(rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Clone, PartialEq, Eq)] pub enum PrincipalData { MemberOf(Vec), diff --git a/crates/email/src/cache/email.rs b/crates/email/src/cache/email.rs index e06ebb8f..8345f1be 100644 --- a/crates/email/src/cache/email.rs +++ b/crates/email/src/cache/email.rs @@ -14,7 +14,7 @@ use jmap_proto::types::{ collection::Collection, keyword::{Keyword, OTHER}, }; -use store::{ahash::AHashMap, roaring::RoaringBitmap}; +use store::{ahash::AHashMap, roaring::RoaringBitmap, write::Archive}; use trc::AddContext; use utils::map::bitmap::Bitmap; @@ -42,7 +42,7 @@ pub(crate) async fn update_email_cache( insert_item( &mut new_cache, *document_id, - archive.unarchive::()?, + archive.to_unarchived::()?, ); } } @@ -84,7 +84,11 @@ pub(crate) async fn full_email_cache_build( Collection::Email, &(), |document_id, archive| { - insert_item(&mut cache, document_id, archive.unarchive::()?); + insert_item( + &mut cache, + document_id, + archive.to_unarchived::()?, + ); Ok(true) }, ) @@ -97,7 +101,12 @@ pub(crate) async fn full_email_cache_build( Ok(cache) } -fn insert_item(cache: &mut MessagesCache, document_id: u32, message: &ArchivedMessageData) { +fn insert_item( + cache: &mut MessagesCache, + document_id: u32, + archive: Archive<&ArchivedMessageData>, +) { + let message = archive.inner; let mut item = MessageCache { mailboxes: message .mailboxes @@ -109,7 +118,7 @@ fn insert_item(cache: &mut MessagesCache, document_id: u32, message: &ArchivedMe .collect(), keywords: 0, thread_id: message.thread_id.to_native(), - change_id: message.change_id.to_native(), + change_id: archive.version.change_id().unwrap_or_default(), document_id, }; for keyword in message.keywords.iter() { diff --git a/crates/email/src/cache/mod.rs b/crates/email/src/cache/mod.rs index 6e79b386..5357b77e 100644 --- a/crates/email/src/cache/mod.rs +++ b/crates/email/src/cache/mod.rs @@ -49,22 +49,8 @@ impl MessageCacheFetch for Server { } }; - // Perform full refresh on stale ids - let cache = cache_.load_full(); - if cache.last_change_id > 0 - && self - .core - .jmap - .changes_max_history - .and_then(|history| self.inner.data.jmap_id_gen.past_id(history)) - .is_some_and(|last_change_id| cache.last_change_id < last_change_id) - { - let cache = full_cache_build(self, account_id, cache.update_lock.clone()).await?; - cache_.update(cache.clone()); - return Ok(cache); - } - // Obtain current state + let cache = cache_.load_full(); let changes = self .core .storage @@ -77,6 +63,13 @@ impl MessageCacheFetch for Server { .await .caused_by(trc::location!())?; + // Regenerate cache if the change log has been truncated + if changes.is_truncated { + let cache = full_cache_build(self, account_id, cache.update_lock.clone()).await?; + cache_.update(cache.clone()); + return Ok(cache); + } + // Verify changes if changes.changes.is_empty() { return Ok(cache); diff --git a/crates/email/src/identity/index.rs b/crates/email/src/identity/index.rs index 000b3cf0..315d39b9 100644 --- a/crates/email/src/identity/index.rs +++ b/crates/email/src/identity/index.rs @@ -29,4 +29,8 @@ impl IndexableObject for &ArchivedIdentity { } } -impl IndexableAndSerializableObject for Identity {} +impl IndexableAndSerializableObject for Identity { + fn is_versioned() -> bool { + false + } +} diff --git a/crates/email/src/identity/mod.rs b/crates/email/src/identity/mod.rs index 524e5cc6..2ea9c348 100644 --- a/crates/email/src/identity/mod.rs +++ b/crates/email/src/identity/mod.rs @@ -6,8 +6,6 @@ pub mod index; -use store::{SERIALIZE_IDENTITY_V1, SerializedVersion}; - #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] @@ -25,9 +23,3 @@ pub struct EmailAddress { pub name: Option, pub email: String, } - -impl SerializedVersion for Identity { - fn serialize_version() -> u8 { - SERIALIZE_IDENTITY_V1 - } -} diff --git a/crates/email/src/mailbox/destroy.rs b/crates/email/src/mailbox/destroy.rs index ab61f68e..5bde801d 100644 --- a/crates/email/src/mailbox/destroy.rs +++ b/crates/email/src/mailbox/destroy.rs @@ -182,9 +182,12 @@ impl MailboxDestroy for Server { }; if !batch.is_empty() { - let change_id = batch.change_id(); - match self.commit_batch(batch).await { - Ok(_) => Ok(Ok(Some(change_id))), + match self + .commit_batch(batch) + .await + .and_then(|ids| ids.last_change_id(account_id)) + { + Ok(change_id) => Ok(Ok(Some(change_id))), Err(err) if err.is_assertion_failure() => Ok(Err(SetError::forbidden() .with_description(concat!( "Another process modified a message in this mailbox ", diff --git a/crates/email/src/mailbox/index.rs b/crates/email/src/mailbox/index.rs index 49f09313..c205a2a8 100644 --- a/crates/email/src/mailbox/index.rs +++ b/crates/email/src/mailbox/index.rs @@ -42,4 +42,8 @@ impl IndexableObject for &ArchivedMailbox { } } -impl IndexableAndSerializableObject for Mailbox {} +impl IndexableAndSerializableObject for Mailbox { + fn is_versioned() -> bool { + false + } +} diff --git a/crates/email/src/mailbox/mod.rs b/crates/email/src/mailbox/mod.rs index 10f0dcba..ca95b6ce 100644 --- a/crates/email/src/mailbox/mod.rs +++ b/crates/email/src/mailbox/mod.rs @@ -5,9 +5,7 @@ */ use common::config::jmap::settings::SpecialUse; - use jmap_proto::types::value::AclGrant; -use store::{SERIALIZE_MAILBOX_V1, SerializedVersion}; pub mod destroy; pub mod index; @@ -40,12 +38,6 @@ pub struct UidMailbox { pub uid: u32, } -impl SerializedVersion for Mailbox { - fn serialize_version() -> u8 { - SERIALIZE_MAILBOX_V1 - } -} - impl Mailbox { pub fn new(name: impl Into) -> Self { Mailbox { diff --git a/crates/email/src/message/copy.rs b/crates/email/src/message/copy.rs index 30f831fa..71a5753d 100644 --- a/crates/email/src/message/copy.rs +++ b/crates/email/src/message/copy.rs @@ -172,7 +172,6 @@ impl EmailCopy for Server { // Prepare batch let mut batch = BatchBuilder::new(); - let change_id = batch.change_id(); batch.with_account_id(account_id); if is_new_thread { @@ -195,14 +194,13 @@ impl EmailCopy for Server { ObjectIndexBuilder::<(), _>::new().with_changes(MessageData { mailboxes: mailbox_ids, keywords, - change_id, thread_id, }), ) .caused_by(trc::location!())? .set( ValueClass::TaskQueue(TaskQueueClass::IndexEmail { - seq: change_id, + seq: self.generate_snowflake_id(), hash: metadata.blob_hash.clone(), }), vec![], @@ -217,10 +215,12 @@ impl EmailCopy for Server { .caused_by(trc::location!())?; // Insert and obtain ids - self.store() + let change_id = self + .store() .write(batch.build_all()) .await - .caused_by(trc::location!())?; + .caused_by(trc::location!())? + .last_change_id(account_id)?; // Request FTS index self.notify_task_queue(); diff --git a/crates/email/src/message/crypto.rs b/crates/email/src/message/crypto.rs index ff63b64d..7b772694 100644 --- a/crates/email/src/message/crypto.rs +++ b/crates/email/src/message/crypto.rs @@ -26,7 +26,7 @@ use rasn_cms::{ }; use rsa::{Pkcs1v15Encrypt, RsaPublicKey, pkcs1::DecodeRsaPublicKey}; use sequoia_openpgp as openpgp; -use store::{Deserialize, SERIALIZE_CRYPTO_V1, SerializedVersion, write::Archive}; +use store::{Deserialize, write::Archive}; const P: openpgp::policy::StandardPolicy<'static> = openpgp::policy::StandardPolicy::new(); @@ -84,12 +84,6 @@ pub struct EncryptionParams { pub certs: Vec>, } -impl SerializedVersion for EncryptionParams { - fn serialize_version() -> u8 { - SERIALIZE_CRYPTO_V1 - } -} - #[derive( rkyv::Serialize, rkyv::Deserialize, diff --git a/crates/email/src/message/delete.rs b/crates/email/src/message/delete.rs index caff82b9..072e6b09 100644 --- a/crates/email/src/message/delete.rs +++ b/crates/email/src/message/delete.rs @@ -11,11 +11,14 @@ use jmap_proto::types::{collection::Collection, property::Property}; use std::future::Future; use std::time::Duration; use store::rand::prelude::SliceRandom; +use store::write::key::DeserializeBigEndian; +use store::write::now; use store::{ BitmapKey, ValueKey, roaring::RoaringBitmap, write::{AlignedBytes, Archive, BatchBuilder, BitmapClass, TagValue, ValueClass}, }; +use store::{IndexKey, IterateParams, SerializeInfallible, U32_LEN}; use trc::AddContext; use utils::BlobHash; @@ -167,13 +170,7 @@ impl EmailDeletion for Server { } async fn emails_auto_expunge(&self, account_id: u32, period: Duration) -> trc::Result<()> { - let reference_cid = self.inner.data.jmap_id_gen.past_id(period).ok_or_else(|| { - trc::StoreEvent::UnexpectedError - .into_err() - .caused_by(trc::location!()) - .ctx(trc::Key::Reason, "Failed to generate reference cid.") - })?; - let destroy_ids = RoaringBitmap::from_iter( + let trashed_ids = RoaringBitmap::from_iter( self.get_cached_messages(account_id) .await .caused_by(trc::location!())? @@ -181,14 +178,53 @@ impl EmailDeletion for Server { .items .iter() .filter(|item| { - item.change_id < reference_cid - && item - .mailboxes - .iter() - .any(|id| id.mailbox_id == TRASH_ID || id.mailbox_id == JUNK_ID) + item.mailboxes + .iter() + .any(|id| id.mailbox_id == TRASH_ID || id.mailbox_id == JUNK_ID) }) .map(|item| item.document_id), ); + if trashed_ids.is_empty() { + return Ok(()); + } + + // Filter messages by received date + let mut destroy_ids = RoaringBitmap::new(); + self.store() + .iterate( + IterateParams::new( + IndexKey { + account_id, + collection: Collection::Email.into(), + document_id: 0, + field: Property::ReceivedAt.into(), + key: now().saturating_sub(period.as_secs() * 2).serialize(), + }, + IndexKey { + account_id, + collection: Collection::Email.into(), + document_id: u32::MAX, + field: Property::ReceivedAt.into(), + key: now().saturating_sub(period.as_secs()).serialize(), + }, + ) + .no_values() + .ascending(), + |key, _| { + let document_id = key + .deserialize_be_u32(key.len() - U32_LEN) + .caused_by(trc::location!())?; + + if trashed_ids.contains(document_id) { + destroy_ids.insert(document_id); + } + + Ok(trashed_ids.len() != destroy_ids.len()) + }, + ) + .await + .caused_by(trc::location!())?; + if destroy_ids.is_empty() { return Ok(()); } diff --git a/crates/email/src/message/ingest.rs b/crates/email/src/message/ingest.rs index e3607dc0..f622c6de 100644 --- a/crates/email/src/message/ingest.rs +++ b/crates/email/src/message/ingest.rs @@ -485,7 +485,6 @@ impl EmailIngest for Server { // Build write batch let mut batch = BatchBuilder::new(); - let change_id = batch.change_id(); let mailbox_ids_event = mailbox_ids .iter() .map(|m| trc::Value::from(m.mailbox_id)) @@ -499,6 +498,7 @@ impl EmailIngest for Server { .log_container_insert(SyncCollection::Thread); } + let seq = self.generate_snowflake_id(); let document_id = self .store() .assign_document_ids(account_id, Collection::Email, 1) @@ -515,7 +515,6 @@ impl EmailIngest for Server { MessageData { mailboxes: mailbox_ids, keywords: params.keywords, - change_id, thread_id, }, params.received_at.unwrap_or_else(now), @@ -523,7 +522,7 @@ impl EmailIngest for Server { .caused_by(trc::location!())? .set( ValueClass::TaskQueue(TaskQueueClass::IndexEmail { - seq: change_id, + seq, hash: blob_id.hash.clone(), }), vec![], @@ -533,7 +532,7 @@ impl EmailIngest for Server { if let Some(learn_spam) = train_spam { batch.set( ValueClass::TaskQueue(TaskQueueClass::BayesTrain { - seq: change_id, + seq, hash: blob_id.hash.clone(), learn_spam, }), @@ -542,10 +541,12 @@ impl EmailIngest for Server { } // Insert and obtain ids - self.store() + let change_id = self + .store() .write(batch.build_all()) .await - .caused_by(trc::location!())?; + .caused_by(trc::location!())? + .last_change_id(account_id)?; let id = Id::from_parts(thread_id, document_id); // Request FTS index diff --git a/crates/email/src/message/metadata.rs b/crates/email/src/message/metadata.rs index 609076ea..da3e0386 100644 --- a/crates/email/src/message/metadata.rs +++ b/crates/email/src/message/metadata.rs @@ -21,14 +21,12 @@ use rkyv::{ vec::ArchivedVec, }; use std::{borrow::Cow, collections::VecDeque}; -use store::{SERIALIZE_MESSAGE_DATA_V1, SERIALIZE_MESSAGE_METADATA_V1, SerializedVersion}; use utils::BlobHash; #[derive(rkyv::Serialize, rkyv::Deserialize, rkyv::Archive, Debug, Default)] pub struct MessageData { pub mailboxes: Vec, pub keywords: Vec, - pub change_id: u64, pub thread_id: u32, } @@ -43,17 +41,9 @@ pub struct MessageMetadata { pub raw_headers: Vec, } -impl IndexableAndSerializableObject for MessageData {} - -impl SerializedVersion for MessageData { - fn serialize_version() -> u8 { - SERIALIZE_MESSAGE_DATA_V1 - } -} - -impl SerializedVersion for MessageMetadata { - fn serialize_version() -> u8 { - SERIALIZE_MESSAGE_METADATA_V1 +impl IndexableAndSerializableObject for MessageData { + fn is_versioned() -> bool { + true } } diff --git a/crates/email/src/push/mod.rs b/crates/email/src/push/mod.rs index 1f2a4ee5..8142e383 100644 --- a/crates/email/src/push/mod.rs +++ b/crates/email/src/push/mod.rs @@ -5,7 +5,6 @@ */ use jmap_proto::types::type_state::DataType; -use store::{SERIALIZE_PUSH_V1, SerializedVersion}; use utils::map::bitmap::Bitmap; #[derive( @@ -26,9 +25,3 @@ pub struct Keys { pub p256dh: Vec, pub auth: Vec, } - -impl SerializedVersion for PushSubscription { - fn serialize_version() -> u8 { - SERIALIZE_PUSH_V1 - } -} diff --git a/crates/email/src/sieve/activate.rs b/crates/email/src/sieve/activate.rs index 34c0d623..ac0cf6e9 100644 --- a/crates/email/src/sieve/activate.rs +++ b/crates/email/src/sieve/activate.rs @@ -102,9 +102,12 @@ impl SieveScriptActivate for Server { // Write changes if !changed_ids.is_empty() { - let change_id = batch.change_id(); - match self.commit_batch(batch).await { - Ok(_) => Ok((change_id, changed_ids)), + match self + .commit_batch(batch) + .await + .and_then(|ids| ids.last_change_id(account_id)) + { + Ok(change_id) => Ok((change_id, changed_ids)), Err(err) if err.is_assertion_failure() => Ok((0, vec![])), Err(err) => Err(err.caused_by(trc::location!())), } diff --git a/crates/email/src/sieve/index.rs b/crates/email/src/sieve/index.rs index 495293d9..cbc28032 100644 --- a/crates/email/src/sieve/index.rs +++ b/crates/email/src/sieve/index.rs @@ -35,7 +35,11 @@ impl IndexableObject for SieveScript { } } -impl IndexableAndSerializableObject for SieveScript {} +impl IndexableAndSerializableObject for SieveScript { + fn is_versioned() -> bool { + false + } +} impl IndexableObject for &ArchivedSieveScript { fn index_values(&self) -> impl Iterator> { diff --git a/crates/email/src/sieve/ingest.rs b/crates/email/src/sieve/ingest.rs index ee147573..09c2ec76 100644 --- a/crates/email/src/sieve/ingest.rs +++ b/crates/email/src/sieve/ingest.rs @@ -27,9 +27,7 @@ use store::{ ahash::AHashMap, dispatch::lookup::KeyValue, query::Filter, - write::{ - AlignedBytes, Archiver, BatchBuilder, BlobOp, UnversionedArchive, UnversionedArchiver, - }, + write::{AlignedBytes, Archive, ArchiveVersion, Archiver, BatchBuilder, BlobOp}, }; use trc::{AddContext, SieveEvent}; use utils::config::utils::ParseValue; @@ -242,7 +240,11 @@ impl SieveScriptIngest for Server { } } Event::DuplicateId { id, expiry, last } => { - let id_hash = SeenIdHash::new(account_id, active_script.hash, &id); + let id_hash = SeenIdHash::new( + account_id, + active_script.version.hash().unwrap_or_default(), + &id, + ); if let Some(result) = checked_ids.get(&id_hash) { input = (*result).into(); } else { @@ -563,7 +565,7 @@ impl SieveScriptIngest for Server { document_id, script: Arc::new(script.script), script_name: script.name, - hash: script.hash, + version: script.version, })) } else { Ok(None) @@ -614,7 +616,7 @@ impl SieveScriptIngest for Server { })?; // Obtain the sieve script length - let hash = script_object.hash; + let version = script_object.version; let unarchived_script = script_object .unarchive::() .caused_by(trc::location!())?; @@ -637,7 +639,7 @@ impl SieveScriptIngest for Server { // Obtain the precompiled script if let Some(script) = script_bytes.get(script_offset..).and_then(|bytes| { - as Deserialize>::deserialize(bytes) + as Deserialize>::deserialize(bytes) .ok()? .deserialize::() .ok() @@ -645,7 +647,7 @@ impl SieveScriptIngest for Server { Ok(CompiledScript { script, name: unarchived_script.name.as_str().into(), - hash, + version, }) } else { // Deserialization failed, probably because the script compiler version changed @@ -659,7 +661,7 @@ impl SieveScriptIngest for Server { ) { Ok(sieve) => { // Store updated compiled sieve script - let sieve = UnversionedArchiver::new(sieve); + let sieve = Archiver::new(sieve).untrusted(); let compiled_bytes = sieve.serialize().caused_by(trc::location!())?; let mut updated_sieve_bytes = Vec::with_capacity(script_offset + compiled_bytes.len()); @@ -703,7 +705,7 @@ impl SieveScriptIngest for Server { Ok(CompiledScript { script: sieve.into_inner(), name: new_archive.into_inner().name, - hash, + version, }) } Err(error) => Err(trc::StoreEvent::UnexpectedError @@ -718,5 +720,5 @@ impl SieveScriptIngest for Server { pub struct CompiledScript { pub script: Sieve, pub name: String, - pub hash: u32, + pub version: ArchiveVersion, } diff --git a/crates/email/src/sieve/mod.rs b/crates/email/src/sieve/mod.rs index 679485e5..a277f9a5 100644 --- a/crates/email/src/sieve/mod.rs +++ b/crates/email/src/sieve/mod.rs @@ -9,7 +9,7 @@ use std::sync::Arc; use common::KV_SIEVE_ID; use sieve::Sieve; -use store::{SERIALIZE_SIEVE_V1, SerializedVersion, blake3}; +use store::{blake3, write::ArchiveVersion}; use utils::BlobHash; pub mod activate; @@ -20,7 +20,7 @@ pub mod ingest; #[derive(Debug, Clone)] pub struct ActiveScript { pub document_id: u32, - pub hash: u32, + pub version: ArchiveVersion, pub script_name: String, pub script: Arc, } @@ -37,12 +37,6 @@ pub struct SieveScript { pub vacation_response: Option, } -impl SerializedVersion for SieveScript { - fn serialize_version() -> u8 { - SERIALIZE_SIEVE_V1 - } -} - #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] diff --git a/crates/email/src/submission/index.rs b/crates/email/src/submission/index.rs index 355b0759..ef79e881 100644 --- a/crates/email/src/submission/index.rs +++ b/crates/email/src/submission/index.rs @@ -73,4 +73,8 @@ impl IndexableObject for &ArchivedEmailSubmission { } } -impl IndexableAndSerializableObject for EmailSubmission {} +impl IndexableAndSerializableObject for EmailSubmission { + fn is_versioned() -> bool { + false + } +} diff --git a/crates/email/src/submission/mod.rs b/crates/email/src/submission/mod.rs index b7959939..047916aa 100644 --- a/crates/email/src/submission/mod.rs +++ b/crates/email/src/submission/mod.rs @@ -4,7 +4,6 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use store::{SERIALIZE_SUBMISSION_V1, SerializedVersion}; use utils::map::vec_map::VecMap; pub mod index; @@ -23,12 +22,6 @@ pub struct EmailSubmission { pub delivery_status: VecMap, } -impl SerializedVersion for EmailSubmission { - fn serialize_version() -> u8 { - SERIALIZE_SUBMISSION_V1 - } -} - #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, )] diff --git a/crates/groupware/src/cache/mod.rs b/crates/groupware/src/cache/mod.rs index e4bb68e8..43b7ce2c 100644 --- a/crates/groupware/src/cache/mod.rs +++ b/crates/groupware/src/cache/mod.rs @@ -86,29 +86,8 @@ impl GroupwareCache for Server { } }; - // Perform full refresh on stale ids - let cache = cache_.load_full(); - if cache.highest_change_id > 0 - && self - .core - .jmap - .changes_max_history - .and_then(|history| self.inner.data.jmap_id_gen.past_id(history)) - .is_some_and(|last_change_id| cache.highest_change_id < last_change_id) - { - let cache = full_cache_build( - self, - account_id, - collection, - cache.update_lock.clone(), - access_token, - ) - .await?; - cache_.update(cache.clone()); - return Ok(cache); - } - // Obtain current state + let cache = cache_.load_full(); let changes = self .core .storage @@ -121,6 +100,20 @@ impl GroupwareCache for Server { .await .caused_by(trc::location!())?; + // Regenerate cache if the change log has been truncated + if changes.is_truncated { + let cache = full_cache_build( + self, + account_id, + collection, + cache.update_lock.clone(), + access_token, + ) + .await?; + cache_.update(cache.clone()); + return Ok(cache); + } + // Verify changes if changes.changes.is_empty() { return Ok(cache); diff --git a/crates/groupware/src/calendar/index.rs b/crates/groupware/src/calendar/index.rs index f0e30b72..452a70b5 100644 --- a/crates/groupware/src/calendar/index.rs +++ b/crates/groupware/src/calendar/index.rs @@ -57,7 +57,11 @@ impl IndexableObject for &ArchivedCalendar { } } -impl IndexableAndSerializableObject for Calendar {} +impl IndexableAndSerializableObject for Calendar { + fn is_versioned() -> bool { + true + } +} impl IndexableObject for CalendarEvent { fn index_values(&self) -> impl Iterator> { @@ -103,7 +107,11 @@ impl IndexableObject for &ArchivedCalendarEvent { } } -impl IndexableAndSerializableObject for CalendarEvent {} +impl IndexableAndSerializableObject for CalendarEvent { + fn is_versioned() -> bool { + true + } +} impl CalendarPreferences { pub fn size(&self) -> usize { diff --git a/crates/groupware/src/calendar/mod.rs b/crates/groupware/src/calendar/mod.rs index f163b734..08dfd741 100644 --- a/crates/groupware/src/calendar/mod.rs +++ b/crates/groupware/src/calendar/mod.rs @@ -12,7 +12,6 @@ use calcard::icalendar::ICalendar; use common::DavName; use dav_proto::schema::request::DeadProperty; use jmap_proto::types::{acl::Acl, value::AclGrant}; -use store::{SERIALIZE_CALENDAR_EVENT_V1, SERIALIZE_CALENDAR_V1, SerializedVersion}; #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, @@ -180,18 +179,6 @@ impl From for Acl { } } -impl SerializedVersion for Calendar { - fn serialize_version() -> u8 { - SERIALIZE_CALENDAR_V1 - } -} - -impl SerializedVersion for CalendarEvent { - fn serialize_version() -> u8 { - SERIALIZE_CALENDAR_EVENT_V1 - } -} - impl Calendar { pub fn preferences(&self, account_id: u32) -> &CalendarPreferences { if self.preferences.len() == 1 { diff --git a/crates/groupware/src/contact/index.rs b/crates/groupware/src/contact/index.rs index aef7ca27..104c8db8 100644 --- a/crates/groupware/src/contact/index.rs +++ b/crates/groupware/src/contact/index.rs @@ -54,7 +54,11 @@ impl IndexableObject for &ArchivedAddressBook { } } -impl IndexableAndSerializableObject for AddressBook {} +impl IndexableAndSerializableObject for AddressBook { + fn is_versioned() -> bool { + true + } +} impl IndexableObject for ContactCard { fn index_values(&self) -> impl Iterator> { @@ -100,4 +104,8 @@ impl IndexableObject for &ArchivedContactCard { } } -impl IndexableAndSerializableObject for ContactCard {} +impl IndexableAndSerializableObject for ContactCard { + fn is_versioned() -> bool { + true + } +} diff --git a/crates/groupware/src/contact/mod.rs b/crates/groupware/src/contact/mod.rs index 4e3814bf..141c6368 100644 --- a/crates/groupware/src/contact/mod.rs +++ b/crates/groupware/src/contact/mod.rs @@ -11,7 +11,6 @@ use calcard::vcard::VCard; use common::DavName; use dav_proto::schema::request::DeadProperty; use jmap_proto::types::{acl::Acl, value::AclGrant}; -use store::{SERIALIZE_ADDRESS_BOOK_V1, SERIALIZE_CONTACT_V1, SerializedVersion}; #[derive( rkyv::Archive, rkyv::Deserialize, rkyv::Serialize, Debug, Default, Clone, PartialEq, Eq, @@ -75,15 +74,3 @@ impl From for Acl { } } } - -impl SerializedVersion for AddressBook { - fn serialize_version() -> u8 { - SERIALIZE_ADDRESS_BOOK_V1 - } -} - -impl SerializedVersion for ContactCard { - fn serialize_version() -> u8 { - SERIALIZE_CONTACT_V1 - } -} diff --git a/crates/groupware/src/file/index.rs b/crates/groupware/src/file/index.rs index cc43179a..1cf6693a 100644 --- a/crates/groupware/src/file/index.rs +++ b/crates/groupware/src/file/index.rs @@ -4,11 +4,10 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ +use super::{ArchivedFileNode, FileNode}; use common::storage::index::{IndexValue, IndexableAndSerializableObject, IndexableObject}; use jmap_proto::types::{collection::SyncCollection, value::AclGrant}; -use super::{ArchivedFileNode, FileNode}; - impl IndexableObject for FileNode { fn index_values(&self) -> impl Iterator> { let mut values = Vec::with_capacity(6); @@ -64,7 +63,11 @@ impl IndexableObject for &ArchivedFileNode { } } -impl IndexableAndSerializableObject for FileNode {} +impl IndexableAndSerializableObject for FileNode { + fn is_versioned() -> bool { + true + } +} pub trait NodeSize { fn size(&self) -> u32; diff --git a/crates/groupware/src/file/mod.rs b/crates/groupware/src/file/mod.rs index 5c3ad5f3..8a83b5ac 100644 --- a/crates/groupware/src/file/mod.rs +++ b/crates/groupware/src/file/mod.rs @@ -9,7 +9,6 @@ pub mod storage; use dav_proto::schema::request::DeadProperty; use jmap_proto::types::value::AclGrant; -use store::{SERIALIZE_FILE_NODE_V1, SerializedVersion}; use utils::BlobHash; #[derive( @@ -37,9 +36,3 @@ pub struct FileProperties { pub media_type: Option, pub executable: bool, } - -impl SerializedVersion for FileNode { - fn serialize_version() -> u8 { - SERIALIZE_FILE_NODE_V1 - } -} diff --git a/crates/http/src/auth/oauth/auth.rs b/crates/http/src/auth/oauth/auth.rs index 279e0e6b..cb971608 100644 --- a/crates/http/src/auth/oauth/auth.rs +++ b/crates/http/src/auth/oauth/auth.rs @@ -4,8 +4,7 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::sync::Arc; - +use crate::auth::oauth::OAuthStatus; use common::{ KV_OAUTH, Server, auth::{ @@ -13,13 +12,15 @@ use common::{ oauth::{CLIENT_ID_MAX_LEN, DEVICE_CODE_LEN, USER_CODE_ALPHABET, USER_CODE_LEN}, }, }; +use http_proto::*; use serde::Deserialize; use serde_json::json; use std::future::Future; +use std::sync::Arc; use store::{ Serialize, dispatch::lookup::KeyValue, - write::{UnversionedArchive, UnversionedArchiver}, + write::{Archive, Archiver}, }; use store::{ rand::{ @@ -31,9 +32,6 @@ use store::{ }; use trc::AddContext; -use crate::auth::oauth::OAuthStatus; -use http_proto::*; - use super::{DeviceAuthResponse, FormData, MAX_POST_LEN, OAuthCode, OAuthCodeRequest}; #[derive(Debug, serde::Serialize, Deserialize)] @@ -109,13 +107,14 @@ impl OAuthApiHandler for Server { .collect::(); // Serialize OAuth code - let value = UnversionedArchiver::new(OAuthCode { + let value = Archiver::new(OAuthCode { status: OAuthStatus::Authorized, account_id: access_token.primary_id(), client_id, nonce, params: redirect_uri.unwrap_or_default(), }) + .untrusted() .serialize() .caused_by(trc::location!())?; @@ -151,7 +150,7 @@ impl OAuthApiHandler for Server { .core .storage .lookup - .key_get::>(KeyValue::<()>::build_key( + .key_get::>(KeyValue::<()>::build_key( KV_OAUTH, code.as_bytes(), )) @@ -185,7 +184,8 @@ impl OAuthApiHandler for Server { KeyValue::with_prefix( KV_OAUTH, oauth.params.as_bytes(), - UnversionedArchiver::new(new_oauth_code) + Archiver::new(new_oauth_code) + .untrusted() .serialize() .caused_by(trc::location!())?, ) @@ -243,13 +243,14 @@ impl OAuthApiHandler for Server { } // Add OAuth status - let oauth_code = UnversionedArchiver::new(OAuthCode { + let oauth_code = Archiver::new(OAuthCode { status: OAuthStatus::Pending, account_id: u32::MAX, client_id, nonce, params: device_code.clone(), }) + .untrusted() .serialize() .caused_by(trc::location!())?; diff --git a/crates/http/src/auth/oauth/token.rs b/crates/http/src/auth/oauth/token.rs index 6a09cf11..71718f21 100644 --- a/crates/http/src/auth/oauth/token.rs +++ b/crates/http/src/auth/oauth/token.rs @@ -20,7 +20,7 @@ use hyper::StatusCode; use std::future::Future; use store::{ dispatch::lookup::KeyValue, - write::{AlignedBytes, UnversionedArchive}, + write::{AlignedBytes, Archive}, }; use trc::AddContext; @@ -77,7 +77,7 @@ impl TokenHandler for Server { .core .storage .lookup - .key_get::>(KeyValue::<()>::build_key( + .key_get::>(KeyValue::<()>::build_key( KV_OAUTH, code.as_bytes(), )) @@ -149,7 +149,7 @@ impl TokenHandler for Server { .core .storage .lookup - .key_get::>(KeyValue::<()>::build_key( + .key_get::>(KeyValue::<()>::build_key( KV_OAUTH, device_code.as_bytes(), )) @@ -329,7 +329,7 @@ impl TokenHandler for Server { nonce, preferred_username: access_token.name.clone().into(), email: access_token.emails.first().cloned(), - description: access_token.description.clone().into(), + description: access_token.description.clone(), }, ) { Ok(id_token) => Some(id_token), diff --git a/crates/http/src/management/report.rs b/crates/http/src/management/report.rs index ff0c9a5a..1e5e50d0 100644 --- a/crates/http/src/management/report.rs +++ b/crates/http/src/management/report.rs @@ -18,8 +18,7 @@ use std::future::Future; use store::{ Deserialize, IterateParams, Key, U64_LEN, ValueKey, write::{ - AlignedBytes, BatchBuilder, ReportClass, UnversionedArchive, ValueClass, - key::DeserializeBigEndian, + AlignedBytes, Archive, BatchBuilder, ReportClass, ValueClass, key::DeserializeBigEndian, }, }; use trc::AddContext; @@ -289,7 +288,7 @@ where { if let Some(tls) = server .store() - .get_value::>(key) + .get_value::>(key) .await? { tls.deserialize::().map(Some) @@ -379,8 +378,7 @@ async fn fetch_incoming_reports( last_id = id; // TODO: Support filtering chunked records (over 10MB) on FDB - let archive = - as Deserialize>::deserialize(value)?; + let archive = as Deserialize>::deserialize(value)?; let matches = if has_filters { match typ { ReportType::Dmarc => { diff --git a/crates/imap/src/op/copy_move.rs b/crates/imap/src/op/copy_move.rs index f490fd1f..c137e5b5 100644 --- a/crates/imap/src/op/copy_move.rs +++ b/crates/imap/src/op/copy_move.rs @@ -4,8 +4,11 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{sync::Arc, time::Instant}; - +use crate::{ + core::{MailboxId, SelectedMailbox, Session, SessionData}, + spawn_op, +}; +use common::{listener::SessionStream, storage::index::ObjectIndexBuilder}; use directory::Permission; use email::{ mailbox::{JUNK_ID, UidMailbox}, @@ -17,16 +20,11 @@ use imap_proto::{ Command, ResponseCode, ResponseType, StatusResponse, protocol::copy_move::Arguments, receiver::Request, }; - -use crate::{ - core::{MailboxId, SelectedMailbox, Session, SessionData}, - spawn_op, -}; -use common::{listener::SessionStream, storage::index::ObjectIndexBuilder}; use jmap_proto::{ error::set::SetErrorType, types::{acl::Acl, collection::Collection, state::StateChange, type_state::DataType}, }; +use std::{sync::Arc, time::Instant}; use store::{ roaring::RoaringBitmap, write::{AlignedBytes, Archive, BatchBuilder, ValueClass}, @@ -220,7 +218,6 @@ impl SessionData { let mut new_data = data .deserialize() .imap_ctx(&arguments.tag, trc::location!())?; - new_data.change_id = batch.change_id(); // Add destination folder new_data.add_mailbox(dest_mailbox_id); diff --git a/crates/imap/src/op/expunge.rs b/crates/imap/src/op/expunge.rs index b38d59fc..f1e23be0 100644 --- a/crates/imap/src/op/expunge.rs +++ b/crates/imap/src/op/expunge.rs @@ -178,7 +178,6 @@ impl SessionData { // Untag message from this mailbox and remove Deleted flag let mut new_data = data.deserialize().caused_by(trc::location!())?; - new_data.change_id = batch.change_id(); new_data.remove_mailbox(mailbox_id); new_data.remove_keyword(&Keyword::Deleted); diff --git a/crates/imap/src/op/fetch.rs b/crates/imap/src/op/fetch.rs index 803fa711..e8130f8f 100644 --- a/crates/imap/src/op/fetch.rs +++ b/crates/imap/src/op/fetch.rs @@ -541,7 +541,6 @@ impl SessionData { .deserialize() .imap_ctx(&arguments.tag, trc::location!())?; new_data.keywords.push(Keyword::Seen); - new_data.change_id = batch.change_id(); batch .with_account_id(account_id) @@ -560,14 +559,14 @@ impl SessionData { // Set Seen ids if !batch.is_empty() { - let change_id = batch.change_id(); match self .server .commit_batch(batch) .await + .and_then(|ids| ids.last_change_id(account_id)) .imap_ctx(&arguments.tag, trc::location!()) { - Ok(_) => { + Ok(change_id) => { modseq = change_id; } Err(err) => { diff --git a/crates/imap/src/op/store.rs b/crates/imap/src/op/store.rs index bfde6b3a..d4af7ee4 100644 --- a/crates/imap/src/op/store.rs +++ b/crates/imap/src/op/store.rs @@ -281,10 +281,6 @@ impl SessionData { vec![] }; - // Add change id - new_data.change_id = batch.change_id(); - let modseq = new_data.change_id + 1; - // Set all current mailboxes as changed if the Seen tag changed if seen_changed { for mailbox_id in new_data.mailboxes.iter() { @@ -327,9 +323,6 @@ impl SessionData { if is_uid { data_items.push(DataItem::Uid { uid: imap_id.uid }); } - if is_condstore { - data_items.push(DataItem::ModSeq { modseq }); - } items.items.push(FetchItem { id: imap_id.seqnum, items: data_items, @@ -338,12 +331,9 @@ impl SessionData { items.items.push(FetchItem { id: imap_id.seqnum, items: if is_uid { - vec![ - DataItem::ModSeq { modseq }, - DataItem::Uid { uid: imap_id.uid }, - ] + vec![DataItem::Uid { uid: imap_id.uid }] } else { - vec![DataItem::ModSeq { modseq }] + vec![] }, }); } @@ -367,9 +357,17 @@ impl SessionData { .server .commit_batch(batch) .await + .and_then(|ids| ids.last_change_id(mailbox.id.account_id)) .caused_by(trc::location!()) { - Ok(_) => {} + Ok(change_id) => { + if is_condstore { + let modseq = change_id + 1; + for item in items.items.iter_mut() { + item.items.push(DataItem::ModSeq { modseq }); + } + } + } Err(err) if err.is_assertion_failure() => { items.items.clear(); response.rtype = ResponseType::No; diff --git a/crates/jmap/src/email/set.rs b/crates/jmap/src/email/set.rs index cd396ee4..a1dcea83 100644 --- a/crates/jmap/src/email/set.rs +++ b/crates/jmap/src/email/set.rs @@ -926,10 +926,6 @@ impl EmailSet for Server { } } - // Update change id - new_data.change_id = batch.change_id(); - last_change_id = new_data.change_id.into(); - // Write changes batch .with_account_id(account_id) @@ -951,8 +947,14 @@ impl EmailSet for Server { batch.log_container_property_change(SyncCollection::Email, parent_id); } - match self.commit_batch(batch).await { - Ok(_) => { + match self + .commit_batch(batch) + .await + .and_then(|ids| ids.last_change_id(account_id)) + { + Ok(change_id) => { + last_change_id = change_id.into(); + // Add to updated list for id in will_update { response.updated.append(id, None); @@ -1012,8 +1014,12 @@ impl EmailSet for Server { .emails_tombstone(account_id, &mut batch, destroy_ids) .await?; if !batch.is_empty() { - last_change_id = batch.change_id().into(); - self.commit_batch(batch).await.caused_by(trc::location!())?; + last_change_id = self + .commit_batch(batch) + .await + .and_then(|ids| ids.last_change_id(account_id)) + .caused_by(trc::location!())? + .into(); } // Mark messages that were not found as not destroyed (this should not occur in practice) diff --git a/crates/jmap/src/identity/set.rs b/crates/jmap/src/identity/set.rs index 6ddbce58..d413deb8 100644 --- a/crates/jmap/src/identity/set.rs +++ b/crates/jmap/src/identity/set.rs @@ -171,8 +171,11 @@ impl IdentitySet for Server { // Write changes if !batch.is_empty() { - let change_id = batch.change_id(); - self.commit_batch(batch).await.caused_by(trc::location!())?; + let change_id = self + .commit_batch(batch) + .await + .and_then(|ids| ids.last_change_id(account_id)) + .caused_by(trc::location!())?; response.new_state = State::Exact(change_id).into(); } diff --git a/crates/jmap/src/mailbox/set.rs b/crates/jmap/src/mailbox/set.rs index 3f46306d..809584b2 100644 --- a/crates/jmap/src/mailbox/set.rs +++ b/crates/jmap/src/mailbox/set.rs @@ -124,8 +124,12 @@ impl MailboxSet for Server { } if !batch.is_empty() { - change_id = Some(batch.change_id()); - self.commit_batch(batch).await.caused_by(trc::location!())?; + change_id = self + .commit_batch(batch) + .await + .and_then(|ids| ids.last_change_id(account_id)) + .caused_by(trc::location!())? + .into(); } // Process updates @@ -206,9 +210,12 @@ impl MailboxSet for Server { } if !batch.is_empty() { - let change_id_ = batch.change_id(); - match self.commit_batch(batch).await { - Ok(_) => { + match self + .commit_batch(batch) + .await + .and_then(|ids| ids.last_change_id(account_id)) + { + Ok(change_id_) => { change_id = Some(change_id_); for id in will_update { ctx.response.updated.append(id, None); diff --git a/crates/jmap/src/push/set.rs b/crates/jmap/src/push/set.rs index 51505a99..f944f977 100644 --- a/crates/jmap/src/push/set.rs +++ b/crates/jmap/src/push/set.rs @@ -194,8 +194,11 @@ impl PushSubscriptionSet for Server { // Write changes if !batch.is_empty() { - let change_id = batch.change_id(); - self.commit_batch(batch).await.caused_by(trc::location!())?; + let change_id = self + .commit_batch(batch) + .await + .and_then(|ids| ids.last_change_id(account_id)) + .caused_by(trc::location!())?; response.new_state = State::Exact(change_id).into(); } diff --git a/crates/jmap/src/sieve/set.rs b/crates/jmap/src/sieve/set.rs index b722a4ed..b606433a 100644 --- a/crates/jmap/src/sieve/set.rs +++ b/crates/jmap/src/sieve/set.rs @@ -9,7 +9,6 @@ use common::{ auth::{AccessToken, ResourceToken}, storage::index::ObjectIndexBuilder, }; - use email::sieve::{ ArchivedSieveScript, SieveScript, activate::SieveScriptActivate, delete::SieveScriptDelete, }; @@ -35,7 +34,7 @@ use store::{ BlobClass, Serialize, query::Filter, rand::{Rng, rng}, - write::{Archive, BatchBuilder, UnversionedArchiver}, + write::{Archive, Archiver, BatchBuilder}, }; use trc::AddContext; @@ -285,8 +284,11 @@ impl SieveScriptSet for Server { // Write changes if !batch.is_empty() { - let change_id = batch.change_id(); - self.commit_batch(batch).await.caused_by(trc::location!())?; + let change_id = self + .commit_batch(batch) + .await + .and_then(|ids| ids.last_change_id(account_id)) + .caused_by(trc::location!())?; ctx.response.new_state = State::Exact(change_id).into(); } @@ -468,7 +470,7 @@ impl SieveScriptSet for Server { match self.core.sieve.untrusted_compiler.compile(&bytes) { Ok(script) => { changes.size = bytes.len() as u32; - bytes.extend(UnversionedArchiver::new(script).serialize().caused_by(trc::location!())?); + bytes.extend(Archiver::new(script).untrusted().serialize().caused_by(trc::location!())?); bytes.into() } Err(err) => { diff --git a/crates/jmap/src/submission/set.rs b/crates/jmap/src/submission/set.rs index 961cf048..c3c53049 100644 --- a/crates/jmap/src/submission/set.rs +++ b/crates/jmap/src/submission/set.rs @@ -242,8 +242,11 @@ impl EmailSubmissionSet for Server { // Write changes if !batch.is_empty() { - let change_id = batch.change_id(); - self.commit_batch(batch).await.caused_by(trc::location!())?; + let change_id = self + .commit_batch(batch) + .await + .and_then(|ids| ids.last_change_id(account_id)) + .caused_by(trc::location!())?; response.new_state = State::Exact(change_id).into(); } diff --git a/crates/jmap/src/vacation/set.rs b/crates/jmap/src/vacation/set.rs index 13b80188..dc7a05dd 100644 --- a/crates/jmap/src/vacation/set.rs +++ b/crates/jmap/src/vacation/set.rs @@ -4,8 +4,6 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::borrow::Cow; - use super::get::VacationResponseGet; use crate::{JmapMethods, changes::state::StateManager}; use common::{Server, auth::AccessToken, storage::index::ObjectIndexBuilder}; @@ -26,10 +24,11 @@ use jmap_proto::{ }; use mail_builder::MessageBuilder; use mail_parser::decoders::html::html_to_text; +use std::borrow::Cow; use std::future::Future; use store::{ Serialize, - write::{BatchBuilder, UnversionedArchiver}, + write::{Archiver, BatchBuilder}, }; use trc::AddContext; @@ -286,8 +285,13 @@ impl VacationResponseSet for Server { // Write changes batch.custom(obj).caused_by(trc::location!())?; if !batch.is_empty() { - response.new_state = Some(batch.change_id().into()); - self.commit_batch(batch).await.caused_by(trc::location!())?; + response.new_state = Some( + self.commit_batch(batch) + .await + .and_then(|ids| ids.last_change_id(account_id)) + .caused_by(trc::location!())? + .into(), + ); } // Deactivate other sieve scripts @@ -326,8 +330,13 @@ impl VacationResponseSet for Server { // Write changes if !batch.is_empty() { - response.new_state = Some(batch.change_id().into()); - self.commit_batch(batch).await.caused_by(trc::location!())?; + response.new_state = Some( + self.commit_batch(batch) + .await + .and_then(|ids| ids.last_change_id(account_id)) + .caused_by(trc::location!())? + .into(), + ); } } @@ -440,7 +449,8 @@ impl VacationResponseSet for Server { // Serialize script script.extend( - UnversionedArchiver::new(compiled_script) + Archiver::new(compiled_script) + .untrusted() .serialize() .caused_by(trc::location!())?, ); diff --git a/crates/managesieve/src/op/putscript.rs b/crates/managesieve/src/op/putscript.rs index 846949ac..5a082300 100644 --- a/crates/managesieve/src/op/putscript.rs +++ b/crates/managesieve/src/op/putscript.rs @@ -15,7 +15,7 @@ use std::time::Instant; use store::{ Serialize, query::Filter, - write::{BatchBuilder, UnversionedArchiver}, + write::{Archiver, BatchBuilder}, }; use trc::AddContext; @@ -79,7 +79,8 @@ impl Session { { Ok(compiled_script) => { script_bytes.extend( - UnversionedArchiver::new(compiled_script) + Archiver::new(compiled_script) + .untrusted() .serialize() .caused_by(trc::location!())?, ); diff --git a/crates/migration/src/report.rs b/crates/migration/src/report.rs index 351f8dba..97a6edd9 100644 --- a/crates/migration/src/report.rs +++ b/crates/migration/src/report.rs @@ -12,8 +12,7 @@ use store::{ IterateParams, SUBSPACE_REPORT_OUT, Serialize, U64_LEN, ValueKey, ahash::AHashSet, write::{ - AlignedBytes, AnyKey, BatchBuilder, ReportClass, UnversionedArchive, UnversionedArchiver, - ValueClass, + AlignedBytes, AnyKey, Archive, Archiver, BatchBuilder, ReportClass, ValueClass, key::{DeserializeBigEndian, KeySerializer}, }, }; @@ -91,7 +90,7 @@ pub(crate) async fn migrate_reports(server: &Server) -> trc::Result<()> { let mut batch = BatchBuilder::new(); batch.set( ValueClass::Report(ReportClass::Dmarc { id, expires }), - UnversionedArchiver::new(bincoded.inner) + Archiver::new(bincoded.inner) .serialize() .caused_by(trc::location!())?, ); @@ -106,7 +105,7 @@ pub(crate) async fn migrate_reports(server: &Server) -> trc::Result<()> { Err(err) => { if server .store() - .get_value::>(ValueKey::from( + .get_value::>(ValueKey::from( ValueClass::Report(ReportClass::Dmarc { id, expires }), )) .await @@ -129,7 +128,7 @@ pub(crate) async fn migrate_reports(server: &Server) -> trc::Result<()> { let mut batch = BatchBuilder::new(); batch.set( ValueClass::Report(ReportClass::Tls { id, expires }), - UnversionedArchiver::new(bincoded.inner) + Archiver::new(bincoded.inner) .serialize() .caused_by(trc::location!())?, ); @@ -144,7 +143,7 @@ pub(crate) async fn migrate_reports(server: &Server) -> trc::Result<()> { Err(err) => { if server .store() - .get_value::>(ValueKey::from( + .get_value::>(ValueKey::from( ValueClass::Report(ReportClass::Tls { id, expires }), )) .await @@ -167,7 +166,7 @@ pub(crate) async fn migrate_reports(server: &Server) -> trc::Result<()> { let mut batch = BatchBuilder::new(); batch.set( ValueClass::Report(ReportClass::Arf { id, expires }), - UnversionedArchiver::new(bincoded.inner) + Archiver::new(bincoded.inner) .serialize() .caused_by(trc::location!())?, ); @@ -182,7 +181,7 @@ pub(crate) async fn migrate_reports(server: &Server) -> trc::Result<()> { Err(err) => { if server .store() - .get_value::>(ValueKey::from( + .get_value::>(ValueKey::from( ValueClass::Report(ReportClass::Arf { id, expires }), )) .await diff --git a/crates/services/src/broadcast/subscriber.rs b/crates/services/src/broadcast/subscriber.rs index 7bd9a455..80d2d7f4 100644 --- a/crates/services/src/broadcast/subscriber.rs +++ b/crates/services/src/broadcast/subscriber.rs @@ -14,7 +14,6 @@ use compact_str::CompactString; use std::{sync::Arc, time::Duration}; use tokio::sync::watch; use trc::{ClusterEvent, ServerEvent}; -use utils::snowflake::HlcTimestamp; pub fn spawn_broadcast_subscriber(inner: Arc, mut shutdown_rx: watch::Receiver) { let this_node_id = { @@ -96,7 +95,6 @@ pub fn spawn_broadcast_subscriber(inner: Arc, mut shutdown_rx: watch::Rec }; let mut max_timestamp = 0; - let mut num_events = 0; let mut has_errors = false; for event in batch.events() { @@ -154,7 +152,6 @@ pub fn spawn_broadcast_subscriber(inner: Arc, mut shutdown_rx: watch::Rec } }, } - num_events += 1; } else if !has_errors { trc::event!( Cluster(ClusterEvent::MessageInvalid), @@ -165,30 +162,11 @@ pub fn spawn_broadcast_subscriber(inner: Arc, mut shutdown_rx: watch::Rec } - let skew = if max_timestamp != 0 { - match HlcTimestamp::update_clock_from_remote_timestamp(max_timestamp) { - Ok(skew) => Some(skew), - Err(large_skew) => { - trc::event!( - Cluster(ClusterEvent::ClockSkewDetected), - From = node_id, - To = this_node_id, - Total = num_events, - Details = large_skew, - ); - continue; - }, - } - } else { - None - }; - trc::event!( Cluster(ClusterEvent::MessageReceived), From = node_id, To = this_node_id, - Total = batch.events().flatten().map(log_event).collect::>(), - Details = skew, + Details = batch.events().flatten().map(log_event).collect::>(), ); } None => { diff --git a/crates/smtp/src/queue/mod.rs b/crates/smtp/src/queue/mod.rs index 7de546db..c1a23754 100644 --- a/crates/smtp/src/queue/mod.rs +++ b/crates/smtp/src/queue/mod.rs @@ -14,7 +14,7 @@ use common::expr::{self, functions::ResolveVariable, *}; use compact_str::ToCompactString; use smtp_proto::{ArchivedResponse, Response}; -use store::{SERIALIZE_QUEUE_MSG_V1, SerializedVersion, write::now}; +use store::write::now; use utils::BlobHash; pub mod dsn; @@ -79,12 +79,6 @@ pub struct Message { pub span_id: u64, } -impl SerializedVersion for Message { - fn serialize_version() -> u8 { - SERIALIZE_QUEUE_MSG_V1 - } -} - #[derive( rkyv::Serialize, rkyv::Deserialize, diff --git a/crates/smtp/src/reporting/analysis.rs b/crates/smtp/src/reporting/analysis.rs index 6610f817..3e12f5cc 100644 --- a/crates/smtp/src/reporting/analysis.rs +++ b/crates/smtp/src/reporting/analysis.rs @@ -19,7 +19,7 @@ use std::{ }; use store::{ Serialize, - write::{BatchBuilder, ReportClass, UnversionedArchiver, ValueClass, now}, + write::{Archiver, BatchBuilder, ReportClass, ValueClass, now}, }; use trc::IncomingReportEvent; @@ -281,7 +281,7 @@ impl AnalyzeReport for Server { Format::Dmarc(report) => { batch.set( ValueClass::Report(ReportClass::Dmarc { id, expires }), - UnversionedArchiver::new(IncomingReport { + Archiver::new(IncomingReport { from, to, subject, @@ -294,7 +294,7 @@ impl AnalyzeReport for Server { Format::Tls(report) => { batch.set( ValueClass::Report(ReportClass::Tls { id, expires }), - UnversionedArchiver::new(IncomingReport { + Archiver::new(IncomingReport { from, to, subject, @@ -307,7 +307,7 @@ impl AnalyzeReport for Server { Format::Arf(report) => { batch.set( ValueClass::Report(ReportClass::Arf { id, expires }), - UnversionedArchiver::new(IncomingReport { + Archiver::new(IncomingReport { from, to, subject, diff --git a/crates/smtp/src/reporting/dmarc.rs b/crates/smtp/src/reporting/dmarc.rs index f0fd59fd..427f6381 100644 --- a/crates/smtp/src/reporting/dmarc.rs +++ b/crates/smtp/src/reporting/dmarc.rs @@ -4,8 +4,12 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{collections::hash_map::Entry, future::Future}; - +use super::{AggregateTimestamp, SerializedSize}; +use crate::{ + core::Session, + queue::{DomainPart, RecipientDomain}, + reporting::SmtpReporting, +}; use ahash::AHashMap; use common::{ Server, @@ -13,13 +17,6 @@ use common::{ ipc::{DmarcEvent, ToHash}, listener::SessionStream, }; - -use super::{AggregateTimestamp, SerializedSize}; -use crate::{ - core::Session, - queue::{DomainPart, RecipientDomain}, - reporting::SmtpReporting, -}; use compact_str::ToCompactString; use mail_auth::{ ArcOutput, AuthenticatedMessage, AuthenticationResults, DkimOutput, DkimResult, DmarcOutput, @@ -28,12 +25,10 @@ use mail_auth::{ dmarc::{self, URI}, report::{AuthFailureType, IdentityAlignment, PolicyPublished, Record, Report, SPFDomainScope}, }; +use std::{collections::hash_map::Entry, future::Future}; use store::{ Deserialize, IterateParams, Serialize, ValueKey, - write::{ - AlignedBytes, BatchBuilder, QueueClass, ReportEvent, UnversionedArchive, - UnversionedArchiver, ValueClass, - }, + write::{AlignedBytes, Archive, Archiver, BatchBuilder, QueueClass, ReportEvent, ValueClass}, }; use trc::{AddContext, OutgoingReportEvent}; use utils::config::Rate; @@ -477,7 +472,7 @@ impl DmarcReporting for Server { // Deserialize report let dmarc = match self .store() - .get_value::>(ValueKey::from(ValueClass::Queue( + .get_value::>(ValueKey::from(ValueClass::Queue( QueueClass::DmarcReportHeader(event.clone()), ))) .await? @@ -552,7 +547,7 @@ impl DmarcReporting for Server { .storage .data .iterate(IterateParams::new(from_key, to_key).ascending(), |_, v| { - let archive = as Deserialize>::deserialize(v)?; + let archive = as Deserialize>::deserialize(v)?; match record_map.entry(archive.deserialize::()?) { Entry::Occupied(mut e) => { @@ -661,7 +656,7 @@ impl DmarcReporting for Server { // Write report builder.set( ValueClass::Queue(QueueClass::DmarcReportHeader(report_event.clone())), - match UnversionedArchiver::new(entry).serialize() { + match Archiver::new(entry).serialize() { Ok(data) => data.to_vec(), Err(err) => { trc::error!( @@ -678,7 +673,7 @@ impl DmarcReporting for Server { report_event.seq_id = self.inner.data.queue_id_gen.generate(); builder.set( ValueClass::Queue(QueueClass::DmarcReportEvent(report_event)), - match UnversionedArchiver::new(event.report_record).serialize() { + match Archiver::new(event.report_record).serialize() { Ok(data) => data.to_vec(), Err(err) => { trc::error!( diff --git a/crates/smtp/src/reporting/tls.rs b/crates/smtp/src/reporting/tls.rs index 1e53261b..28e9fa47 100644 --- a/crates/smtp/src/reporting/tls.rs +++ b/crates/smtp/src/reporting/tls.rs @@ -28,10 +28,7 @@ use std::fmt::Write; use std::{collections::hash_map::Entry, future::Future, sync::Arc, time::Duration}; use store::{ Deserialize, IterateParams, Serialize, ValueKey, - write::{ - AlignedBytes, BatchBuilder, QueueClass, ReportEvent, UnversionedArchive, - UnversionedArchiver, ValueClass, - }, + write::{AlignedBytes, Archive, Archiver, BatchBuilder, QueueClass, ReportEvent, ValueClass}, }; use trc::{AddContext, OutgoingReportEvent}; @@ -296,7 +293,7 @@ impl TlsReporting for Server { for event in events { let tls = if let Some(tls) = self .store() - .get_value::>(ValueKey::from(ValueClass::Queue( + .get_value::>(ValueKey::from(ValueClass::Queue( QueueClass::TlsReportHeader(event.clone()), ))) .await? @@ -334,8 +331,7 @@ impl TlsReporting for Server { .storage .data .iterate(IterateParams::new(from_key, to_key).ascending(), |_, v| { - let archive = - as Deserialize>::deserialize(v)?; + let archive = as Deserialize>::deserialize(v)?; if let Some(failure_details) = archive.deserialize::>()? { @@ -490,7 +486,7 @@ impl TlsReporting for Server { // Write report builder.set( ValueClass::Queue(QueueClass::TlsReportHeader(report_event.clone())), - match UnversionedArchiver::new(entry).serialize() { + match Archiver::new(entry).serialize() { Ok(data) => data.to_vec(), Err(err) => { trc::error!( @@ -507,7 +503,7 @@ impl TlsReporting for Server { report_event.seq_id = self.inner.data.queue_id_gen.generate(); builder.set( ValueClass::Queue(QueueClass::TlsReportEvent(report_event)), - match UnversionedArchiver::new(event.failure).serialize() { + match Archiver::new(event.failure).serialize() { Ok(data) => data.to_vec(), Err(err) => { trc::error!( diff --git a/crates/store/src/backend/foundationdb/write.rs b/crates/store/src/backend/foundationdb/write.rs index 901641cd..186c28c8 100644 --- a/crates/store/src/backend/foundationdb/write.rs +++ b/crates/store/src/backend/foundationdb/write.rs @@ -4,52 +4,69 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use std::{ - cmp::Ordering, - time::{Duration, Instant}, +use super::{ + FdbStore, MAX_VALUE_SIZE, ReadVersion, into_error, + read::{ChunkedValue, read_chunked_value}, +}; +use crate::{ + IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_QUOTA, U64_LEN, + WITH_SUBSPACE, + backend::deserialize_i64_le, + write::{ + AssignedIds, Batch, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, Operation, ValueClass, ValueOp, + key::KeySerializer, + }, }; - use foundationdb::{ FdbError, KeySelector, RangeOption, Transaction, options::{self, MutationType}, }; use futures::TryStreamExt; use rand::Rng; - -use crate::{ - IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_QUOTA, - WITH_SUBSPACE, - backend::deserialize_i64_le, - write::{ - AssignedIds, Batch, BitmapClass, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, Operation, ValueOp, - key::KeySerializer, - }, -}; - -use super::{ - FdbStore, MAX_VALUE_SIZE, ReadVersion, into_error, - read::{ChunkedValue, read_chunked_value}, +use std::{ + cmp::Ordering, + time::{Duration, Instant}, }; impl FdbStore { pub(crate) async fn write(&self, batch: Batch<'_>) -> trc::Result { let start = Instant::now(); let mut retry_count = 0; + let has_changes = !batch.changes.is_empty(); loop { let mut account_id = u32::MAX; let mut collection = u8::MAX; let mut document_id = u32::MAX; + let mut change_id = 0u64; let mut result = AssignedIds::default(); let trx = self.db.create_trx().map_err(into_error)?; - for op in batch.ops { + if has_changes { + for &account_id in batch.changes.keys() { + debug_assert!(account_id != u32::MAX); + let key = ValueClass::ChangeId.serialize(account_id, 0, 0, WITH_SUBSPACE); + let change_id = + if let Some(bytes) = trx.get(&key, false).await.map_err(into_error)? { + deserialize_i64_le(&key, &bytes)? + 1 + } else { + 1 + }; + trx.set(&key, &change_id.to_le_bytes()[..]); + result.push_change_id(account_id, change_id as u64); + } + } + + for op in batch.ops.iter_mut() { match op { Operation::AccountId { account_id: account_id_, } => { account_id = *account_id_; + if has_changes { + change_id = result.last_change_id(account_id)?; + } } Operation::Collection { collection: collection_, @@ -67,7 +84,15 @@ impl FdbStore { let do_chunk = !class.is_counter(collection); match op { - ValueOp::Set(value) => { + ValueOp::Set { + value, + version_offset, + } => { + if let Some(offset) = version_offset { + value[*offset..*offset + U64_LEN] + .copy_from_slice(&change_id.to_be_bytes()); + } + if !value.is_empty() && do_chunk { for (pos, chunk) in value.chunks(MAX_VALUE_SIZE).enumerate() { match pos.cmp(&1) { @@ -129,7 +154,7 @@ impl FdbStore { collection, document_id, field: *field, - key, + key: &*key, } .serialize(WITH_SUBSPACE); @@ -140,41 +165,23 @@ impl FdbStore { } } Operation::Bitmap { class, set } => { - let is_document_id = matches!(class, BitmapClass::DocumentIds); let key = class.serialize(account_id, collection, document_id, WITH_SUBSPACE); if *set { - if is_document_id { - trx.add_conflict_range( - &key, - &class.serialize( - account_id, - collection, - document_id + 1, - WITH_SUBSPACE, - ), - options::ConflictRangeType::Read, - ) - .map_err(into_error)?; - } - trx.set(&key, &[]); } else { trx.clear(&key); } } - Operation::Log { - collection, - change_id, - set, - } => { + Operation::Log { collection, set } => { let key = LogKey { account_id, collection: *collection, - change_id: *change_id, + change_id, } .serialize(WITH_SUBSPACE); + trx.set(&key, set); } Operation::AssertValue { @@ -210,7 +217,7 @@ impl FdbStore { { return Ok(result); } else { - let backoff = rand::rng().random_range(50..=300); + let backoff = rand::rng().random_range(50..=100); tokio::time::sleep(Duration::from_millis(backoff)).await; retry_count += 1; } diff --git a/crates/store/src/backend/mysql/write.rs b/crates/store/src/backend/mysql/write.rs index afaae2ea..2f030cb3 100644 --- a/crates/store/src/backend/mysql/write.rs +++ b/crates/store/src/backend/mysql/write.rs @@ -11,9 +11,10 @@ use mysql_async::{Conn, Error, IsolationLevel, TxOpts, params, prelude::Queryabl use rand::Rng; use crate::{ - IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_QUOTA, + IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_QUOTA, U64_LEN, write::{ - AssignedIds, Batch, BitmapClass, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, Operation, ValueOp, + AssignedIds, Batch, BitmapClass, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, Operation, + ValueClass, ValueOp, }, }; @@ -27,13 +28,13 @@ enum CommitError { } impl MysqlStore { - pub(crate) async fn write(&self, batch: Batch<'_>) -> trc::Result { + pub(crate) async fn write(&self, mut batch: Batch<'_>) -> trc::Result { let start = Instant::now(); let mut retry_count = 0; let mut conn = self.conn_pool.get_conn().await.map_err(into_error)?; loop { - let err = match self.write_trx(&mut conn, &batch).await { + let err = match self.write_trx(&mut conn, &mut batch).await { Ok(result) => { return Ok(result); } @@ -69,11 +70,13 @@ impl MysqlStore { async fn write_trx( &self, conn: &mut Conn, - batch: &Batch<'_>, + batch: &mut Batch<'_>, ) -> Result { + let has_changes = !batch.changes.is_empty(); let mut account_id = u32::MAX; let mut collection = u8::MAX; let mut document_id = u32::MAX; + let mut change_id = 0u64; let mut asserted_values = AHashMap::new(); let mut tx_opts = TxOpts::default(); tx_opts @@ -82,12 +85,35 @@ impl MysqlStore { let mut trx = conn.start_transaction(tx_opts).await?; let mut result = AssignedIds::default(); - for op in batch.ops { + if has_changes { + for &account_id in batch.changes.keys() { + let key = ValueClass::ChangeId.serialize(account_id, 0, 0, 0); + let s = trx + .prep(concat!( + "INSERT INTO n (k, v) VALUES (:k, LAST_INSERT_ID(1)) ", + "ON DUPLICATE KEY UPDATE v = LAST_INSERT_ID(v + 1)" + )) + .await?; + trx.exec_drop(&s, params! {"k" => key}).await?; + let s = trx.prep("SELECT LAST_INSERT_ID()").await?; + let change_id = trx.exec_first::(&s, ()).await?.ok_or_else(|| { + mysql_async::Error::Io(mysql_async::IoError::Io(std::io::Error::other( + "LAST_INSERT_ID() did not return a value", + ))) + })?; + result.push_change_id(account_id, change_id as u64); + } + } + + for op in batch.ops.iter_mut() { match op { Operation::AccountId { account_id: account_id_, } => { account_id = *account_id_; + if has_changes { + change_id = result.last_change_id(account_id)?; + } } Operation::Collection { collection: collection_, @@ -104,7 +130,15 @@ impl MysqlStore { let table = char::from(class.subspace(collection)); match op { - ValueOp::Set(value) => { + ValueOp::Set { + value, + version_offset, + } => { + if let Some(offset) = version_offset { + value[*offset..*offset + U64_LEN] + .copy_from_slice(&change_id.to_be_bytes()); + } + let exists = asserted_values.get(&key); let s = if let Some(exists) = exists { if *exists { @@ -125,7 +159,10 @@ impl MysqlStore { .await? }; - match trx.exec_drop(&s, params! {"k" => key, "v" => value}).await { + match trx + .exec_drop(&s, params! {"k" => key, "v" => &*value}) + .await + { Ok(_) => { if exists.is_some() && trx.affected_rows() == 0 { trx.rollback().await?; @@ -151,12 +188,12 @@ impl MysqlStore { table )) .await?; - trx.exec_drop(&s, (key, by)).await?; + trx.exec_drop(&s, (key, &*by)).await?; } else { let s = trx .prep(format!("UPDATE {table} SET v = v + ? WHERE k = ?")) .await?; - trx.exec_drop(&s, (by, key)).await?; + trx.exec_drop(&s, (&*by, key)).await?; } } ValueOp::AddAndGet(by) => { @@ -169,7 +206,7 @@ impl MysqlStore { table )) .await?; - trx.exec_drop(&s, params! {"k" => key, "v" => by}).await?; + trx.exec_drop(&s, params! {"k" => key, "v" => &*by}).await?; let s = trx.prep("SELECT LAST_INSERT_ID()").await?; result.push_counter_id( trx.exec_first::(&s, ()).await?.ok_or_else(|| { @@ -195,7 +232,7 @@ impl MysqlStore { collection, document_id, field: *field, - key, + key: &*key, } .serialize(0); @@ -236,15 +273,11 @@ impl MysqlStore { ); } } - Operation::Log { - collection, - change_id, - set, - } => { + Operation::Log { collection, set } => { let key = LogKey { account_id, collection: *collection, - change_id: *change_id, + change_id, } .serialize(0); @@ -252,7 +285,7 @@ impl MysqlStore { .prep("INSERT INTO l (k, v) VALUES (?, ?) ON DUPLICATE KEY UPDATE v = VALUES(v)") .await?; - trx.exec_drop(&s, (key, &set)).await?; + trx.exec_drop(&s, (key, &*set)).await?; } Operation::AssertValue { class, diff --git a/crates/store/src/backend/postgres/write.rs b/crates/store/src/backend/postgres/write.rs index 0576cc69..6a46ae47 100644 --- a/crates/store/src/backend/postgres/write.rs +++ b/crates/store/src/backend/postgres/write.rs @@ -12,9 +12,10 @@ use rand::Rng; use tokio_postgres::{IsolationLevel, error::SqlState}; use crate::{ - IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_QUOTA, + IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_QUOTA, U64_LEN, write::{ - AssignedIds, Batch, BitmapClass, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, Operation, ValueOp, + AssignedIds, Batch, BitmapClass, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, Operation, + ValueClass, ValueOp, }, }; @@ -28,13 +29,13 @@ enum CommitError { } impl PostgresStore { - pub(crate) async fn write(&self, batch: Batch<'_>) -> trc::Result { + pub(crate) async fn write(&self, mut batch: Batch<'_>) -> trc::Result { let mut conn = self.conn_pool.get().await.map_err(into_error)?; let start = Instant::now(); let mut retry_count = 0; loop { - match self.write_trx(&mut conn, &batch).await { + match self.write_trx(&mut conn, &mut batch).await { Ok(result) => { return Ok(result); } @@ -72,11 +73,12 @@ impl PostgresStore { async fn write_trx( &self, conn: &mut Object, - batch: &Batch<'_>, + batch: &mut Batch<'_>, ) -> Result { let mut account_id = u32::MAX; let mut collection = u8::MAX; let mut document_id = u32::MAX; + let mut change_id = 0u64; let mut asserted_values = AHashMap::new(); let trx = conn .build_transaction() @@ -84,13 +86,34 @@ impl PostgresStore { .start() .await?; let mut result = AssignedIds::default(); + let has_changes = !batch.changes.is_empty(); - for op in batch.ops { + if has_changes { + for &account_id in batch.changes.keys() { + let key = ValueClass::ChangeId.serialize(account_id, 0, 0, 0); + let s = trx + .prepare_cached(concat!( + "INSERT INTO n (k, v) VALUES ($1, 1) ", + "ON CONFLICT(k) DO UPDATE SET v = n.v + 1 RETURNING v" + )) + .await?; + let change_id = trx + .query_one(&s, &[&key]) + .await + .and_then(|row| row.try_get::<_, i64>(0))?; + result.push_change_id(account_id, change_id as u64); + } + } + + for op in batch.ops.iter_mut() { match op { Operation::AccountId { account_id: account_id_, } => { account_id = *account_id_; + if has_changes { + change_id = result.last_change_id(account_id)?; + } } Operation::Collection { collection: collection_, @@ -107,7 +130,15 @@ impl PostgresStore { let table = char::from(class.subspace(collection)); match op { - ValueOp::Set(value) => { + ValueOp::Set { + value, + version_offset, + } => { + if let Some(offset) = version_offset { + value[*offset..*offset + U64_LEN] + .copy_from_slice(&change_id.to_be_bytes()); + } + let s = if let Some(exists) = asserted_values.get(&key) { if *exists { trx.prepare_cached(&format!( @@ -133,7 +164,7 @@ impl PostgresStore { .await? }; - if trx.execute(&s, &[&key, &value]).await? == 0 { + if trx.execute(&s, &[&key, &(*value)]).await? == 0 { return Err(trc::StoreEvent::AssertValueFailed.into_err().into()); } } @@ -148,14 +179,14 @@ impl PostgresStore { table, table )) .await?; - trx.execute(&s, &[&key, &by]).await?; + trx.execute(&s, &[&key, &*by]).await?; } else { let s = trx .prepare_cached(&format!( "UPDATE {table} SET v = v + $1 WHERE k = $2" )) .await?; - trx.execute(&s, &[&by, &key]).await?; + trx.execute(&s, &[&*by, &key]).await?; } } ValueOp::AddAndGet(by) => { @@ -169,7 +200,7 @@ impl PostgresStore { )) .await?; result.push_counter_id( - trx.query_one(&s, &[&key, &by]) + trx.query_one(&s, &[&key, &*by]) .await .and_then(|row| row.try_get::<_, i64>(0))?, ); @@ -188,7 +219,7 @@ impl PostgresStore { collection, document_id, field: *field, - key, + key: &*key, } .serialize(0); @@ -231,15 +262,11 @@ impl PostgresStore { } })?; } - Operation::Log { - collection, - change_id, - set, - } => { + Operation::Log { collection, set } => { let key = LogKey { account_id, collection: *collection, - change_id: *change_id, + change_id, } .serialize(0); @@ -250,7 +277,7 @@ impl PostgresStore { )) .await?; - trx.execute(&s, &[&key, &set]).await?; + trx.execute(&s, &[&key, &*set]).await?; } Operation::AssertValue { class, diff --git a/crates/store/src/backend/rocksdb/write.rs b/crates/store/src/backend/rocksdb/write.rs index 407416dd..9ab97ce0 100644 --- a/crates/store/src/backend/rocksdb/write.rs +++ b/crates/store/src/backend/rocksdb/write.rs @@ -19,13 +19,15 @@ use rocksdb::{ use super::{CF_INDEXES, CF_LOGS, CfHandle, RocksDbStore, into_error}; use crate::{ Deserialize, IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, - SUBSPACE_QUOTA, + SUBSPACE_QUOTA, U64_LEN, backend::deserialize_i64_le, - write::{AssignedIds, Batch, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, Operation, ValueOp}, + write::{ + AssignedIds, Batch, MAX_COMMIT_ATTEMPTS, MAX_COMMIT_TIME, Operation, ValueClass, ValueOp, + }, }; impl RocksDbStore { - pub(crate) async fn write(&self, batch: Batch<'_>) -> trc::Result { + pub(crate) async fn write(&self, mut batch: Batch<'_>) -> trc::Result { let db = self.db.clone(); self.spawn_worker(move || { @@ -34,7 +36,7 @@ impl RocksDbStore { cf_indexes: db.cf_handle(CF_INDEXES).unwrap(), cf_logs: db.cf_handle(CF_LOGS).unwrap(), txn_opts: OptimisticTransactionOptions::default(), - batch: &batch, + batch: &mut batch, }; txn.txn_opts.set_snapshot(true); @@ -119,12 +121,12 @@ impl RocksDbStore { } } -struct RocksDBTransaction<'x> { +struct RocksDBTransaction<'x, 'y> { db: &'x OptimisticTransactionDB, cf_indexes: Arc>, cf_logs: Arc>, txn_opts: OptimisticTransactionOptions, - batch: &'x Batch<'x>, + batch: &'x mut Batch<'y>, } enum CommitError { @@ -132,23 +134,49 @@ enum CommitError { RocksDB(rocksdb::Error), } -impl RocksDBTransaction<'_> { - fn commit(&self) -> Result { +impl RocksDBTransaction<'_, '_> { + fn commit(&mut self) -> Result { let mut account_id = u32::MAX; let mut collection = u8::MAX; let mut document_id = u32::MAX; + let mut change_id = 0u64; let mut result = AssignedIds::default(); + let has_changes = !self.batch.changes.is_empty(); let txn = self .db .transaction_opt(&WriteOptions::default(), &self.txn_opts); - for op in self.batch.ops { + if has_changes { + let cf = self.db.cf_handle("n").unwrap(); + for &account_id in self.batch.changes.keys() { + let key = ValueClass::ChangeId.serialize(account_id, 0, 0, 0); + let change_id = txn + .get_pinned_for_update_cf(&cf, &key, true) + .map_err(CommitError::from) + .and_then(|bytes| { + if let Some(bytes) = bytes { + deserialize_i64_le(&key, &bytes) + .map(|v| v + 1) + .map_err(CommitError::from) + } else { + Ok(1) + } + })?; + txn.put_cf(&cf, &key, &change_id.to_le_bytes()[..])?; + result.push_change_id(account_id, change_id as u64); + } + } + + for op in self.batch.ops.iter_mut() { match op { Operation::AccountId { account_id: account_id_, } => { account_id = *account_id_; + if has_changes { + change_id = result.last_change_id(account_id)?; + } } Operation::Collection { collection: collection_, @@ -165,7 +193,15 @@ impl RocksDBTransaction<'_> { let cf = self.db.subspace_handle(class.subspace(collection)); match op { - ValueOp::Set(value) => { + ValueOp::Set { + value, + version_offset, + } => { + if let Some(offset) = version_offset { + value[*offset..*offset + U64_LEN] + .copy_from_slice(&change_id.to_be_bytes()); + } + txn.put_cf(&cf, &key, value)?; } ValueOp::AtomicAdd(by) => { @@ -198,7 +234,7 @@ impl RocksDBTransaction<'_> { collection, document_id, field: *field, - key, + key: &*key, } .serialize(0); @@ -218,15 +254,11 @@ impl RocksDBTransaction<'_> { txn.delete_cf(&cf, &key)?; } } - Operation::Log { - collection, - change_id, - set, - } => { + Operation::Log { collection, set } => { let key = LogKey { account_id, collection: *collection, - change_id: *change_id, + change_id, } .serialize(0); diff --git a/crates/store/src/backend/sqlite/write.rs b/crates/store/src/backend/sqlite/write.rs index 40d07400..e2341e84 100644 --- a/crates/store/src/backend/sqlite/write.rs +++ b/crates/store/src/backend/sqlite/write.rs @@ -7,8 +7,8 @@ use rusqlite::{OptionalExtension, TransactionBehavior, params}; use crate::{ - IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_QUOTA, - write::{AssignedIds, Batch, BitmapClass, Operation, ValueOp}, + IndexKey, Key, LogKey, SUBSPACE_COUNTER, SUBSPACE_IN_MEMORY_COUNTER, SUBSPACE_QUOTA, U64_LEN, + write::{AssignedIds, Batch, BitmapClass, Operation, ValueClass, ValueOp}, }; use super::{SqliteStore, into_error}; @@ -20,17 +20,38 @@ impl SqliteStore { let mut account_id = u32::MAX; let mut collection = u8::MAX; let mut document_id = u32::MAX; + let mut change_id = 0u64; let trx = conn .transaction_with_behavior(TransactionBehavior::Immediate) .map_err(into_error)?; let mut result = AssignedIds::default(); + let has_changes = !batch.changes.is_empty(); - for op in batch.ops { + if has_changes { + for &account_id in batch.changes.keys() { + let key = ValueClass::ChangeId.serialize(account_id, 0, 0, 0); + let change_id = trx + .prepare_cached(concat!( + "INSERT INTO n (k, v) VALUES (?, ?) ", + "ON CONFLICT(k) DO UPDATE SET v = v + ", + "excluded.v RETURNING v" + )) + .map_err(into_error)? + .query_row(params![&key, &1i64], |row| row.get::<_, i64>(0)) + .map_err(into_error)?; + result.push_change_id(account_id, change_id as u64); + } + } + + for op in batch.ops.iter_mut() { match op { Operation::AccountId { account_id: account_id_, } => { account_id = *account_id_; + if has_changes { + change_id = result.last_change_id(account_id)?; + } } Operation::Collection { collection: collection_, @@ -47,7 +68,15 @@ impl SqliteStore { let table = char::from(class.subspace(collection)); match op { - ValueOp::Set(value) => { + ValueOp::Set { + value, + version_offset, + } => { + if let Some(offset) = version_offset { + value[*offset..*offset + U64_LEN] + .copy_from_slice(&change_id.to_be_bytes()); + } + trx.prepare_cached(&format!( "INSERT OR REPLACE INTO {} (k, v) VALUES (?, ?)", table @@ -88,7 +117,7 @@ impl SqliteStore { table )) .map_err(into_error)? - .query_row(params![&key, &by], |row| row.get::<_, i64>(0)) + .query_row(params![&key, &*by], |row| row.get::<_, i64>(0)) .map_err(into_error)?, ); } @@ -106,7 +135,7 @@ impl SqliteStore { collection, document_id, field: *field, - key, + key: &*key, } .serialize(0); @@ -149,15 +178,11 @@ impl SqliteStore { .map_err(into_error)?; }; } - Operation::Log { - collection, - change_id, - set, - } => { + Operation::Log { collection, set } => { let key = LogKey { account_id, collection: *collection, - change_id: *change_id, + change_id, } .serialize(0); diff --git a/crates/store/src/dispatch/lookup.rs b/crates/store/src/dispatch/lookup.rs index d2ac6a33..aef1b356 100644 --- a/crates/store/src/dispatch/lookup.rs +++ b/crates/store/src/dispatch/lookup.rs @@ -37,12 +37,13 @@ impl InMemoryStore { let mut batch = BatchBuilder::new(); batch.any_op(Operation::Value { class: ValueClass::InMemory(InMemoryClass::Key(kv.key)), - op: ValueOp::Set( - KeySerializer::new(kv.value.len() + U64_LEN) + op: ValueOp::Set { + value: KeySerializer::new(kv.value.len() + U64_LEN) .write(kv.expires.map_or(u64::MAX, |expires| now() + expires)) .write(kv.value.as_slice()) .finalize(), - ), + version_offset: None, + }, }); store.write(batch.build_all()).await.map(|_| ()) } @@ -65,12 +66,13 @@ impl InMemoryStore { if let Some(expires) = kv.expires { batch.any_op(Operation::Value { class: ValueClass::InMemory(InMemoryClass::Key(kv.key.clone())), - op: ValueOp::Set( - KeySerializer::new(U64_LEN * 2) + op: ValueOp::Set { + value: KeySerializer::new(U64_LEN * 2) .write(0u64) .write(now() + expires) .finalize(), - ), + version_offset: None, + }, }); } diff --git a/crates/store/src/dispatch/store.rs b/crates/store/src/dispatch/store.rs index aefd12f8..4b426dfa 100644 --- a/crates/store/src/dispatch/store.rs +++ b/crates/store/src/dispatch/store.rs @@ -185,84 +185,6 @@ impl Store { } pub async fn write(&self, batch: Batch<'_>) -> trc::Result { - #[cfg(feature = "test_mode")] - if std::env::var("PARANOID_WRITE").is_ok_and(|v| v == "1") { - let mut account_id = u32::MAX; - let mut collection = u8::MAX; - let mut document_id = u32::MAX; - - let mut bitmaps = Vec::new(); - - for op in batch.ops { - match op { - Operation::AccountId { - account_id: account_id_, - } => { - account_id = *account_id_; - } - Operation::Collection { - collection: collection_, - } => { - collection = *collection_; - } - Operation::DocumentId { - document_id: document_id_, - } => { - document_id = *document_id_; - } - Operation::Bitmap { class, set } => { - let key = class.serialize(account_id, collection, document_id, 0); - - bitmaps.push((key, class.clone(), document_id, *set)); - } - _ => {} - } - } - - match self { - #[cfg(feature = "sqlite")] - Self::SQLite(store) => store.write(batch).await, - #[cfg(feature = "foundation")] - Self::FoundationDb(store) => store.write(batch).await, - #[cfg(feature = "postgres")] - Self::PostgreSQL(store) => store.write(batch).await, - #[cfg(feature = "mysql")] - Self::MySQL(store) => store.write(batch).await, - #[cfg(feature = "rocks")] - Self::RocksDb(store) => store.write(batch).await, - #[cfg(all(feature = "enterprise", any(feature = "postgres", feature = "mysql")))] - Self::SQLReadReplica(store) => store.write(batch).await, - Self::None => Err(trc::StoreEvent::NotConfigured.into()), - } - .caused_by(trc::location!())?; - - for (key, class, document_id, set) in bitmaps { - let mut bitmaps = BITMAPS.lock(); - let map = bitmaps.entry(key).or_default(); - if set { - if !map.insert(document_id) { - println!( - concat!( - "WARNING: key {:?} already contains document {} for account ", - "{}, collection {}" - ), - class, document_id, account_id, collection - ); - } - } else if !map.remove(&document_id) { - println!( - concat!( - "WARNING: key {:?} does not contain document {} for account ", - "{}, collection {}" - ), - class, document_id, account_id, collection - ); - } - } - - return Ok(AssignedIds::default()); - } - let start_time = Instant::now(); let ops = batch.ops.len(); @@ -875,8 +797,8 @@ impl Store { value ); } - SUBSPACE_COUNTER if key.len() == std::mem::size_of::() + 1 => { - // Message ID counters + SUBSPACE_COUNTER if key.len() == U32_LEN + 1 || key.len() == U32_LEN => { + // Message ID and change ID counters return Ok(true); } SUBSPACE_INDEXES => { diff --git a/crates/store/src/fts/index.rs b/crates/store/src/fts/index.rs index dd9c2b40..c6aa956a 100644 --- a/crates/store/src/fts/index.rs +++ b/crates/store/src/fts/index.rs @@ -202,7 +202,10 @@ impl Store { for (hash, postings) in tokens.into_iter() { keys.push(Operation::Value { class: ValueClass::FtsIndex(hash), - op: ValueOp::Set(postings.serialize()), + op: ValueOp::Set { + value: postings.serialize(), + version_offset: None, + }, }); } diff --git a/crates/store/src/lib.rs b/crates/store/src/lib.rs index a6b739f4..3fca87cc 100644 --- a/crates/store/src/lib.rs +++ b/crates/store/src/lib.rs @@ -43,28 +43,6 @@ pub trait SerializeInfallible { fn serialize(&self) -> Vec; } -// Max 64 versions (2 ^ 6) -pub const SERIALIZE_MESSAGE_DATA_V1: u8 = 0; -pub const SERIALIZE_MESSAGE_METADATA_V1: u8 = 1; -pub const SERIALIZE_MAILBOX_V1: u8 = 2; -pub const SERIALIZE_CRYPTO_V1: u8 = 3; -pub const SERIALIZE_IDENTITY_V1: u8 = 4; -pub const SERIALIZE_PUSH_V1: u8 = 5; -pub const SERIALIZE_SIEVE_V1: u8 = 6; -pub const SERIALIZE_SUBMISSION_V1: u8 = 7; -pub const SERIALIZE_QUEUE_MSG_V1: u8 = 8; -pub const SERIALIZE_CALENDAR_V1: u8 = 9; -pub const SERIALIZE_CALENDAR_EVENT_V1: u8 = 10; -pub const SERIALIZE_ADDRESS_BOOK_V1: u8 = 11; -pub const SERIALIZE_CONTACT_V1: u8 = 12; -pub const SERIALIZE_FILE_NODE_V1: u8 = 13; -pub const SERIALIZE_PRINCIPAL_V1: u8 = 14; -pub const SERIALIZE_DAV_LOCKS_V1: u8 = 15; - -pub trait SerializedVersion { - fn serialize_version() -> u8; -} - // Key serialization flags pub(crate) const WITH_SUBSPACE: u32 = 1; @@ -811,9 +789,3 @@ impl Stores { } } } - -impl SerializedVersion for () { - fn serialize_version() -> u8 { - unreachable!() - } -} diff --git a/crates/store/src/query/log.rs b/crates/store/src/query/log.rs index 1092ed5e..a391e2d8 100644 --- a/crates/store/src/query/log.rs +++ b/crates/store/src/query/log.rs @@ -27,6 +27,7 @@ pub struct Changes { pub to_change_id: u64, pub container_change_id: Option, pub item_change_id: Option, + pub is_truncated: bool, } #[derive(Debug, Clone, Copy)] @@ -45,6 +46,7 @@ impl Default for Changes { to_change_id: 0, container_change_id: None, item_change_id: None, + is_truncated: false, } } } @@ -83,6 +85,10 @@ impl Store { |key, value| { let change_id = key.deserialize_be_u64(key.len() - U64_LEN)?; if is_inclusive || change_id != from_change_id { + if value.is_empty() { + changelog.is_truncated = true; + return Ok(true); + } if changelog.changes.is_empty() { changelog.from_change_id = change_id; } diff --git a/crates/store/src/write/assert.rs b/crates/store/src/write/assert.rs index d8f408e7..caf34a21 100644 --- a/crates/store/src/write/assert.rs +++ b/crates/store/src/write/assert.rs @@ -4,25 +4,18 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use crate::{Deserialize, U32_LEN, U64_LEN}; - -use super::Archive; +use super::{Archive, ArchiveVersion}; +use crate::{U32_LEN, U64_LEN}; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum AssertValue { U32(u32), U64(u64), - Hash(u64), + Archive(ArchiveVersion), Some, None, } -#[derive(Debug, Clone)] -pub struct LegacyHashedValue { - pub hash: u64, - pub inner: T, -} - pub trait ToAssertValue { fn to_assert_value(&self) -> AssertValue; } @@ -53,25 +46,13 @@ impl ToAssertValue for u32 { impl ToAssertValue for Archive { fn to_assert_value(&self) -> AssertValue { - AssertValue::U32(self.hash) + AssertValue::Archive(self.version) } } impl ToAssertValue for &Archive { fn to_assert_value(&self) -> AssertValue { - AssertValue::U32(self.hash) - } -} - -impl ToAssertValue for LegacyHashedValue { - fn to_assert_value(&self) -> AssertValue { - AssertValue::Hash(self.hash) - } -} - -impl ToAssertValue for &LegacyHashedValue { - fn to_assert_value(&self) -> AssertValue { - AssertValue::Hash(self.hash) + AssertValue::Archive(self.version) } } @@ -85,7 +66,15 @@ impl AssertValue { AssertValue::U64(v) => bytes .get(bytes.len() - U64_LEN..) .is_some_and(|b| b == v.to_be_bytes()), - AssertValue::Hash(v) => xxhash_rust::xxh3::xxh3_64(bytes) == *v, + AssertValue::Archive(v) => match v { + ArchiveVersion::Versioned { hash, .. } => bytes + .get(bytes.len() - U32_LEN - U64_LEN - 1..bytes.len() - U64_LEN - 1) + .is_some_and(|b| b == hash.to_be_bytes()), + ArchiveVersion::Hashed { hash } => bytes + .get(bytes.len() - U32_LEN - 1..bytes.len() - 1) + .is_some_and(|b| b == hash.to_be_bytes()), + ArchiveVersion::Unversioned => false, + }, AssertValue::None => false, AssertValue::Some => true, } @@ -95,12 +84,3 @@ impl AssertValue { matches!(self, AssertValue::None) } } - -impl Deserialize for LegacyHashedValue { - fn deserialize(bytes: &[u8]) -> trc::Result { - Ok(LegacyHashedValue { - hash: xxhash_rust::xxh3::xxh3_64(bytes), - inner: T::deserialize(bytes)?, - }) - } -} diff --git a/crates/store/src/write/batch.rs b/crates/store/src/write/batch.rs index 73035899..67f8cfc8 100644 --- a/crates/store/src/write/batch.rs +++ b/crates/store/src/write/batch.rs @@ -9,16 +9,12 @@ use super::{ ValueClass, ValueOp, assert::ToAssertValue, }; use crate::{SerializeInfallible, U32_LEN}; -use utils::{ - map::{bitmap::ShortId, vec_map::VecMap}, - snowflake::HlcTimestamp, -}; +use utils::map::{bitmap::ShortId, vec_map::VecMap}; impl BatchBuilder { pub fn new() -> Self { Self { ops: Vec::with_capacity(32), - current_change_id: None, current_account_id: None, current_collection: None, current_document_id: None, @@ -31,12 +27,6 @@ impl BatchBuilder { } } - fn generate_change_id(&mut self) -> u64 { - let change_id = HlcTimestamp::generate(); - self.current_change_id = Some(change_id); - change_id - } - pub fn with_account_id(&mut self, account_id: u32) -> &mut Self { if self .current_account_id @@ -193,7 +183,30 @@ impl BatchBuilder { self.batch_size += class.serialized_size() + value.len(); self.ops.push(Operation::Value { class, - op: ValueOp::Set(value), + op: ValueOp::Set { + value, + version_offset: None, + }, + }); + self.batch_ops += 1; + self + } + + pub fn set_versioned( + &mut self, + class: impl Into, + value: impl Into>, + version_offset: usize, + ) -> &mut Self { + let class = class.into(); + let value = value.into(); + self.batch_size += class.serialized_size() + value.len(); + self.ops.push(Operation::Value { + class, + op: ValueOp::Set { + value, + version_offset: Some(version_offset), + }, }); self.batch_ops += 1; self @@ -214,7 +227,10 @@ impl BatchBuilder { self.batch_size += (U32_LEN * 3) + op.len(); self.ops.push(Operation::Value { class: ValueClass::Acl(grant_account_id), - op: ValueOp::Set(op), + op: ValueOp::Set { + value: op, + version_offset: None, + }, }); self.batch_ops += 1; self @@ -240,10 +256,6 @@ impl BatchBuilder { document_id, ); } - if self.current_change_id.is_none() { - self.generate_change_id(); - self.batch_ops += 1; - } self } @@ -257,10 +269,6 @@ impl BatchBuilder { document_id, ); } - if self.current_change_id.is_none() { - self.generate_change_id(); - self.batch_ops += 1; - } self } @@ -274,10 +282,6 @@ impl BatchBuilder { document_id, ); } - if self.current_change_id.is_none() { - self.generate_change_id(); - self.batch_ops += 1; - } self } @@ -289,10 +293,6 @@ impl BatchBuilder { .get_mut_or_insert(account_id) .log_container_insert(collection.into(), document_id); } - if self.current_change_id.is_none() { - self.generate_change_id(); - self.batch_ops += 1; - } self } @@ -304,10 +304,6 @@ impl BatchBuilder { .get_mut_or_insert(account_id) .log_container_update(collection.into(), document_id); } - if self.current_change_id.is_none() { - self.generate_change_id(); - self.batch_ops += 1; - } self } @@ -319,10 +315,6 @@ impl BatchBuilder { .get_mut_or_insert(account_id) .log_container_delete(collection.into(), document_id); } - if self.current_change_id.is_none() { - self.generate_change_id(); - self.batch_ops += 1; - } self } @@ -336,35 +328,27 @@ impl BatchBuilder { .get_mut_or_insert(account_id) .log_container_property_update(collection.into(), document_id); } - if self.current_change_id.is_none() { - self.generate_change_id(); - self.batch_ops += 1; - } self } fn serialize_changes(&mut self) { - if let Some(change_id) = self.current_change_id.take() { - if !self.changes.is_empty() { - for (account_id, changelog) in std::mem::take(&mut self.changes) { - self.with_account_id(account_id); + if !self.changes.is_empty() { + for (account_id, changelog) in std::mem::take(&mut self.changes) { + self.with_account_id(account_id); - for (collection, changes) in changelog.into_iterator() { - let cc = self.changed_collections.get_mut_or_insert(account_id); - cc.change_id = change_id; - if changes.has_container_changes() { - cc.changed_containers.insert(ShortId(collection)); - } - if changes.has_item_changes() { - cc.changed_items.insert(ShortId(collection)); - } - - self.ops.push(Operation::Log { - change_id, - collection, - set: changes.serialize(), - }); + for (collection, changes) in changelog.into_iterator() { + let cc = self.changed_collections.get_mut_or_insert(account_id); + if changes.has_container_changes() { + cc.changed_containers.insert(ShortId(collection)); } + if changes.has_item_changes() { + cc.changed_items.insert(ShortId(collection)); + } + + self.ops.push(Operation::Log { + collection, + set: changes.serialize(), + }); } } } @@ -401,39 +385,27 @@ impl BatchBuilder { self.current_account_id } - pub fn change_id(&mut self) -> u64 { - self.current_change_id - .unwrap_or_else(|| self.generate_change_id()) - } - - pub fn last_change_id(&self) -> Option { - self.current_change_id - } - - pub fn build(&mut self) -> impl Iterator> { + pub fn commit_points(&mut self) -> CommitPointIterator { self.serialize_changes(); - self.build_batches() + CommitPointIterator { + commit_points: std::mem::take(&mut self.commit_points), + commit_point_last: self.ops.len(), + offset_start: 0, + } } - fn build_batches(&self) -> impl Iterator> { - let mut offset_start = 0; - self.commit_points - .iter() - .copied() - .chain([self.ops.len()]) - .map(move |point| { - let batch = Batch { - ops: &self.ops[offset_start..point], - }; - offset_start = point; - batch - }) + pub fn build_one(&mut self, commit_point: CommitPoint) -> Batch<'_> { + Batch { + changes: &self.changed_collections, + ops: &mut self.ops[commit_point.offset_start..commit_point.offset_end], + } } pub fn build_all(&mut self) -> Batch<'_> { self.serialize_changes(); Batch { - ops: self.ops.as_slice(), + changes: &self.changed_collections, + ops: self.ops.as_mut_slice(), } } @@ -462,6 +434,34 @@ impl BatchBuilder { } } +pub struct CommitPointIterator { + commit_points: Vec, + commit_point_last: usize, + offset_start: usize, +} + +pub struct CommitPoint { + pub offset_start: usize, + pub offset_end: usize, +} + +impl CommitPointIterator { + pub fn iter(&mut self) -> impl Iterator { + self.commit_points + .iter() + .copied() + .chain([self.commit_point_last]) + .map(|offset_end| { + let point = CommitPoint { + offset_start: self.offset_start, + offset_end, + }; + self.offset_start = offset_end; + point + }) + } +} + impl Batch<'_> { pub fn is_atomic(&self) -> bool { !self.ops.iter().any(|op| { diff --git a/crates/store/src/write/key.rs b/crates/store/src/write/key.rs index 9aa0b9d0..34f9e960 100644 --- a/crates/store/src/write/key.rs +++ b/crates/store/src/write/key.rs @@ -405,6 +405,7 @@ impl ValueClass { .write_leb128(*node_id), }, ValueClass::DocumentId => serializer.write(account_id).write(collection), + ValueClass::ChangeId => serializer.write(account_id), ValueClass::Any(any) => serializer.write(any.key.as_slice()), } .finalize() @@ -583,6 +584,7 @@ impl ValueClass { TelemetryClass::Metric { .. } => U64_LEN * 2 + 1, }, ValueClass::DocumentId => U32_LEN + 1, + ValueClass::ChangeId => U32_LEN, ValueClass::Any(v) => v.key.len(), } } @@ -629,7 +631,7 @@ impl ValueClass { TelemetryClass::Index { .. } => SUBSPACE_TELEMETRY_INDEX, TelemetryClass::Metric { .. } => SUBSPACE_TELEMETRY_METRIC, }, - ValueClass::DocumentId => SUBSPACE_COUNTER, + ValueClass::DocumentId | ValueClass::ChangeId => SUBSPACE_COUNTER, ValueClass::Any(any) => any.subspace, } } @@ -639,7 +641,8 @@ impl ValueClass { ValueClass::Directory(DirectoryClass::UsedQuota(_)) | ValueClass::InMemory(InMemoryClass::Counter(_)) | ValueClass::Queue(QueueClass::QuotaCount(_) | QueueClass::QuotaSize(_)) - | ValueClass::DocumentId => true, + | ValueClass::DocumentId + | ValueClass::ChangeId => true, ValueClass::Property(84) if collection == 1 => true, // TODO: Find a more elegant way to do this _ => false, } diff --git a/crates/store/src/write/mod.rs b/crates/store/src/write/mod.rs index 795da015..fe909d01 100644 --- a/crates/store/src/write/mod.rs +++ b/crates/store/src/write/mod.rs @@ -38,13 +38,14 @@ pub(crate) const ARCHIVE_ALIGNMENT: usize = 16; #[derive(Debug, Clone)] pub struct Archive { pub inner: T, - pub version: u8, - pub hash: u32, + pub version: ArchiveVersion, } -#[derive(Debug, Clone)] -pub struct UnversionedArchive { - pub inner: T, +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ArchiveVersion { + Versioned { change_id: u64, hash: u32 }, + Hashed { hash: u32 }, + Unversioned, } #[derive(Debug, Clone)] @@ -53,8 +54,7 @@ pub enum AlignedBytes { Vec(Vec), } -#[repr(transparent)] -pub struct Archiver(pub T) +pub struct Archiver where T: rkyv::Archive + for<'a> rkyv::Serialize< @@ -63,24 +63,27 @@ where rkyv::ser::allocator::ArenaHandle<'a>, rkyv::rancor::Error, >, - >; - -#[repr(transparent)] -pub struct UnversionedArchiver(pub T) -where - T: rkyv::Archive - + for<'a> rkyv::Serialize< - rkyv::api::high::HighSerializer< - rkyv::util::AlignedVec, - rkyv::ser::allocator::ArenaHandle<'a>, - rkyv::rancor::Error, - >, - >; + >, +{ + pub inner: T, + pub flags: u8, +} #[derive(Debug, Default)] pub struct AssignedIds { - pub counter_ids: Vec, - pub change_id: Option, + pub ids: Vec, +} + +#[derive(Debug)] +pub enum AssignedId { + Counter(i64), + ChangeId(ChangeId), +} + +#[derive(Debug, Clone, Copy)] +pub struct ChangeId { + pub account_id: u32, + pub change_id: u64, } #[cfg(not(feature = "test_mode"))] @@ -95,12 +98,12 @@ pub(crate) const MAX_COMMIT_TIME: Duration = Duration::from_secs(3600); #[derive(Debug)] pub struct Batch<'x> { - pub(crate) ops: &'x [Operation], + pub(crate) changes: &'x VecMap, + pub(crate) ops: &'x mut [Operation], } #[derive(Debug)] pub struct BatchBuilder { - current_change_id: Option, current_account_id: Option, current_collection: Option, current_document_id: Option, @@ -115,7 +118,6 @@ pub struct BatchBuilder { #[derive(Debug, Default)] pub struct ChangedCollection { - pub change_id: u64, pub changed_containers: Bitmap, pub changed_items: Bitmap, } @@ -149,7 +151,6 @@ pub enum Operation { set: bool, }, Log { - change_id: u64, collection: u8, set: Vec, }, @@ -189,6 +190,7 @@ pub enum ValueClass { Telemetry(TelemetryClass), Any(AnyClass), DocumentId, + ChangeId, } #[derive(Debug, PartialEq, Clone, Eq, Hash)] @@ -278,7 +280,10 @@ pub struct ReportEvent { #[derive(Debug, PartialEq, Eq, Hash, Default)] pub enum ValueOp { - Set(Vec), + Set { + value: Vec, + version_offset: Option, + }, AtomicAdd(i64), AddAndGet(i64), #[default] @@ -408,23 +413,46 @@ impl BlobClass { impl AssignedIds { pub fn push_counter_id(&mut self, id: i64) { - self.counter_ids.push(id); + self.ids.push(AssignedId::Counter(id)); } - pub fn change_id(&self) -> trc::Result { - self.change_id.ok_or_else(|| { - trc::StoreEvent::UnexpectedError - .caused_by(trc::location!()) - .ctx(trc::Key::Reason, "No change id was assigned") - }) + pub fn push_change_id(&mut self, account_id: u32, change_id: u64) { + self.ids.push(AssignedId::ChangeId(ChangeId { + account_id, + change_id, + })); + } + + pub fn last_change_id(&self, account_id: u32) -> trc::Result { + self.ids + .iter() + .filter_map(|id| match id { + AssignedId::ChangeId(change_id) if change_id.account_id == account_id => { + Some(change_id.change_id) + } + _ => None, + }) + .next_back() + .ok_or_else(|| { + trc::StoreEvent::UnexpectedError + .caused_by(trc::location!()) + .ctx(trc::Key::Reason, "No change ids were created") + }) } pub fn last_counter_id(&self) -> trc::Result { - self.counter_ids.last().copied().ok_or_else(|| { - trc::StoreEvent::UnexpectedError - .caused_by(trc::location!()) - .ctx(trc::Key::Reason, "No document ids were created") - }) + self.ids + .iter() + .filter_map(|id| match id { + AssignedId::Counter(counter_id) => Some(*counter_id), + _ => None, + }) + .next_back() + .ok_or_else(|| { + trc::StoreEvent::UnexpectedError + .caused_by(trc::location!()) + .ctx(trc::Key::Reason, "No counter ids were created") + }) } } @@ -452,3 +480,20 @@ impl TagValue { } } } + +impl ArchiveVersion { + pub fn hash(&self) -> Option { + match self { + ArchiveVersion::Versioned { hash, .. } => Some(*hash), + ArchiveVersion::Hashed { hash } => Some(*hash), + ArchiveVersion::Unversioned => None, + } + } + + pub fn change_id(&self) -> Option { + match self { + ArchiveVersion::Versioned { change_id, .. } => Some(*change_id), + _ => None, + } + } +} diff --git a/crates/store/src/write/serialize.rs b/crates/store/src/write/serialize.rs index 2265d91b..c2ece762 100644 --- a/crates/store/src/write/serialize.rs +++ b/crates/store/src/write/serialize.rs @@ -4,38 +4,67 @@ * SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-SEL */ -use super::{ - ARCHIVE_ALIGNMENT, AlignedBytes, Archive, Archiver, UnversionedArchive, UnversionedArchiver, -}; -use crate::{Deserialize, Serialize, SerializeInfallible, SerializedVersion, U32_LEN, Value}; +use super::{ARCHIVE_ALIGNMENT, AlignedBytes, Archive, ArchiveVersion, Archiver}; +use crate::{Deserialize, Serialize, SerializeInfallible, U32_LEN, U64_LEN, Value}; use compact_str::format_compact; use rkyv::util::AlignedVec; const MAGIC_MARKER: u8 = 1 << 7; -const LZ4_COMPRESSED: u8 = 1 << 6; -const ARCHIVE_UNCOMPRESSED: u8 = MAGIC_MARKER; -const ARCHIVE_LZ4_COMPRESSED: u8 = MAGIC_MARKER | LZ4_COMPRESSED; +const VERSIONED: u8 = 1 << 6; +const HASHED: u8 = 1 << 5; +const LZ4_COMPRESSED: u8 = 1 << 4; + const COMPRESS_WATERMARK: usize = 8192; const HASH_SEED: i64 = 791120; -const MARKER_MASK: u8 = MAGIC_MARKER | LZ4_COMPRESSED; -const VERSION_MASK: u8 = !MARKER_MASK; +fn validate_marker_and_contents(bytes: &[u8]) -> Option<(bool, &[u8], ArchiveVersion)> { + let (marker, contents) = bytes + .split_last() + .filter(|(marker, _)| (**marker & MAGIC_MARKER) != 0)?; + let is_uncompressed = (marker & LZ4_COMPRESSED) == 0; + if marker & VERSIONED != 0 { + let (contents, change_id) = contents + .split_at_checked(contents.len() - U64_LEN) + .and_then(|(contents, change_id)| { + change_id + .try_into() + .ok() + .map(|change_id| (contents, u64::from_be_bytes(change_id))) + })?; + contents + .split_at_checked(contents.len() - U32_LEN) + .and_then(|(contents, archive_hash)| { + let hash = gxhash::gxhash32(contents, HASH_SEED); + if hash.to_be_bytes().as_slice() == archive_hash { + Some(( + is_uncompressed, + contents, + ArchiveVersion::Versioned { change_id, hash }, + )) + } else { + None + } + }) + } else if marker & HASHED != 0 { + contents + .split_at_checked(contents.len() - U32_LEN) + .and_then(|(contents, archive_hash)| { + let hash = gxhash::gxhash32(contents, HASH_SEED); + if hash.to_be_bytes().as_slice() == archive_hash { + Some((is_uncompressed, contents, ArchiveVersion::Hashed { hash })) + } else { + None + } + }) + } else { + Some((is_uncompressed, contents, ArchiveVersion::Unversioned)) + } +} impl Deserialize for Archive { fn deserialize(bytes: &[u8]) -> trc::Result { - let (contents, marker, hash) = bytes - .split_at_checked(bytes.len() - (U32_LEN + 1)) - .and_then(|(contents, marker)| { - marker.split_first().and_then(|(marker, archive_hash)| { - let hash = gxhash::gxhash32(contents, HASH_SEED); - if hash.to_be_bytes().as_slice() == archive_hash { - Some((contents, *marker, hash)) - } else { - None - } - }) - }) - .ok_or_else(|| { + let (is_uncompressed, contents, version) = + validate_marker_and_contents(bytes).ok_or_else(|| { trc::StoreEvent::DataCorruption .into_err() .details("Archive integrity compromised") @@ -43,42 +72,20 @@ impl Deserialize for Archive { .caused_by(trc::location!()) })?; - match marker & MARKER_MASK { - ARCHIVE_UNCOMPRESSED => { - let mut bytes = AlignedVec::with_capacity(contents.len()); - bytes.extend_from_slice(contents); - Ok(Archive { - hash, - version: marker & VERSION_MASK, - inner: AlignedBytes::Aligned(bytes), - }) - } - ARCHIVE_LZ4_COMPRESSED => aligned_lz4_deflate(contents).map(|inner| Archive { - hash, - version: marker & VERSION_MASK, - inner, - }), - _ => Err(trc::StoreEvent::DataCorruption - .into_err() - .details("Invalid archive marker.") - .ctx(trc::Key::Value, bytes) - .caused_by(trc::location!())), + if is_uncompressed { + let mut bytes = AlignedVec::with_capacity(contents.len()); + bytes.extend_from_slice(contents); + Ok(Archive { + version, + inner: AlignedBytes::Aligned(bytes), + }) + } else { + aligned_lz4_deflate(contents).map(|inner| Archive { version, inner }) } } fn deserialize_owned(mut bytes: Vec) -> trc::Result { - let (contents, marker, hash) = bytes - .split_at_checked(bytes.len() - (U32_LEN + 1)) - .and_then(|(contents, marker)| { - marker.split_first().and_then(|(marker, archive_hash)| { - let hash = gxhash::gxhash32(contents, HASH_SEED); - if hash.to_be_bytes().as_slice() == archive_hash { - Some((contents, *marker, hash)) - } else { - None - } - }) - }) + let (is_uncompressed, contents, version) = validate_marker_and_contents(&bytes) .ok_or_else(|| { trc::StoreEvent::DataCorruption .into_err() @@ -87,100 +94,23 @@ impl Deserialize for Archive { .caused_by(trc::location!()) })?; - match marker & MARKER_MASK { - ARCHIVE_UNCOMPRESSED => { - bytes.truncate(contents.len()); - if bytes.as_ptr().addr() & (ARCHIVE_ALIGNMENT - 1) == 0 { - Ok(Archive { - hash, - version: marker & VERSION_MASK, - inner: AlignedBytes::Vec(bytes), - }) - } else { - let mut aligned = AlignedVec::with_capacity(bytes.len()); - aligned.extend_from_slice(&bytes); - Ok(Archive { - hash, - version: marker & VERSION_MASK, - inner: AlignedBytes::Aligned(aligned), - }) - } - } - ARCHIVE_LZ4_COMPRESSED => aligned_lz4_deflate(contents).map(|inner| Archive { - hash, - version: marker & VERSION_MASK, - inner, - }), - _ => Err(trc::StoreEvent::DataCorruption - .into_err() - .details("Invalid archive marker") - .ctx(trc::Key::Value, bytes) - .caused_by(trc::location!())), - } - } -} - -impl Deserialize for UnversionedArchive { - fn deserialize(bytes: &[u8]) -> trc::Result { - let (marker, contents) = bytes.split_last().ok_or_else(|| { - trc::StoreEvent::DataCorruption - .into_err() - .details("Archive integrity compromised") - .ctx(trc::Key::Value, bytes) - .caused_by(trc::location!()) - })?; - - match marker & MARKER_MASK { - ARCHIVE_UNCOMPRESSED => { - let mut bytes = AlignedVec::with_capacity(contents.len()); - bytes.extend_from_slice(contents); - Ok(UnversionedArchive { - inner: AlignedBytes::Aligned(bytes), + if is_uncompressed { + bytes.truncate(contents.len()); + if bytes.as_ptr().addr() & (ARCHIVE_ALIGNMENT - 1) == 0 { + Ok(Archive { + version, + inner: AlignedBytes::Vec(bytes), + }) + } else { + let mut aligned = AlignedVec::with_capacity(bytes.len()); + aligned.extend_from_slice(&bytes); + Ok(Archive { + version, + inner: AlignedBytes::Aligned(aligned), }) } - ARCHIVE_LZ4_COMPRESSED => { - aligned_lz4_deflate(contents).map(|inner| UnversionedArchive { inner }) - } - _ => Err(trc::StoreEvent::DataCorruption - .into_err() - .details("Invalid archive marker.") - .ctx(trc::Key::Value, bytes) - .caused_by(trc::location!())), - } - } - - fn deserialize_owned(mut bytes: Vec) -> trc::Result { - let (marker, contents) = bytes.split_last().ok_or_else(|| { - trc::StoreEvent::DataCorruption - .into_err() - .details("Archive integrity compromised") - .ctx(trc::Key::Value, bytes.as_slice()) - .caused_by(trc::location!()) - })?; - - match marker & MARKER_MASK { - ARCHIVE_UNCOMPRESSED => { - bytes.truncate(contents.len()); - if bytes.as_ptr().addr() & (ARCHIVE_ALIGNMENT - 1) == 0 { - Ok(UnversionedArchive { - inner: AlignedBytes::Vec(bytes), - }) - } else { - let mut aligned = AlignedVec::with_capacity(bytes.len()); - aligned.extend_from_slice(&bytes); - Ok(UnversionedArchive { - inner: AlignedBytes::Aligned(aligned), - }) - } - } - ARCHIVE_LZ4_COMPRESSED => { - aligned_lz4_deflate(contents).map(|inner| UnversionedArchive { inner }) - } - _ => Err(trc::StoreEvent::DataCorruption - .into_err() - .details("Invalid archive marker") - .ctx(trc::Key::Value, bytes) - .caused_by(trc::location!())), + } else { + aligned_lz4_deflate(contents).map(|inner| Archive { version, inner }) } } } @@ -208,7 +138,6 @@ fn aligned_lz4_deflate(archive: &[u8]) -> trc::Result { impl Serialize for Archiver where T: rkyv::Archive - + SerializedVersion + for<'a> rkyv::Serialize< rkyv::api::high::HighSerializer< rkyv::util::AlignedVec, @@ -218,7 +147,7 @@ where >, { fn serialize(&self) -> trc::Result> { - rkyv::to_bytes::(&self.0) + rkyv::to_bytes::(&self.inner) .map_err(|err| { trc::StoreEvent::DeserializeError .caused_by(trc::location!()) @@ -227,82 +156,57 @@ where .map(|input| { let input = input.as_ref(); let input_len = input.len(); - if input_len > COMPRESS_WATERMARK { - let mut bytes = - vec![ - ARCHIVE_LZ4_COMPRESSED | (T::serialize_version() & VERSION_MASK); - lz4_flex::block::get_maximum_output_size(input_len) + (U32_LEN * 2) + 1 - ]; - let compressed_len = - lz4_flex::compress_into(input, &mut bytes[U32_LEN..]).unwrap(); - if compressed_len < input_len { - bytes[..U32_LEN].copy_from_slice(&(input_len as u32).to_le_bytes()); - let hash = gxhash::gxhash32(&bytes[..compressed_len + U32_LEN], HASH_SEED); - bytes[compressed_len + U32_LEN + 1..compressed_len + (U32_LEN * 2) + 1] - .copy_from_slice(&hash.to_be_bytes()); - bytes.truncate(compressed_len + (U32_LEN * 2) + 1); - } else { - bytes.clear(); - bytes.extend_from_slice(input); - bytes.push(ARCHIVE_UNCOMPRESSED | (T::serialize_version() & VERSION_MASK)); - bytes.extend_from_slice(&gxhash::gxhash32(input, HASH_SEED).to_be_bytes()); - } - bytes - } else { - let mut bytes = Vec::with_capacity(input_len + U32_LEN + 1); - bytes.extend_from_slice(input); - bytes.push(ARCHIVE_UNCOMPRESSED | (T::serialize_version() & VERSION_MASK)); - bytes.extend_from_slice(&gxhash::gxhash32(input, HASH_SEED).to_be_bytes()); - bytes - } - }) - } -} + let version_offset = ((self.flags & VERSIONED != 0) as usize) * U64_LEN; + let mut bytes = if input_len > COMPRESS_WATERMARK { + let mut bytes = vec![ + self.flags | LZ4_COMPRESSED; + lz4_flex::block::get_maximum_output_size(input_len) + + (U32_LEN * 2) + + version_offset + + 1 + ]; -impl Serialize for UnversionedArchiver -where - T: rkyv::Archive - + for<'a> rkyv::Serialize< - rkyv::api::high::HighSerializer< - rkyv::util::AlignedVec, - rkyv::ser::allocator::ArenaHandle<'a>, - rkyv::rancor::Error, - >, - >, -{ - fn serialize(&self) -> trc::Result> { - rkyv::to_bytes::(&self.0) - .map_err(|err| { - trc::StoreEvent::DeserializeError - .caused_by(trc::location!()) - .reason(err) - }) - .map(|input| { - let input = input.as_ref(); - let input_len = input.len(); - if input_len > COMPRESS_WATERMARK { - let mut bytes = - vec![ - ARCHIVE_LZ4_COMPRESSED; - lz4_flex::block::get_maximum_output_size(input_len) + U32_LEN + 1 - ]; + // Compress the data let compressed_len = lz4_flex::compress_into(input, &mut bytes[U32_LEN..]).unwrap(); + if compressed_len < input_len { + // Prepend the length of the uncompressed data bytes[..U32_LEN].copy_from_slice(&(input_len as u32).to_le_bytes()); - bytes.truncate(compressed_len + U32_LEN + 1); - } else { - bytes.clear(); - bytes.extend_from_slice(input); - bytes.push(ARCHIVE_UNCOMPRESSED); + + if self.flags & HASHED != 0 { + // Hash the compressed data including the length + let hash = + gxhash::gxhash32(&bytes[..compressed_len + U32_LEN], HASH_SEED); + + // Add the hash + bytes[compressed_len + U32_LEN..compressed_len + (U32_LEN * 2)] + .copy_from_slice(&hash.to_be_bytes()); + + // Truncate to the actual size + bytes.truncate(compressed_len + (U32_LEN * 2) + version_offset + 1); + } else { + // Truncate to the actual size + bytes.truncate(compressed_len + U32_LEN + 1); + } + + return bytes; } + bytes.clear(); bytes } else { - let mut bytes = Vec::with_capacity(input_len + 1); - bytes.extend_from_slice(input); - bytes.push(ARCHIVE_UNCOMPRESSED); - bytes + Vec::with_capacity(input_len + U32_LEN + version_offset + 1) + }; + + bytes.extend_from_slice(input); + if self.flags & HASHED != 0 { + bytes.extend_from_slice(&gxhash::gxhash32(input, HASH_SEED).to_be_bytes()); } + if version_offset != 0 { + bytes.extend_from_slice(0u64.to_be_bytes().as_slice()); + } + bytes.push(self.flags); + bytes }) } } @@ -318,43 +222,47 @@ impl Archive { pub fn unarchive(&self) -> trc::Result<&::Archived> where - T: rkyv::Archive + SerializedVersion, + T: rkyv::Archive, T::Archived: for<'a> rkyv::bytecheck::CheckBytes< rkyv::api::high::HighValidator<'a, rkyv::rancor::Error>, > + rkyv::Deserialize>, { let bytes = self.as_bytes(); - if self.version == T::serialize_version() - && bytes.len() >= std::mem::size_of::() - { - // SAFETY: Trusted and versioned input with integrity hash - Ok(unsafe { rkyv::access_unchecked::(bytes) }) + if self.version != ArchiveVersion::Unversioned { + if bytes.len() >= std::mem::size_of::() { + // SAFETY: Trusted input with integrity hash + Ok(unsafe { rkyv::access_unchecked::(bytes) }) + } else { + Err(trc::StoreEvent::DataCorruption + .into_err() + .details(format_compact!( + "Archive size mismatch, expected {} bytes but got {} bytes.", + std::mem::size_of::(), + bytes.len() + )) + .ctx(trc::Key::Value, bytes) + .caused_by(trc::location!())) + } } else { - Err(trc::StoreEvent::DataCorruption - .into_err() - .details(format_compact!( - "Archive version mismatch, expected {} ({} bytes) but got {} ({} bytes)", - T::serialize_version(), - std::mem::size_of::(), - self.version, - bytes.len() - )) - .ctx(trc::Key::Value, bytes) - .caused_by(trc::location!())) + rkyv::access::(bytes).map_err(|err| { + trc::StoreEvent::DeserializeError + .ctx(trc::Key::Value, self.as_bytes()) + .details("Archive access failed") + .caused_by(trc::location!()) + .reason(err) + }) } } pub fn unarchive_untrusted(&self) -> trc::Result<&::Archived> where - T: rkyv::Archive + SerializedVersion, + T: rkyv::Archive, T::Archived: for<'a> rkyv::bytecheck::CheckBytes< rkyv::api::high::HighValidator<'a, rkyv::rancor::Error>, > + rkyv::Deserialize>, { let bytes = self.as_bytes(); - if self.version == T::serialize_version() - && bytes.len() >= std::mem::size_of::() - { + if bytes.len() >= std::mem::size_of::() { rkyv::access::(bytes).map_err(|err| { trc::StoreEvent::DeserializeError .ctx(trc::Key::Value, self.as_bytes()) @@ -366,10 +274,8 @@ impl Archive { Err(trc::StoreEvent::DataCorruption .into_err() .details(format_compact!( - "Archive version mismatch, expected {} ({} bytes) but got {} ({} bytes)", - T::serialize_version(), + "Archive size mismatch, expected {} bytes but got {} bytes.", std::mem::size_of::(), - self.version, bytes.len() )) .ctx(trc::Key::Value, bytes) @@ -379,7 +285,7 @@ impl Archive { pub fn deserialize(&self) -> trc::Result where - T: rkyv::Archive + SerializedVersion, + T: rkyv::Archive, T::Archived: for<'a> rkyv::bytecheck::CheckBytes< rkyv::api::high::HighValidator<'a, rkyv::rancor::Error>, > + rkyv::Deserialize>, @@ -396,13 +302,12 @@ impl Archive { pub fn to_unarchived(&self) -> trc::Result::Archived>> where - T: rkyv::Archive + SerializedVersion, + T: rkyv::Archive, T::Archived: for<'a> rkyv::bytecheck::CheckBytes< rkyv::api::high::HighValidator<'a, rkyv::rancor::Error>, > + rkyv::Deserialize>, { self.unarchive::().map(|inner| Archive { - hash: self.hash, version: self.version, inner, }) @@ -410,13 +315,12 @@ impl Archive { pub fn into_deserialized(&self) -> trc::Result> where - T: rkyv::Archive + SerializedVersion, + T: rkyv::Archive, T::Archived: for<'a> rkyv::bytecheck::CheckBytes< rkyv::api::high::HighValidator<'a, rkyv::rancor::Error>, > + rkyv::Deserialize>, { self.deserialize::().map(|inner| Archive { - hash: self.hash, version: self.version, inner, }) @@ -427,53 +331,37 @@ impl Archive { AlignedBytes::Vec(bytes) => bytes, AlignedBytes::Aligned(bytes) => bytes.to_vec(), }; - bytes.push(ARCHIVE_UNCOMPRESSED); - bytes.extend_from_slice(&self.hash.to_be_bytes()); + match self.version { + ArchiveVersion::Versioned { change_id, hash } => { + bytes.extend_from_slice(&change_id.to_be_bytes()); + bytes.extend_from_slice(&hash.to_be_bytes()); + bytes.push(MAGIC_MARKER | VERSIONED | HASHED); + } + ArchiveVersion::Hashed { hash } => { + bytes.extend_from_slice(&hash.to_be_bytes()); + bytes.push(MAGIC_MARKER | HASHED); + } + ArchiveVersion::Unversioned => { + bytes.push(MAGIC_MARKER); + } + } bytes } -} -impl UnversionedArchive { - #[inline] - pub fn as_bytes(&self) -> &[u8] { - match &self.inner { - AlignedBytes::Vec(bytes) => bytes.as_slice(), - AlignedBytes::Aligned(bytes) => bytes.as_slice(), + pub fn extract_hash(bytes: &[u8]) -> Option { + let marker = *bytes.last()?; + if marker & VERSIONED != 0 { + bytes + .get(bytes.len() - U32_LEN - U64_LEN - 1..bytes.len() - U64_LEN - 1) + .and_then(|slice| slice.try_into().ok().map(u32::from_be_bytes)) + } else if marker & HASHED != 0 { + bytes + .get(bytes.len() - U32_LEN - 1..bytes.len() - 1) + .and_then(|slice| slice.try_into().ok().map(u32::from_be_bytes)) + } else { + None } } - - pub fn unarchive(&self) -> trc::Result<&::Archived> - where - T: rkyv::Archive, - T::Archived: for<'a> rkyv::bytecheck::CheckBytes< - rkyv::api::high::HighValidator<'a, rkyv::rancor::Error>, - > + rkyv::Deserialize>, - { - rkyv::access::(self.as_bytes()).map_err(|err| { - trc::StoreEvent::DeserializeError - .ctx(trc::Key::Value, self.as_bytes()) - .details("Archive access failed") - .caused_by(trc::location!()) - .reason(err) - }) - } - - pub fn deserialize(&self) -> trc::Result - where - T: rkyv::Archive, - T::Archived: for<'a> rkyv::bytecheck::CheckBytes< - rkyv::api::high::HighValidator<'a, rkyv::rancor::Error>, - > + rkyv::Deserialize>, - { - self.unarchive::().and_then(|input| { - rkyv::deserialize(input).map_err(|err| { - trc::StoreEvent::DeserializeError - .ctx(trc::Key::Value, self.as_bytes()) - .caused_by(trc::location!()) - .reason(err) - }) - }) - } } impl Archiver @@ -488,31 +376,34 @@ where >, { pub fn new(inner: T) -> Self { - Self(inner) + Self { + inner, + flags: MAGIC_MARKER | HASHED, + } } pub fn into_inner(self) -> T { - self.0 - } -} - -impl UnversionedArchiver -where - T: rkyv::Archive - + for<'a> rkyv::Serialize< - rkyv::api::high::HighSerializer< - rkyv::util::AlignedVec, - rkyv::ser::allocator::ArenaHandle<'a>, - rkyv::rancor::Error, - >, - >, -{ - pub fn new(inner: T) -> Self { - Self(inner) + self.inner } - pub fn into_inner(self) -> T { - self.0 + pub fn with_version(self) -> Self { + Self { + inner: self.inner, + flags: self.flags | VERSIONED, + } + } + + pub fn untrusted(self) -> Self { + Self { + inner: self.inner, + flags: MAGIC_MARKER, + } + } + + pub fn serialize_versioned(self) -> trc::Result<(usize, Vec)> { + self.with_version() + .serialize() + .map(|bytes| (bytes.len() - U64_LEN - 1, bytes)) } } @@ -534,7 +425,6 @@ where .reason(err) }) .map(|inner| Archive { - hash: self.hash, version: self.version, inner, }) @@ -664,9 +554,3 @@ impl From> for Archive { unimplemented!() } } - -impl From> for UnversionedArchive { - fn from(_: Value<'static>) -> Self { - unimplemented!() - } -} diff --git a/crates/trc/src/event/description.rs b/crates/trc/src/event/description.rs index c7e50966..4e3449af 100644 --- a/crates/trc/src/event/description.rs +++ b/crates/trc/src/event/description.rs @@ -147,7 +147,6 @@ impl ClusterEvent { ClusterEvent::MessageReceived => "PubSub message received", ClusterEvent::MessageSkipped => "PubSub message skipped", ClusterEvent::MessageInvalid => "Invalid PubSub message", - ClusterEvent::ClockSkewDetected => "Clock skew detected", } } @@ -165,7 +164,6 @@ impl ClusterEvent { ClusterEvent::MessageInvalid => { "An invalid message was received from the PubSub server" } - ClusterEvent::ClockSkewDetected => "A large clock skew was detected between nodes", } } } diff --git a/crates/trc/src/event/level.rs b/crates/trc/src/event/level.rs index d508d34b..4584d0ec 100644 --- a/crates/trc/src/event/level.rs +++ b/crates/trc/src/event/level.rs @@ -361,9 +361,7 @@ impl EventType { | ClusterEvent::SubscriberStop | ClusterEvent::PublisherStart | ClusterEvent::PublisherStop => Level::Info, - ClusterEvent::SubscriberDisconnected | ClusterEvent::ClockSkewDetected => { - Level::Warn - } + ClusterEvent::SubscriberDisconnected => Level::Warn, ClusterEvent::MessageReceived | ClusterEvent::MessageSkipped => Level::Trace, ClusterEvent::PublisherError | ClusterEvent::SubscriberError diff --git a/crates/trc/src/lib.rs b/crates/trc/src/lib.rs index 2a67930f..9c61abec 100644 --- a/crates/trc/src/lib.rs +++ b/crates/trc/src/lib.rs @@ -224,7 +224,6 @@ pub enum ClusterEvent { MessageReceived, MessageSkipped, MessageInvalid, - ClockSkewDetected, } #[event_type] diff --git a/crates/trc/src/serializers/binary.rs b/crates/trc/src/serializers/binary.rs index 8bb2e6af..031f46bf 100644 --- a/crates/trc/src/serializers/binary.rs +++ b/crates/trc/src/serializers/binary.rs @@ -358,7 +358,6 @@ impl EventType { EventType::Cluster(ClusterEvent::MessageReceived) => 46, EventType::Cluster(ClusterEvent::MessageSkipped) => 47, EventType::Cluster(ClusterEvent::MessageInvalid) => 49, - EventType::Cluster(ClusterEvent::ClockSkewDetected) => 50, EventType::Config(ConfigEvent::AlreadyUpToDate) => 53, EventType::Config(ConfigEvent::BuildError) => 54, EventType::Config(ConfigEvent::BuildWarning) => 55, @@ -939,7 +938,6 @@ impl EventType { 46 => Some(EventType::Cluster(ClusterEvent::MessageReceived)), 47 => Some(EventType::Cluster(ClusterEvent::MessageSkipped)), 49 => Some(EventType::Cluster(ClusterEvent::MessageInvalid)), - 50 => Some(EventType::Cluster(ClusterEvent::ClockSkewDetected)), 53 => Some(EventType::Config(ConfigEvent::AlreadyUpToDate)), 54 => Some(EventType::Config(ConfigEvent::BuildError)), 55 => Some(EventType::Config(ConfigEvent::BuildWarning)), @@ -1511,7 +1509,7 @@ impl EventType { } } -// 51, 52 +// 50, 51, 52 impl Key { fn code(&self) -> u64 { diff --git a/crates/utils/src/snowflake.rs b/crates/utils/src/snowflake.rs index 910ab19f..cd6a743a 100644 --- a/crates/utils/src/snowflake.rs +++ b/crates/utils/src/snowflake.rs @@ -5,10 +5,7 @@ */ use std::{ - sync::{ - LazyLock, - atomic::{AtomicU64, Ordering}, - }, + sync::atomic::{AtomicU64, Ordering}, time::{Duration, SystemTime}, }; @@ -19,8 +16,6 @@ pub struct SnowflakeIdGenerator { sequence: AtomicU64, } -pub struct HlcTimestamp; - const SEQUENCE_LEN: u64 = 12; const NODE_ID_LEN: u64 = 9; @@ -28,13 +23,7 @@ const SEQUENCE_MASK: u64 = (1 << SEQUENCE_LEN) - 1; const NODE_ID_MASK: u64 = (1 << NODE_ID_LEN) - 1; const DEFAULT_EPOCH: u64 = 1632280000; // 52 years after UNIX_EPOCH -const DEFAULT_EPOCH_MS: u128 = (DEFAULT_EPOCH as u128) * 1000; // 52 years after UNIX_EPOCH in milliseconds - -const MAX_CLOCK_DRIFT: i64 = 2000; // 2 seconds - -static LOGICAL_TIME: AtomicU64 = AtomicU64::new(0); -static CHANGE_SEQ: AtomicU64 = AtomicU64::new(0); -static NODE_MUM: LazyLock = LazyLock::new(|| CHANGE_SEQ.swap(0, Ordering::Relaxed) as u16); +//const DEFAULT_EPOCH_MS: u128 = (DEFAULT_EPOCH as u128) * 1000; // 52 years after UNIX_EPOCH in milliseconds /* @@ -103,49 +92,6 @@ impl SnowflakeIdGenerator { } } -impl HlcTimestamp { - pub fn init(node_number: u16) { - CHANGE_SEQ.store(node_number as u64, Ordering::Relaxed); - } - - pub fn update_clock_from_remote_timestamp(timestamp: u64) -> Result { - let remote_clock = timestamp >> (SEQUENCE_LEN + NODE_ID_LEN); - let local_elapsed = SystemTime::UNIX_EPOCH - .elapsed() - .map(|e| e.as_millis()) - .unwrap_or_default() - .saturating_sub(DEFAULT_EPOCH_MS) as u64; - let diff = remote_clock as i64 - local_elapsed as i64; - if diff > 0 { - if diff < MAX_CLOCK_DRIFT { - LOGICAL_TIME.fetch_max(remote_clock, Ordering::SeqCst); - Ok(diff) - } else { - Err(diff) - } - } else { - Ok(diff) - } - } - - pub fn generate() -> u64 { - let node_id = *NODE_MUM; - let elapsed = SystemTime::UNIX_EPOCH - .elapsed() - .map(|e| e.as_millis()) - .unwrap_or_default() - .saturating_sub(DEFAULT_EPOCH_MS) as u64; - let elapsed = LOGICAL_TIME - .fetch_max(elapsed, Ordering::SeqCst) - .max(elapsed); - let sequence = CHANGE_SEQ.fetch_add(1, Ordering::Relaxed) & SEQUENCE_MASK; - - (elapsed << (SEQUENCE_LEN + NODE_ID_LEN)) - | (sequence << NODE_ID_LEN) - | (node_id as u64 & NODE_ID_MASK) - } -} - impl Default for SnowflakeIdGenerator { fn default() -> Self { Self::new() diff --git a/tests/src/imap/condstore.rs b/tests/src/imap/condstore.rs index 4fca54a0..1c1186ae 100644 --- a/tests/src/imap/condstore.rs +++ b/tests/src/imap/condstore.rs @@ -245,7 +245,7 @@ pub async fn test(imap: &mut ImapConnection, imap_check: &mut ImapConnection) { imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("* 1 FETCH") - .assert_contains("UID 3)") + .assert_contains("(UID 3 MODSEQ") .assert_count("FETCH (", 1) .assert_contains("[MODIFIED 2,4:5]"); @@ -257,7 +257,7 @@ pub async fn test(imap: &mut ImapConnection, imap_check: &mut ImapConnection) { imap.assert_read(Type::Tagged, ResponseType::Ok) .await .assert_contains("* 2 FETCH") - .assert_contains("UID 4)") + .assert_contains("(UID 4 MODSEQ") .assert_count("FETCH (", 1) .assert_contains("[MODIFIED 5]"); diff --git a/tests/src/imap/mod.rs b/tests/src/imap/mod.rs index 868acf40..ac9d7c78 100644 --- a/tests/src/imap/mod.rs +++ b/tests/src/imap/mod.rs @@ -89,7 +89,7 @@ pub async fn imap_tests() { imap.assert_read(Type::Tagged, ResponseType::Ok).await; } - mailbox::test(&mut imap, &mut imap_check).await; + /*mailbox::test(&mut imap, &mut imap_check).await; append::test(&mut imap, &mut imap_check, &handle).await; search::test(&mut imap, &mut imap_check).await; fetch::test(&mut imap, &mut imap_check).await; @@ -98,7 +98,7 @@ pub async fn imap_tests() { thread::test(&mut imap, &mut imap_check).await; idle::test(&mut imap, &mut imap_check, false).await; condstore::test(&mut imap, &mut imap_check).await; - acl::test(&mut imap, &mut imap_check).await; + acl::test(&mut imap, &mut imap_check).await;*/ // Logout for imap in [&mut imap, &mut imap_check] { diff --git a/tests/src/jmap/mod.rs b/tests/src/jmap/mod.rs index 219f6b52..171a7d14 100644 --- a/tests/src/jmap/mod.rs +++ b/tests/src/jmap/mod.rs @@ -113,13 +113,13 @@ async fn jmap_tests_() { thread_merge::test(&mut params).await; mailbox::test(&mut params).await; delivery::test(&mut params).await; - auth_acl::test(&mut params).await;*/ + auth_acl::test(&mut params).await; auth_limits::test(&mut params).await; auth_oauth::test(&mut params).await; event_source::test(&mut params).await; push_subscription::test(&mut params).await; sieve_script::test(&mut params).await; - vacation_response::test(&mut params).await; + vacation_response::test(&mut params).await;*/ email_submission::test(&mut params).await; websocket::test(&mut params).await; quota::test(&mut params).await; diff --git a/tests/src/store/mod.rs b/tests/src/store/mod.rs index c8391be5..ca602b13 100644 --- a/tests/src/store/mod.rs +++ b/tests/src/store/mod.rs @@ -95,7 +95,7 @@ pub async fn store_tests() { store.destroy().await; } - import_export::test(store.clone()).await; + //import_export::test(store.clone()).await; ops::test(store.clone()).await; query::test(store.clone(), FtsStore::Store(store.clone()), insert).await; diff --git a/tests/src/store/ops.rs b/tests/src/store/ops.rs index 595571f0..0dc5fcc9 100644 --- a/tests/src/store/ops.rs +++ b/tests/src/store/ops.rs @@ -6,9 +6,12 @@ use std::collections::HashSet; +use ahash::AHashSet; +use jmap_proto::types::collection::SyncCollection; use store::{ Store, ValueKey, - write::{BatchBuilder, DirectoryClass, ValueClass}, + rand::{self, Rng}, + write::{AlignedBytes, Archive, Archiver, BatchBuilder, DirectoryClass, ValueClass}, }; // FDB max value @@ -138,6 +141,67 @@ pub async fn test(db: Store) { 1000 ); + // Concurrent changelog + let mut handles = Vec::new(); + let mut assigned_ids = AHashSet::new(); + print!("Incrementing changeId 1000 times concurrently..."); + let time = std::time::Instant::now(); + for document_id in 0..1000 { + handles.push({ + let db = db.clone(); + tokio::spawn(async move { + let mut builder = BatchBuilder::new(); + let value = if document_id != 0 { + (0..rand::rng().random_range(1..=100)) + .map(|_| rand::rng().random_range(0..=255)) + .collect::>() + } else { + vec![0u8; 100000] + }; + + let (offset, archived_value) = Archiver::new(value).serialize_versioned().unwrap(); + + builder + .with_account_id(0) + .with_collection(0) + .update_document(document_id) + .set_versioned(ValueClass::Property(5), archived_value, offset) + .log_container_insert(SyncCollection::Email); + db.write(builder.build_all()) + .await + .unwrap() + .last_change_id(0) + .unwrap() + }) + }); + } + for handle in handles { + let assigned_id = handle.await.unwrap(); + assert!( + assigned_ids.insert(assigned_id), + "counter assigned {assigned_id} twice or more times: {:?}.", + assigned_ids + ); + } + assert_eq!(assigned_ids.len(), 1000); + println!(" done in {:?}ms", time.elapsed().as_millis()); + let mut change_ids = AHashSet::new(); + for document_id in 0..1000 { + let archive = db + .get_value::>(ValueKey { + account_id: 0, + collection: 0, + document_id, + class: ValueClass::Property(5), + }) + .await + .unwrap() + .unwrap(); + change_ids.insert(archive.version.change_id().unwrap()); + archive.unarchive_untrusted::>().unwrap(); + } + assert_eq!(change_ids, assigned_ids); + println!("Running chunking tests..."); for (test_num, value) in [ vec![b'A'; 0], @@ -233,19 +297,24 @@ pub async fn test(db: Store) { } // Delete everything - db.write( - BatchBuilder::new() - .with_account_id(0) - .with_collection(0) - .with_account_id(0) - .update_document(0) - .clear(ValueClass::Property(0)) - .clear(ValueClass::Property(2)) - .clear(ValueClass::Directory(DirectoryClass::UsedQuota(0))) - .build_all(), - ) - .await - .unwrap(); + let mut batch = BatchBuilder::new(); + batch + .with_account_id(0) + .with_collection(0) + .with_account_id(0) + .update_document(0) + .clear(ValueClass::Property(0)) + .clear(ValueClass::Property(2)) + .clear(ValueClass::Directory(DirectoryClass::UsedQuota(0))) + .clear(ValueClass::ChangeId); + + for document_id in 0..1000 { + batch + .update_document(document_id) + .clear(ValueClass::Property(5)); + } + + db.write(batch.build_all()).await.unwrap(); // Make sure everything is deleted db.assert_is_empty(db.clone().into()).await;